projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
net/mlx5e: TC, Restore tunnel info for sample offload
[linux-block.git] / drivers / net / ethernet / mellanox / mlx5 / core / eswitch_offloads.c
CommitLineData
69697b6e
OG		 1/*
2 * Copyright (c) 2016, Mellanox Technologies. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33#include <linux/etherdevice.h>
133dcfc5 34#include <linux/idr.h>
69697b6e
OG		 35#include <linux/mlx5/driver.h>
36#include <linux/mlx5/mlx5_ifc.h>
37#include <linux/mlx5/vport.h>
38#include <linux/mlx5/fs.h>
39#include "mlx5_core.h"
40#include "eswitch.h"
34ca6535 41#include "esw/indir_table.h"
ea651a86 42#include "esw/acl/ofld.h"
80f09dfc 43#include "rdma.h"
e52c2802
PB		 44#include "en.h"
45#include "fs_core.h"
ac004b83 46#include "lib/devcom.h"
a3888f33 47#include "lib/eq.h"
ae430332 48#include "lib/fs_chains.h"
c620b772 49#include "en_tc.h"
c9355682 50#include "en/mapping.h"
c85a6b8f 51#include "devlink.h"
69697b6e 52
47dd7e60
PP		 53#define mlx5_esw_for_each_rep(esw, i, rep) \
54 xa_for_each(&((esw)->offloads.vport_reps), i, rep)
55
56#define mlx5_esw_for_each_sf_rep(esw, i, rep) \
57 xa_for_each_marked(&((esw)->offloads.vport_reps), i, rep, MLX5_ESW_VPT_SF)
58
59#define mlx5_esw_for_each_vf_rep(esw, index, rep) \
60 mlx5_esw_for_each_entry_marked(&((esw)->offloads.vport_reps), index, \
61 rep, (esw)->esw_funcs.num_vfs, MLX5_ESW_VPT_VF)
62
cd7e4186
BW		 63/* There are two match-all miss flows, one for unicast dst mac and
64 * one for multicast.
65 */
66#define MLX5_ESW_MISS_FLOWS (2)
c9b99abc
BW		 67#define UPLINK_REP_INDEX 0
68
c796bb7c
CM		 69#define MLX5_ESW_VPORT_TBL_SIZE 128
70#define MLX5_ESW_VPORT_TBL_NUM_GROUPS 4
71
72static const struct esw_vport_tbl_namespace mlx5_esw_vport_tbl_mirror_ns = {
73 .max_fte = MLX5_ESW_VPORT_TBL_SIZE,
74 .max_num_groups = MLX5_ESW_VPORT_TBL_NUM_GROUPS,
75 .flags = 0,
76};
77
879c8f84
BW		 78static struct mlx5_eswitch_rep *mlx5_eswitch_get_rep(struct mlx5_eswitch *esw,
79 u16 vport_num)
80{
47dd7e60 81 return xa_load(&esw->offloads.vport_reps, vport_num);
879c8f84
BW		 82}
83
6f7bbad1
JL		 84static void
85mlx5_eswitch_set_rule_flow_source(struct mlx5_eswitch *esw,
86 struct mlx5_flow_spec *spec,
87 struct mlx5_esw_flow_attr *attr)
88{
89 if (MLX5_CAP_ESW_FLOWTABLE(esw->dev, flow_source) &&
036e19b9
HI		 90 attr && attr->in_rep)
91 spec->flow_context.flow_source =
92 attr->in_rep->vport == MLX5_VPORT_UPLINK ?
93 MLX5_FLOW_CONTEXT_FLOW_SOURCE_UPLINK :
94 MLX5_FLOW_CONTEXT_FLOW_SOURCE_LOCAL_VPORT;
6f7bbad1 95}
b7826076 96
f94d6389
CM		 97/* Actually only the upper 16 bits of reg c0 need to be cleared, but the lower 16 bits
98 * are not needed as well in the following process. So clear them all for simplicity.
99 */
100void
101mlx5_eswitch_clear_rule_source_port(struct mlx5_eswitch *esw, struct mlx5_flow_spec *spec)
102{
103 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
104 void *misc2;
105
106 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2);
107 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, 0);
108
109 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2);
110 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, 0);
111
112 if (!memchr_inv(misc2, 0, MLX5_ST_SZ_BYTES(fte_match_set_misc2)))
113 spec->match_criteria_enable &= ~MLX5_MATCH_MISC_PARAMETERS_2;
114 }
115}
116
c01cfd0f
JL		 117static void
118mlx5_eswitch_set_rule_source_port(struct mlx5_eswitch *esw,
119 struct mlx5_flow_spec *spec,
a508728a 120 struct mlx5_flow_attr *attr,
b055ecf5
MB		 121 struct mlx5_eswitch *src_esw,
122 u16 vport)
c01cfd0f
JL		 123{
124 void *misc2;
125 void *misc;
126
127 /* Use metadata matching because vport is not represented by single
128 * VHCA in dual-port RoCE mode, and matching on source vport may fail.
129 */
130 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
a508728a
VB		 131 if (mlx5_esw_indir_table_decap_vport(attr))
132 vport = mlx5_esw_indir_table_decap_vport(attr);
c01cfd0f
JL		 133 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2);
134 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0,
b055ecf5
MB		 135 mlx5_eswitch_get_vport_metadata_for_match(src_esw,
136 vport));
c01cfd0f
JL		 137
138 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2);
0f0d3827
PB		 139 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0,
140 mlx5_eswitch_get_vport_metadata_mask());
c01cfd0f
JL		 141
142 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2;
c01cfd0f
JL		 143 } else {
144 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters);
b055ecf5 145 MLX5_SET(fte_match_set_misc, misc, source_port, vport);
c01cfd0f
JL		 146
147 if (MLX5_CAP_ESW(esw->dev, merged_eswitch))
148 MLX5_SET(fte_match_set_misc, misc,
149 source_eswitch_owner_vhca_id,
b055ecf5 150 MLX5_CAP_GEN(src_esw->dev, vhca_id));
c01cfd0f
JL		 151
152 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters);
153 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
154 if (MLX5_CAP_ESW(esw->dev, merged_eswitch))
155 MLX5_SET_TO_ONES(fte_match_set_misc, misc,
156 source_eswitch_owner_vhca_id);
157
158 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS;
159 }
c01cfd0f
JL		 160}
161
a508728a
VB		 162static int
163esw_setup_decap_indir(struct mlx5_eswitch *esw,
164 struct mlx5_flow_attr *attr,
165 struct mlx5_flow_spec *spec)
166{
167 struct mlx5_flow_table *ft;
168
169 if (!(attr->flags & MLX5_ESW_ATTR_FLAG_SRC_REWRITE))
170 return -EOPNOTSUPP;
171
172 ft = mlx5_esw_indir_table_get(esw, attr, spec,
173 mlx5_esw_indir_table_decap_vport(attr), true);
174 return PTR_ERR_OR_ZERO(ft);
175}
176
9e51c0a6 177static void
a508728a
VB		 178esw_cleanup_decap_indir(struct mlx5_eswitch *esw,
179 struct mlx5_flow_attr *attr)
180{
181 if (mlx5_esw_indir_table_decap_vport(attr))
182 mlx5_esw_indir_table_put(esw, attr,
183 mlx5_esw_indir_table_decap_vport(attr),
184 true);
185}
186
f94d6389
CM		 187static int
188esw_setup_sampler_dest(struct mlx5_flow_destination *dest,
189 struct mlx5_flow_act *flow_act,
bcd6740c 190 struct mlx5_flow_attr *attr,
f94d6389
CM		 191 int i)
192{
193 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
194 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_SAMPLER;
bcd6740c 195 dest[i].sampler_id = attr->sample_attr->sampler_id;
f94d6389
CM		 196
197 return 0;
198}
199
a508728a 200static int
9e51c0a6
VB		 201esw_setup_ft_dest(struct mlx5_flow_destination *dest,
202 struct mlx5_flow_act *flow_act,
a508728a 203 struct mlx5_eswitch *esw,
9e51c0a6 204 struct mlx5_flow_attr *attr,
a508728a 205 struct mlx5_flow_spec *spec,
9e51c0a6
VB		 206 int i)
207{
208 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
209 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
210 dest[i].ft = attr->dest_ft;
a508728a
VB		 211
212 if (mlx5_esw_indir_table_decap_vport(attr))
213 return esw_setup_decap_indir(esw, attr, spec);
214 return 0;
9e51c0a6
VB		 215}
216
217static void
218esw_setup_slow_path_dest(struct mlx5_flow_destination *dest,
219 struct mlx5_flow_act *flow_act,
220 struct mlx5_fs_chains *chains,
221 int i)
222{
2a2c84fa
RD		 223 if (mlx5_chains_ignore_flow_level_supported(chains))
224 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
9e51c0a6
VB		 225 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
226 dest[i].ft = mlx5_chains_get_tc_end_ft(chains);
227}
228
229static int
230esw_setup_chain_dest(struct mlx5_flow_destination *dest,
231 struct mlx5_flow_act *flow_act,
232 struct mlx5_fs_chains *chains,
233 u32 chain, u32 prio, u32 level,
234 int i)
235{
236 struct mlx5_flow_table *ft;
237
238 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
239 ft = mlx5_chains_get_table(chains, chain, prio, level);
240 if (IS_ERR(ft))
241 return PTR_ERR(ft);
242
243 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
244 dest[i].ft = ft;
245 return 0;
246}
247
10742efc
VB		 248static void esw_put_dest_tables_loop(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr,
249 int from, int to)
250{
251 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
252 struct mlx5_fs_chains *chains = esw_chains(esw);
253 int i;
254
255 for (i = from; i < to; i++)
256 if (esw_attr->dests[i].flags & MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE)
257 mlx5_chains_put_table(chains, 0, 1, 0);
a508728a
VB		 258 else if (mlx5_esw_indir_table_needed(esw, attr, esw_attr->dests[i].rep->vport,
259 esw_attr->dests[i].mdev))
260 mlx5_esw_indir_table_put(esw, attr, esw_attr->dests[i].rep->vport,
261 false);
10742efc
VB		 262}
263
264static bool
265esw_is_chain_src_port_rewrite(struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr)
266{
267 int i;
268
269 for (i = esw_attr->split_count; i < esw_attr->out_count; i++)
270 if (esw_attr->dests[i].flags & MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE)
271 return true;
272 return false;
273}
274
275static int
276esw_setup_chain_src_port_rewrite(struct mlx5_flow_destination *dest,
277 struct mlx5_flow_act *flow_act,
278 struct mlx5_eswitch *esw,
279 struct mlx5_fs_chains *chains,
280 struct mlx5_flow_attr *attr,
281 int *i)
282{
283 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
284 int j, err;
285
286 if (!(attr->flags & MLX5_ESW_ATTR_FLAG_SRC_REWRITE))
287 return -EOPNOTSUPP;
288
289 for (j = esw_attr->split_count; j < esw_attr->out_count; j++, (*i)++) {
290 err = esw_setup_chain_dest(dest, flow_act, chains, attr->dest_chain, 1, 0, *i);
291 if (err)
292 goto err_setup_chain;
293 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
294 flow_act->pkt_reformat = esw_attr->dests[j].pkt_reformat;
295 }
296 return 0;
297
298err_setup_chain:
299 esw_put_dest_tables_loop(esw, attr, esw_attr->split_count, j);
300 return err;
301}
302
303static void esw_cleanup_chain_src_port_rewrite(struct mlx5_eswitch *esw,
304 struct mlx5_flow_attr *attr)
305{
306 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
307
308 esw_put_dest_tables_loop(esw, attr, esw_attr->split_count, esw_attr->out_count);
309}
310
a508728a
VB		 311static bool
312esw_is_indir_table(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr)
313{
314 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
315 int i;
316
317 for (i = esw_attr->split_count; i < esw_attr->out_count; i++)
318 if (mlx5_esw_indir_table_needed(esw, attr, esw_attr->dests[i].rep->vport,
319 esw_attr->dests[i].mdev))
320 return true;
321 return false;
322}
323
324static int
325esw_setup_indir_table(struct mlx5_flow_destination *dest,
326 struct mlx5_flow_act *flow_act,
327 struct mlx5_eswitch *esw,
328 struct mlx5_flow_attr *attr,
329 struct mlx5_flow_spec *spec,
330 bool ignore_flow_lvl,
331 int *i)
332{
333 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
334 int j, err;
335
336 if (!(attr->flags & MLX5_ESW_ATTR_FLAG_SRC_REWRITE))
337 return -EOPNOTSUPP;
338
339 for (j = esw_attr->split_count; j < esw_attr->out_count; j++, (*i)++) {
340 if (ignore_flow_lvl)
341 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
342 dest[*i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
343
344 dest[*i].ft = mlx5_esw_indir_table_get(esw, attr, spec,
345 esw_attr->dests[j].rep->vport, false);
346 if (IS_ERR(dest[*i].ft)) {
347 err = PTR_ERR(dest[*i].ft);
348 goto err_indir_tbl_get;
349 }
350 }
351
352 if (mlx5_esw_indir_table_decap_vport(attr)) {
353 err = esw_setup_decap_indir(esw, attr, spec);
354 if (err)
355 goto err_indir_tbl_get;
356 }
357
358 return 0;
359
360err_indir_tbl_get:
361 esw_put_dest_tables_loop(esw, attr, esw_attr->split_count, j);
362 return err;
363}
364
365static void esw_cleanup_indir_table(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr)
366{
367 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
368
369 esw_put_dest_tables_loop(esw, attr, esw_attr->split_count, esw_attr->out_count);
370 esw_cleanup_decap_indir(esw, attr);
371}
372
9e51c0a6
VB		 373static void
374esw_cleanup_chain_dest(struct mlx5_fs_chains *chains, u32 chain, u32 prio, u32 level)
375{
376 mlx5_chains_put_table(chains, chain, prio, level);
377}
378
379static void
380esw_setup_vport_dest(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act,
381 struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr,
382 int attr_idx, int dest_idx, bool pkt_reformat)
383{
384 dest[dest_idx].type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
385 dest[dest_idx].vport.num = esw_attr->dests[attr_idx].rep->vport;
c6719725
MD		 386 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) {
387 dest[dest_idx].vport.vhca_id =
388 MLX5_CAP_GEN(esw_attr->dests[attr_idx].mdev, vhca_id);
9e51c0a6 389 dest[dest_idx].vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID;
c6719725 390 }
9e51c0a6
VB		 391 if (esw_attr->dests[attr_idx].flags & MLX5_ESW_DEST_ENCAP) {
392 if (pkt_reformat) {
393 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT;
394 flow_act->pkt_reformat = esw_attr->dests[attr_idx].pkt_reformat;
395 }
396 dest[dest_idx].vport.flags |= MLX5_FLOW_DEST_VPORT_REFORMAT_ID;
397 dest[dest_idx].vport.pkt_reformat = esw_attr->dests[attr_idx].pkt_reformat;
398 }
399}
400
401static int
402esw_setup_vport_dests(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act,
403 struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr,
404 int i)
405{
406 int j;
407
408 for (j = esw_attr->split_count; j < esw_attr->out_count; j++, i++)
409 esw_setup_vport_dest(dest, flow_act, esw, esw_attr, j, i, true);
410 return i;
411}
412
e929e3da
MD		 413static bool
414esw_src_port_rewrite_supported(struct mlx5_eswitch *esw)
415{
416 return MLX5_CAP_GEN(esw->dev, reg_c_preserve) &&
417 mlx5_eswitch_vport_match_metadata_enabled(esw) &&
418 MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, ignore_flow_level);
419}
420
9e51c0a6
VB		 421static int
422esw_setup_dests(struct mlx5_flow_destination *dest,
423 struct mlx5_flow_act *flow_act,
424 struct mlx5_eswitch *esw,
425 struct mlx5_flow_attr *attr,
10742efc 426 struct mlx5_flow_spec *spec,
9e51c0a6
VB		 427 int *i)
428{
429 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
430 struct mlx5_fs_chains *chains = esw_chains(esw);
431 int err = 0;
432
10742efc 433 if (!mlx5_eswitch_termtbl_required(esw, attr, flow_act, spec) &&
e929e3da 434 esw_src_port_rewrite_supported(esw))
10742efc
VB		 435 attr->flags |= MLX5_ESW_ATTR_FLAG_SRC_REWRITE;
436
f94d6389 437 if (attr->flags & MLX5_ESW_ATTR_FLAG_SAMPLE) {
bcd6740c 438 esw_setup_sampler_dest(dest, flow_act, attr, *i);
f94d6389
CM		 439 (*i)++;
440 } else if (attr->dest_ft) {
a508728a 441 esw_setup_ft_dest(dest, flow_act, esw, attr, spec, *i);
9e51c0a6
VB		 442 (*i)++;
443 } else if (attr->flags & MLX5_ESW_ATTR_FLAG_SLOW_PATH) {
444 esw_setup_slow_path_dest(dest, flow_act, chains, *i);
445 (*i)++;
446 } else if (attr->dest_chain) {
447 err = esw_setup_chain_dest(dest, flow_act, chains, attr->dest_chain,
448 1, 0, *i);
449 (*i)++;
a508728a
VB		 450 } else if (esw_is_indir_table(esw, attr)) {
451 err = esw_setup_indir_table(dest, flow_act, esw, attr, spec, true, i);
10742efc
VB		 452 } else if (esw_is_chain_src_port_rewrite(esw, esw_attr)) {
453 err = esw_setup_chain_src_port_rewrite(dest, flow_act, esw, chains, attr, i);
9e51c0a6
VB		 454 } else {
455 *i = esw_setup_vport_dests(dest, flow_act, esw, esw_attr, *i);
456 }
457
458 return err;
459}
460
461static void
462esw_cleanup_dests(struct mlx5_eswitch *esw,
463 struct mlx5_flow_attr *attr)
464{
10742efc 465 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
9e51c0a6
VB		 466 struct mlx5_fs_chains *chains = esw_chains(esw);
467
a508728a
VB		 468 if (attr->dest_ft) {
469 esw_cleanup_decap_indir(esw, attr);
470 } else if (!(attr->flags & MLX5_ESW_ATTR_FLAG_SLOW_PATH)) {
10742efc
VB		 471 if (attr->dest_chain)
472 esw_cleanup_chain_dest(chains, attr->dest_chain, 1, 0);
a508728a
VB		 473 else if (esw_is_indir_table(esw, attr))
474 esw_cleanup_indir_table(esw, attr);
10742efc
VB		 475 else if (esw_is_chain_src_port_rewrite(esw, esw_attr))
476 esw_cleanup_chain_src_port_rewrite(esw, attr);
477 }
9e51c0a6
VB		 478}
479
74491de9 480struct mlx5_flow_handle *
3d80d1a2
OG		 481mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch *esw,
482 struct mlx5_flow_spec *spec,
c620b772 483 struct mlx5_flow_attr *attr)
3d80d1a2 484{
592d3651 485 struct mlx5_flow_destination dest[MLX5_MAX_FLOW_FWD_VPORTS + 1] = {};
42f7ad67 486 struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, };
c620b772 487 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
ae430332 488 struct mlx5_fs_chains *chains = esw_chains(esw);
c620b772
AL		 489 bool split = !!(esw_attr->split_count);
490 struct mlx5_vport_tbl_attr fwd_attr;
74491de9 491 struct mlx5_flow_handle *rule;
e52c2802 492 struct mlx5_flow_table *fdb;
9e51c0a6 493 int i = 0;
3d80d1a2 494
f6455de0 495 if (esw->mode != MLX5_ESWITCH_OFFLOADS)
3d80d1a2
OG		 496 return ERR_PTR(-EOPNOTSUPP);
497
6acfbf38
OG		 498 flow_act.action = attr->action;
499 /* if per flow vlan pop/push is emulated, don't set that into the firmware */
cc495188 500 if (!mlx5_eswitch_vlan_actions_supported(esw->dev, 1))
6acfbf38
OG		 501 flow_act.action &= ~(MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH |
502 MLX5_FLOW_CONTEXT_ACTION_VLAN_POP);
503 else if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH) {
c620b772
AL		 504 flow_act.vlan[0].ethtype = ntohs(esw_attr->vlan_proto[0]);
505 flow_act.vlan[0].vid = esw_attr->vlan_vid[0];
506 flow_act.vlan[0].prio = esw_attr->vlan_prio[0];
cc495188 507 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2) {
c620b772
AL		 508 flow_act.vlan[1].ethtype = ntohs(esw_attr->vlan_proto[1]);
509 flow_act.vlan[1].vid = esw_attr->vlan_vid[1];
510 flow_act.vlan[1].prio = esw_attr->vlan_prio[1];
cc495188 511 }
6acfbf38 512 }
776b12b6 513
10742efc
VB		 514 mlx5_eswitch_set_rule_flow_source(esw, spec, esw_attr);
515
66958ed9 516 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) {
9e51c0a6
VB		 517 int err;
518
10742efc 519 err = esw_setup_dests(dest, &flow_act, esw, attr, spec, &i);
9e51c0a6
VB		 520 if (err) {
521 rule = ERR_PTR(err);
522 goto err_create_goto_table;
56e858df 523 }
e37a79e5 524 }
14e6b038 525
c620b772
AL		 526 if (esw_attr->decap_pkt_reformat)
527 flow_act.pkt_reformat = esw_attr->decap_pkt_reformat;
14e6b038 528
66958ed9 529 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_COUNT) {
e37a79e5 530 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
171c7625 531 dest[i].counter_id = mlx5_fc_id(attr->counter);
e37a79e5 532 i++;
3d80d1a2
OG		 533 }
534
93b3586e 535 if (attr->outer_match_level != MLX5_MATCH_NONE)
6363651d 536 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS;
93b3586e
HN		 537 if (attr->inner_match_level != MLX5_MATCH_NONE)
538 spec->match_criteria_enable |= MLX5_MATCH_INNER_HEADERS;
3d80d1a2 539
aa24670e 540 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR)
2b688ea5 541 flow_act.modify_hdr = attr->modify_hdr;
d7e75a32 542
bcd6740c
CM		 543 /* sample_attr is allocated only when there is a sample action */
544 if (attr->sample_attr && attr->sample_attr->sample_default_tbl) {
545 fdb = attr->sample_attr->sample_default_tbl;
f94d6389 546 } else if (split) {
c620b772
AL		 547 fwd_attr.chain = attr->chain;
548 fwd_attr.prio = attr->prio;
549 fwd_attr.vport = esw_attr->in_rep->vport;
c796bb7c 550 fwd_attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns;
c620b772 551
0a9e2307 552 fdb = mlx5_esw_vporttbl_get(esw, &fwd_attr);
96e32687 553 } else {
d18296ff 554 if (attr->chain || attr->prio)
ae430332
AL		 555 fdb = mlx5_chains_get_table(chains, attr->chain,
556 attr->prio, 0);
d18296ff 557 else
c620b772 558 fdb = attr->ft;
6fb0701a
PB		 559
560 if (!(attr->flags & MLX5_ESW_ATTR_FLAG_NO_IN_PORT))
a508728a 561 mlx5_eswitch_set_rule_source_port(esw, spec, attr,
b055ecf5
MB		 562 esw_attr->in_mdev->priv.eswitch,
563 esw_attr->in_rep->vport);
96e32687 564 }
e52c2802
PB		 565 if (IS_ERR(fdb)) {
566 rule = ERR_CAST(fdb);
567 goto err_esw_get;
568 }
569
84be2fda 570 if (mlx5_eswitch_termtbl_required(esw, attr, &flow_act, spec))
c620b772 571 rule = mlx5_eswitch_add_termtbl_rule(esw, fdb, spec, esw_attr,
10caabda 572 &flow_act, dest, i);
84be2fda 573 else
10caabda 574 rule = mlx5_add_flow_rules(fdb, spec, &flow_act, dest, i);
3d80d1a2 575 if (IS_ERR(rule))
e52c2802 576 goto err_add_rule;
375f51e2 577 else
525e84be 578 atomic64_inc(&esw->offloads.num_flows);
3d80d1a2 579
e52c2802
PB		 580 return rule;
581
582err_add_rule:
96e32687 583 if (split)
0a9e2307 584 mlx5_esw_vporttbl_put(esw, &fwd_attr);
d18296ff 585 else if (attr->chain || attr->prio)
ae430332 586 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0);
e52c2802 587err_esw_get:
9e51c0a6 588 esw_cleanup_dests(esw, attr);
e52c2802 589err_create_goto_table:
aa0cbbae 590 return rule;
3d80d1a2
OG		 591}
592
e4ad91f2
CM		 593struct mlx5_flow_handle *
594mlx5_eswitch_add_fwd_rule(struct mlx5_eswitch *esw,
595 struct mlx5_flow_spec *spec,
c620b772 596 struct mlx5_flow_attr *attr)
e4ad91f2
CM		 597{
598 struct mlx5_flow_destination dest[MLX5_MAX_FLOW_FWD_VPORTS + 1] = {};
42f7ad67 599 struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, };
c620b772 600 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
ae430332 601 struct mlx5_fs_chains *chains = esw_chains(esw);
c620b772 602 struct mlx5_vport_tbl_attr fwd_attr;
e52c2802
PB		 603 struct mlx5_flow_table *fast_fdb;
604 struct mlx5_flow_table *fwd_fdb;
e4ad91f2 605 struct mlx5_flow_handle *rule;
10742efc 606 int i, err = 0;
e4ad91f2 607
ae430332 608 fast_fdb = mlx5_chains_get_table(chains, attr->chain, attr->prio, 0);
e52c2802
PB		 609 if (IS_ERR(fast_fdb)) {
610 rule = ERR_CAST(fast_fdb);
611 goto err_get_fast;
612 }
613
c620b772
AL		 614 fwd_attr.chain = attr->chain;
615 fwd_attr.prio = attr->prio;
616 fwd_attr.vport = esw_attr->in_rep->vport;
c796bb7c 617 fwd_attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns;
0a9e2307 618 fwd_fdb = mlx5_esw_vporttbl_get(esw, &fwd_attr);
e52c2802
PB		 619 if (IS_ERR(fwd_fdb)) {
620 rule = ERR_CAST(fwd_fdb);
621 goto err_get_fwd;
622 }
623
e4ad91f2 624 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
10742efc 625 for (i = 0; i < esw_attr->split_count; i++) {
a508728a
VB		 626 if (esw_is_indir_table(esw, attr))
627 err = esw_setup_indir_table(dest, &flow_act, esw, attr, spec, false, &i);
628 else if (esw_is_chain_src_port_rewrite(esw, esw_attr))
10742efc
VB		 629 err = esw_setup_chain_src_port_rewrite(dest, &flow_act, esw, chains, attr,
630 &i);
631 else
632 esw_setup_vport_dest(dest, &flow_act, esw, esw_attr, i, i, false);
633
634 if (err) {
635 rule = ERR_PTR(err);
636 goto err_chain_src_rewrite;
637 }
638 }
e4ad91f2 639 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
873d2f12 640 dest[i].ft = fwd_fdb;
e4ad91f2
CM		 641 i++;
642
a508728a 643 mlx5_eswitch_set_rule_source_port(esw, spec, attr,
b055ecf5
MB		 644 esw_attr->in_mdev->priv.eswitch,
645 esw_attr->in_rep->vport);
e4ad91f2 646
93b3586e 647 if (attr->outer_match_level != MLX5_MATCH_NONE)
c01cfd0f 648 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS;
e4ad91f2 649
278d51f2 650 flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL;
e52c2802 651 rule = mlx5_add_flow_rules(fast_fdb, spec, &flow_act, dest, i);
e4ad91f2 652
10742efc
VB		 653 if (IS_ERR(rule)) {
654 i = esw_attr->split_count;
655 goto err_chain_src_rewrite;
656 }
e4ad91f2 657
525e84be 658 atomic64_inc(&esw->offloads.num_flows);
e52c2802
PB		 659
660 return rule;
10742efc
VB		 661err_chain_src_rewrite:
662 esw_put_dest_tables_loop(esw, attr, 0, i);
0a9e2307 663 mlx5_esw_vporttbl_put(esw, &fwd_attr);
e52c2802 664err_get_fwd:
ae430332 665 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0);
e52c2802 666err_get_fast:
e4ad91f2
CM		 667 return rule;
668}
669
e52c2802
PB		 670static void
671__mlx5_eswitch_del_rule(struct mlx5_eswitch *esw,
672 struct mlx5_flow_handle *rule,
c620b772 673 struct mlx5_flow_attr *attr,
e52c2802
PB		 674 bool fwd_rule)
675{
c620b772 676 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
ae430332 677 struct mlx5_fs_chains *chains = esw_chains(esw);
c620b772
AL		 678 bool split = (esw_attr->split_count > 0);
679 struct mlx5_vport_tbl_attr fwd_attr;
10caabda 680 int i;
e52c2802
PB		 681
682 mlx5_del_flow_rules(rule);
10caabda 683
84be2fda 684 if (!(attr->flags & MLX5_ESW_ATTR_FLAG_SLOW_PATH)) {
d8a2034f
EC		 685 /* unref the term table */
686 for (i = 0; i < MLX5_MAX_FLOW_FWD_VPORTS; i++) {
c620b772
AL		 687 if (esw_attr->dests[i].termtbl)
688 mlx5_eswitch_termtbl_put(esw, esw_attr->dests[i].termtbl);
d8a2034f 689 }
10caabda
OS		 690 }
691
525e84be 692 atomic64_dec(&esw->offloads.num_flows);
e52c2802 693
c620b772
AL		 694 if (fwd_rule || split) {
695 fwd_attr.chain = attr->chain;
696 fwd_attr.prio = attr->prio;
697 fwd_attr.vport = esw_attr->in_rep->vport;
c796bb7c 698 fwd_attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns;
c620b772
AL		 699 }
700
e52c2802 701 if (fwd_rule) {
0a9e2307 702 mlx5_esw_vporttbl_put(esw, &fwd_attr);
ae430332 703 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0);
10742efc 704 esw_put_dest_tables_loop(esw, attr, 0, esw_attr->split_count);
e52c2802 705 } else {
96e32687 706 if (split)
0a9e2307 707 mlx5_esw_vporttbl_put(esw, &fwd_attr);
d18296ff 708 else if (attr->chain || attr->prio)
ae430332 709 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0);
9e51c0a6 710 esw_cleanup_dests(esw, attr);
e52c2802
PB		 711 }
712}
713
d85cdccb
OG		 714void
715mlx5_eswitch_del_offloaded_rule(struct mlx5_eswitch *esw,
716 struct mlx5_flow_handle *rule,
c620b772 717 struct mlx5_flow_attr *attr)
d85cdccb 718{
e52c2802 719 __mlx5_eswitch_del_rule(esw, rule, attr, false);
d85cdccb
OG		 720}
721
48265006
OG		 722void
723mlx5_eswitch_del_fwd_rule(struct mlx5_eswitch *esw,
724 struct mlx5_flow_handle *rule,
c620b772 725 struct mlx5_flow_attr *attr)
48265006 726{
e52c2802 727 __mlx5_eswitch_del_rule(esw, rule, attr, true);
48265006
OG		 728}
729
f5f82476
OG		 730static int esw_set_global_vlan_pop(struct mlx5_eswitch *esw, u8 val)
731{
732 struct mlx5_eswitch_rep *rep;
47dd7e60
PP		 733 unsigned long i;
734 int err = 0;
f5f82476
OG		 735
736 esw_debug(esw->dev, "%s applying global %s policy\n", __func__, val ? "pop" : "none");
47dd7e60 737 mlx5_esw_for_each_host_func_vport(esw, i, rep, esw->esw_funcs.num_vfs) {
8693115a 738 if (atomic_read(&rep->rep_data[REP_ETH].state) != REP_LOADED)
f5f82476
OG		 739 continue;
740
741 err = __mlx5_eswitch_set_vport_vlan(esw, rep->vport, 0, 0, val);
742 if (err)
743 goto out;
744 }
745
746out:
747 return err;
748}
749
750static struct mlx5_eswitch_rep *
751esw_vlan_action_get_vport(struct mlx5_esw_flow_attr *attr, bool push, bool pop)
752{
753 struct mlx5_eswitch_rep *in_rep, *out_rep, *vport = NULL;
754
755 in_rep = attr->in_rep;
df65a573 756 out_rep = attr->dests[0].rep;
f5f82476
OG		 757
758 if (push)
759 vport = in_rep;
760 else if (pop)
761 vport = out_rep;
762 else
763 vport = in_rep;
764
765 return vport;
766}
767
768static int esw_add_vlan_action_check(struct mlx5_esw_flow_attr *attr,
769 bool push, bool pop, bool fwd)
770{
771 struct mlx5_eswitch_rep *in_rep, *out_rep;
772
773 if ((push || pop) && !fwd)
774 goto out_notsupp;
775
776 in_rep = attr->in_rep;
df65a573 777 out_rep = attr->dests[0].rep;
f5f82476 778
b05af6aa 779 if (push && in_rep->vport == MLX5_VPORT_UPLINK)
f5f82476
OG		 780 goto out_notsupp;
781
b05af6aa 782 if (pop && out_rep->vport == MLX5_VPORT_UPLINK)
f5f82476
OG		 783 goto out_notsupp;
784
785 /* vport has vlan push configured, can't offload VF --> wire rules w.o it */
786 if (!push && !pop && fwd)
b05af6aa 787 if (in_rep->vlan && out_rep->vport == MLX5_VPORT_UPLINK)
f5f82476
OG		 788 goto out_notsupp;
789
790 /* protects against (1) setting rules with different vlans to push and
791 * (2) setting rules w.o vlans (attr->vlan = 0) && w. vlans to push (!= 0)
792 */
1482bd3d 793 if (push && in_rep->vlan_refcount && (in_rep->vlan != attr->vlan_vid[0]))
f5f82476
OG		 794 goto out_notsupp;
795
796 return 0;
797
798out_notsupp:
9eb78923 799 return -EOPNOTSUPP;
f5f82476
OG		 800}
801
802int mlx5_eswitch_add_vlan_action(struct mlx5_eswitch *esw,
c620b772 803 struct mlx5_flow_attr *attr)
f5f82476
OG		 804{
805 struct offloads_fdb *offloads = &esw->fdb_table.offloads;
c620b772 806 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
f5f82476
OG		 807 struct mlx5_eswitch_rep *vport = NULL;
808 bool push, pop, fwd;
809 int err = 0;
810
6acfbf38 811 /* nop if we're on the vlan push/pop non emulation mode */
cc495188 812 if (mlx5_eswitch_vlan_actions_supported(esw->dev, 1))
6acfbf38
OG		 813 return 0;
814
f5f82476
OG		 815 push = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH);
816 pop = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_POP);
e52c2802
PB		 817 fwd = !!((attr->action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) &&
818 !attr->dest_chain);
f5f82476 819
0e18134f
VB		 820 mutex_lock(&esw->state_lock);
821
c620b772 822 err = esw_add_vlan_action_check(esw_attr, push, pop, fwd);
f5f82476 823 if (err)
0e18134f 824 goto unlock;
f5f82476 825
39ac237c 826 attr->flags &= ~MLX5_ESW_ATTR_FLAG_VLAN_HANDLED;
f5f82476 827
c620b772 828 vport = esw_vlan_action_get_vport(esw_attr, push, pop);
f5f82476
OG		 829
830 if (!push && !pop && fwd) {
831 /* tracks VF --> wire rules without vlan push action */
c620b772 832 if (esw_attr->dests[0].rep->vport == MLX5_VPORT_UPLINK) {
f5f82476 833 vport->vlan_refcount++;
39ac237c 834 attr->flags |= MLX5_ESW_ATTR_FLAG_VLAN_HANDLED;
f5f82476
OG		 835 }
836
0e18134f 837 goto unlock;
f5f82476
OG		 838 }
839
840 if (!push && !pop)
0e18134f 841 goto unlock;
f5f82476
OG		 842
843 if (!(offloads->vlan_push_pop_refcount)) {
844 /* it's the 1st vlan rule, apply global vlan pop policy */
845 err = esw_set_global_vlan_pop(esw, SET_VLAN_STRIP);
846 if (err)
847 goto out;
848 }
849 offloads->vlan_push_pop_refcount++;
850
851 if (push) {
852 if (vport->vlan_refcount)
853 goto skip_set_push;
854
c620b772
AL		 855 err = __mlx5_eswitch_set_vport_vlan(esw, vport->vport, esw_attr->vlan_vid[0],
856 0, SET_VLAN_INSERT | SET_VLAN_STRIP);
f5f82476
OG		 857 if (err)
858 goto out;
c620b772 859 vport->vlan = esw_attr->vlan_vid[0];
f5f82476
OG		 860skip_set_push:
861 vport->vlan_refcount++;
862 }
863out:
864 if (!err)
39ac237c 865 attr->flags |= MLX5_ESW_ATTR_FLAG_VLAN_HANDLED;
0e18134f
VB		 866unlock:
867 mutex_unlock(&esw->state_lock);
f5f82476
OG		 868 return err;
869}
870
871int mlx5_eswitch_del_vlan_action(struct mlx5_eswitch *esw,
c620b772 872 struct mlx5_flow_attr *attr)
f5f82476
OG		 873{
874 struct offloads_fdb *offloads = &esw->fdb_table.offloads;
c620b772 875 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr;
f5f82476
OG		 876 struct mlx5_eswitch_rep *vport = NULL;
877 bool push, pop, fwd;
878 int err = 0;
879
6acfbf38 880 /* nop if we're on the vlan push/pop non emulation mode */
cc495188 881 if (mlx5_eswitch_vlan_actions_supported(esw->dev, 1))
6acfbf38
OG		 882 return 0;
883
39ac237c 884 if (!(attr->flags & MLX5_ESW_ATTR_FLAG_VLAN_HANDLED))
f5f82476
OG		 885 return 0;
886
887 push = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH);
888 pop = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_VLAN_POP);
889 fwd = !!(attr->action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST);
890
0e18134f
VB		 891 mutex_lock(&esw->state_lock);
892
c620b772 893 vport = esw_vlan_action_get_vport(esw_attr, push, pop);
f5f82476
OG		 894
895 if (!push && !pop && fwd) {
896 /* tracks VF --> wire rules without vlan push action */
c620b772 897 if (esw_attr->dests[0].rep->vport == MLX5_VPORT_UPLINK)
f5f82476
OG		 898 vport->vlan_refcount--;
899
0e18134f 900 goto out;
f5f82476
OG		 901 }
902
903 if (push) {
904 vport->vlan_refcount--;
905 if (vport->vlan_refcount)
906 goto skip_unset_push;
907
908 vport->vlan = 0;
909 err = __mlx5_eswitch_set_vport_vlan(esw, vport->vport,
910 0, 0, SET_VLAN_STRIP);
911 if (err)
912 goto out;
913 }
914
915skip_unset_push:
916 offloads->vlan_push_pop_refcount--;
917 if (offloads->vlan_push_pop_refcount)
0e18134f 918 goto out;
f5f82476
OG		 919
920 /* no more vlan rules, stop global vlan pop policy */
921 err = esw_set_global_vlan_pop(esw, 0);
922
923out:
0e18134f 924 mutex_unlock(&esw->state_lock);
f5f82476
OG		 925 return err;
926}
927
f7a68945 928struct mlx5_flow_handle *
3a46f4fb 929mlx5_eswitch_add_send_to_vport_rule(struct mlx5_eswitch *on_esw,
979bf468 930 struct mlx5_eswitch *from_esw,
3a46f4fb 931 struct mlx5_eswitch_rep *rep,
02f3afd9 932 u32 sqn)
ab22be9b 933{
66958ed9 934 struct mlx5_flow_act flow_act = {0};
4c5009c5 935 struct mlx5_flow_destination dest = {};
74491de9 936 struct mlx5_flow_handle *flow_rule;
c5bb1730 937 struct mlx5_flow_spec *spec;
ab22be9b
OG		 938 void *misc;
939
1b9a07ee 940 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
c5bb1730 941 if (!spec) {
ab22be9b
OG		 942 flow_rule = ERR_PTR(-ENOMEM);
943 goto out;
944 }
945
c5bb1730 946 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters);
ab22be9b 947 MLX5_SET(fte_match_set_misc, misc, source_sqn, sqn);
a1b3839a 948 /* source vport is the esw manager */
979bf468 949 MLX5_SET(fte_match_set_misc, misc, source_port, from_esw->manager_vport);
3a46f4fb 950 if (MLX5_CAP_ESW(on_esw->dev, merged_eswitch))
7d97822a 951 MLX5_SET(fte_match_set_misc, misc, source_eswitch_owner_vhca_id,
979bf468 952 MLX5_CAP_GEN(from_esw->dev, vhca_id));
ab22be9b 953
c5bb1730 954 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters);
ab22be9b
OG		 955 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_sqn);
956 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
3a46f4fb 957 if (MLX5_CAP_ESW(on_esw->dev, merged_eswitch))
7d97822a
MB		 958 MLX5_SET_TO_ONES(fte_match_set_misc, misc,
959 source_eswitch_owner_vhca_id);
ab22be9b 960
c5bb1730 961 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS;
ab22be9b 962 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
3a46f4fb
MB		 963 dest.vport.num = rep->vport;
964 dest.vport.vhca_id = MLX5_CAP_GEN(rep->esw->dev, vhca_id);
965 dest.vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID;
66958ed9 966 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
ab22be9b 967
d0444254
AL		 968 if (rep->vport == MLX5_VPORT_UPLINK)
969 spec->flow_context.flow_source = MLX5_FLOW_CONTEXT_FLOW_SOURCE_LOCAL_VPORT;
970
3a46f4fb 971 flow_rule = mlx5_add_flow_rules(on_esw->fdb_table.offloads.slow_fdb,
39ac237c 972 spec, &flow_act, &dest, 1);
ab22be9b 973 if (IS_ERR(flow_rule))
3a46f4fb
MB		 974 esw_warn(on_esw->dev, "FDB: Failed to add send to vport rule err %ld\n",
975 PTR_ERR(flow_rule));
ab22be9b 976out:
c5bb1730 977 kvfree(spec);
ab22be9b
OG		 978 return flow_rule;
979}
57cbd893 980EXPORT_SYMBOL(mlx5_eswitch_add_send_to_vport_rule);
ab22be9b 981
159fe639
MB		 982void mlx5_eswitch_del_send_to_vport_rule(struct mlx5_flow_handle *rule)
983{
984 mlx5_del_flow_rules(rule);
985}
986
8e404fef
VB		 987static void mlx5_eswitch_del_send_to_vport_meta_rules(struct mlx5_eswitch *esw)
988{
989 struct mlx5_flow_handle **flows = esw->fdb_table.offloads.send_to_vport_meta_rules;
47dd7e60 990 int i = 0, num_vfs = esw->esw_funcs.num_vfs;
8e404fef
VB		 991
992 if (!num_vfs || !flows)
993 return;
994
47dd7e60
PP		 995 for (i = 0; i < num_vfs; i++)
996 mlx5_del_flow_rules(flows[i]);
8e404fef
VB		 997
998 kvfree(flows);
999}
1000
1001static int
1002mlx5_eswitch_add_send_to_vport_meta_rules(struct mlx5_eswitch *esw)
1003{
8e404fef
VB		 1004 struct mlx5_flow_destination dest = {};
1005 struct mlx5_flow_act flow_act = {0};
6308a5f0 1006 int num_vfs, rule_idx = 0, err = 0;
8e404fef
VB		 1007 struct mlx5_flow_handle *flow_rule;
1008 struct mlx5_flow_handle **flows;
1009 struct mlx5_flow_spec *spec;
47dd7e60
PP		 1010 struct mlx5_vport *vport;
1011 unsigned long i;
6308a5f0 1012 u16 vport_num;
8e404fef
VB		 1013
1014 num_vfs = esw->esw_funcs.num_vfs;
1015 flows = kvzalloc(num_vfs * sizeof(*flows), GFP_KERNEL);
1016 if (!flows)
1017 return -ENOMEM;
1018
1019 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
1020 if (!spec) {
1021 err = -ENOMEM;
1022 goto alloc_err;
1023 }
1024
1025 MLX5_SET(fte_match_param, spec->match_criteria,
1026 misc_parameters_2.metadata_reg_c_0, mlx5_eswitch_get_vport_metadata_mask());
1027 MLX5_SET(fte_match_param, spec->match_criteria,
1028 misc_parameters_2.metadata_reg_c_1, ESW_TUN_MASK);
1029 MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_1,
1030 ESW_TUN_SLOW_TABLE_GOTO_VPORT_MARK);
1031
1032 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2;
1033 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
1034 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
1035
47dd7e60
PP		 1036 mlx5_esw_for_each_vf_vport(esw, i, vport, num_vfs) {
1037 vport_num = vport->vport;
8e404fef
VB		 1038 MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_0,
1039 mlx5_eswitch_get_vport_metadata_for_match(esw, vport_num));
1040 dest.vport.num = vport_num;
1041
1042 flow_rule = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb,
1043 spec, &flow_act, &dest, 1);
1044 if (IS_ERR(flow_rule)) {
1045 err = PTR_ERR(flow_rule);
1046 esw_warn(esw->dev, "FDB: Failed to add send to vport meta rule idx %d, err %ld\n",
1047 rule_idx, PTR_ERR(flow_rule));
1048 goto rule_err;
1049 }
1050 flows[rule_idx++] = flow_rule;
1051 }
1052
1053 esw->fdb_table.offloads.send_to_vport_meta_rules = flows;
1054 kvfree(spec);
1055 return 0;
1056
1057rule_err:
1058 while (--rule_idx >= 0)
1059 mlx5_del_flow_rules(flows[rule_idx]);
1060 kvfree(spec);
1061alloc_err:
1062 kvfree(flows);
1063 return err;
1064}
1065
5b7cb745
PB		 1066static bool mlx5_eswitch_reg_c1_loopback_supported(struct mlx5_eswitch *esw)
1067{
1068 return MLX5_CAP_ESW_FLOWTABLE(esw->dev, fdb_to_vport_reg_c_id) &
1069 MLX5_FDB_TO_VPORT_REG_C_1;
1070}
1071
332bd3a5 1072static int esw_set_passing_vport_metadata(struct mlx5_eswitch *esw, bool enable)
c1286050
JL		 1073{
1074 u32 out[MLX5_ST_SZ_DW(query_esw_vport_context_out)] = {};
e08a6832
LR		 1075 u32 min[MLX5_ST_SZ_DW(modify_esw_vport_context_in)] = {};
1076 u32 in[MLX5_ST_SZ_DW(query_esw_vport_context_in)] = {};
5b7cb745 1077 u8 curr, wanted;
c1286050
JL		 1078 int err;
1079
5b7cb745
PB		 1080 if (!mlx5_eswitch_reg_c1_loopback_supported(esw) &&
1081 !mlx5_eswitch_vport_match_metadata_enabled(esw))
332bd3a5 1082 return 0;
c1286050 1083
e08a6832
LR		 1084 MLX5_SET(query_esw_vport_context_in, in, opcode,
1085 MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT);
1086 err = mlx5_cmd_exec_inout(esw->dev, query_esw_vport_context, in, out);
c1286050
JL		 1087 if (err)
1088 return err;
1089
5b7cb745
PB		 1090 curr = MLX5_GET(query_esw_vport_context_out, out,
1091 esw_vport_context.fdb_to_vport_reg_c_id);
1092 wanted = MLX5_FDB_TO_VPORT_REG_C_0;
1093 if (mlx5_eswitch_reg_c1_loopback_supported(esw))
1094 wanted |= MLX5_FDB_TO_VPORT_REG_C_1;
c1286050 1095
332bd3a5 1096 if (enable)
5b7cb745 1097 curr |= wanted;
332bd3a5 1098 else
5b7cb745 1099 curr &= ~wanted;
c1286050 1100
e08a6832 1101 MLX5_SET(modify_esw_vport_context_in, min,
5b7cb745 1102 esw_vport_context.fdb_to_vport_reg_c_id, curr);
e08a6832 1103 MLX5_SET(modify_esw_vport_context_in, min,
c1286050
JL		 1104 field_select.fdb_to_vport_reg_c_id, 1);
1105
e08a6832 1106 err = mlx5_eswitch_modify_esw_vport_context(esw->dev, 0, false, min);
5b7cb745
PB		 1107 if (!err) {
1108 if (enable && (curr & MLX5_FDB_TO_VPORT_REG_C_1))
1109 esw->flags |= MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED;
1110 else
1111 esw->flags &= ~MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED;
1112 }
1113
1114 return err;
c1286050
JL		 1115}
1116
a5641cb5
JL		 1117static void peer_miss_rules_setup(struct mlx5_eswitch *esw,
1118 struct mlx5_core_dev *peer_dev,
ac004b83
RD		 1119 struct mlx5_flow_spec *spec,
1120 struct mlx5_flow_destination *dest)
1121{
a5641cb5 1122 void *misc;
ac004b83 1123
a5641cb5
JL		 1124 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
1125 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
1126 misc_parameters_2);
0f0d3827
PB		 1127 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
1128 mlx5_eswitch_get_vport_metadata_mask());
ac004b83 1129
a5641cb5
JL		 1130 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2;
1131 } else {
1132 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1133 misc_parameters);
ac004b83 1134
a5641cb5
JL		 1135 MLX5_SET(fte_match_set_misc, misc, source_eswitch_owner_vhca_id,
1136 MLX5_CAP_GEN(peer_dev, vhca_id));
1137
1138 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS;
1139
1140 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
1141 misc_parameters);
1142 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
1143 MLX5_SET_TO_ONES(fte_match_set_misc, misc,
1144 source_eswitch_owner_vhca_id);
1145 }
ac004b83
RD		 1146
1147 dest->type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
a1b3839a 1148 dest->vport.num = peer_dev->priv.eswitch->manager_vport;
ac004b83 1149 dest->vport.vhca_id = MLX5_CAP_GEN(peer_dev, vhca_id);
04de7dda 1150 dest->vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID;
ac004b83
RD		 1151}
1152
a5641cb5
JL		 1153static void esw_set_peer_miss_rule_source_port(struct mlx5_eswitch *esw,
1154 struct mlx5_eswitch *peer_esw,
1155 struct mlx5_flow_spec *spec,
1156 u16 vport)
1157{
1158 void *misc;
1159
1160 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
1161 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1162 misc_parameters_2);
1163 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
1164 mlx5_eswitch_get_vport_metadata_for_match(peer_esw,
1165 vport));
1166 } else {
1167 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1168 misc_parameters);
1169 MLX5_SET(fte_match_set_misc, misc, source_port, vport);
1170 }
1171}
1172
ac004b83
RD		 1173static int esw_add_fdb_peer_miss_rules(struct mlx5_eswitch *esw,
1174 struct mlx5_core_dev *peer_dev)
1175{
1176 struct mlx5_flow_destination dest = {};
1177 struct mlx5_flow_act flow_act = {0};
1178 struct mlx5_flow_handle **flows;
ac004b83
RD		 1179 /* total vports is the same for both e-switches */
1180 int nvports = esw->total_vports;
47dd7e60
PP		 1181 struct mlx5_flow_handle *flow;
1182 struct mlx5_flow_spec *spec;
1183 struct mlx5_vport *vport;
1184 unsigned long i;
ac004b83 1185 void *misc;
47dd7e60 1186 int err;
ac004b83
RD		 1187
1188 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
1189 if (!spec)
1190 return -ENOMEM;
1191
a5641cb5 1192 peer_miss_rules_setup(esw, peer_dev, spec, &dest);
ac004b83
RD		 1193
1194 flows = kvzalloc(nvports * sizeof(*flows), GFP_KERNEL);
1195 if (!flows) {
1196 err = -ENOMEM;
1197 goto alloc_flows_err;
1198 }
1199
1200 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
1201 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1202 misc_parameters);
1203
81cd229c 1204 if (mlx5_core_is_ecpf_esw_manager(esw->dev)) {
47dd7e60 1205 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_PF);
a5641cb5
JL		 1206 esw_set_peer_miss_rule_source_port(esw, peer_dev->priv.eswitch,
1207 spec, MLX5_VPORT_PF);
1208
81cd229c
BW		 1209 flow = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb,
1210 spec, &flow_act, &dest, 1);
1211 if (IS_ERR(flow)) {
1212 err = PTR_ERR(flow);
1213 goto add_pf_flow_err;
1214 }
47dd7e60 1215 flows[vport->index] = flow;
81cd229c
BW		 1216 }
1217
1218 if (mlx5_ecpf_vport_exists(esw->dev)) {
47dd7e60 1219 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_ECPF);
81cd229c
BW		 1220 MLX5_SET(fte_match_set_misc, misc, source_port, MLX5_VPORT_ECPF);
1221 flow = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb,
1222 spec, &flow_act, &dest, 1);
1223 if (IS_ERR(flow)) {
1224 err = PTR_ERR(flow);
1225 goto add_ecpf_flow_err;
1226 }
47dd7e60 1227 flows[vport->index] = flow;
81cd229c
BW		 1228 }
1229
47dd7e60 1230 mlx5_esw_for_each_vf_vport(esw, i, vport, mlx5_core_max_vfs(esw->dev)) {
a5641cb5
JL		 1231 esw_set_peer_miss_rule_source_port(esw,
1232 peer_dev->priv.eswitch,
47dd7e60 1233 spec, vport->vport);
a5641cb5 1234
ac004b83
RD		 1235 flow = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb,
1236 spec, &flow_act, &dest, 1);
1237 if (IS_ERR(flow)) {
1238 err = PTR_ERR(flow);
81cd229c 1239 goto add_vf_flow_err;
ac004b83 1240 }
47dd7e60 1241 flows[vport->index] = flow;
ac004b83
RD		 1242 }
1243
1244 esw->fdb_table.offloads.peer_miss_rules = flows;
1245
1246 kvfree(spec);
1247 return 0;
1248
81cd229c 1249add_vf_flow_err:
47dd7e60
PP		 1250 mlx5_esw_for_each_vf_vport(esw, i, vport, mlx5_core_max_vfs(esw->dev)) {
1251 if (!flows[vport->index])
1252 continue;
1253 mlx5_del_flow_rules(flows[vport->index]);
1254 }
1255 if (mlx5_ecpf_vport_exists(esw->dev)) {
1256 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_ECPF);
1257 mlx5_del_flow_rules(flows[vport->index]);
1258 }
81cd229c 1259add_ecpf_flow_err:
47dd7e60
PP		 1260 if (mlx5_core_is_ecpf_esw_manager(esw->dev)) {
1261 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_PF);
1262 mlx5_del_flow_rules(flows[vport->index]);
1263 }
81cd229c
BW		 1264add_pf_flow_err:
1265 esw_warn(esw->dev, "FDB: Failed to add peer miss flow rule err %d\n", err);
ac004b83
RD		 1266 kvfree(flows);
1267alloc_flows_err:
1268 kvfree(spec);
1269 return err;
1270}
1271
1272static void esw_del_fdb_peer_miss_rules(struct mlx5_eswitch *esw)
1273{
1274 struct mlx5_flow_handle **flows;
47dd7e60
PP		 1275 struct mlx5_vport *vport;
1276 unsigned long i;
ac004b83
RD		 1277
1278 flows = esw->fdb_table.offloads.peer_miss_rules;
1279
47dd7e60
PP		 1280 mlx5_esw_for_each_vf_vport(esw, i, vport, mlx5_core_max_vfs(esw->dev))
1281 mlx5_del_flow_rules(flows[vport->index]);
ac004b83 1282
47dd7e60
PP		 1283 if (mlx5_ecpf_vport_exists(esw->dev)) {
1284 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_ECPF);
1285 mlx5_del_flow_rules(flows[vport->index]);
1286 }
81cd229c 1287
47dd7e60
PP		 1288 if (mlx5_core_is_ecpf_esw_manager(esw->dev)) {
1289 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_PF);
1290 mlx5_del_flow_rules(flows[vport->index]);
1291 }
ac004b83
RD		 1292 kvfree(flows);
1293}
1294
3aa33572
OG		 1295static int esw_add_fdb_miss_rule(struct mlx5_eswitch *esw)
1296{
66958ed9 1297 struct mlx5_flow_act flow_act = {0};
4c5009c5 1298 struct mlx5_flow_destination dest = {};
74491de9 1299 struct mlx5_flow_handle *flow_rule = NULL;
c5bb1730 1300 struct mlx5_flow_spec *spec;
f80be543
MB		 1301 void *headers_c;
1302 void *headers_v;
3aa33572 1303 int err = 0;
f80be543
MB		 1304 u8 *dmac_c;
1305 u8 *dmac_v;
3aa33572 1306
1b9a07ee 1307 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
c5bb1730 1308 if (!spec) {
3aa33572
OG		 1309 err = -ENOMEM;
1310 goto out;
1311 }
1312
f80be543
MB		 1313 spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
1314 headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
1315 outer_headers);
1316 dmac_c = MLX5_ADDR_OF(fte_match_param, headers_c,
1317 outer_headers.dmac_47_16);
1318 dmac_c[0] = 0x01;
1319
3aa33572 1320 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
a1b3839a 1321 dest.vport.num = esw->manager_vport;
66958ed9 1322 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
3aa33572 1323
39ac237c
PB		 1324 flow_rule = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb,
1325 spec, &flow_act, &dest, 1);
3aa33572
OG		 1326 if (IS_ERR(flow_rule)) {
1327 err = PTR_ERR(flow_rule);
f80be543 1328 esw_warn(esw->dev, "FDB: Failed to add unicast miss flow rule err %d\n", err);
3aa33572
OG		 1329 goto out;
1330 }
1331
f80be543
MB		 1332 esw->fdb_table.offloads.miss_rule_uni = flow_rule;
1333
1334 headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1335 outer_headers);
1336 dmac_v = MLX5_ADDR_OF(fte_match_param, headers_v,
1337 outer_headers.dmac_47_16);
1338 dmac_v[0] = 0x01;
39ac237c
PB		 1339 flow_rule = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb,
1340 spec, &flow_act, &dest, 1);
f80be543
MB		 1341 if (IS_ERR(flow_rule)) {
1342 err = PTR_ERR(flow_rule);
1343 esw_warn(esw->dev, "FDB: Failed to add multicast miss flow rule err %d\n", err);
1344 mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_uni);
1345 goto out;
1346 }
1347
1348 esw->fdb_table.offloads.miss_rule_multi = flow_rule;
1349
3aa33572 1350out:
c5bb1730 1351 kvfree(spec);
3aa33572
OG		 1352 return err;
1353}
1354
11b717d6
PB		 1355struct mlx5_flow_handle *
1356esw_add_restore_rule(struct mlx5_eswitch *esw, u32 tag)
1357{
1358 struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, };
1359 struct mlx5_flow_table *ft = esw->offloads.ft_offloads_restore;
1360 struct mlx5_flow_context *flow_context;
1361 struct mlx5_flow_handle *flow_rule;
1362 struct mlx5_flow_destination dest;
1363 struct mlx5_flow_spec *spec;
1364 void *misc;
1365
60acc105
PB		 1366 if (!mlx5_eswitch_reg_c1_loopback_supported(esw))
1367 return ERR_PTR(-EOPNOTSUPP);
1368
9f4d9283 1369 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
11b717d6
PB		 1370 if (!spec)
1371 return ERR_PTR(-ENOMEM);
1372
1373 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
1374 misc_parameters_2);
1375 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
a91d98a0 1376 ESW_REG_C0_USER_DATA_METADATA_MASK);
11b717d6
PB		 1377 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
1378 misc_parameters_2);
1379 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, tag);
1380 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2;
6724e66b
PB		 1381 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST |
1382 MLX5_FLOW_CONTEXT_ACTION_MOD_HDR;
1383 flow_act.modify_hdr = esw->offloads.restore_copy_hdr_id;
11b717d6
PB		 1384
1385 flow_context = &spec->flow_context;
1386 flow_context->flags |= FLOW_CONTEXT_HAS_TAG;
1387 flow_context->flow_tag = tag;
1388 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
1389 dest.ft = esw->offloads.ft_offloads;
1390
1391 flow_rule = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
9f4d9283 1392 kvfree(spec);
11b717d6
PB		 1393
1394 if (IS_ERR(flow_rule))
1395 esw_warn(esw->dev,
1396 "Failed to create restore rule for tag: %d, err(%d)\n",
1397 tag, (int)PTR_ERR(flow_rule));
1398
1399 return flow_rule;
1400}
1401
1967ce6e 1402#define MAX_PF_SQ 256
cd3d07e7 1403#define MAX_SQ_NVPORTS 32
1967ce6e 1404
a5641cb5
JL		 1405static void esw_set_flow_group_source_port(struct mlx5_eswitch *esw,
1406 u32 *flow_group_in)
1407{
1408 void *match_criteria = MLX5_ADDR_OF(create_flow_group_in,
1409 flow_group_in,
1410 match_criteria);
1411
1412 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
1413 MLX5_SET(create_flow_group_in, flow_group_in,
1414 match_criteria_enable,
1415 MLX5_MATCH_MISC_PARAMETERS_2);
1416
0f0d3827
PB		 1417 MLX5_SET(fte_match_param, match_criteria,
1418 misc_parameters_2.metadata_reg_c_0,
1419 mlx5_eswitch_get_vport_metadata_mask());
a5641cb5
JL		 1420 } else {
1421 MLX5_SET(create_flow_group_in, flow_group_in,
1422 match_criteria_enable,
1423 MLX5_MATCH_MISC_PARAMETERS);
1424
1425 MLX5_SET_TO_ONES(fte_match_param, match_criteria,
1426 misc_parameters.source_port);
1427 }
1428}
1429
ae430332 1430#if IS_ENABLED(CONFIG_MLX5_CLS_ACT)
0a9e2307 1431static void esw_vport_tbl_put(struct mlx5_eswitch *esw)
4c7f4028
CM		 1432{
1433 struct mlx5_vport_tbl_attr attr;
1434 struct mlx5_vport *vport;
47dd7e60 1435 unsigned long i;
4c7f4028
CM		 1436
1437 attr.chain = 0;
1438 attr.prio = 1;
47dd7e60 1439 mlx5_esw_for_each_vport(esw, i, vport) {
4c7f4028 1440 attr.vport = vport->vport;
c796bb7c 1441 attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns;
0a9e2307 1442 mlx5_esw_vporttbl_put(esw, &attr);
4c7f4028
CM		 1443 }
1444}
1445
0a9e2307 1446static int esw_vport_tbl_get(struct mlx5_eswitch *esw)
4c7f4028
CM		 1447{
1448 struct mlx5_vport_tbl_attr attr;
1449 struct mlx5_flow_table *fdb;
1450 struct mlx5_vport *vport;
47dd7e60 1451 unsigned long i;
4c7f4028
CM		 1452
1453 attr.chain = 0;
1454 attr.prio = 1;
47dd7e60 1455 mlx5_esw_for_each_vport(esw, i, vport) {
4c7f4028 1456 attr.vport = vport->vport;
c796bb7c 1457 attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns;
0a9e2307 1458 fdb = mlx5_esw_vporttbl_get(esw, &attr);
4c7f4028
CM		 1459 if (IS_ERR(fdb))
1460 goto out;
1461 }
1462 return 0;
1463
1464out:
0a9e2307 1465 esw_vport_tbl_put(esw);
4c7f4028
CM		 1466 return PTR_ERR(fdb);
1467}
1468
ae430332
AL		 1469#define fdb_modify_header_fwd_to_table_supported(esw) \
1470 (MLX5_CAP_ESW_FLOWTABLE((esw)->dev, fdb_modify_header_fwd_to_table))
1471static void esw_init_chains_offload_flags(struct mlx5_eswitch *esw, u32 *flags)
1472{
1473 struct mlx5_core_dev *dev = esw->dev;
1474
1475 if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev, ignore_flow_level))
1476 *flags |= MLX5_CHAINS_IGNORE_FLOW_LEVEL_SUPPORTED;
1477
1478 if (!MLX5_CAP_ESW_FLOWTABLE(dev, multi_fdb_encap) &&
1479 esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE) {
1480 *flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED;
1481 esw_warn(dev, "Tc chains and priorities offload aren't supported, update firmware if needed\n");
1482 } else if (!mlx5_eswitch_reg_c1_loopback_enabled(esw)) {
1483 *flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED;
1484 esw_warn(dev, "Tc chains and priorities offload aren't supported\n");
1485 } else if (!fdb_modify_header_fwd_to_table_supported(esw)) {
1486 /* Disabled when ttl workaround is needed, e.g
1487 * when ESWITCH_IPV4_TTL_MODIFY_ENABLE = true in mlxconfig
1488 */
1489 esw_warn(dev,
1490 "Tc chains and priorities offload aren't supported, check firmware version, or mlxconfig settings\n");
1491 *flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED;
1492 } else {
1493 *flags |= MLX5_CHAINS_AND_PRIOS_SUPPORTED;
1494 esw_info(dev, "Supported tc chains and prios offload\n");
1495 }
1496
1497 if (esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE)
1498 *flags |= MLX5_CHAINS_FT_TUNNEL_SUPPORTED;
1499}
1500
1501static int
1502esw_chains_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *miss_fdb)
1503{
1504 struct mlx5_core_dev *dev = esw->dev;
1505 struct mlx5_flow_table *nf_ft, *ft;
1506 struct mlx5_chains_attr attr = {};
1507 struct mlx5_fs_chains *chains;
1508 u32 fdb_max;
1509 int err;
1510
1511 fdb_max = 1 << MLX5_CAP_ESW_FLOWTABLE_FDB(dev, log_max_ft_size);
1512
1513 esw_init_chains_offload_flags(esw, &attr.flags);
1514 attr.ns = MLX5_FLOW_NAMESPACE_FDB;
1515 attr.max_ft_sz = fdb_max;
1516 attr.max_grp_num = esw->params.large_group_num;
1517 attr.default_ft = miss_fdb;
c9355682 1518 attr.mapping = esw->offloads.reg_c0_obj_pool;
ae430332
AL		 1519
1520 chains = mlx5_chains_create(dev, &attr);
1521 if (IS_ERR(chains)) {
1522 err = PTR_ERR(chains);
1523 esw_warn(dev, "Failed to create fdb chains err(%d)\n", err);
1524 return err;
1525 }
1526
1527 esw->fdb_table.offloads.esw_chains_priv = chains;
1528
1529 /* Create tc_end_ft which is the always created ft chain */
1530 nf_ft = mlx5_chains_get_table(chains, mlx5_chains_get_nf_ft_chain(chains),
1531 1, 0);
1532 if (IS_ERR(nf_ft)) {
1533 err = PTR_ERR(nf_ft);
1534 goto nf_ft_err;
1535 }
1536
1537 /* Always open the root for fast path */
1538 ft = mlx5_chains_get_table(chains, 0, 1, 0);
1539 if (IS_ERR(ft)) {
1540 err = PTR_ERR(ft);
1541 goto level_0_err;
1542 }
1543
1544 /* Open level 1 for split fdb rules now if prios isn't supported */
1545 if (!mlx5_chains_prios_supported(chains)) {
0a9e2307 1546 err = esw_vport_tbl_get(esw);
ae430332
AL		 1547 if (err)
1548 goto level_1_err;
1549 }
1550
1551 mlx5_chains_set_end_ft(chains, nf_ft);
1552
1553 return 0;
1554
1555level_1_err:
1556 mlx5_chains_put_table(chains, 0, 1, 0);
1557level_0_err:
1558 mlx5_chains_put_table(chains, mlx5_chains_get_nf_ft_chain(chains), 1, 0);
1559nf_ft_err:
1560 mlx5_chains_destroy(chains);
1561 esw->fdb_table.offloads.esw_chains_priv = NULL;
1562
1563 return err;
1564}
1565
1566static void
1567esw_chains_destroy(struct mlx5_eswitch *esw, struct mlx5_fs_chains *chains)
1568{
1569 if (!mlx5_chains_prios_supported(chains))
0a9e2307 1570 esw_vport_tbl_put(esw);
ae430332
AL		 1571 mlx5_chains_put_table(chains, 0, 1, 0);
1572 mlx5_chains_put_table(chains, mlx5_chains_get_nf_ft_chain(chains), 1, 0);
1573 mlx5_chains_destroy(chains);
1574}
1575
1576#else /* CONFIG_MLX5_CLS_ACT */
1577
1578static int
1579esw_chains_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *miss_fdb)
1580{ return 0; }
1581
1582static void
1583esw_chains_destroy(struct mlx5_eswitch *esw, struct mlx5_fs_chains *chains)
1584{}
1585
1586#endif
1587
0da3c12d 1588static int esw_create_offloads_fdb_tables(struct mlx5_eswitch *esw)
1967ce6e
OG		 1589{
1590 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1591 struct mlx5_flow_table_attr ft_attr = {};
8e404fef 1592 int num_vfs, table_size, ix, err = 0;
1967ce6e
OG		 1593 struct mlx5_core_dev *dev = esw->dev;
1594 struct mlx5_flow_namespace *root_ns;
1595 struct mlx5_flow_table *fdb = NULL;
39ac237c 1596 u32 flags = 0, *flow_group_in;
1967ce6e
OG		 1597 struct mlx5_flow_group *g;
1598 void *match_criteria;
f80be543 1599 u8 *dmac;
1967ce6e
OG		 1600
1601 esw_debug(esw->dev, "Create offloads FDB Tables\n");
39ac237c 1602
1b9a07ee 1603 flow_group_in = kvzalloc(inlen, GFP_KERNEL);
1967ce6e
OG		 1604 if (!flow_group_in)
1605 return -ENOMEM;
1606
1607 root_ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_FDB);
1608 if (!root_ns) {
1609 esw_warn(dev, "Failed to get FDB flow namespace\n");
1610 err = -EOPNOTSUPP;
1611 goto ns_err;
1612 }
8463daf1
MG		 1613 esw->fdb_table.offloads.ns = root_ns;
1614 err = mlx5_flow_namespace_set_mode(root_ns,
1615 esw->dev->priv.steering->mode);
1616 if (err) {
1617 esw_warn(dev, "Failed to set FDB namespace steering mode\n");
1618 goto ns_err;
1619 }
1967ce6e 1620
898b0786
MB		 1621 /* To be strictly correct:
1622 * MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ)
1623 * should be:
1624 * esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ +
1625 * peer_esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ
1626 * but as the peer device might not be in switchdev mode it's not
1627 * possible. We use the fact that by default FW sets max vfs and max sfs
1628 * to the same value on both devices. If it needs to be changed in the future note
1629 * the peer miss group should also be created based on the number of
1630 * total vports of the peer (currently is also uses esw->total_vports).
1631 */
1632 table_size = MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ) +
8e404fef 1633 MLX5_ESW_MISS_FLOWS + esw->total_vports + esw->esw_funcs.num_vfs;
b3ba5149 1634
e52c2802
PB		 1635 /* create the slow path fdb with encap set, so further table instances
1636 * can be created at run time while VFs are probed if the FW allows that.
1637 */
1638 if (esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE)
1639 flags |= (MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT |
1640 MLX5_FLOW_TABLE_TUNNEL_EN_DECAP);
1641
1642 ft_attr.flags = flags;
b3ba5149
ES		 1643 ft_attr.max_fte = table_size;
1644 ft_attr.prio = FDB_SLOW_PATH;
1645
1646 fdb = mlx5_create_flow_table(root_ns, &ft_attr);
1033665e
OG		 1647 if (IS_ERR(fdb)) {
1648 err = PTR_ERR(fdb);
1649 esw_warn(dev, "Failed to create slow path FDB Table err %d\n", err);
1650 goto slow_fdb_err;
1651 }
52fff327 1652 esw->fdb_table.offloads.slow_fdb = fdb;
1033665e 1653
ec3be887
VB		 1654 /* Create empty TC-miss managed table. This allows plugging in following
1655 * priorities without directly exposing their level 0 table to
1656 * eswitch_offloads and passing it as miss_fdb to following call to
1657 * esw_chains_create().
1658 */
1659 memset(&ft_attr, 0, sizeof(ft_attr));
1660 ft_attr.prio = FDB_TC_MISS;
1661 esw->fdb_table.offloads.tc_miss_table = mlx5_create_flow_table(root_ns, &ft_attr);
1662 if (IS_ERR(esw->fdb_table.offloads.tc_miss_table)) {
1663 err = PTR_ERR(esw->fdb_table.offloads.tc_miss_table);
1664 esw_warn(dev, "Failed to create TC miss FDB Table err %d\n", err);
1665 goto tc_miss_table_err;
1666 }
1667
1668 err = esw_chains_create(esw, esw->fdb_table.offloads.tc_miss_table);
39ac237c 1669 if (err) {
ae430332 1670 esw_warn(dev, "Failed to open fdb chains err(%d)\n", err);
39ac237c 1671 goto fdb_chains_err;
e52c2802
PB		 1672 }
1673
69697b6e 1674 /* create send-to-vport group */
69697b6e
OG		 1675 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
1676 MLX5_MATCH_MISC_PARAMETERS);
1677
1678 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, match_criteria);
1679
1680 MLX5_SET_TO_ONES(fte_match_param, match_criteria, misc_parameters.source_sqn);
1681 MLX5_SET_TO_ONES(fte_match_param, match_criteria, misc_parameters.source_port);
7d97822a
MB		 1682 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) {
1683 MLX5_SET_TO_ONES(fte_match_param, match_criteria,
1684 misc_parameters.source_eswitch_owner_vhca_id);
1685 MLX5_SET(create_flow_group_in, flow_group_in,
1686 source_eswitch_owner_vhca_id_valid, 1);
1687 }
69697b6e 1688
898b0786
MB		 1689 /* See comment above table_size calculation */
1690 ix = MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ);
69697b6e
OG		 1691 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0);
1692 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, ix - 1);
1693
1694 g = mlx5_create_flow_group(fdb, flow_group_in);
1695 if (IS_ERR(g)) {
1696 err = PTR_ERR(g);
1697 esw_warn(dev, "Failed to create send-to-vport flow group err(%d)\n", err);
1698 goto send_vport_err;
1699 }
1700 esw->fdb_table.offloads.send_to_vport_grp = g;
1701
e929e3da
MD		 1702 if (esw_src_port_rewrite_supported(esw)) {
1703 /* meta send to vport */
1704 memset(flow_group_in, 0, inlen);
1705 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
1706 MLX5_MATCH_MISC_PARAMETERS_2);
8e404fef 1707
e929e3da 1708 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, match_criteria);
8e404fef 1709
e929e3da
MD		 1710 MLX5_SET(fte_match_param, match_criteria,
1711 misc_parameters_2.metadata_reg_c_0,
1712 mlx5_eswitch_get_vport_metadata_mask());
1713 MLX5_SET(fte_match_param, match_criteria,
1714 misc_parameters_2.metadata_reg_c_1, ESW_TUN_MASK);
1715
1716 num_vfs = esw->esw_funcs.num_vfs;
1717 if (num_vfs) {
1718 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, ix);
1719 MLX5_SET(create_flow_group_in, flow_group_in,
1720 end_flow_index, ix + num_vfs - 1);
1721 ix += num_vfs;
1722
1723 g = mlx5_create_flow_group(fdb, flow_group_in);
1724 if (IS_ERR(g)) {
1725 err = PTR_ERR(g);
1726 esw_warn(dev, "Failed to create send-to-vport meta flow group err(%d)\n",
1727 err);
1728 goto send_vport_meta_err;
1729 }
1730 esw->fdb_table.offloads.send_to_vport_meta_grp = g;
1731
1732 err = mlx5_eswitch_add_send_to_vport_meta_rules(esw);
1733 if (err)
1734 goto meta_rule_err;
8e404fef 1735 }
8e404fef
VB		 1736 }
1737
6cec0229
MD		 1738 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) {
1739 /* create peer esw miss group */
1740 memset(flow_group_in, 0, inlen);
ac004b83 1741
6cec0229 1742 esw_set_flow_group_source_port(esw, flow_group_in);
a5641cb5 1743
6cec0229
MD		 1744 if (!mlx5_eswitch_vport_match_metadata_enabled(esw)) {
1745 match_criteria = MLX5_ADDR_OF(create_flow_group_in,
1746 flow_group_in,
1747 match_criteria);
ac004b83 1748
6cec0229
MD		 1749 MLX5_SET_TO_ONES(fte_match_param, match_criteria,
1750 misc_parameters.source_eswitch_owner_vhca_id);
a5641cb5 1751
6cec0229
MD		 1752 MLX5_SET(create_flow_group_in, flow_group_in,
1753 source_eswitch_owner_vhca_id_valid, 1);
1754 }
ac004b83 1755
6cec0229
MD		 1756 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, ix);
1757 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index,
1758 ix + esw->total_vports - 1);
1759 ix += esw->total_vports;
ac004b83 1760
6cec0229
MD		 1761 g = mlx5_create_flow_group(fdb, flow_group_in);
1762 if (IS_ERR(g)) {
1763 err = PTR_ERR(g);
1764 esw_warn(dev, "Failed to create peer miss flow group err(%d)\n", err);
1765 goto peer_miss_err;
1766 }
1767 esw->fdb_table.offloads.peer_miss_grp = g;
ac004b83 1768 }
ac004b83 1769
69697b6e
OG		 1770 /* create miss group */
1771 memset(flow_group_in, 0, inlen);
f80be543
MB		 1772 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
1773 MLX5_MATCH_OUTER_HEADERS);
1774 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in,
1775 match_criteria);
1776 dmac = MLX5_ADDR_OF(fte_match_param, match_criteria,
1777 outer_headers.dmac_47_16);
1778 dmac[0] = 0x01;
69697b6e
OG		 1779
1780 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, ix);
cd7e4186
BW		 1781 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index,
1782 ix + MLX5_ESW_MISS_FLOWS);
69697b6e
OG		 1783
1784 g = mlx5_create_flow_group(fdb, flow_group_in);
1785 if (IS_ERR(g)) {
1786 err = PTR_ERR(g);
1787 esw_warn(dev, "Failed to create miss flow group err(%d)\n", err);
1788 goto miss_err;
1789 }
1790 esw->fdb_table.offloads.miss_grp = g;
1791
3aa33572
OG		 1792 err = esw_add_fdb_miss_rule(esw);
1793 if (err)
1794 goto miss_rule_err;
1795
c88a026e 1796 kvfree(flow_group_in);
69697b6e
OG		 1797 return 0;
1798
3aa33572
OG		 1799miss_rule_err:
1800 mlx5_destroy_flow_group(esw->fdb_table.offloads.miss_grp);
69697b6e 1801miss_err:
6cec0229
MD		 1802 if (MLX5_CAP_ESW(esw->dev, merged_eswitch))
1803 mlx5_destroy_flow_group(esw->fdb_table.offloads.peer_miss_grp);
ac004b83 1804peer_miss_err:
8e404fef
VB		 1805 mlx5_eswitch_del_send_to_vport_meta_rules(esw);
1806meta_rule_err:
1807 if (esw->fdb_table.offloads.send_to_vport_meta_grp)
1808 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_meta_grp);
1809send_vport_meta_err:
69697b6e
OG		 1810 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_grp);
1811send_vport_err:
ae430332 1812 esw_chains_destroy(esw, esw_chains(esw));
39ac237c 1813fdb_chains_err:
ec3be887
VB		 1814 mlx5_destroy_flow_table(esw->fdb_table.offloads.tc_miss_table);
1815tc_miss_table_err:
52fff327 1816 mlx5_destroy_flow_table(esw->fdb_table.offloads.slow_fdb);
1033665e 1817slow_fdb_err:
8463daf1
MG		 1818 /* Holds true only as long as DMFS is the default */
1819 mlx5_flow_namespace_set_mode(root_ns, MLX5_FLOW_STEERING_MODE_DMFS);
69697b6e
OG		 1820ns_err:
1821 kvfree(flow_group_in);
1822 return err;
1823}
1824
1967ce6e 1825static void esw_destroy_offloads_fdb_tables(struct mlx5_eswitch *esw)
69697b6e 1826{
e52c2802 1827 if (!esw->fdb_table.offloads.slow_fdb)
69697b6e
OG		 1828 return;
1829
1967ce6e 1830 esw_debug(esw->dev, "Destroy offloads FDB Tables\n");
f80be543
MB		 1831 mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_multi);
1832 mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_uni);
8e404fef 1833 mlx5_eswitch_del_send_to_vport_meta_rules(esw);
69697b6e 1834 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_grp);
8e404fef
VB		 1835 if (esw->fdb_table.offloads.send_to_vport_meta_grp)
1836 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_meta_grp);
6cec0229
MD		 1837 if (MLX5_CAP_ESW(esw->dev, merged_eswitch))
1838 mlx5_destroy_flow_group(esw->fdb_table.offloads.peer_miss_grp);
69697b6e
OG		 1839 mlx5_destroy_flow_group(esw->fdb_table.offloads.miss_grp);
1840
ae430332
AL		 1841 esw_chains_destroy(esw, esw_chains(esw));
1842
ec3be887 1843 mlx5_destroy_flow_table(esw->fdb_table.offloads.tc_miss_table);
52fff327 1844 mlx5_destroy_flow_table(esw->fdb_table.offloads.slow_fdb);
8463daf1
MG		 1845 /* Holds true only as long as DMFS is the default */
1846 mlx5_flow_namespace_set_mode(esw->fdb_table.offloads.ns,
1847 MLX5_FLOW_STEERING_MODE_DMFS);
7dc84de9 1848 atomic64_set(&esw->user_count, 0);
69697b6e 1849}
c116c6ee 1850
8d6bd3c3 1851static int esw_create_offloads_table(struct mlx5_eswitch *esw)
c116c6ee 1852{
b3ba5149 1853 struct mlx5_flow_table_attr ft_attr = {};
c116c6ee 1854 struct mlx5_core_dev *dev = esw->dev;
b3ba5149
ES		 1855 struct mlx5_flow_table *ft_offloads;
1856 struct mlx5_flow_namespace *ns;
c116c6ee
OG		 1857 int err = 0;
1858
1859 ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_OFFLOADS);
1860 if (!ns) {
1861 esw_warn(esw->dev, "Failed to get offloads flow namespace\n");
eff596da 1862 return -EOPNOTSUPP;
c116c6ee
OG		 1863 }
1864
8d6bd3c3 1865 ft_attr.max_fte = esw->total_vports + MLX5_ESW_MISS_FLOWS;
11b717d6 1866 ft_attr.prio = 1;
b3ba5149
ES		 1867
1868 ft_offloads = mlx5_create_flow_table(ns, &ft_attr);
c116c6ee
OG		 1869 if (IS_ERR(ft_offloads)) {
1870 err = PTR_ERR(ft_offloads);
1871 esw_warn(esw->dev, "Failed to create offloads table, err %d\n", err);
1872 return err;
1873 }
1874
1875 esw->offloads.ft_offloads = ft_offloads;
1876 return 0;
1877}
1878
1879static void esw_destroy_offloads_table(struct mlx5_eswitch *esw)
1880{
1881 struct mlx5_esw_offload *offloads = &esw->offloads;
1882
1883 mlx5_destroy_flow_table(offloads->ft_offloads);
1884}
fed9ce22 1885
8d6bd3c3 1886static int esw_create_vport_rx_group(struct mlx5_eswitch *esw)
fed9ce22
OG		 1887{
1888 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
1889 struct mlx5_flow_group *g;
fed9ce22 1890 u32 *flow_group_in;
8d6bd3c3 1891 int nvports;
fed9ce22 1892 int err = 0;
fed9ce22 1893
8d6bd3c3 1894 nvports = esw->total_vports + MLX5_ESW_MISS_FLOWS;
1b9a07ee 1895 flow_group_in = kvzalloc(inlen, GFP_KERNEL);
fed9ce22
OG		 1896 if (!flow_group_in)
1897 return -ENOMEM;
1898
1899 /* create vport rx group */
a5641cb5 1900 esw_set_flow_group_source_port(esw, flow_group_in);
fed9ce22
OG		 1901
1902 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0);
1903 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, nvports - 1);
1904
1905 g = mlx5_create_flow_group(esw->offloads.ft_offloads, flow_group_in);
1906
1907 if (IS_ERR(g)) {
1908 err = PTR_ERR(g);
1909 mlx5_core_warn(esw->dev, "Failed to create vport rx group err %d\n", err);
1910 goto out;
1911 }
1912
1913 esw->offloads.vport_rx_group = g;
1914out:
e574978a 1915 kvfree(flow_group_in);
fed9ce22
OG		 1916 return err;
1917}
1918
1919static void esw_destroy_vport_rx_group(struct mlx5_eswitch *esw)
1920{
1921 mlx5_destroy_flow_group(esw->offloads.vport_rx_group);
1922}
1923
74491de9 1924struct mlx5_flow_handle *
02f3afd9 1925mlx5_eswitch_create_vport_rx_rule(struct mlx5_eswitch *esw, u16 vport,
c966f7d5 1926 struct mlx5_flow_destination *dest)
fed9ce22 1927{
66958ed9 1928 struct mlx5_flow_act flow_act = {0};
74491de9 1929 struct mlx5_flow_handle *flow_rule;
c5bb1730 1930 struct mlx5_flow_spec *spec;
fed9ce22
OG		 1931 void *misc;
1932
1b9a07ee 1933 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
c5bb1730 1934 if (!spec) {
fed9ce22
OG		 1935 flow_rule = ERR_PTR(-ENOMEM);
1936 goto out;
1937 }
1938
a5641cb5
JL		 1939 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) {
1940 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2);
1941 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
1942 mlx5_eswitch_get_vport_metadata_for_match(esw, vport));
fed9ce22 1943
a5641cb5 1944 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2);
0f0d3827
PB		 1945 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
1946 mlx5_eswitch_get_vport_metadata_mask());
fed9ce22 1947
a5641cb5
JL		 1948 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2;
1949 } else {
1950 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters);
1951 MLX5_SET(fte_match_set_misc, misc, source_port, vport);
1952
1953 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters);
1954 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
1955
1956 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS;
1957 }
fed9ce22 1958
66958ed9 1959 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
74491de9 1960 flow_rule = mlx5_add_flow_rules(esw->offloads.ft_offloads, spec,
c966f7d5 1961 &flow_act, dest, 1);
fed9ce22
OG		 1962 if (IS_ERR(flow_rule)) {
1963 esw_warn(esw->dev, "fs offloads: Failed to add vport rx rule err %ld\n", PTR_ERR(flow_rule));
1964 goto out;
1965 }
1966
1967out:
c5bb1730 1968 kvfree(spec);
fed9ce22
OG		 1969 return flow_rule;
1970}
feae9087 1971
47dd7e60 1972static int mlx5_eswitch_inline_mode_get(struct mlx5_eswitch *esw, u8 *mode)
cc617ced
PP		 1973{
1974 u8 prev_mlx5_mode, mlx5_mode = MLX5_INLINE_MODE_L2;
1975 struct mlx5_core_dev *dev = esw->dev;
47dd7e60
PP		 1976 struct mlx5_vport *vport;
1977 unsigned long i;
cc617ced
PP		 1978
1979 if (!MLX5_CAP_GEN(dev, vport_group_manager))
1980 return -EOPNOTSUPP;
1981
1982 if (esw->mode == MLX5_ESWITCH_NONE)
1983 return -EOPNOTSUPP;
1984
1985 switch (MLX5_CAP_ETH(dev, wqe_inline_mode)) {
1986 case MLX5_CAP_INLINE_MODE_NOT_REQUIRED:
1987 mlx5_mode = MLX5_INLINE_MODE_NONE;
1988 goto out;
1989 case MLX5_CAP_INLINE_MODE_L2:
1990 mlx5_mode = MLX5_INLINE_MODE_L2;
1991 goto out;
1992 case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT:
1993 goto query_vports;
1994 }
1995
1996query_vports:
1997 mlx5_query_nic_vport_min_inline(dev, esw->first_host_vport, &prev_mlx5_mode);
47dd7e60
PP		 1998 mlx5_esw_for_each_host_func_vport(esw, i, vport, esw->esw_funcs.num_vfs) {
1999 mlx5_query_nic_vport_min_inline(dev, vport->vport, &mlx5_mode);
cc617ced
PP		 2000 if (prev_mlx5_mode != mlx5_mode)
2001 return -EINVAL;
2002 prev_mlx5_mode = mlx5_mode;
2003 }
2004
2005out:
2006 *mode = mlx5_mode;
2007 return 0;
e08a6832 2008}
bf3347c4 2009
11b717d6
PB		 2010static void esw_destroy_restore_table(struct mlx5_eswitch *esw)
2011{
2012 struct mlx5_esw_offload *offloads = &esw->offloads;
2013
60acc105
PB		 2014 if (!mlx5_eswitch_reg_c1_loopback_supported(esw))
2015 return;
2016
6724e66b 2017 mlx5_modify_header_dealloc(esw->dev, offloads->restore_copy_hdr_id);
11b717d6
PB		 2018 mlx5_destroy_flow_group(offloads->restore_group);
2019 mlx5_destroy_flow_table(offloads->ft_offloads_restore);
2020}
2021
2022static int esw_create_restore_table(struct mlx5_eswitch *esw)
2023{
d65dbedf 2024 u8 modact[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {};
11b717d6
PB		 2025 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
2026 struct mlx5_flow_table_attr ft_attr = {};
2027 struct mlx5_core_dev *dev = esw->dev;
2028 struct mlx5_flow_namespace *ns;
6724e66b 2029 struct mlx5_modify_hdr *mod_hdr;
11b717d6
PB		 2030 void *match_criteria, *misc;
2031 struct mlx5_flow_table *ft;
2032 struct mlx5_flow_group *g;
2033 u32 *flow_group_in;
2034 int err = 0;
2035
60acc105
PB		 2036 if (!mlx5_eswitch_reg_c1_loopback_supported(esw))
2037 return 0;
2038
11b717d6
PB		 2039 ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_OFFLOADS);
2040 if (!ns) {
2041 esw_warn(esw->dev, "Failed to get offloads flow namespace\n");
2042 return -EOPNOTSUPP;
2043 }
2044
2045 flow_group_in = kvzalloc(inlen, GFP_KERNEL);
2046 if (!flow_group_in) {
2047 err = -ENOMEM;
2048 goto out_free;
2049 }
2050
a91d98a0 2051 ft_attr.max_fte = 1 << ESW_REG_C0_USER_DATA_METADATA_BITS;
11b717d6
PB		 2052 ft = mlx5_create_flow_table(ns, &ft_attr);
2053 if (IS_ERR(ft)) {
2054 err = PTR_ERR(ft);
2055 esw_warn(esw->dev, "Failed to create restore table, err %d\n",
2056 err);
2057 goto out_free;
2058 }
2059
11b717d6
PB		 2060 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in,
2061 match_criteria);
2062 misc = MLX5_ADDR_OF(fte_match_param, match_criteria,
2063 misc_parameters_2);
2064
2065 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0,
a91d98a0 2066 ESW_REG_C0_USER_DATA_METADATA_MASK);
11b717d6
PB		 2067 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0);
2068 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index,
2069 ft_attr.max_fte - 1);
2070 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
2071 MLX5_MATCH_MISC_PARAMETERS_2);
2072 g = mlx5_create_flow_group(ft, flow_group_in);
2073 if (IS_ERR(g)) {
2074 err = PTR_ERR(g);
2075 esw_warn(dev, "Failed to create restore flow group, err: %d\n",
2076 err);
2077 goto err_group;
2078 }
2079
6724e66b
PB		 2080 MLX5_SET(copy_action_in, modact, action_type, MLX5_ACTION_TYPE_COPY);
2081 MLX5_SET(copy_action_in, modact, src_field,
2082 MLX5_ACTION_IN_FIELD_METADATA_REG_C_1);
2083 MLX5_SET(copy_action_in, modact, dst_field,
2084 MLX5_ACTION_IN_FIELD_METADATA_REG_B);
2085 mod_hdr = mlx5_modify_header_alloc(esw->dev,
2086 MLX5_FLOW_NAMESPACE_KERNEL, 1,
2087 modact);
2088 if (IS_ERR(mod_hdr)) {
e9864539 2089 err = PTR_ERR(mod_hdr);
6724e66b
PB		 2090 esw_warn(dev, "Failed to create restore mod header, err: %d\n",
2091 err);
6724e66b
PB		 2092 goto err_mod_hdr;
2093 }
2094
11b717d6
PB		 2095 esw->offloads.ft_offloads_restore = ft;
2096 esw->offloads.restore_group = g;
6724e66b 2097 esw->offloads.restore_copy_hdr_id = mod_hdr;
11b717d6 2098
c8508713
RD		 2099 kvfree(flow_group_in);
2100
11b717d6
PB		 2101 return 0;
2102
6724e66b
PB		 2103err_mod_hdr:
2104 mlx5_destroy_flow_group(g);
11b717d6
PB		 2105err_group:
2106 mlx5_destroy_flow_table(ft);
2107out_free:
2108 kvfree(flow_group_in);
2109
2110 return err;
cc617ced
PP		 2111}
2112
db7ff19e
EB		 2113static int esw_offloads_start(struct mlx5_eswitch *esw,
2114 struct netlink_ext_ack *extack)
c930a3ad 2115{
062f4bf4 2116 int err, err1;
c930a3ad 2117
8e0aa4bc
PP		 2118 mlx5_eswitch_disable_locked(esw, false);
2119 err = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_OFFLOADS,
2120 esw->dev->priv.sriov.num_vfs);
6c419ba8 2121 if (err) {
8c98ee77
EB		 2122 NL_SET_ERR_MSG_MOD(extack,
2123 "Failed setting eswitch to offloads");
8e0aa4bc
PP		 2124 err1 = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_LEGACY,
2125 MLX5_ESWITCH_IGNORE_NUM_VFS);
8c98ee77
EB		 2126 if (err1) {
2127 NL_SET_ERR_MSG_MOD(extack,
2128 "Failed setting eswitch back to legacy");
2129 }
6c419ba8 2130 }
bffaa916
RD		 2131 if (esw->offloads.inline_mode == MLX5_INLINE_MODE_NONE) {
2132 if (mlx5_eswitch_inline_mode_get(esw,
bffaa916
RD		 2133 &esw->offloads.inline_mode)) {
2134 esw->offloads.inline_mode = MLX5_INLINE_MODE_L2;
8c98ee77
EB		 2135 NL_SET_ERR_MSG_MOD(extack,
2136 "Inline mode is different between vports");
bffaa916
RD		 2137 }
2138 }
c930a3ad
OG		 2139 return err;
2140}
2141
47dd7e60
PP		 2142static void mlx5_esw_offloads_rep_mark_set(struct mlx5_eswitch *esw,
2143 struct mlx5_eswitch_rep *rep,
2144 xa_mark_t mark)
e8d31c4d 2145{
47dd7e60
PP		 2146 bool mark_set;
2147
2148 /* Copy the mark from vport to its rep */
2149 mark_set = xa_get_mark(&esw->vports, rep->vport, mark);
2150 if (mark_set)
2151 xa_set_mark(&esw->offloads.vport_reps, rep->vport, mark);
e8d31c4d
MB		 2152}
2153
47dd7e60 2154static int mlx5_esw_offloads_rep_init(struct mlx5_eswitch *esw, const struct mlx5_vport *vport)
e8d31c4d 2155{
e8d31c4d 2156 struct mlx5_eswitch_rep *rep;
47dd7e60
PP		 2157 int rep_type;
2158 int err;
e8d31c4d 2159
47dd7e60
PP		 2160 rep = kzalloc(sizeof(*rep), GFP_KERNEL);
2161 if (!rep)
e8d31c4d
MB		 2162 return -ENOMEM;
2163
47dd7e60
PP		 2164 rep->vport = vport->vport;
2165 rep->vport_index = vport->index;
2166 for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++)
2167 atomic_set(&rep->rep_data[rep_type].state, REP_UNREGISTERED);
f121e0ea 2168
47dd7e60
PP		 2169 err = xa_insert(&esw->offloads.vport_reps, rep->vport, rep, GFP_KERNEL);
2170 if (err)
2171 goto insert_err;
2172
2173 mlx5_esw_offloads_rep_mark_set(esw, rep, MLX5_ESW_VPT_HOST_FN);
2174 mlx5_esw_offloads_rep_mark_set(esw, rep, MLX5_ESW_VPT_VF);
2175 mlx5_esw_offloads_rep_mark_set(esw, rep, MLX5_ESW_VPT_SF);
2176 return 0;
2177
2178insert_err:
2179 kfree(rep);
2180 return err;
2181}
2182
2183static void mlx5_esw_offloads_rep_cleanup(struct mlx5_eswitch *esw,
2184 struct mlx5_eswitch_rep *rep)
2185{
2186 xa_erase(&esw->offloads.vport_reps, rep->vport);
2187 kfree(rep);
2188}
2189
2190void esw_offloads_cleanup_reps(struct mlx5_eswitch *esw)
2191{
2192 struct mlx5_eswitch_rep *rep;
2193 unsigned long i;
e8d31c4d 2194
47dd7e60
PP		 2195 mlx5_esw_for_each_rep(esw, i, rep)
2196 mlx5_esw_offloads_rep_cleanup(esw, rep);
2197 xa_destroy(&esw->offloads.vport_reps);
2198}
2199
2200int esw_offloads_init_reps(struct mlx5_eswitch *esw)
2201{
2202 struct mlx5_vport *vport;
2203 unsigned long i;
2204 int err;
2205
2206 xa_init(&esw->offloads.vport_reps);
2207
2208 mlx5_esw_for_each_vport(esw, i, vport) {
2209 err = mlx5_esw_offloads_rep_init(esw, vport);
2210 if (err)
2211 goto err;
2212 }
e8d31c4d 2213 return 0;
47dd7e60
PP		 2214
2215err:
2216 esw_offloads_cleanup_reps(esw);
2217 return err;
e8d31c4d
MB		 2218}
2219
c9b99abc
BW		 2220static void __esw_offloads_unload_rep(struct mlx5_eswitch *esw,
2221 struct mlx5_eswitch_rep *rep, u8 rep_type)
2222{
8693115a 2223 if (atomic_cmpxchg(&rep->rep_data[rep_type].state,
6f4e0219 2224 REP_LOADED, REP_REGISTERED) == REP_LOADED)
8693115a 2225 esw->offloads.rep_ops[rep_type]->unload(rep);
c9b99abc
BW		 2226}
2227
d7f33a45
VP		 2228static void __unload_reps_sf_vport(struct mlx5_eswitch *esw, u8 rep_type)
2229{
2230 struct mlx5_eswitch_rep *rep;
47dd7e60 2231 unsigned long i;
d7f33a45
VP		 2232
2233 mlx5_esw_for_each_sf_rep(esw, i, rep)
2234 __esw_offloads_unload_rep(esw, rep, rep_type);
2235}
2236
4110fc59 2237static void __unload_reps_all_vport(struct mlx5_eswitch *esw, u8 rep_type)
6ed1803a
MB		 2238{
2239 struct mlx5_eswitch_rep *rep;
47dd7e60 2240 unsigned long i;
4110fc59 2241
d7f33a45
VP		 2242 __unload_reps_sf_vport(esw, rep_type);
2243
47dd7e60 2244 mlx5_esw_for_each_vf_rep(esw, i, rep)
4110fc59 2245 __esw_offloads_unload_rep(esw, rep, rep_type);
c9b99abc 2246
81cd229c
BW		 2247 if (mlx5_ecpf_vport_exists(esw->dev)) {
2248 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_ECPF);
2249 __esw_offloads_unload_rep(esw, rep, rep_type);
2250 }
2251
2252 if (mlx5_core_is_ecpf_esw_manager(esw->dev)) {
2253 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_PF);
2254 __esw_offloads_unload_rep(esw, rep, rep_type);
2255 }
2256
879c8f84 2257 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_UPLINK);
c9b99abc 2258 __esw_offloads_unload_rep(esw, rep, rep_type);
6ed1803a
MB		 2259}
2260
d970812b 2261int mlx5_esw_offloads_rep_load(struct mlx5_eswitch *esw, u16 vport_num)
a4b97ab4 2262{
c2d7712c
BW		 2263 struct mlx5_eswitch_rep *rep;
2264 int rep_type;
a4b97ab4
MB		 2265 int err;
2266
c2d7712c
BW		 2267 rep = mlx5_eswitch_get_rep(esw, vport_num);
2268 for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++)
2269 if (atomic_cmpxchg(&rep->rep_data[rep_type].state,
2270 REP_REGISTERED, REP_LOADED) == REP_REGISTERED) {
2271 err = esw->offloads.rep_ops[rep_type]->load(esw->dev, rep);
2272 if (err)
2273 goto err_reps;
2274 }
2275
2276 return 0;
a4b97ab4
MB		 2277
2278err_reps:
c2d7712c
BW		 2279 atomic_set(&rep->rep_data[rep_type].state, REP_REGISTERED);
2280 for (--rep_type; rep_type >= 0; rep_type--)
2281 __esw_offloads_unload_rep(esw, rep, rep_type);
6ed1803a
MB		 2282 return err;
2283}
2284
d970812b 2285void mlx5_esw_offloads_rep_unload(struct mlx5_eswitch *esw, u16 vport_num)
c2d7712c
BW		 2286{
2287 struct mlx5_eswitch_rep *rep;
2288 int rep_type;
2289
c2d7712c
BW		 2290 rep = mlx5_eswitch_get_rep(esw, vport_num);
2291 for (rep_type = NUM_REP_TYPES - 1; rep_type >= 0; rep_type--)
2292 __esw_offloads_unload_rep(esw, rep, rep_type);
2293}
2294
38679b5a
PP		 2295int esw_offloads_load_rep(struct mlx5_eswitch *esw, u16 vport_num)
2296{
2297 int err;
2298
2299 if (esw->mode != MLX5_ESWITCH_OFFLOADS)
2300 return 0;
2301
865d6d1c
RD		 2302 if (vport_num != MLX5_VPORT_UPLINK) {
2303 err = mlx5_esw_offloads_devlink_port_register(esw, vport_num);
2304 if (err)
2305 return err;
2306 }
c7eddc60 2307
38679b5a 2308 err = mlx5_esw_offloads_rep_load(esw, vport_num);
c7eddc60
PP		 2309 if (err)
2310 goto load_err;
2311 return err;
2312
2313load_err:
865d6d1c
RD		 2314 if (vport_num != MLX5_VPORT_UPLINK)
2315 mlx5_esw_offloads_devlink_port_unregister(esw, vport_num);
38679b5a
PP		 2316 return err;
2317}
2318
2319void esw_offloads_unload_rep(struct mlx5_eswitch *esw, u16 vport_num)
2320{
2321 if (esw->mode != MLX5_ESWITCH_OFFLOADS)
2322 return;
2323
2324 mlx5_esw_offloads_rep_unload(esw, vport_num);
865d6d1c
RD		 2325
2326 if (vport_num != MLX5_VPORT_UPLINK)
2327 mlx5_esw_offloads_devlink_port_unregister(esw, vport_num);
38679b5a
PP		 2328}
2329
db202995
MB		 2330static int esw_set_uplink_slave_ingress_root(struct mlx5_core_dev *master,
2331 struct mlx5_core_dev *slave)
2332{
2333 u32 in[MLX5_ST_SZ_DW(set_flow_table_root_in)] = {};
2334 u32 out[MLX5_ST_SZ_DW(set_flow_table_root_out)] = {};
2335 struct mlx5_eswitch *esw;
2336 struct mlx5_flow_root_namespace *root;
2337 struct mlx5_flow_namespace *ns;
2338 struct mlx5_vport *vport;
2339 int err;
2340
2341 MLX5_SET(set_flow_table_root_in, in, opcode,
2342 MLX5_CMD_OP_SET_FLOW_TABLE_ROOT);
2343 MLX5_SET(set_flow_table_root_in, in, table_type, FS_FT_ESW_INGRESS_ACL);
2344 MLX5_SET(set_flow_table_root_in, in, other_vport, 1);
2345 MLX5_SET(set_flow_table_root_in, in, vport_number, MLX5_VPORT_UPLINK);
2346
2347 if (master) {
2348 esw = master->priv.eswitch;
2349 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_UPLINK);
2350 MLX5_SET(set_flow_table_root_in, in, table_of_other_vport, 1);
2351 MLX5_SET(set_flow_table_root_in, in, table_vport_number,
2352 MLX5_VPORT_UPLINK);
2353
2354 ns = mlx5_get_flow_vport_acl_namespace(master,
2355 MLX5_FLOW_NAMESPACE_ESW_INGRESS,
2356 vport->index);
2357 root = find_root(&ns->node);
2358 mutex_lock(&root->chain_lock);
2359
2360 MLX5_SET(set_flow_table_root_in, in,
2361 table_eswitch_owner_vhca_id_valid, 1);
2362 MLX5_SET(set_flow_table_root_in, in,
2363 table_eswitch_owner_vhca_id,
2364 MLX5_CAP_GEN(master, vhca_id));
2365 MLX5_SET(set_flow_table_root_in, in, table_id,
2366 root->root_ft->id);
2367 } else {
2368 esw = slave->priv.eswitch;
2369 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_UPLINK);
2370 ns = mlx5_get_flow_vport_acl_namespace(slave,
2371 MLX5_FLOW_NAMESPACE_ESW_INGRESS,
2372 vport->index);
2373 root = find_root(&ns->node);
2374 mutex_lock(&root->chain_lock);
2375 MLX5_SET(set_flow_table_root_in, in, table_id, root->root_ft->id);
2376 }
2377
2378 err = mlx5_cmd_exec(slave, in, sizeof(in), out, sizeof(out));
2379 mutex_unlock(&root->chain_lock);
2380
2381 return err;
2382}
2383
2384static int esw_set_slave_root_fdb(struct mlx5_core_dev *master,
2385 struct mlx5_core_dev *slave)
2386{
2387 u32 in[MLX5_ST_SZ_DW(set_flow_table_root_in)] = {};
2388 u32 out[MLX5_ST_SZ_DW(set_flow_table_root_out)] = {};
2389 struct mlx5_flow_root_namespace *root;
2390 struct mlx5_flow_namespace *ns;
2391 int err;
2392
2393 MLX5_SET(set_flow_table_root_in, in, opcode,
2394 MLX5_CMD_OP_SET_FLOW_TABLE_ROOT);
2395 MLX5_SET(set_flow_table_root_in, in, table_type,
2396 FS_FT_FDB);
2397
2398 if (master) {
2399 ns = mlx5_get_flow_namespace(master,
2400 MLX5_FLOW_NAMESPACE_FDB);
2401 root = find_root(&ns->node);
2402 mutex_lock(&root->chain_lock);
2403 MLX5_SET(set_flow_table_root_in, in,
2404 table_eswitch_owner_vhca_id_valid, 1);
2405 MLX5_SET(set_flow_table_root_in, in,
2406 table_eswitch_owner_vhca_id,
2407 MLX5_CAP_GEN(master, vhca_id));
2408 MLX5_SET(set_flow_table_root_in, in, table_id,
2409 root->root_ft->id);
2410 } else {
2411 ns = mlx5_get_flow_namespace(slave,
2412 MLX5_FLOW_NAMESPACE_FDB);
2413 root = find_root(&ns->node);
2414 mutex_lock(&root->chain_lock);
2415 MLX5_SET(set_flow_table_root_in, in, table_id,
2416 root->root_ft->id);
2417 }
2418
2419 err = mlx5_cmd_exec(slave, in, sizeof(in), out, sizeof(out));
2420 mutex_unlock(&root->chain_lock);
2421
2422 return err;
2423}
2424
2425static int __esw_set_master_egress_rule(struct mlx5_core_dev *master,
2426 struct mlx5_core_dev *slave,
2427 struct mlx5_vport *vport,
2428 struct mlx5_flow_table *acl)
2429{
2430 struct mlx5_flow_handle *flow_rule = NULL;
2431 struct mlx5_flow_destination dest = {};
2432 struct mlx5_flow_act flow_act = {};
2433 struct mlx5_flow_spec *spec;
2434 int err = 0;
2435 void *misc;
2436
2437 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
2438 if (!spec)
2439 return -ENOMEM;
2440
2441 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS;
2442 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value,
2443 misc_parameters);
2444 MLX5_SET(fte_match_set_misc, misc, source_port, MLX5_VPORT_UPLINK);
2445 MLX5_SET(fte_match_set_misc, misc, source_eswitch_owner_vhca_id,
2446 MLX5_CAP_GEN(slave, vhca_id));
2447
2448 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters);
2449 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
2450 MLX5_SET_TO_ONES(fte_match_set_misc, misc,
2451 source_eswitch_owner_vhca_id);
2452
2453 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
2454 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
2455 dest.vport.num = slave->priv.eswitch->manager_vport;
2456 dest.vport.vhca_id = MLX5_CAP_GEN(slave, vhca_id);
2457 dest.vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID;
2458
2459 flow_rule = mlx5_add_flow_rules(acl, spec, &flow_act,
2460 &dest, 1);
2461 if (IS_ERR(flow_rule))
2462 err = PTR_ERR(flow_rule);
2463 else
2464 vport->egress.offloads.bounce_rule = flow_rule;
2465
2466 kvfree(spec);
2467 return err;
2468}
2469
2470static int esw_set_master_egress_rule(struct mlx5_core_dev *master,
2471 struct mlx5_core_dev *slave)
2472{
2473 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
2474 struct mlx5_eswitch *esw = master->priv.eswitch;
2475 struct mlx5_flow_table_attr ft_attr = {
2476 .max_fte = 1, .prio = 0, .level = 0,
2477 };
2478 struct mlx5_flow_namespace *egress_ns;
2479 struct mlx5_flow_table *acl;
2480 struct mlx5_flow_group *g;
2481 struct mlx5_vport *vport;
2482 void *match_criteria;
2483 u32 *flow_group_in;
2484 int err;
2485
2486 vport = mlx5_eswitch_get_vport(esw, esw->manager_vport);
2487 if (IS_ERR(vport))
2488 return PTR_ERR(vport);
2489
2490 egress_ns = mlx5_get_flow_vport_acl_namespace(master,
2491 MLX5_FLOW_NAMESPACE_ESW_EGRESS,
2492 vport->index);
2493 if (!egress_ns)
2494 return -EINVAL;
2495
2496 if (vport->egress.acl)
2497 return -EINVAL;
2498
2499 flow_group_in = kvzalloc(inlen, GFP_KERNEL);
2500 if (!flow_group_in)
2501 return -ENOMEM;
2502
2503 acl = mlx5_create_vport_flow_table(egress_ns, &ft_attr, vport->vport);
2504 if (IS_ERR(acl)) {
2505 err = PTR_ERR(acl);
2506 goto out;
2507 }
2508
2509 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in,
2510 match_criteria);
2511 MLX5_SET_TO_ONES(fte_match_param, match_criteria,
2512 misc_parameters.source_port);
2513 MLX5_SET_TO_ONES(fte_match_param, match_criteria,
2514 misc_parameters.source_eswitch_owner_vhca_id);
2515 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
2516 MLX5_MATCH_MISC_PARAMETERS);
2517
2518 MLX5_SET(create_flow_group_in, flow_group_in,
2519 source_eswitch_owner_vhca_id_valid, 1);
2520 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0);
2521 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 0);
2522
2523 g = mlx5_create_flow_group(acl, flow_group_in);
2524 if (IS_ERR(g)) {
2525 err = PTR_ERR(g);
2526 goto err_group;
2527 }
2528
2529 err = __esw_set_master_egress_rule(master, slave, vport, acl);
2530 if (err)
2531 goto err_rule;
2532
2533 vport->egress.acl = acl;
2534 vport->egress.offloads.bounce_grp = g;
2535
2536 kvfree(flow_group_in);
2537
2538 return 0;
2539
2540err_rule:
2541 mlx5_destroy_flow_group(g);
2542err_group:
2543 mlx5_destroy_flow_table(acl);
2544out:
2545 kvfree(flow_group_in);
2546 return err;
2547}
2548
2549static void esw_unset_master_egress_rule(struct mlx5_core_dev *dev)
2550{
2551 struct mlx5_vport *vport;
2552
2553 vport = mlx5_eswitch_get_vport(dev->priv.eswitch,
2554 dev->priv.eswitch->manager_vport);
2555
2556 esw_acl_egress_ofld_cleanup(vport);
2557}
2558
2559int mlx5_eswitch_offloads_config_single_fdb(struct mlx5_eswitch *master_esw,
2560 struct mlx5_eswitch *slave_esw)
2561{
2562 int err;
2563
2564 err = esw_set_uplink_slave_ingress_root(master_esw->dev,
2565 slave_esw->dev);
2566 if (err)
2567 return -EINVAL;
2568
2569 err = esw_set_slave_root_fdb(master_esw->dev,
2570 slave_esw->dev);
2571 if (err)
2572 goto err_fdb;
2573
2574 err = esw_set_master_egress_rule(master_esw->dev,
2575 slave_esw->dev);
2576 if (err)
2577 goto err_acl;
2578
2579 return err;
2580
2581err_acl:
2582 esw_set_slave_root_fdb(NULL, slave_esw->dev);
2583
2584err_fdb:
2585 esw_set_uplink_slave_ingress_root(NULL, slave_esw->dev);
2586
2587 return err;
2588}
2589
2590void mlx5_eswitch_offloads_destroy_single_fdb(struct mlx5_eswitch *master_esw,
2591 struct mlx5_eswitch *slave_esw)
2592{
2593 esw_unset_master_egress_rule(master_esw->dev);
2594 esw_set_slave_root_fdb(NULL, slave_esw->dev);
2595 esw_set_uplink_slave_ingress_root(NULL, slave_esw->dev);
2596}
2597
ac004b83
RD		 2598#define ESW_OFFLOADS_DEVCOM_PAIR (0)
2599#define ESW_OFFLOADS_DEVCOM_UNPAIR (1)
2600
c8e6a9e6 2601static void mlx5_esw_offloads_rep_event_unpair(struct mlx5_eswitch *esw)
ac004b83 2602{
c8e6a9e6
MB		 2603 const struct mlx5_eswitch_rep_ops *ops;
2604 struct mlx5_eswitch_rep *rep;
2605 unsigned long i;
2606 u8 rep_type;
2607
2608 mlx5_esw_for_each_rep(esw, i, rep) {
2609 rep_type = NUM_REP_TYPES;
2610 while (rep_type--) {
2611 ops = esw->offloads.rep_ops[rep_type];
2612 if (atomic_read(&rep->rep_data[rep_type].state) == REP_LOADED &&
2613 ops->event)
2614 ops->event(esw, rep, MLX5_SWITCHDEV_EVENT_UNPAIR, NULL);
2615 }
2616 }
ac004b83
RD		 2617}
2618
2619static void mlx5_esw_offloads_unpair(struct mlx5_eswitch *esw)
2620{
d956873f 2621#if IS_ENABLED(CONFIG_MLX5_CLS_ACT)
04de7dda 2622 mlx5e_tc_clean_fdb_peer_flows(esw);
d956873f 2623#endif
c8e6a9e6 2624 mlx5_esw_offloads_rep_event_unpair(esw);
ac004b83
RD		 2625 esw_del_fdb_peer_miss_rules(esw);
2626}
2627
c8e6a9e6
MB		 2628static int mlx5_esw_offloads_pair(struct mlx5_eswitch *esw,
2629 struct mlx5_eswitch *peer_esw)
2630{
2631 const struct mlx5_eswitch_rep_ops *ops;
2632 struct mlx5_eswitch_rep *rep;
2633 unsigned long i;
2634 u8 rep_type;
2635 int err;
2636
2637 err = esw_add_fdb_peer_miss_rules(esw, peer_esw->dev);
2638 if (err)
2639 return err;
2640
2641 mlx5_esw_for_each_rep(esw, i, rep) {
2642 for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++) {
2643 ops = esw->offloads.rep_ops[rep_type];
2644 if (atomic_read(&rep->rep_data[rep_type].state) == REP_LOADED &&
2645 ops->event) {
2646 err = ops->event(esw, rep, MLX5_SWITCHDEV_EVENT_PAIR, peer_esw);
2647 if (err)
2648 goto err_out;
2649 }
2650 }
2651 }
2652
2653 return 0;
2654
2655err_out:
2656 mlx5_esw_offloads_unpair(esw);
2657 return err;
2658}
2659
8463daf1
MG		 2660static int mlx5_esw_offloads_set_ns_peer(struct mlx5_eswitch *esw,
2661 struct mlx5_eswitch *peer_esw,
2662 bool pair)
2663{
2664 struct mlx5_flow_root_namespace *peer_ns;
2665 struct mlx5_flow_root_namespace *ns;
2666 int err;
2667
2668 peer_ns = peer_esw->dev->priv.steering->fdb_root_ns;
2669 ns = esw->dev->priv.steering->fdb_root_ns;
2670
2671 if (pair) {
2672 err = mlx5_flow_namespace_set_peer(ns, peer_ns);
2673 if (err)
2674 return err;
2675
e53e6655 2676 err = mlx5_flow_namespace_set_peer(peer_ns, ns);
8463daf1
MG		 2677 if (err) {
2678 mlx5_flow_namespace_set_peer(ns, NULL);
2679 return err;
2680 }
2681 } else {
2682 mlx5_flow_namespace_set_peer(ns, NULL);
2683 mlx5_flow_namespace_set_peer(peer_ns, NULL);
2684 }
2685
2686 return 0;
2687}
2688
ac004b83
RD		 2689static int mlx5_esw_offloads_devcom_event(int event,
2690 void *my_data,
2691 void *event_data)
2692{
2693 struct mlx5_eswitch *esw = my_data;
ac004b83 2694 struct mlx5_devcom *devcom = esw->dev->priv.devcom;
8463daf1 2695 struct mlx5_eswitch *peer_esw = event_data;
ac004b83
RD		 2696 int err;
2697
2698 switch (event) {
2699 case ESW_OFFLOADS_DEVCOM_PAIR:
dd3fddb8
RD		 2700 if (mlx5_get_next_phys_dev(esw->dev) != peer_esw->dev)
2701 break;
2702
a5641cb5
JL		 2703 if (mlx5_eswitch_vport_match_metadata_enabled(esw) !=
2704 mlx5_eswitch_vport_match_metadata_enabled(peer_esw))
2705 break;
2706
8463daf1 2707 err = mlx5_esw_offloads_set_ns_peer(esw, peer_esw, true);
ac004b83
RD		 2708 if (err)
2709 goto err_out;
8463daf1
MG		 2710 err = mlx5_esw_offloads_pair(esw, peer_esw);
2711 if (err)
2712 goto err_peer;
ac004b83
RD		 2713
2714 err = mlx5_esw_offloads_pair(peer_esw, esw);
2715 if (err)
2716 goto err_pair;
2717
2718 mlx5_devcom_set_paired(devcom, MLX5_DEVCOM_ESW_OFFLOADS, true);
2719 break;
2720
2721 case ESW_OFFLOADS_DEVCOM_UNPAIR:
2722 if (!mlx5_devcom_is_paired(devcom, MLX5_DEVCOM_ESW_OFFLOADS))
2723 break;
2724
2725 mlx5_devcom_set_paired(devcom, MLX5_DEVCOM_ESW_OFFLOADS, false);
2726 mlx5_esw_offloads_unpair(peer_esw);
2727 mlx5_esw_offloads_unpair(esw);
8463daf1 2728 mlx5_esw_offloads_set_ns_peer(esw, peer_esw, false);
ac004b83
RD		 2729 break;
2730 }
2731
2732 return 0;
2733
2734err_pair:
2735 mlx5_esw_offloads_unpair(esw);
8463daf1
MG		 2736err_peer:
2737 mlx5_esw_offloads_set_ns_peer(esw, peer_esw, false);
ac004b83
RD		 2738err_out:
2739 mlx5_core_err(esw->dev, "esw offloads devcom event failure, event %u err %d",
2740 event, err);
2741 return err;
2742}
2743
2744static void esw_offloads_devcom_init(struct mlx5_eswitch *esw)
2745{
2746 struct mlx5_devcom *devcom = esw->dev->priv.devcom;
2747
04de7dda
RD		 2748 INIT_LIST_HEAD(&esw->offloads.peer_flows);
2749 mutex_init(&esw->offloads.peer_mutex);
2750
ac004b83
RD		 2751 if (!MLX5_CAP_ESW(esw->dev, merged_eswitch))
2752 return;
2753
2754 mlx5_devcom_register_component(devcom,
2755 MLX5_DEVCOM_ESW_OFFLOADS,
2756 mlx5_esw_offloads_devcom_event,
2757 esw);
2758
2759 mlx5_devcom_send_event(devcom,
2760 MLX5_DEVCOM_ESW_OFFLOADS,
2761 ESW_OFFLOADS_DEVCOM_PAIR, esw);
2762}
2763
2764static void esw_offloads_devcom_cleanup(struct mlx5_eswitch *esw)
2765{
2766 struct mlx5_devcom *devcom = esw->dev->priv.devcom;
2767
2768 if (!MLX5_CAP_ESW(esw->dev, merged_eswitch))
2769 return;
2770
2771 mlx5_devcom_send_event(devcom, MLX5_DEVCOM_ESW_OFFLOADS,
2772 ESW_OFFLOADS_DEVCOM_UNPAIR, esw);
2773
2774 mlx5_devcom_unregister_component(devcom, MLX5_DEVCOM_ESW_OFFLOADS);
2775}
2776
7bf481d7 2777bool mlx5_esw_vport_match_metadata_supported(const struct mlx5_eswitch *esw)
92ab1eb3
JL		 2778{
2779 if (!MLX5_CAP_ESW(esw->dev, esw_uplink_ingress_acl))
2780 return false;
2781
2782 if (!(MLX5_CAP_ESW_FLOWTABLE(esw->dev, fdb_to_vport_reg_c_id) &
2783 MLX5_FDB_TO_VPORT_REG_C_0))
2784 return false;
2785
2786 if (!MLX5_CAP_ESW_FLOWTABLE(esw->dev, flow_source))
2787 return false;
2788
2789 if (mlx5_core_is_ecpf_esw_manager(esw->dev) ||
2790 mlx5_ecpf_vport_exists(esw->dev))
2791 return false;
2792
2793 return true;
2794}
2795
133dcfc5
VP		 2796u32 mlx5_esw_match_metadata_alloc(struct mlx5_eswitch *esw)
2797{
7cd7becd 2798 u32 vport_end_ida = (1 << ESW_VPORT_BITS) - 1;
2799 u32 max_pf_num = (1 << ESW_PFNUM_BITS) - 1;
2800 u32 pf_num;
133dcfc5
VP		 2801 int id;
2802
7cd7becd 2803 /* Only 4 bits of pf_num */
2804 pf_num = PCI_FUNC(esw->dev->pdev->devfn);
2805 if (pf_num > max_pf_num)
2806 return 0;
133dcfc5 2807
7cd7becd 2808 /* Metadata is 4 bits of PFNUM and 12 bits of unique id */
2809 /* Use only non-zero vport_id (1-4095) for all PF's */
2810 id = ida_alloc_range(&esw->offloads.vport_metadata_ida, 1, vport_end_ida, GFP_KERNEL);
2811 if (id < 0)
2812 return 0;
2813 id = (pf_num << ESW_VPORT_BITS) | id;
2814 return id;
133dcfc5
VP		 2815}
2816
2817void mlx5_esw_match_metadata_free(struct mlx5_eswitch *esw, u32 metadata)
2818{
7cd7becd 2819 u32 vport_bit_mask = (1 << ESW_VPORT_BITS) - 1;
2820
2821 /* Metadata contains only 12 bits of actual ida id */
2822 ida_free(&esw->offloads.vport_metadata_ida, metadata & vport_bit_mask);
133dcfc5
VP		 2823}
2824
2825static int esw_offloads_vport_metadata_setup(struct mlx5_eswitch *esw,
2826 struct mlx5_vport *vport)
2827{
133dcfc5
VP		 2828 vport->default_metadata = mlx5_esw_match_metadata_alloc(esw);
2829 vport->metadata = vport->default_metadata;
2830 return vport->metadata ? 0 : -ENOSPC;
2831}
2832
2833static void esw_offloads_vport_metadata_cleanup(struct mlx5_eswitch *esw,
2834 struct mlx5_vport *vport)
2835{
406493a5 2836 if (!vport->default_metadata)
133dcfc5
VP		 2837 return;
2838
2839 WARN_ON(vport->metadata != vport->default_metadata);
2840 mlx5_esw_match_metadata_free(esw, vport->default_metadata);
2841}
2842
fc99c3d6
VP		 2843static void esw_offloads_metadata_uninit(struct mlx5_eswitch *esw)
2844{
2845 struct mlx5_vport *vport;
47dd7e60 2846 unsigned long i;
fc99c3d6
VP		 2847
2848 if (!mlx5_eswitch_vport_match_metadata_enabled(esw))
2849 return;
2850
47dd7e60 2851 mlx5_esw_for_each_vport(esw, i, vport)
fc99c3d6
VP		 2852 esw_offloads_vport_metadata_cleanup(esw, vport);
2853}
2854
2855static int esw_offloads_metadata_init(struct mlx5_eswitch *esw)
2856{
2857 struct mlx5_vport *vport;
47dd7e60 2858 unsigned long i;
fc99c3d6 2859 int err;
fc99c3d6
VP		 2860
2861 if (!mlx5_eswitch_vport_match_metadata_enabled(esw))
2862 return 0;
2863
47dd7e60 2864 mlx5_esw_for_each_vport(esw, i, vport) {
fc99c3d6
VP		 2865 err = esw_offloads_vport_metadata_setup(esw, vport);
2866 if (err)
2867 goto metadata_err;
2868 }
2869
2870 return 0;
2871
2872metadata_err:
2873 esw_offloads_metadata_uninit(esw);
2874 return err;
2875}
2876
7bf481d7
PP		 2877int mlx5_esw_offloads_vport_metadata_set(struct mlx5_eswitch *esw, bool enable)
2878{
2879 int err = 0;
2880
2881 down_write(&esw->mode_lock);
2882 if (esw->mode != MLX5_ESWITCH_NONE) {
2883 err = -EBUSY;
2884 goto done;
2885 }
2886 if (!mlx5_esw_vport_match_metadata_supported(esw)) {
2887 err = -EOPNOTSUPP;
2888 goto done;
2889 }
2890 if (enable)
2891 esw->flags |= MLX5_ESWITCH_VPORT_MATCH_METADATA;
2892 else
2893 esw->flags &= ~MLX5_ESWITCH_VPORT_MATCH_METADATA;
2894done:
2895 up_write(&esw->mode_lock);
2896 return err;
2897}
2898
748da30b 2899int
89a0f1fb
PP		 2900esw_vport_create_offloads_acl_tables(struct mlx5_eswitch *esw,
2901 struct mlx5_vport *vport)
7445cfb1 2902{
7445cfb1
JL		 2903 int err;
2904
07bab950 2905 err = esw_acl_ingress_ofld_setup(esw, vport);
89a0f1fb 2906 if (err)
fc99c3d6 2907 return err;
7445cfb1 2908
2c40db2f
PP		 2909 err = esw_acl_egress_ofld_setup(esw, vport);
2910 if (err)
2911 goto egress_err;
07bab950
VP		 2912
2913 return 0;
2914
2915egress_err:
2916 esw_acl_ingress_ofld_cleanup(esw, vport);
89a0f1fb
PP		 2917 return err;
2918}
18486737 2919
748da30b 2920void
89a0f1fb
PP		 2921esw_vport_destroy_offloads_acl_tables(struct mlx5_eswitch *esw,
2922 struct mlx5_vport *vport)
2923{
ea651a86 2924 esw_acl_egress_ofld_cleanup(vport);
07bab950 2925 esw_acl_ingress_ofld_cleanup(esw, vport);
89a0f1fb 2926}
7445cfb1 2927
748da30b 2928static int esw_create_uplink_offloads_acl_tables(struct mlx5_eswitch *esw)
7445cfb1
JL		 2929{
2930 struct mlx5_vport *vport;
18486737 2931
748da30b 2932 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_UPLINK);
7bef147a
SM		 2933 if (IS_ERR(vport))
2934 return PTR_ERR(vport);
2935
4e9a9ef7 2936 return esw_vport_create_offloads_acl_tables(esw, vport);
18486737
EB		 2937}
2938
748da30b 2939static void esw_destroy_uplink_offloads_acl_tables(struct mlx5_eswitch *esw)
18486737 2940{
786ef904 2941 struct mlx5_vport *vport;
7445cfb1 2942
748da30b 2943 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_UPLINK);
7bef147a
SM		 2944 if (IS_ERR(vport))
2945 return;
2946
748da30b 2947 esw_vport_destroy_offloads_acl_tables(esw, vport);
18486737
EB		 2948}
2949
db202995
MB		 2950int mlx5_eswitch_reload_reps(struct mlx5_eswitch *esw)
2951{
2952 struct mlx5_eswitch_rep *rep;
2953 unsigned long i;
2954 int ret;
2955
2956 if (!esw || esw->mode != MLX5_ESWITCH_OFFLOADS)
2957 return 0;
2958
2959 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_UPLINK);
2960 if (atomic_read(&rep->rep_data[REP_ETH].state) != REP_LOADED)
2961 return 0;
2962
2963 ret = mlx5_esw_offloads_rep_load(esw, MLX5_VPORT_UPLINK);
2964 if (ret)
2965 return ret;
2966
2967 mlx5_esw_for_each_rep(esw, i, rep) {
2968 if (atomic_read(&rep->rep_data[REP_ETH].state) == REP_LOADED)
2969 mlx5_esw_offloads_rep_load(esw, rep->vport);
2970 }
2971
2972 return 0;
2973}
2974
062f4bf4 2975static int esw_offloads_steering_init(struct mlx5_eswitch *esw)
6ed1803a 2976{
34ca6535 2977 struct mlx5_esw_indir_table *indir;
6ed1803a
MB		 2978 int err;
2979
5c1d260e 2980 memset(&esw->fdb_table.offloads, 0, sizeof(struct offloads_fdb));
f8d1edda
PP		 2981 mutex_init(&esw->fdb_table.offloads.vports.lock);
2982 hash_init(esw->fdb_table.offloads.vports.table);
7dc84de9 2983 atomic64_set(&esw->user_count, 0);
e52c2802 2984
34ca6535
VB		 2985 indir = mlx5_esw_indir_table_init();
2986 if (IS_ERR(indir)) {
2987 err = PTR_ERR(indir);
2988 goto create_indir_err;
2989 }
2990 esw->fdb_table.offloads.indir = indir;
2991
748da30b 2992 err = esw_create_uplink_offloads_acl_tables(esw);
7445cfb1 2993 if (err)
f8d1edda 2994 goto create_acl_err;
18486737 2995
8d6bd3c3 2996 err = esw_create_offloads_table(esw);
c930a3ad 2997 if (err)
11b717d6 2998 goto create_offloads_err;
c930a3ad 2999
11b717d6 3000 err = esw_create_restore_table(esw);
c930a3ad 3001 if (err)
11b717d6
PB		 3002 goto create_restore_err;
3003
0da3c12d 3004 err = esw_create_offloads_fdb_tables(esw);
11b717d6
PB		 3005 if (err)
3006 goto create_fdb_err;
c930a3ad 3007
8d6bd3c3 3008 err = esw_create_vport_rx_group(esw);
c930a3ad
OG		 3009 if (err)
3010 goto create_fg_err;
3011
3012 return 0;
3013
3014create_fg_err:
1967ce6e 3015 esw_destroy_offloads_fdb_tables(esw);
7445cfb1 3016create_fdb_err:
11b717d6
PB		 3017 esw_destroy_restore_table(esw);
3018create_restore_err:
3019 esw_destroy_offloads_table(esw);
3020create_offloads_err:
748da30b 3021 esw_destroy_uplink_offloads_acl_tables(esw);
f8d1edda 3022create_acl_err:
34ca6535
VB		 3023 mlx5_esw_indir_table_destroy(esw->fdb_table.offloads.indir);
3024create_indir_err:
f8d1edda 3025 mutex_destroy(&esw->fdb_table.offloads.vports.lock);
c930a3ad
OG		 3026 return err;
3027}
3028
eca8cc38
BW		 3029static void esw_offloads_steering_cleanup(struct mlx5_eswitch *esw)
3030{
3031 esw_destroy_vport_rx_group(esw);
eca8cc38 3032 esw_destroy_offloads_fdb_tables(esw);
11b717d6
PB		 3033 esw_destroy_restore_table(esw);
3034 esw_destroy_offloads_table(esw);
748da30b 3035 esw_destroy_uplink_offloads_acl_tables(esw);
34ca6535 3036 mlx5_esw_indir_table_destroy(esw->fdb_table.offloads.indir);
f8d1edda 3037 mutex_destroy(&esw->fdb_table.offloads.vports.lock);
eca8cc38
BW		 3038}
3039
7e736f9a
PP		 3040static void
3041esw_vfs_changed_event_handler(struct mlx5_eswitch *esw, const u32 *out)
a3888f33 3042{
5ccf2770 3043 bool host_pf_disabled;
7e736f9a 3044 u16 new_num_vfs;
a3888f33 3045
7e736f9a
PP		 3046 new_num_vfs = MLX5_GET(query_esw_functions_out, out,
3047 host_params_context.host_num_of_vfs);
5ccf2770
BW		 3048 host_pf_disabled = MLX5_GET(query_esw_functions_out, out,
3049 host_params_context.host_pf_disabled);
a3888f33 3050
7e736f9a
PP		 3051 if (new_num_vfs == esw->esw_funcs.num_vfs || host_pf_disabled)
3052 return;
a3888f33
BW		 3053
3054 /* Number of VFs can only change from "0 to x" or "x to 0". */
cd56f929 3055 if (esw->esw_funcs.num_vfs > 0) {
23bb50cf 3056 mlx5_eswitch_unload_vf_vports(esw, esw->esw_funcs.num_vfs);
a3888f33 3057 } else {
7e736f9a 3058 int err;
a3888f33 3059
23bb50cf
BW		 3060 err = mlx5_eswitch_load_vf_vports(esw, new_num_vfs,
3061 MLX5_VPORT_UC_ADDR_CHANGE);
a3888f33 3062 if (err)
7e736f9a 3063 return;
a3888f33 3064 }
7e736f9a 3065 esw->esw_funcs.num_vfs = new_num_vfs;
a3888f33
BW		 3066}
3067
7e736f9a 3068static void esw_functions_changed_event_handler(struct work_struct *work)
ac35dcd6 3069{
7e736f9a
PP		 3070 struct mlx5_host_work *host_work;
3071 struct mlx5_eswitch *esw;
dd28087c 3072 const u32 *out;
ac35dcd6 3073
7e736f9a
PP		 3074 host_work = container_of(work, struct mlx5_host_work, work);
3075 esw = host_work->esw;
a3888f33 3076
dd28087c
PP		 3077 out = mlx5_esw_query_functions(esw->dev);
3078 if (IS_ERR(out))
7e736f9a 3079 goto out;
a3888f33 3080
7e736f9a 3081 esw_vfs_changed_event_handler(esw, out);
dd28087c 3082 kvfree(out);
a3888f33 3083out:
ac35dcd6
VP		 3084 kfree(host_work);
3085}
3086
16fff98a 3087int mlx5_esw_funcs_changed_handler(struct notifier_block *nb, unsigned long type, void *data)
a3888f33 3088{
cd56f929 3089 struct mlx5_esw_functions *esw_funcs;
a3888f33 3090 struct mlx5_host_work *host_work;
a3888f33
BW		 3091 struct mlx5_eswitch *esw;
3092
3093 host_work = kzalloc(sizeof(*host_work), GFP_ATOMIC);
3094 if (!host_work)
3095 return NOTIFY_DONE;
3096
cd56f929
VP		 3097 esw_funcs = mlx5_nb_cof(nb, struct mlx5_esw_functions, nb);
3098 esw = container_of(esw_funcs, struct mlx5_eswitch, esw_funcs);
a3888f33
BW		 3099
3100 host_work->esw = esw;
3101
062f4bf4 3102 INIT_WORK(&host_work->work, esw_functions_changed_event_handler);
a3888f33
BW		 3103 queue_work(esw->work_queue, &host_work->work);
3104
3105 return NOTIFY_OK;
3106}
3107
a53cf949
PP		 3108static int mlx5_esw_host_number_init(struct mlx5_eswitch *esw)
3109{
3110 const u32 *query_host_out;
3111
3112 if (!mlx5_core_is_ecpf_esw_manager(esw->dev))
3113 return 0;
3114
3115 query_host_out = mlx5_esw_query_functions(esw->dev);
3116 if (IS_ERR(query_host_out))
3117 return PTR_ERR(query_host_out);
3118
3119 /* Mark non local controller with non zero controller number. */
3120 esw->offloads.host_number = MLX5_GET(query_esw_functions_out, query_host_out,
3121 host_params_context.host_number);
3122 kvfree(query_host_out);
3123 return 0;
3124}
3125
f1b9acd3
PP		 3126bool mlx5_esw_offloads_controller_valid(const struct mlx5_eswitch *esw, u32 controller)
3127{
3128 /* Local controller is always valid */
3129 if (controller == 0)
3130 return true;
3131
3132 if (!mlx5_core_is_ecpf_esw_manager(esw->dev))
3133 return false;
3134
3135 /* External host number starts with zero in device */
3136 return (controller == esw->offloads.host_number + 1);
3137}
3138
5896b972 3139int esw_offloads_enable(struct mlx5_eswitch *esw)
eca8cc38 3140{
c9355682 3141 struct mapping_ctx *reg_c0_obj_pool;
3b83b6c2 3142 struct mlx5_vport *vport;
47dd7e60 3143 unsigned long i;
2198b932 3144 u64 mapping_id;
47dd7e60 3145 int err;
eca8cc38 3146
9a64144d
MG		 3147 if (MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, reformat) &&
3148 MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, decap))
3149 esw->offloads.encap = DEVLINK_ESWITCH_ENCAP_MODE_BASIC;
3150 else
3151 esw->offloads.encap = DEVLINK_ESWITCH_ENCAP_MODE_NONE;
3152
2bb72e7e 3153 mutex_init(&esw->offloads.termtbl_mutex);
8463daf1 3154 mlx5_rdma_enable_roce(esw->dev);
eca8cc38 3155
a53cf949
PP		 3156 err = mlx5_esw_host_number_init(esw);
3157 if (err)
cd1ef966 3158 goto err_metadata;
a53cf949 3159
fc99c3d6
VP		 3160 err = esw_offloads_metadata_init(esw);
3161 if (err)
3162 goto err_metadata;
3163
332bd3a5
PP		 3164 err = esw_set_passing_vport_metadata(esw, true);
3165 if (err)
3166 goto err_vport_metadata;
c1286050 3167
2198b932
RD		 3168 mapping_id = mlx5_query_nic_system_image_guid(esw->dev);
3169
3170 reg_c0_obj_pool = mapping_create_for_id(mapping_id, MAPPING_TYPE_CHAIN,
3171 sizeof(struct mlx5_mapped_obj),
3172 ESW_REG_C0_USER_DATA_METADATA_MASK,
3173 true);
3174
c9355682
CM		 3175 if (IS_ERR(reg_c0_obj_pool)) {
3176 err = PTR_ERR(reg_c0_obj_pool);
3177 goto err_pool;
3178 }
3179 esw->offloads.reg_c0_obj_pool = reg_c0_obj_pool;
3180
7983a675
PB		 3181 err = esw_offloads_steering_init(esw);
3182 if (err)
3183 goto err_steering_init;
3184
3b83b6c2
DL		 3185 /* Representor will control the vport link state */
3186 mlx5_esw_for_each_vf_vport(esw, i, vport, esw->esw_funcs.num_vfs)
3187 vport->info.link_state = MLX5_VPORT_ADMIN_STATE_DOWN;
3188
c2d7712c
BW		 3189 /* Uplink vport rep must load first. */
3190 err = esw_offloads_load_rep(esw, MLX5_VPORT_UPLINK);
925a6acc 3191 if (err)
c2d7712c 3192 goto err_uplink;
c1286050 3193
c2d7712c 3194 err = mlx5_eswitch_enable_pf_vf_vports(esw, MLX5_VPORT_UC_ADDR_CHANGE);
eca8cc38 3195 if (err)
c2d7712c 3196 goto err_vports;
eca8cc38
BW		 3197
3198 esw_offloads_devcom_init(esw);
a3888f33 3199
eca8cc38
BW		 3200 return 0;
3201
925a6acc 3202err_vports:
c2d7712c
BW		 3203 esw_offloads_unload_rep(esw, MLX5_VPORT_UPLINK);
3204err_uplink:
7983a675 3205 esw_offloads_steering_cleanup(esw);
79949985 3206err_steering_init:
c9355682
CM		 3207 mapping_destroy(reg_c0_obj_pool);
3208err_pool:
79949985 3209 esw_set_passing_vport_metadata(esw, false);
7983a675 3210err_vport_metadata:
fc99c3d6
VP		 3211 esw_offloads_metadata_uninit(esw);
3212err_metadata:
8463daf1 3213 mlx5_rdma_disable_roce(esw->dev);
2bb72e7e 3214 mutex_destroy(&esw->offloads.termtbl_mutex);
eca8cc38
BW		 3215 return err;
3216}
3217
db7ff19e
EB		 3218static int esw_offloads_stop(struct mlx5_eswitch *esw,
3219 struct netlink_ext_ack *extack)
c930a3ad 3220{
062f4bf4 3221 int err, err1;
c930a3ad 3222
8e0aa4bc
PP		 3223 mlx5_eswitch_disable_locked(esw, false);
3224 err = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_LEGACY,
3225 MLX5_ESWITCH_IGNORE_NUM_VFS);
6c419ba8 3226 if (err) {
8c98ee77 3227 NL_SET_ERR_MSG_MOD(extack, "Failed setting eswitch to legacy");
8e0aa4bc
PP		 3228 err1 = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_OFFLOADS,
3229 MLX5_ESWITCH_IGNORE_NUM_VFS);
8c98ee77
EB		 3230 if (err1) {
3231 NL_SET_ERR_MSG_MOD(extack,
3232 "Failed setting eswitch back to offloads");
3233 }
6c419ba8 3234 }
c930a3ad
OG		 3235
3236 return err;
3237}
3238
5896b972 3239void esw_offloads_disable(struct mlx5_eswitch *esw)
c930a3ad 3240{
ac004b83 3241 esw_offloads_devcom_cleanup(esw);
5896b972 3242 mlx5_eswitch_disable_pf_vf_vports(esw);
c2d7712c 3243 esw_offloads_unload_rep(esw, MLX5_VPORT_UPLINK);
332bd3a5 3244 esw_set_passing_vport_metadata(esw, false);
eca8cc38 3245 esw_offloads_steering_cleanup(esw);
c9355682 3246 mapping_destroy(esw->offloads.reg_c0_obj_pool);
fc99c3d6 3247 esw_offloads_metadata_uninit(esw);
8463daf1 3248 mlx5_rdma_disable_roce(esw->dev);
2bb72e7e 3249 mutex_destroy(&esw->offloads.termtbl_mutex);
9a64144d 3250 esw->offloads.encap = DEVLINK_ESWITCH_ENCAP_MODE_NONE;
c930a3ad
OG		 3251}
3252
ef78618b 3253static int esw_mode_from_devlink(u16 mode, u16 *mlx5_mode)
c930a3ad
OG		 3254{
3255 switch (mode) {
3256 case DEVLINK_ESWITCH_MODE_LEGACY:
f6455de0 3257 *mlx5_mode = MLX5_ESWITCH_LEGACY;
c930a3ad
OG		 3258 break;
3259 case DEVLINK_ESWITCH_MODE_SWITCHDEV:
f6455de0 3260 *mlx5_mode = MLX5_ESWITCH_OFFLOADS;
c930a3ad
OG		 3261 break;
3262 default:
3263 return -EINVAL;
3264 }
3265
3266 return 0;
3267}
3268
ef78618b
OG		 3269static int esw_mode_to_devlink(u16 mlx5_mode, u16 *mode)
3270{
3271 switch (mlx5_mode) {
f6455de0 3272 case MLX5_ESWITCH_LEGACY:
ef78618b
OG		 3273 *mode = DEVLINK_ESWITCH_MODE_LEGACY;
3274 break;
f6455de0 3275 case MLX5_ESWITCH_OFFLOADS:
ef78618b
OG		 3276 *mode = DEVLINK_ESWITCH_MODE_SWITCHDEV;
3277 break;
3278 default:
3279 return -EINVAL;
3280 }
3281
3282 return 0;
3283}
3284
bffaa916
RD		 3285static int esw_inline_mode_from_devlink(u8 mode, u8 *mlx5_mode)
3286{
3287 switch (mode) {
3288 case DEVLINK_ESWITCH_INLINE_MODE_NONE:
3289 *mlx5_mode = MLX5_INLINE_MODE_NONE;
3290 break;
3291 case DEVLINK_ESWITCH_INLINE_MODE_LINK:
3292 *mlx5_mode = MLX5_INLINE_MODE_L2;
3293 break;
3294 case DEVLINK_ESWITCH_INLINE_MODE_NETWORK:
3295 *mlx5_mode = MLX5_INLINE_MODE_IP;
3296 break;
3297 case DEVLINK_ESWITCH_INLINE_MODE_TRANSPORT:
3298 *mlx5_mode = MLX5_INLINE_MODE_TCP_UDP;
3299 break;
3300 default:
3301 return -EINVAL;
3302 }
3303
3304 return 0;
3305}
3306
3307static int esw_inline_mode_to_devlink(u8 mlx5_mode, u8 *mode)
3308{
3309 switch (mlx5_mode) {
3310 case MLX5_INLINE_MODE_NONE:
3311 *mode = DEVLINK_ESWITCH_INLINE_MODE_NONE;
3312 break;
3313 case MLX5_INLINE_MODE_L2:
3314 *mode = DEVLINK_ESWITCH_INLINE_MODE_LINK;
3315 break;
3316 case MLX5_INLINE_MODE_IP:
3317 *mode = DEVLINK_ESWITCH_INLINE_MODE_NETWORK;
3318 break;
3319 case MLX5_INLINE_MODE_TCP_UDP:
3320 *mode = DEVLINK_ESWITCH_INLINE_MODE_TRANSPORT;
3321 break;
3322 default:
3323 return -EINVAL;
3324 }
3325
3326 return 0;
3327}
3328
ae24432c
PP		 3329static int eswitch_devlink_esw_mode_check(const struct mlx5_eswitch *esw)
3330{
3331 /* devlink commands in NONE eswitch mode are currently supported only
3332 * on ECPF.
3333 */
3334 return (esw->mode == MLX5_ESWITCH_NONE &&
3335 !mlx5_core_is_ecpf_esw_manager(esw->dev)) ? -EOPNOTSUPP : 0;
3336}
3337
db7ff19e
EB		 3338int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode,
3339 struct netlink_ext_ack *extack)
9d1cef19 3340{
9d1cef19 3341 u16 cur_mlx5_mode, mlx5_mode = 0;
bd939753 3342 struct mlx5_eswitch *esw;
ea2128fd 3343 int err = 0;
9d1cef19 3344
bd939753
PP		 3345 esw = mlx5_devlink_eswitch_get(devlink);
3346 if (IS_ERR(esw))
3347 return PTR_ERR(esw);
9d1cef19 3348
ef78618b 3349 if (esw_mode_from_devlink(mode, &mlx5_mode))
c930a3ad
OG		 3350 return -EINVAL;
3351
cac1eb2c 3352 mlx5_lag_disable_change(esw->dev);
7dc84de9
RD		 3353 err = mlx5_esw_try_lock(esw);
3354 if (err < 0) {
3355 NL_SET_ERR_MSG_MOD(extack, "Can't change mode, E-Switch is busy");
cac1eb2c 3356 goto enable_lag;
7dc84de9
RD		 3357 }
3358 cur_mlx5_mode = err;
3359 err = 0;
3360
c930a3ad 3361 if (cur_mlx5_mode == mlx5_mode)
8e0aa4bc 3362 goto unlock;
c930a3ad 3363
c85a6b8f
AL		 3364 if (mode == DEVLINK_ESWITCH_MODE_SWITCHDEV) {
3365 if (mlx5_devlink_trap_get_num_active(esw->dev)) {
3366 NL_SET_ERR_MSG_MOD(extack,
3367 "Can't change mode while devlink traps are active");
3368 err = -EOPNOTSUPP;
3369 goto unlock;
3370 }
8e0aa4bc 3371 err = esw_offloads_start(esw, extack);
c85a6b8f 3372 } else if (mode == DEVLINK_ESWITCH_MODE_LEGACY) {
8e0aa4bc 3373 err = esw_offloads_stop(esw, extack);
c85a6b8f 3374 } else {
8e0aa4bc 3375 err = -EINVAL;
c85a6b8f 3376 }
8e0aa4bc
PP		 3377
3378unlock:
7dc84de9 3379 mlx5_esw_unlock(esw);
cac1eb2c
MB		 3380enable_lag:
3381 mlx5_lag_enable_change(esw->dev);
8e0aa4bc 3382 return err;
feae9087
OG		 3383}
3384
3385int mlx5_devlink_eswitch_mode_get(struct devlink *devlink, u16 *mode)
3386{
bd939753 3387 struct mlx5_eswitch *esw;
9d1cef19 3388 int err;
c930a3ad 3389
bd939753
PP		 3390 esw = mlx5_devlink_eswitch_get(devlink);
3391 if (IS_ERR(esw))
3392 return PTR_ERR(esw);
c930a3ad 3393
c55479d0 3394 down_write(&esw->mode_lock);
bd939753 3395 err = eswitch_devlink_esw_mode_check(esw);
ae24432c 3396 if (err)
8e0aa4bc 3397 goto unlock;
ae24432c 3398
8e0aa4bc
PP		 3399 err = esw_mode_to_devlink(esw->mode, mode);
3400unlock:
c55479d0 3401 up_write(&esw->mode_lock);
8e0aa4bc 3402 return err;
feae9087 3403}
127ea380 3404
47dd7e60
PP		 3405static int mlx5_esw_vports_inline_set(struct mlx5_eswitch *esw, u8 mlx5_mode,
3406 struct netlink_ext_ack *extack)
3407{
3408 struct mlx5_core_dev *dev = esw->dev;
3409 struct mlx5_vport *vport;
3410 u16 err_vport_num = 0;
3411 unsigned long i;
3412 int err = 0;
3413
3414 mlx5_esw_for_each_host_func_vport(esw, i, vport, esw->esw_funcs.num_vfs) {
3415 err = mlx5_modify_nic_vport_min_inline(dev, vport->vport, mlx5_mode);
3416 if (err) {
3417 err_vport_num = vport->vport;
3418 NL_SET_ERR_MSG_MOD(extack,
3419 "Failed to set min inline on vport");
3420 goto revert_inline_mode;
3421 }
3422 }
3423 return 0;
3424
3425revert_inline_mode:
3426 mlx5_esw_for_each_host_func_vport(esw, i, vport, esw->esw_funcs.num_vfs) {
3427 if (vport->vport == err_vport_num)
3428 break;
3429 mlx5_modify_nic_vport_min_inline(dev,
3430 vport->vport,
3431 esw->offloads.inline_mode);
3432 }
3433 return err;
3434}
3435
db7ff19e
EB		 3436int mlx5_devlink_eswitch_inline_mode_set(struct devlink *devlink, u8 mode,
3437 struct netlink_ext_ack *extack)
bffaa916
RD		 3438{
3439 struct mlx5_core_dev *dev = devlink_priv(devlink);
bd939753 3440 struct mlx5_eswitch *esw;
bffaa916 3441 u8 mlx5_mode;
47dd7e60 3442 int err;
bffaa916 3443
bd939753
PP		 3444 esw = mlx5_devlink_eswitch_get(devlink);
3445 if (IS_ERR(esw))
3446 return PTR_ERR(esw);
bffaa916 3447
c55479d0 3448 down_write(&esw->mode_lock);
ae24432c
PP		 3449 err = eswitch_devlink_esw_mode_check(esw);
3450 if (err)
8e0aa4bc 3451 goto out;
ae24432c 3452
c415f704
OG		 3453 switch (MLX5_CAP_ETH(dev, wqe_inline_mode)) {
3454 case MLX5_CAP_INLINE_MODE_NOT_REQUIRED:
bcd68c04
JC		 3455 if (mode == DEVLINK_ESWITCH_INLINE_MODE_NONE) {
3456 err = 0;
8e0aa4bc 3457 goto out;
bcd68c04
JC		 3458 }
3459
c8b838d1 3460 fallthrough;
c415f704 3461 case MLX5_CAP_INLINE_MODE_L2:
8c98ee77 3462 NL_SET_ERR_MSG_MOD(extack, "Inline mode can't be set");
8e0aa4bc
PP		 3463 err = -EOPNOTSUPP;
3464 goto out;
c415f704
OG		 3465 case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT:
3466 break;
3467 }
bffaa916 3468
525e84be 3469 if (atomic64_read(&esw->offloads.num_flows) > 0) {
8c98ee77
EB		 3470 NL_SET_ERR_MSG_MOD(extack,
3471 "Can't set inline mode when flows are configured");
8e0aa4bc
PP		 3472 err = -EOPNOTSUPP;
3473 goto out;
375f51e2
RD		 3474 }
3475
bffaa916
RD		 3476 err = esw_inline_mode_from_devlink(mode, &mlx5_mode);
3477 if (err)
3478 goto out;
3479
47dd7e60
PP		 3480 err = mlx5_esw_vports_inline_set(esw, mlx5_mode, extack);
3481 if (err)
3482 goto out;
bffaa916
RD		 3483
3484 esw->offloads.inline_mode = mlx5_mode;
c55479d0 3485 up_write(&esw->mode_lock);
bffaa916
RD		 3486 return 0;
3487
bffaa916 3488out:
c55479d0 3489 up_write(&esw->mode_lock);
bffaa916
RD		 3490 return err;
3491}
3492
3493int mlx5_devlink_eswitch_inline_mode_get(struct devlink *devlink, u8 *mode)
3494{
bd939753 3495 struct mlx5_eswitch *esw;
9d1cef19 3496 int err;
bffaa916 3497
bd939753
PP		 3498 esw = mlx5_devlink_eswitch_get(devlink);
3499 if (IS_ERR(esw))
3500 return PTR_ERR(esw);
bffaa916 3501
c55479d0 3502 down_write(&esw->mode_lock);
ae24432c
PP		 3503 err = eswitch_devlink_esw_mode_check(esw);
3504 if (err)
8e0aa4bc 3505 goto unlock;
ae24432c 3506
8e0aa4bc
PP		 3507 err = esw_inline_mode_to_devlink(esw->offloads.inline_mode, mode);
3508unlock:
c55479d0 3509 up_write(&esw->mode_lock);
8e0aa4bc 3510 return err;
bffaa916
RD		 3511}
3512
98fdbea5
LR		 3513int mlx5_devlink_eswitch_encap_mode_set(struct devlink *devlink,
3514 enum devlink_eswitch_encap_mode encap,
db7ff19e 3515 struct netlink_ext_ack *extack)
7768d197
RD		 3516{
3517 struct mlx5_core_dev *dev = devlink_priv(devlink);
bd939753 3518 struct mlx5_eswitch *esw;
7768d197
RD		 3519 int err;
3520
bd939753
PP		 3521 esw = mlx5_devlink_eswitch_get(devlink);
3522 if (IS_ERR(esw))
3523 return PTR_ERR(esw);
7768d197 3524
c55479d0 3525 down_write(&esw->mode_lock);
ae24432c
PP		 3526 err = eswitch_devlink_esw_mode_check(esw);
3527 if (err)
8e0aa4bc 3528 goto unlock;
ae24432c 3529
7768d197 3530 if (encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE &&
60786f09 3531 (!MLX5_CAP_ESW_FLOWTABLE_FDB(dev, reformat) ||
8e0aa4bc
PP		 3532 !MLX5_CAP_ESW_FLOWTABLE_FDB(dev, decap))) {
3533 err = -EOPNOTSUPP;
3534 goto unlock;
3535 }
7768d197 3536
8e0aa4bc
PP		 3537 if (encap && encap != DEVLINK_ESWITCH_ENCAP_MODE_BASIC) {
3538 err = -EOPNOTSUPP;
3539 goto unlock;
3540 }
7768d197 3541
f6455de0 3542 if (esw->mode == MLX5_ESWITCH_LEGACY) {
7768d197 3543 esw->offloads.encap = encap;
8e0aa4bc 3544 goto unlock;
7768d197
RD		 3545 }
3546
3547 if (esw->offloads.encap == encap)
8e0aa4bc 3548 goto unlock;
7768d197 3549
525e84be 3550 if (atomic64_read(&esw->offloads.num_flows) > 0) {
8c98ee77
EB		 3551 NL_SET_ERR_MSG_MOD(extack,
3552 "Can't set encapsulation when flows are configured");
8e0aa4bc
PP		 3553 err = -EOPNOTSUPP;
3554 goto unlock;
7768d197
RD		 3555 }
3556
e52c2802 3557 esw_destroy_offloads_fdb_tables(esw);
7768d197
RD		 3558
3559 esw->offloads.encap = encap;
e52c2802 3560
0da3c12d 3561 err = esw_create_offloads_fdb_tables(esw);
e52c2802 3562
7768d197 3563 if (err) {
8c98ee77
EB		 3564 NL_SET_ERR_MSG_MOD(extack,
3565 "Failed re-creating fast FDB table");
7768d197 3566 esw->offloads.encap = !encap;
0da3c12d 3567 (void)esw_create_offloads_fdb_tables(esw);
7768d197 3568 }
e52c2802 3569
8e0aa4bc 3570unlock:
c55479d0 3571 up_write(&esw->mode_lock);
7768d197
RD		 3572 return err;
3573}
3574
98fdbea5
LR		 3575int mlx5_devlink_eswitch_encap_mode_get(struct devlink *devlink,
3576 enum devlink_eswitch_encap_mode *encap)
7768d197 3577{
bd939753 3578 struct mlx5_eswitch *esw;
9d1cef19 3579 int err;
7768d197 3580
bd939753
PP		 3581 esw = mlx5_devlink_eswitch_get(devlink);
3582 if (IS_ERR(esw))
3583 return PTR_ERR(esw);
3584
7768d197 3585
c55479d0 3586 down_write(&esw->mode_lock);
ae24432c
PP		 3587 err = eswitch_devlink_esw_mode_check(esw);
3588 if (err)
8e0aa4bc 3589 goto unlock;
ae24432c 3590
7768d197 3591 *encap = esw->offloads.encap;
8e0aa4bc 3592unlock:
c55479d0 3593 up_write(&esw->mode_lock);
7768d197
RD		 3594 return 0;
3595}
3596
c2d7712c
BW		 3597static bool
3598mlx5_eswitch_vport_has_rep(const struct mlx5_eswitch *esw, u16 vport_num)
3599{
3600 /* Currently, only ECPF based device has representor for host PF. */
3601 if (vport_num == MLX5_VPORT_PF &&
3602 !mlx5_core_is_ecpf_esw_manager(esw->dev))
3603 return false;
3604
3605 if (vport_num == MLX5_VPORT_ECPF &&
3606 !mlx5_ecpf_vport_exists(esw->dev))
3607 return false;
3608
3609 return true;
3610}
3611
f8e8fa02 3612void mlx5_eswitch_register_vport_reps(struct mlx5_eswitch *esw,
8693115a 3613 const struct mlx5_eswitch_rep_ops *ops,
f8e8fa02 3614 u8 rep_type)
127ea380 3615{
8693115a 3616 struct mlx5_eswitch_rep_data *rep_data;
f8e8fa02 3617 struct mlx5_eswitch_rep *rep;
47dd7e60 3618 unsigned long i;
9deb2241 3619
8693115a 3620 esw->offloads.rep_ops[rep_type] = ops;
47dd7e60
PP		 3621 mlx5_esw_for_each_rep(esw, i, rep) {
3622 if (likely(mlx5_eswitch_vport_has_rep(esw, rep->vport))) {
59c904c8 3623 rep->esw = esw;
c2d7712c
BW		 3624 rep_data = &rep->rep_data[rep_type];
3625 atomic_set(&rep_data->state, REP_REGISTERED);
3626 }
f8e8fa02 3627 }
127ea380 3628}
f8e8fa02 3629EXPORT_SYMBOL(mlx5_eswitch_register_vport_reps);
127ea380 3630
f8e8fa02 3631void mlx5_eswitch_unregister_vport_reps(struct mlx5_eswitch *esw, u8 rep_type)
127ea380 3632{
cb67b832 3633 struct mlx5_eswitch_rep *rep;
47dd7e60 3634 unsigned long i;
cb67b832 3635
f6455de0 3636 if (esw->mode == MLX5_ESWITCH_OFFLOADS)
062f4bf4 3637 __unload_reps_all_vport(esw, rep_type);
127ea380 3638
47dd7e60 3639 mlx5_esw_for_each_rep(esw, i, rep)
8693115a 3640 atomic_set(&rep->rep_data[rep_type].state, REP_UNREGISTERED);
127ea380 3641}
f8e8fa02 3642EXPORT_SYMBOL(mlx5_eswitch_unregister_vport_reps);
726293f1 3643
a4b97ab4 3644void *mlx5_eswitch_get_uplink_priv(struct mlx5_eswitch *esw, u8 rep_type)
726293f1 3645{
726293f1
HHZ		 3646 struct mlx5_eswitch_rep *rep;
3647
879c8f84 3648 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_UPLINK);
8693115a 3649 return rep->rep_data[rep_type].priv;
726293f1 3650}
22215908
MB		 3651
3652void *mlx5_eswitch_get_proto_dev(struct mlx5_eswitch *esw,
02f3afd9 3653 u16 vport,
22215908
MB		 3654 u8 rep_type)
3655{
22215908
MB		 3656 struct mlx5_eswitch_rep *rep;
3657
879c8f84 3658 rep = mlx5_eswitch_get_rep(esw, vport);
22215908 3659
8693115a
PP		 3660 if (atomic_read(&rep->rep_data[rep_type].state) == REP_LOADED &&
3661 esw->offloads.rep_ops[rep_type]->get_proto_dev)
3662 return esw->offloads.rep_ops[rep_type]->get_proto_dev(rep);
22215908
MB		 3663 return NULL;
3664}
57cbd893 3665EXPORT_SYMBOL(mlx5_eswitch_get_proto_dev);
22215908
MB		 3666
3667void *mlx5_eswitch_uplink_get_proto_dev(struct mlx5_eswitch *esw, u8 rep_type)
3668{
879c8f84 3669 return mlx5_eswitch_get_proto_dev(esw, MLX5_VPORT_UPLINK, rep_type);
22215908 3670}
57cbd893
MB		 3671EXPORT_SYMBOL(mlx5_eswitch_uplink_get_proto_dev);
3672
3673struct mlx5_eswitch_rep *mlx5_eswitch_vport_rep(struct mlx5_eswitch *esw,
02f3afd9 3674 u16 vport)
57cbd893 3675{
879c8f84 3676 return mlx5_eswitch_get_rep(esw, vport);
57cbd893
MB		 3677}
3678EXPORT_SYMBOL(mlx5_eswitch_vport_rep);
91d6291c 3679
5b7cb745
PB		 3680bool mlx5_eswitch_reg_c1_loopback_enabled(const struct mlx5_eswitch *esw)
3681{
3682 return !!(esw->flags & MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED);
3683}
3684EXPORT_SYMBOL(mlx5_eswitch_reg_c1_loopback_enabled);
3685
7445cfb1
JL		 3686bool mlx5_eswitch_vport_match_metadata_enabled(const struct mlx5_eswitch *esw)
3687{
3688 return !!(esw->flags & MLX5_ESWITCH_VPORT_MATCH_METADATA);
3689}
3690EXPORT_SYMBOL(mlx5_eswitch_vport_match_metadata_enabled);
3691
0f0d3827 3692u32 mlx5_eswitch_get_vport_metadata_for_match(struct mlx5_eswitch *esw,
7445cfb1
JL		 3693 u16 vport_num)
3694{
133dcfc5 3695 struct mlx5_vport *vport = mlx5_eswitch_get_vport(esw, vport_num);
0f0d3827 3696
133dcfc5
VP		 3697 if (WARN_ON_ONCE(IS_ERR(vport)))
3698 return 0;
0f0d3827 3699
133dcfc5 3700 return vport->metadata << (32 - ESW_SOURCE_PORT_METADATA_BITS);
7445cfb1
JL		 3701}
3702EXPORT_SYMBOL(mlx5_eswitch_get_vport_metadata_for_match);
d970812b
PP		 3703
3704int mlx5_esw_offloads_sf_vport_enable(struct mlx5_eswitch *esw, struct devlink_port *dl_port,
f1b9acd3 3705 u16 vport_num, u32 controller, u32 sfnum)
d970812b
PP		 3706{
3707 int err;
3708
3709 err = mlx5_esw_vport_enable(esw, vport_num, MLX5_VPORT_UC_ADDR_CHANGE);
3710 if (err)
3711 return err;
3712
f1b9acd3 3713 err = mlx5_esw_devlink_sf_port_register(esw, dl_port, vport_num, controller, sfnum);
d970812b
PP		 3714 if (err)
3715 goto devlink_err;
3716
3717 err = mlx5_esw_offloads_rep_load(esw, vport_num);
3718 if (err)
3719 goto rep_err;
3720 return 0;
3721
3722rep_err:
3723 mlx5_esw_devlink_sf_port_unregister(esw, vport_num);
3724devlink_err:
3725 mlx5_esw_vport_disable(esw, vport_num);
3726 return err;
3727}
3728
3729void mlx5_esw_offloads_sf_vport_disable(struct mlx5_eswitch *esw, u16 vport_num)
3730{
3731 mlx5_esw_offloads_rep_unload(esw, vport_num);
3732 mlx5_esw_devlink_sf_port_unregister(esw, vport_num);
3733 mlx5_esw_vport_disable(esw, vport_num);
3734}
84ae9c1f
VB		 3735
3736static int mlx5_esw_query_vport_vhca_id(struct mlx5_eswitch *esw, u16 vport_num, u16 *vhca_id)
3737{
3738 int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out);
3739 void *query_ctx;
3740 void *hca_caps;
3741 int err;
3742
3743 *vhca_id = 0;
3744 if (mlx5_esw_is_manager_vport(esw, vport_num) ||
3745 !MLX5_CAP_GEN(esw->dev, vhca_resource_manager))
3746 return -EPERM;
3747
3748 query_ctx = kzalloc(query_out_sz, GFP_KERNEL);
3749 if (!query_ctx)
3750 return -ENOMEM;
3751
3752 err = mlx5_vport_get_other_func_cap(esw->dev, vport_num, query_ctx);
3753 if (err)
3754 goto out_free;
3755
3756 hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability);
3757 *vhca_id = MLX5_GET(cmd_hca_cap, hca_caps, vhca_id);
3758
3759out_free:
3760 kfree(query_ctx);
3761 return err;
3762}
3763
3764int mlx5_esw_vport_vhca_id_set(struct mlx5_eswitch *esw, u16 vport_num)
3765{
3766 u16 *old_entry, *vhca_map_entry, vhca_id;
3767 int err;
3768
3769 err = mlx5_esw_query_vport_vhca_id(esw, vport_num, &vhca_id);
3770 if (err) {
3771 esw_warn(esw->dev, "Getting vhca_id for vport failed (vport=%u,err=%d)\n",
3772 vport_num, err);
3773 return err;
3774 }
3775
3776 vhca_map_entry = kmalloc(sizeof(*vhca_map_entry), GFP_KERNEL);
3777 if (!vhca_map_entry)
3778 return -ENOMEM;
3779
3780 *vhca_map_entry = vport_num;
3781 old_entry = xa_store(&esw->offloads.vhca_map, vhca_id, vhca_map_entry, GFP_KERNEL);
3782 if (xa_is_err(old_entry)) {
3783 kfree(vhca_map_entry);
3784 return xa_err(old_entry);
3785 }
3786 kfree(old_entry);
3787 return 0;
3788}
3789
3790void mlx5_esw_vport_vhca_id_clear(struct mlx5_eswitch *esw, u16 vport_num)
3791{
3792 u16 *vhca_map_entry, vhca_id;
3793 int err;
3794
3795 err = mlx5_esw_query_vport_vhca_id(esw, vport_num, &vhca_id);
3796 if (err)
3797 esw_warn(esw->dev, "Getting vhca_id for vport failed (vport=%hu,err=%d)\n",
3798 vport_num, err);
3799
3800 vhca_map_entry = xa_erase(&esw->offloads.vhca_map, vhca_id);
3801 kfree(vhca_map_entry);
3802}
3803
3804int mlx5_eswitch_vhca_id_to_vport(struct mlx5_eswitch *esw, u16 vhca_id, u16 *vport_num)
3805{
3806 u16 *res = xa_load(&esw->offloads.vhca_map, vhca_id);
3807
3808 if (!res)
3809 return -ENOENT;
3810
3811 *vport_num = *res;
3812 return 0;
3813}
10742efc
VB		 3814
3815u32 mlx5_eswitch_get_vport_metadata_for_set(struct mlx5_eswitch *esw,
3816 u16 vport_num)
3817{
3818 struct mlx5_vport *vport = mlx5_eswitch_get_vport(esw, vport_num);
3819
3820 if (WARN_ON_ONCE(IS_ERR(vport)))
3821 return 0;
3822
3823 return vport->metadata;
3824}
3825EXPORT_SYMBOL(mlx5_eswitch_get_vport_metadata_for_set);
Linux 6.x block layer and io_uring tree(s)