2 * Copyright (c) 2016, Mellanox Technologies. All rights reserved.
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 #include <linux/etherdevice.h>
34 #include <linux/mlx5/driver.h>
35 #include <linux/mlx5/mlx5_ifc.h>
36 #include <linux/mlx5/vport.h>
37 #include <linux/mlx5/fs.h>
38 #include "mlx5_core.h"
46 struct mlx5_flow_rule *
47 mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch *esw,
48 struct mlx5_flow_spec *spec,
49 u32 action, u32 src_vport, u32 dst_vport)
51 struct mlx5_flow_destination dest = { 0 };
52 struct mlx5_fc *counter = NULL;
53 struct mlx5_flow_rule *rule;
56 if (esw->mode != SRIOV_OFFLOADS)
57 return ERR_PTR(-EOPNOTSUPP);
59 if (action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) {
60 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
61 dest.vport_num = dst_vport;
62 action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST;
63 } else if (action & MLX5_FLOW_CONTEXT_ACTION_COUNT) {
64 counter = mlx5_fc_create(esw->dev, true);
66 return ERR_CAST(counter);
67 dest.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER;
68 dest.counter = counter;
71 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters);
72 MLX5_SET(fte_match_set_misc, misc, source_port, src_vport);
74 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters);
75 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
77 spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS |
78 MLX5_MATCH_MISC_PARAMETERS;
80 rule = mlx5_add_flow_rule((struct mlx5_flow_table *)esw->fdb_table.fdb,
81 spec, action, 0, &dest);
84 mlx5_fc_destroy(esw->dev, counter);
89 static struct mlx5_flow_rule *
90 mlx5_eswitch_add_send_to_vport_rule(struct mlx5_eswitch *esw, int vport, u32 sqn)
92 struct mlx5_flow_destination dest;
93 struct mlx5_flow_rule *flow_rule;
94 struct mlx5_flow_spec *spec;
97 spec = mlx5_vzalloc(sizeof(*spec));
99 esw_warn(esw->dev, "FDB: Failed to alloc match parameters\n");
100 flow_rule = ERR_PTR(-ENOMEM);
104 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters);
105 MLX5_SET(fte_match_set_misc, misc, source_sqn, sqn);
106 MLX5_SET(fte_match_set_misc, misc, source_port, 0x0); /* source vport is 0 */
108 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters);
109 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_sqn);
110 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
112 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS;
113 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
114 dest.vport_num = vport;
116 flow_rule = mlx5_add_flow_rule(esw->fdb_table.offloads.fdb, spec,
117 MLX5_FLOW_CONTEXT_ACTION_FWD_DEST,
119 if (IS_ERR(flow_rule))
120 esw_warn(esw->dev, "FDB: Failed to add send to vport rule err %ld\n", PTR_ERR(flow_rule));
126 void mlx5_eswitch_sqs2vport_stop(struct mlx5_eswitch *esw,
127 struct mlx5_eswitch_rep *rep)
129 struct mlx5_esw_sq *esw_sq, *tmp;
131 if (esw->mode != SRIOV_OFFLOADS)
134 list_for_each_entry_safe(esw_sq, tmp, &rep->vport_sqs_list, list) {
135 mlx5_del_flow_rule(esw_sq->send_to_vport_rule);
136 list_del(&esw_sq->list);
141 int mlx5_eswitch_sqs2vport_start(struct mlx5_eswitch *esw,
142 struct mlx5_eswitch_rep *rep,
143 u16 *sqns_array, int sqns_num)
145 struct mlx5_flow_rule *flow_rule;
146 struct mlx5_esw_sq *esw_sq;
151 if (esw->mode != SRIOV_OFFLOADS)
154 vport = rep->vport == 0 ?
155 FDB_UPLINK_VPORT : rep->vport;
157 for (i = 0; i < sqns_num; i++) {
158 esw_sq = kzalloc(sizeof(*esw_sq), GFP_KERNEL);
164 /* Add re-inject rule to the PF/representor sqs */
165 flow_rule = mlx5_eswitch_add_send_to_vport_rule(esw,
168 if (IS_ERR(flow_rule)) {
169 err = PTR_ERR(flow_rule);
173 esw_sq->send_to_vport_rule = flow_rule;
174 list_add(&esw_sq->list, &rep->vport_sqs_list);
179 mlx5_eswitch_sqs2vport_stop(esw, rep);
183 static int esw_add_fdb_miss_rule(struct mlx5_eswitch *esw)
185 struct mlx5_flow_destination dest;
186 struct mlx5_flow_rule *flow_rule = NULL;
187 struct mlx5_flow_spec *spec;
190 spec = mlx5_vzalloc(sizeof(*spec));
192 esw_warn(esw->dev, "FDB: Failed to alloc match parameters\n");
197 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT;
200 flow_rule = mlx5_add_flow_rule(esw->fdb_table.offloads.fdb, spec,
201 MLX5_FLOW_CONTEXT_ACTION_FWD_DEST,
203 if (IS_ERR(flow_rule)) {
204 err = PTR_ERR(flow_rule);
205 esw_warn(esw->dev, "FDB: Failed to add miss flow rule err %d\n", err);
209 esw->fdb_table.offloads.miss_rule = flow_rule;
215 #define MAX_PF_SQ 256
216 #define ESW_OFFLOADS_NUM_ENTRIES (1 << 13) /* 8K */
217 #define ESW_OFFLOADS_NUM_GROUPS 4
219 static int esw_create_offloads_fdb_table(struct mlx5_eswitch *esw, int nvports)
221 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
222 struct mlx5_core_dev *dev = esw->dev;
223 struct mlx5_flow_namespace *root_ns;
224 struct mlx5_flow_table *fdb = NULL;
225 struct mlx5_flow_group *g;
227 void *match_criteria;
228 int table_size, ix, err = 0;
230 flow_group_in = mlx5_vzalloc(inlen);
234 root_ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_FDB);
236 esw_warn(dev, "Failed to get FDB flow namespace\n");
240 esw_debug(dev, "Create offloads FDB table, log_max_size(%d)\n",
241 MLX5_CAP_ESW_FLOWTABLE_FDB(dev, log_max_ft_size));
243 fdb = mlx5_create_auto_grouped_flow_table(root_ns, FDB_FAST_PATH,
244 ESW_OFFLOADS_NUM_ENTRIES,
245 ESW_OFFLOADS_NUM_GROUPS, 0);
248 esw_warn(dev, "Failed to create Fast path FDB Table err %d\n", err);
251 esw->fdb_table.fdb = fdb;
253 table_size = nvports + MAX_PF_SQ + 1;
254 fdb = mlx5_create_flow_table(root_ns, FDB_SLOW_PATH, table_size, 0);
257 esw_warn(dev, "Failed to create slow path FDB Table err %d\n", err);
260 esw->fdb_table.offloads.fdb = fdb;
262 /* create send-to-vport group */
263 memset(flow_group_in, 0, inlen);
264 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
265 MLX5_MATCH_MISC_PARAMETERS);
267 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, match_criteria);
269 MLX5_SET_TO_ONES(fte_match_param, match_criteria, misc_parameters.source_sqn);
270 MLX5_SET_TO_ONES(fte_match_param, match_criteria, misc_parameters.source_port);
272 ix = nvports + MAX_PF_SQ;
273 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0);
274 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, ix - 1);
276 g = mlx5_create_flow_group(fdb, flow_group_in);
279 esw_warn(dev, "Failed to create send-to-vport flow group err(%d)\n", err);
282 esw->fdb_table.offloads.send_to_vport_grp = g;
284 /* create miss group */
285 memset(flow_group_in, 0, inlen);
286 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, 0);
288 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, ix);
289 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, ix + 1);
291 g = mlx5_create_flow_group(fdb, flow_group_in);
294 esw_warn(dev, "Failed to create miss flow group err(%d)\n", err);
297 esw->fdb_table.offloads.miss_grp = g;
299 err = esw_add_fdb_miss_rule(esw);
306 mlx5_destroy_flow_group(esw->fdb_table.offloads.miss_grp);
308 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_grp);
310 mlx5_destroy_flow_table(esw->fdb_table.offloads.fdb);
312 mlx5_destroy_flow_table(esw->fdb_table.fdb);
315 kvfree(flow_group_in);
319 static void esw_destroy_offloads_fdb_table(struct mlx5_eswitch *esw)
321 if (!esw->fdb_table.fdb)
324 esw_debug(esw->dev, "Destroy offloads FDB Table\n");
325 mlx5_del_flow_rule(esw->fdb_table.offloads.miss_rule);
326 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_grp);
327 mlx5_destroy_flow_group(esw->fdb_table.offloads.miss_grp);
329 mlx5_destroy_flow_table(esw->fdb_table.offloads.fdb);
330 mlx5_destroy_flow_table(esw->fdb_table.fdb);
333 static int esw_create_offloads_table(struct mlx5_eswitch *esw)
335 struct mlx5_flow_namespace *ns;
336 struct mlx5_flow_table *ft_offloads;
337 struct mlx5_core_dev *dev = esw->dev;
340 ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_OFFLOADS);
342 esw_warn(esw->dev, "Failed to get offloads flow namespace\n");
346 ft_offloads = mlx5_create_flow_table(ns, 0, dev->priv.sriov.num_vfs + 2, 0);
347 if (IS_ERR(ft_offloads)) {
348 err = PTR_ERR(ft_offloads);
349 esw_warn(esw->dev, "Failed to create offloads table, err %d\n", err);
353 esw->offloads.ft_offloads = ft_offloads;
357 static void esw_destroy_offloads_table(struct mlx5_eswitch *esw)
359 struct mlx5_esw_offload *offloads = &esw->offloads;
361 mlx5_destroy_flow_table(offloads->ft_offloads);
364 static int esw_create_vport_rx_group(struct mlx5_eswitch *esw)
366 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
367 struct mlx5_flow_group *g;
368 struct mlx5_priv *priv = &esw->dev->priv;
370 void *match_criteria, *misc;
372 int nvports = priv->sriov.num_vfs + 2;
374 flow_group_in = mlx5_vzalloc(inlen);
378 /* create vport rx group */
379 memset(flow_group_in, 0, inlen);
380 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable,
381 MLX5_MATCH_MISC_PARAMETERS);
383 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, match_criteria);
384 misc = MLX5_ADDR_OF(fte_match_param, match_criteria, misc_parameters);
385 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
387 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0);
388 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, nvports - 1);
390 g = mlx5_create_flow_group(esw->offloads.ft_offloads, flow_group_in);
394 mlx5_core_warn(esw->dev, "Failed to create vport rx group err %d\n", err);
398 esw->offloads.vport_rx_group = g;
400 kfree(flow_group_in);
404 static void esw_destroy_vport_rx_group(struct mlx5_eswitch *esw)
406 mlx5_destroy_flow_group(esw->offloads.vport_rx_group);
409 struct mlx5_flow_rule *
410 mlx5_eswitch_create_vport_rx_rule(struct mlx5_eswitch *esw, int vport, u32 tirn)
412 struct mlx5_flow_destination dest;
413 struct mlx5_flow_rule *flow_rule;
414 struct mlx5_flow_spec *spec;
417 spec = mlx5_vzalloc(sizeof(*spec));
419 esw_warn(esw->dev, "Failed to alloc match parameters\n");
420 flow_rule = ERR_PTR(-ENOMEM);
424 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters);
425 MLX5_SET(fte_match_set_misc, misc, source_port, vport);
427 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters);
428 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port);
430 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS;
431 dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR;
434 flow_rule = mlx5_add_flow_rule(esw->offloads.ft_offloads, spec,
435 MLX5_FLOW_CONTEXT_ACTION_FWD_DEST,
437 if (IS_ERR(flow_rule)) {
438 esw_warn(esw->dev, "fs offloads: Failed to add vport rx rule err %ld\n", PTR_ERR(flow_rule));
447 static int esw_offloads_start(struct mlx5_eswitch *esw)
449 int err, err1, num_vfs = esw->dev->priv.sriov.num_vfs;
451 if (esw->mode != SRIOV_LEGACY) {
452 esw_warn(esw->dev, "Can't set offloads mode, SRIOV legacy not enabled\n");
456 mlx5_eswitch_disable_sriov(esw);
457 err = mlx5_eswitch_enable_sriov(esw, num_vfs, SRIOV_OFFLOADS);
459 esw_warn(esw->dev, "Failed setting eswitch to offloads, err %d\n", err);
460 err1 = mlx5_eswitch_enable_sriov(esw, num_vfs, SRIOV_LEGACY);
462 esw_warn(esw->dev, "Failed setting eswitch back to legacy, err %d\n", err);
467 int esw_offloads_init(struct mlx5_eswitch *esw, int nvports)
469 struct mlx5_eswitch_rep *rep;
473 err = esw_create_offloads_fdb_table(esw, nvports);
477 err = esw_create_offloads_table(esw);
481 err = esw_create_vport_rx_group(esw);
485 for (vport = 0; vport < nvports; vport++) {
486 rep = &esw->offloads.vport_reps[vport];
490 err = rep->load(esw, rep);
497 for (vport--; vport >= 0; vport--) {
498 rep = &esw->offloads.vport_reps[vport];
501 rep->unload(esw, rep);
503 esw_destroy_vport_rx_group(esw);
506 esw_destroy_offloads_table(esw);
509 esw_destroy_offloads_fdb_table(esw);
513 static int esw_offloads_stop(struct mlx5_eswitch *esw)
515 int err, err1, num_vfs = esw->dev->priv.sriov.num_vfs;
517 mlx5_eswitch_disable_sriov(esw);
518 err = mlx5_eswitch_enable_sriov(esw, num_vfs, SRIOV_LEGACY);
520 esw_warn(esw->dev, "Failed setting eswitch to legacy, err %d\n", err);
521 err1 = mlx5_eswitch_enable_sriov(esw, num_vfs, SRIOV_OFFLOADS);
523 esw_warn(esw->dev, "Failed setting eswitch back to offloads, err %d\n", err);
529 void esw_offloads_cleanup(struct mlx5_eswitch *esw, int nvports)
531 struct mlx5_eswitch_rep *rep;
534 for (vport = 0; vport < nvports; vport++) {
535 rep = &esw->offloads.vport_reps[vport];
538 rep->unload(esw, rep);
541 esw_destroy_vport_rx_group(esw);
542 esw_destroy_offloads_table(esw);
543 esw_destroy_offloads_fdb_table(esw);
546 static int esw_mode_from_devlink(u16 mode, u16 *mlx5_mode)
549 case DEVLINK_ESWITCH_MODE_LEGACY:
550 *mlx5_mode = SRIOV_LEGACY;
552 case DEVLINK_ESWITCH_MODE_SWITCHDEV:
553 *mlx5_mode = SRIOV_OFFLOADS;
562 static int esw_mode_to_devlink(u16 mlx5_mode, u16 *mode)
566 *mode = DEVLINK_ESWITCH_MODE_LEGACY;
569 *mode = DEVLINK_ESWITCH_MODE_SWITCHDEV;
578 int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode)
580 struct mlx5_core_dev *dev;
581 u16 cur_mlx5_mode, mlx5_mode = 0;
583 dev = devlink_priv(devlink);
585 if (!MLX5_CAP_GEN(dev, vport_group_manager))
588 cur_mlx5_mode = dev->priv.eswitch->mode;
590 if (cur_mlx5_mode == SRIOV_NONE)
593 if (esw_mode_from_devlink(mode, &mlx5_mode))
596 if (cur_mlx5_mode == mlx5_mode)
599 if (mode == DEVLINK_ESWITCH_MODE_SWITCHDEV)
600 return esw_offloads_start(dev->priv.eswitch);
601 else if (mode == DEVLINK_ESWITCH_MODE_LEGACY)
602 return esw_offloads_stop(dev->priv.eswitch);
607 int mlx5_devlink_eswitch_mode_get(struct devlink *devlink, u16 *mode)
609 struct mlx5_core_dev *dev;
611 dev = devlink_priv(devlink);
613 if (!MLX5_CAP_GEN(dev, vport_group_manager))
616 if (dev->priv.eswitch->mode == SRIOV_NONE)
619 return esw_mode_to_devlink(dev->priv.eswitch->mode, mode);
622 void mlx5_eswitch_register_vport_rep(struct mlx5_eswitch *esw,
623 struct mlx5_eswitch_rep *rep)
625 struct mlx5_esw_offload *offloads = &esw->offloads;
627 memcpy(&offloads->vport_reps[rep->vport], rep,
628 sizeof(struct mlx5_eswitch_rep));
630 INIT_LIST_HEAD(&offloads->vport_reps[rep->vport].vport_sqs_list);
631 offloads->vport_reps[rep->vport].valid = true;
634 void mlx5_eswitch_unregister_vport_rep(struct mlx5_eswitch *esw,
637 struct mlx5_esw_offload *offloads = &esw->offloads;
638 struct mlx5_eswitch_rep *rep;
640 rep = &offloads->vport_reps[vport];
642 if (esw->mode == SRIOV_OFFLOADS && esw->vports[vport].enabled)
643 rep->unload(esw, rep);
645 offloads->vport_reps[vport].valid = false;