mlx5_core-y := main.o cmd.o debugfs.o fw.o eq.o uar.o pagealloc.o \
health.o mcg.o cq.o alloc.o port.o mr.o pd.o \
transobj.o vport.o sriov.o fs_cmd.o fs_core.o pci_irq.o \
- fs_counters.o rl.o lag.o dev.o events.o wq.o lib/gid.o \
+ fs_counters.o fs_ft_pool.o rl.o lag.o dev.o events.o wq.o lib/gid.o \
lib/devcom.o lib/pci_vsc.o lib/dm.o diag/fs_tracepoint.o \
diag/fw_tracer.o diag/crdump.o devlink.o diag/rsc_dump.o \
fw_reset.o qos.o
struct mlx5e_tc_update_priv *tc_priv)
{
struct mlx5e_priv *priv = netdev_priv(skb->dev);
- u32 tunnel_id = reg_c1 >> ESW_TUN_OFFSET;
+ u32 tunnel_id = (reg_c1 >> ESW_TUN_OFFSET) & TUNNEL_ID_MASK;
if (chain) {
struct mlx5_rep_uplink_priv *uplink_priv;
#include "en_tc.h"
#include "en_rep.h"
-#define MLX5_CT_ZONE_BITS (mlx5e_tc_attr_to_reg_mappings[ZONE_TO_REG].mlen * 8)
+#define MLX5_CT_ZONE_BITS (mlx5e_tc_attr_to_reg_mappings[ZONE_TO_REG].mlen)
#define MLX5_CT_ZONE_MASK GENMASK(MLX5_CT_ZONE_BITS - 1, 0)
#define MLX5_CT_STATE_ESTABLISHED_BIT BIT(1)
#define MLX5_CT_STATE_TRK_BIT BIT(2)
#define MLX5_CT_STATE_RELATED_BIT BIT(5)
#define MLX5_CT_STATE_INVALID_BIT BIT(6)
-#define MLX5_FTE_ID_BITS (mlx5e_tc_attr_to_reg_mappings[FTEID_TO_REG].mlen * 8)
+#define MLX5_FTE_ID_BITS (mlx5e_tc_attr_to_reg_mappings[FTEID_TO_REG].mlen)
#define MLX5_FTE_ID_MAX GENMASK(MLX5_FTE_ID_BITS - 1, 0)
#define MLX5_FTE_ID_MASK MLX5_FTE_ID_MAX
-#define MLX5_CT_LABELS_BITS (mlx5e_tc_attr_to_reg_mappings[LABELS_TO_REG].mlen * 8)
+#define MLX5_CT_LABELS_BITS (mlx5e_tc_attr_to_reg_mappings[LABELS_TO_REG].mlen)
#define MLX5_CT_LABELS_MASK GENMASK(MLX5_CT_LABELS_BITS - 1, 0)
#define ct_dbg(fmt, args...)\
unsigned long flags;
};
+static void
+mlx5_tc_ct_entry_destroy_mod_hdr(struct mlx5_tc_ct_priv *ct_priv,
+ struct mlx5_flow_attr *attr,
+ struct mlx5e_mod_hdr_handle *mh);
+
static const struct rhashtable_params cts_ht_params = {
.head_offset = offsetof(struct mlx5_ct_entry, node),
.key_offset = offsetof(struct mlx5_ct_entry, cookie),
ct_dbg("Deleting ct entry rule in zone %d", entry->tuple.zone);
mlx5_tc_rule_delete(netdev_priv(ct_priv->netdev), zone_rule->rule, attr);
- mlx5e_mod_hdr_detach(ct_priv->dev,
- ct_priv->mod_hdr_tbl, zone_rule->mh);
+ mlx5_tc_ct_entry_destroy_mod_hdr(ct_priv, zone_rule->attr, zone_rule->mh);
mlx5_put_label_mapping(ct_priv, attr->ct_attr.ct_labels_id);
kfree(attr);
}
if (err)
goto err_mapping;
- *mh = mlx5e_mod_hdr_attach(ct_priv->dev,
- ct_priv->mod_hdr_tbl,
- ct_priv->ns_type,
- &mod_acts);
- if (IS_ERR(*mh)) {
- err = PTR_ERR(*mh);
- goto err_mapping;
+ if (nat) {
+ attr->modify_hdr = mlx5_modify_header_alloc(ct_priv->dev, ct_priv->ns_type,
+ mod_acts.num_actions,
+ mod_acts.actions);
+ if (IS_ERR(attr->modify_hdr)) {
+ err = PTR_ERR(attr->modify_hdr);
+ goto err_mapping;
+ }
+
+ *mh = NULL;
+ } else {
+ *mh = mlx5e_mod_hdr_attach(ct_priv->dev,
+ ct_priv->mod_hdr_tbl,
+ ct_priv->ns_type,
+ &mod_acts);
+ if (IS_ERR(*mh)) {
+ err = PTR_ERR(*mh);
+ goto err_mapping;
+ }
+ attr->modify_hdr = mlx5e_mod_hdr_get(*mh);
}
- attr->modify_hdr = mlx5e_mod_hdr_get(*mh);
dealloc_mod_hdr_actions(&mod_acts);
return 0;
return err;
}
+static void
+mlx5_tc_ct_entry_destroy_mod_hdr(struct mlx5_tc_ct_priv *ct_priv,
+ struct mlx5_flow_attr *attr,
+ struct mlx5e_mod_hdr_handle *mh)
+{
+ if (mh)
+ mlx5e_mod_hdr_detach(ct_priv->dev, ct_priv->mod_hdr_tbl, mh);
+ else
+ mlx5_modify_header_dealloc(ct_priv->dev, attr->modify_hdr);
+}
+
static int
mlx5_tc_ct_entry_add_rule(struct mlx5_tc_ct_priv *ct_priv,
struct flow_rule *flow_rule,
return 0;
err_rule:
- mlx5e_mod_hdr_detach(ct_priv->dev,
- ct_priv->mod_hdr_tbl, zone_rule->mh);
+ mlx5_tc_ct_entry_destroy_mod_hdr(ct_priv, zone_rule->attr, zone_rule->mh);
mlx5_put_label_mapping(ct_priv, attr->ct_attr.ct_labels_id);
err_mod_hdr:
kfree(attr);
}
if (rev_entry && refcount_inc_not_zero(&rev_entry->counter->refcount)) {
- ct_dbg("Using shared counter entry=0x%p rev=0x%p\n", entry, rev_entry);
+ ct_dbg("Using shared counter entry=0x%p rev=0x%p", entry, rev_entry);
shared_counter = rev_entry->counter;
spin_unlock_bh(&ct_priv->ht_lock);
#define zone_to_reg_ct {\
.mfield = MLX5_ACTION_IN_FIELD_METADATA_REG_C_2,\
.moffset = 0,\
- .mlen = 2,\
+ .mlen = 16,\
.soffset = MLX5_BYTE_OFF(fte_match_param,\
- misc_parameters_2.metadata_reg_c_2) + 2,\
+ misc_parameters_2.metadata_reg_c_2),\
}
#define ctstate_to_reg_ct {\
.mfield = MLX5_ACTION_IN_FIELD_METADATA_REG_C_2,\
- .moffset = 2,\
- .mlen = 2,\
+ .moffset = 16,\
+ .mlen = 16,\
.soffset = MLX5_BYTE_OFF(fte_match_param,\
misc_parameters_2.metadata_reg_c_2),\
}
#define mark_to_reg_ct {\
.mfield = MLX5_ACTION_IN_FIELD_METADATA_REG_C_3,\
.moffset = 0,\
- .mlen = 4,\
+ .mlen = 32,\
.soffset = MLX5_BYTE_OFF(fte_match_param,\
misc_parameters_2.metadata_reg_c_3),\
}
#define labels_to_reg_ct {\
.mfield = MLX5_ACTION_IN_FIELD_METADATA_REG_C_4,\
.moffset = 0,\
- .mlen = 4,\
+ .mlen = 32,\
.soffset = MLX5_BYTE_OFF(fte_match_param,\
misc_parameters_2.metadata_reg_c_4),\
}
#define fteid_to_reg_ct {\
.mfield = MLX5_ACTION_IN_FIELD_METADATA_REG_C_5,\
.moffset = 0,\
- .mlen = 4,\
+ .mlen = 32,\
.soffset = MLX5_BYTE_OFF(fte_match_param,\
misc_parameters_2.metadata_reg_c_5),\
}
#define zone_restore_to_reg_ct {\
.mfield = MLX5_ACTION_IN_FIELD_METADATA_REG_C_1,\
.moffset = 0,\
- .mlen = (ESW_ZONE_ID_BITS / 8),\
+ .mlen = ESW_ZONE_ID_BITS,\
.soffset = MLX5_BYTE_OFF(fte_match_param,\
- misc_parameters_2.metadata_reg_c_1) + 3,\
+ misc_parameters_2.metadata_reg_c_1),\
}
#define nic_zone_restore_to_reg_ct {\
.mfield = MLX5_ACTION_IN_FIELD_METADATA_REG_B,\
- .moffset = 2,\
- .mlen = (ESW_ZONE_ID_BITS / 8),\
+ .moffset = 16,\
+ .mlen = ESW_ZONE_ID_BITS,\
}
#define REG_MAPPING_MLEN(reg) (mlx5e_tc_attr_to_reg_mappings[reg].mlen)
#define REG_MAPPING_MOFFSET(reg) (mlx5e_tc_attr_to_reg_mappings[reg].moffset)
-#define REG_MAPPING_SHIFT(reg) (REG_MAPPING_MOFFSET(reg) * 8)
#if IS_ENABLED(CONFIG_MLX5_TC_CT)
if (rep->vlan && skb_vlan_tag_present(skb))
skb_vlan_pop(skb);
- if (!mlx5e_rep_tc_update_skb(cqe, skb, &tc_priv)) {
+ if (unlikely(!mlx5_ipsec_is_rx_flow(cqe) &&
+ !mlx5e_rep_tc_update_skb(cqe, skb, &tc_priv))) {
dev_kfree_skb_any(skb);
goto free_wqe;
}
mlx5e_complete_rx_cqe(rq, cqe, cqe_bcnt, skb);
- if (!mlx5e_rep_tc_update_skb(cqe, skb, &tc_priv)) {
+ if (unlikely(!mlx5_ipsec_is_rx_flow(cqe) &&
+ !mlx5e_rep_tc_update_skb(cqe, skb, &tc_priv))) {
dev_kfree_skb_any(skb);
goto mpwrq_cqe_out;
}
if (rq->cqd.left) {
work_done += mlx5e_decompress_cqes_cont(rq, cqwq, 0, budget);
- if (rq->cqd.left || work_done >= budget)
+ if (work_done >= budget)
goto out;
}
[CHAIN_TO_REG] = {
.mfield = MLX5_ACTION_IN_FIELD_METADATA_REG_C_0,
.moffset = 0,
- .mlen = 2,
+ .mlen = 16,
},
[VPORT_TO_REG] = {
.mfield = MLX5_ACTION_IN_FIELD_METADATA_REG_C_0,
- .moffset = 2,
- .mlen = 2,
+ .moffset = 16,
+ .mlen = 16,
},
[TUNNEL_TO_REG] = {
.mfield = MLX5_ACTION_IN_FIELD_METADATA_REG_C_1,
- .moffset = 1,
- .mlen = ((ESW_TUN_OPTS_BITS + ESW_TUN_ID_BITS) / 8),
+ .moffset = 8,
+ .mlen = ESW_TUN_OPTS_BITS + ESW_TUN_ID_BITS,
.soffset = MLX5_BYTE_OFF(fte_match_param,
misc_parameters_2.metadata_reg_c_1),
},
[NIC_CHAIN_TO_REG] = {
.mfield = MLX5_ACTION_IN_FIELD_METADATA_REG_B,
.moffset = 0,
- .mlen = 2,
+ .mlen = 16,
},
[NIC_ZONE_RESTORE_TO_REG] = nic_zone_restore_to_reg_ct,
};
void
mlx5e_tc_match_to_reg_match(struct mlx5_flow_spec *spec,
enum mlx5e_tc_attr_to_reg type,
- u32 data,
+ u32 val,
u32 mask)
{
+ void *headers_c = spec->match_criteria, *headers_v = spec->match_value, *fmask, *fval;
int soffset = mlx5e_tc_attr_to_reg_mappings[type].soffset;
+ int moffset = mlx5e_tc_attr_to_reg_mappings[type].moffset;
int match_len = mlx5e_tc_attr_to_reg_mappings[type].mlen;
- void *headers_c = spec->match_criteria;
- void *headers_v = spec->match_value;
- void *fmask, *fval;
+ u32 max_mask = GENMASK(match_len - 1, 0);
+ __be32 curr_mask_be, curr_val_be;
+ u32 curr_mask, curr_val;
fmask = headers_c + soffset;
fval = headers_v + soffset;
- mask = (__force u32)(cpu_to_be32(mask)) >> (32 - (match_len * 8));
- data = (__force u32)(cpu_to_be32(data)) >> (32 - (match_len * 8));
+ memcpy(&curr_mask_be, fmask, 4);
+ memcpy(&curr_val_be, fval, 4);
+
+ curr_mask = be32_to_cpu(curr_mask_be);
+ curr_val = be32_to_cpu(curr_val_be);
+
+ //move to correct offset
+ WARN_ON(mask > max_mask);
+ mask <<= moffset;
+ val <<= moffset;
+ max_mask <<= moffset;
+
+ //zero val and mask
+ curr_mask &= ~max_mask;
+ curr_val &= ~max_mask;
- memcpy(fmask, &mask, match_len);
- memcpy(fval, &data, match_len);
+ //add current to mask
+ curr_mask |= mask;
+ curr_val |= val;
+
+ //back to be32 and write
+ curr_mask_be = cpu_to_be32(curr_mask);
+ curr_val_be = cpu_to_be32(curr_val);
+
+ memcpy(fmask, &curr_mask_be, 4);
+ memcpy(fval, &curr_val_be, 4);
spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2;
}
void
mlx5e_tc_match_to_reg_get_match(struct mlx5_flow_spec *spec,
enum mlx5e_tc_attr_to_reg type,
- u32 *data,
+ u32 *val,
u32 *mask)
{
+ void *headers_c = spec->match_criteria, *headers_v = spec->match_value, *fmask, *fval;
int soffset = mlx5e_tc_attr_to_reg_mappings[type].soffset;
+ int moffset = mlx5e_tc_attr_to_reg_mappings[type].moffset;
int match_len = mlx5e_tc_attr_to_reg_mappings[type].mlen;
- void *headers_c = spec->match_criteria;
- void *headers_v = spec->match_value;
- void *fmask, *fval;
+ u32 max_mask = GENMASK(match_len - 1, 0);
+ __be32 curr_mask_be, curr_val_be;
+ u32 curr_mask, curr_val;
fmask = headers_c + soffset;
fval = headers_v + soffset;
- memcpy(mask, fmask, match_len);
- memcpy(data, fval, match_len);
+ memcpy(&curr_mask_be, fmask, 4);
+ memcpy(&curr_val_be, fval, 4);
+
+ curr_mask = be32_to_cpu(curr_mask_be);
+ curr_val = be32_to_cpu(curr_val_be);
- *mask = be32_to_cpu((__force __be32)(*mask << (32 - (match_len * 8))));
- *data = be32_to_cpu((__force __be32)(*data << (32 - (match_len * 8))));
+ *mask = (curr_mask >> moffset) & max_mask;
+ *val = (curr_val >> moffset) & max_mask;
}
int
(mod_hdr_acts->num_actions * MLX5_MH_ACT_SZ);
/* Firmware has 5bit length field and 0 means 32bits */
- if (mlen == 4)
+ if (mlen == 32)
mlen = 0;
MLX5_SET(set_action_in, modact, action_type, MLX5_ACTION_TYPE_SET);
MLX5_SET(set_action_in, modact, field, mfield);
- MLX5_SET(set_action_in, modact, offset, moffset * 8);
- MLX5_SET(set_action_in, modact, length, mlen * 8);
+ MLX5_SET(set_action_in, modact, offset, moffset);
+ MLX5_SET(set_action_in, modact, length, mlen);
MLX5_SET(set_action_in, modact, data, data);
err = mod_hdr_acts->num_actions;
mod_hdr_acts->num_actions++;
modact = mod_hdr_acts->actions + (act_id * MLX5_MH_ACT_SZ);
/* Firmware has 5bit length field and 0 means 32bits */
- if (mlen == 4)
+ if (mlen == 32)
mlen = 0;
MLX5_SET(set_action_in, modact, action_type, MLX5_ACTION_TYPE_SET);
MLX5_SET(set_action_in, modact, field, mfield);
- MLX5_SET(set_action_in, modact, offset, moffset * 8);
- MLX5_SET(set_action_in, modact, length, mlen * 8);
+ MLX5_SET(set_action_in, modact, offset, moffset);
+ MLX5_SET(set_action_in, modact, length, mlen);
MLX5_SET(set_action_in, modact, data, data);
}
tc_skb_ext->chain = chain;
- zone_restore_id = (reg_b >> REG_MAPPING_SHIFT(NIC_ZONE_RESTORE_TO_REG)) &
+ zone_restore_id = (reg_b >> REG_MAPPING_MOFFSET(NIC_ZONE_RESTORE_TO_REG)) &
ESW_ZONE_ID_MASK;
if (!mlx5e_tc_ct_restore_flow(tc->ct, skb,
*/
#define TUNNEL_INFO_BITS 12
#define TUNNEL_INFO_BITS_MASK GENMASK(TUNNEL_INFO_BITS - 1, 0)
-#define ENC_OPTS_BITS 12
+#define ENC_OPTS_BITS 11
#define ENC_OPTS_BITS_MASK GENMASK(ENC_OPTS_BITS - 1, 0)
#define TUNNEL_ID_BITS (TUNNEL_INFO_BITS + ENC_OPTS_BITS)
#define TUNNEL_ID_MASK GENMASK(TUNNEL_ID_BITS - 1, 0)
struct mlx5e_tc_attr_to_reg_mapping {
int mfield; /* rewrite field */
- int moffset; /* offset of mfield */
- int mlen; /* bytes to rewrite/match */
+ int moffset; /* bit offset of mfield */
+ int mlen; /* bits to rewrite/match */
- int soffset; /* offset of spec for match */
+ int soffset; /* byte offset of spec for match */
};
extern struct mlx5e_tc_attr_to_reg_mapping mlx5e_tc_attr_to_reg_mappings[];
#include "fs_core.h"
#include "fs_cmd.h"
+#include "fs_ft_pool.h"
#include "mlx5_core.h"
#include "eswitch.h"
static int mlx5_cmd_stub_create_flow_table(struct mlx5_flow_root_namespace *ns,
struct mlx5_flow_table *ft,
- unsigned int log_size,
+ unsigned int size,
struct mlx5_flow_table *next_ft)
{
+ ft->max_fte = size ? roundup_pow_of_two(size) : 1;
+
return 0;
}
static int mlx5_cmd_create_flow_table(struct mlx5_flow_root_namespace *ns,
struct mlx5_flow_table *ft,
- unsigned int log_size,
+ unsigned int size,
struct mlx5_flow_table *next_ft)
{
int en_encap = !!(ft->flags & MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT);
struct mlx5_core_dev *dev = ns->dev;
int err;
+ if (size != POOL_NEXT_SIZE)
+ size = roundup_pow_of_two(size);
+ size = mlx5_ft_pool_get_avail_sz(dev, ft->type, size);
+ if (!size)
+ return -ENOSPC;
+
MLX5_SET(create_flow_table_in, in, opcode,
MLX5_CMD_OP_CREATE_FLOW_TABLE);
MLX5_SET(create_flow_table_in, in, table_type, ft->type);
MLX5_SET(create_flow_table_in, in, flow_table_context.level, ft->level);
- MLX5_SET(create_flow_table_in, in, flow_table_context.log_size, log_size);
+ MLX5_SET(create_flow_table_in, in, flow_table_context.log_size, size ? ilog2(size) : 0);
MLX5_SET(create_flow_table_in, in, vport_number, ft->vport);
MLX5_SET(create_flow_table_in, in, other_vport,
!!(ft->flags & MLX5_FLOW_TABLE_OTHER_VPORT));
}
err = mlx5_cmd_exec_inout(dev, create_flow_table, in, out);
- if (!err)
+ if (!err) {
ft->id = MLX5_GET(create_flow_table_out, out,
table_id);
+ ft->max_fte = size;
+ } else {
+ mlx5_ft_pool_put_sz(ns->dev, size);
+ }
+
return err;
}
{
u32 in[MLX5_ST_SZ_DW(destroy_flow_table_in)] = {};
struct mlx5_core_dev *dev = ns->dev;
+ int err;
MLX5_SET(destroy_flow_table_in, in, opcode,
MLX5_CMD_OP_DESTROY_FLOW_TABLE);
MLX5_SET(destroy_flow_table_in, in, other_vport,
!!(ft->flags & MLX5_FLOW_TABLE_OTHER_VPORT));
- return mlx5_cmd_exec_in(dev, destroy_flow_table, in);
+ err = mlx5_cmd_exec_in(dev, destroy_flow_table, in);
+ if (!err)
+ mlx5_ft_pool_put_sz(ns->dev, ft->max_fte);
+
+ return err;
}
static int mlx5_cmd_modify_flow_table(struct mlx5_flow_root_namespace *ns,
struct mlx5_flow_cmds {
int (*create_flow_table)(struct mlx5_flow_root_namespace *ns,
struct mlx5_flow_table *ft,
- unsigned int log_size,
+ unsigned int size,
struct mlx5_flow_table *next_ft);
int (*destroy_flow_table)(struct mlx5_flow_root_namespace *ns,
struct mlx5_flow_table *ft);
#include "mlx5_core.h"
#include "fs_core.h"
#include "fs_cmd.h"
+#include "fs_ft_pool.h"
#include "diag/fs_tracepoint.h"
#include "accel/ipsec.h"
#include "fpga/ipsec.h"
return fg;
}
-static struct mlx5_flow_table *alloc_flow_table(int level, u16 vport, int max_fte,
+static struct mlx5_flow_table *alloc_flow_table(int level, u16 vport,
enum fs_flow_table_type table_type,
enum fs_flow_table_op_mod op_mod,
u32 flags)
ft->op_mod = op_mod;
ft->type = table_type;
ft->vport = vport;
- ft->max_fte = max_fte;
ft->flags = flags;
INIT_LIST_HEAD(&ft->fwd_rules);
mutex_init(&ft->lock);
struct mlx5_flow_table *next_ft;
struct fs_prio *fs_prio = NULL;
struct mlx5_flow_table *ft;
- int log_table_sz;
int err;
if (!root) {
*/
ft = alloc_flow_table(ft_attr->level,
vport,
- ft_attr->max_fte ? roundup_pow_of_two(ft_attr->max_fte) : 0,
root->table_type,
op_mod, ft_attr->flags);
if (IS_ERR(ft)) {
}
tree_init_node(&ft->node, del_hw_flow_table, del_sw_flow_table);
- log_table_sz = ft->max_fte ? ilog2(ft->max_fte) : 0;
next_ft = unmanaged ? ft_attr->next_ft :
find_next_chained_ft(fs_prio);
ft->def_miss_action = ns->def_miss_action;
ft->ns = ns;
- err = root->cmds->create_flow_table(root, ft, log_table_sz, next_ft);
+ err = root->cmds->create_flow_table(root, ft, ft_attr->max_fte, next_ft);
if (err)
goto free_ft;
ft_attr.level = level;
ft_attr.prio = prio;
+ ft_attr.max_fte = 1;
+
return __mlx5_create_flow_table(ns, &ft_attr, FS_FT_OP_MOD_LAG_DEMUX, 0);
}
EXPORT_SYMBOL(mlx5_create_lag_demux_flow_table);
+#define MAX_FLOW_GROUP_SIZE BIT(24)
struct mlx5_flow_table*
mlx5_create_auto_grouped_flow_table(struct mlx5_flow_namespace *ns,
struct mlx5_flow_table_attr *ft_attr)
{
int num_reserved_entries = ft_attr->autogroup.num_reserved_entries;
- int autogroups_max_fte = ft_attr->max_fte - num_reserved_entries;
int max_num_groups = ft_attr->autogroup.max_num_groups;
struct mlx5_flow_table *ft;
-
- if (max_num_groups > autogroups_max_fte)
- return ERR_PTR(-EINVAL);
- if (num_reserved_entries > ft_attr->max_fte)
- return ERR_PTR(-EINVAL);
+ int autogroups_max_fte;
ft = mlx5_create_flow_table(ns, ft_attr);
if (IS_ERR(ft))
return ft;
+ autogroups_max_fte = ft->max_fte - num_reserved_entries;
+ if (max_num_groups > autogroups_max_fte)
+ goto err_validate;
+ if (num_reserved_entries > ft->max_fte)
+ goto err_validate;
+
+ /* Align the number of groups according to the largest group size */
+ if (autogroups_max_fte / (max_num_groups + 1) > MAX_FLOW_GROUP_SIZE)
+ max_num_groups = (autogroups_max_fte / MAX_FLOW_GROUP_SIZE) - 1;
+
ft->autogroup.active = true;
ft->autogroup.required_groups = max_num_groups;
ft->autogroup.max_fte = autogroups_max_fte;
ft->autogroup.group_size = autogroups_max_fte / (max_num_groups + 1);
return ft;
+
+err_validate:
+ mlx5_destroy_flow_table(ft);
+ return ERR_PTR(-ENOSPC);
}
EXPORT_SYMBOL(mlx5_create_auto_grouped_flow_table);
mlx5_cleanup_fc_stats(dev);
kmem_cache_destroy(steering->ftes_cache);
kmem_cache_destroy(steering->fgs_cache);
+ mlx5_ft_pool_destroy(dev);
kfree(steering);
}
if (err)
return err;
+ err = mlx5_ft_pool_init(dev);
+ if (err)
+ return err;
+
steering = kzalloc(sizeof(*steering), GFP_KERNEL);
if (!steering)
- return -ENOMEM;
+ goto err;
steering->dev = dev;
dev->priv.steering = steering;
#define MLX5_CAP_FLOWTABLE_TYPE(mdev, cap, type) ( \
(type == FS_FT_NIC_RX) ? MLX5_CAP_FLOWTABLE_NIC_RX(mdev, cap) : \
+ (type == FS_FT_NIC_TX) ? MLX5_CAP_FLOWTABLE_NIC_TX(mdev, cap) : \
(type == FS_FT_ESW_EGRESS_ACL) ? MLX5_CAP_ESW_EGRESS_ACL(mdev, cap) : \
(type == FS_FT_ESW_INGRESS_ACL) ? MLX5_CAP_ESW_INGRESS_ACL(mdev, cap) : \
(type == FS_FT_FDB) ? MLX5_CAP_ESW_FLOWTABLE_FDB(mdev, cap) : \
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
+/* Copyright (c) 2021 Mellanox Technologies. */
+
+#include "fs_ft_pool.h"
+
+/* Firmware currently has 4 pool of 4 sizes that it supports (FT_POOLS),
+ * and a virtual memory region of 16M (MLX5_FT_SIZE), this region is duplicated
+ * for each flow table pool. We can allocate up to 16M of each pool,
+ * and we keep track of how much we used via mlx5_ft_pool_get_avail_sz.
+ * Firmware doesn't report any of this for now.
+ * ESW_POOL is expected to be sorted from large to small and match firmware
+ * pools.
+ */
+#define FT_SIZE (16 * 1024 * 1024)
+static const unsigned int FT_POOLS[] = { 4 * 1024 * 1024,
+ 1 * 1024 * 1024,
+ 64 * 1024,
+ 128,
+ 1 /* size for termination tables */ };
+struct mlx5_ft_pool {
+ int ft_left[ARRAY_SIZE(FT_POOLS)];
+};
+
+int mlx5_ft_pool_init(struct mlx5_core_dev *dev)
+{
+ struct mlx5_ft_pool *ft_pool;
+ int i;
+
+ ft_pool = kzalloc(sizeof(*ft_pool), GFP_KERNEL);
+
+ for (i = ARRAY_SIZE(FT_POOLS) - 1; i >= 0; i--)
+ ft_pool->ft_left[i] = FT_SIZE / FT_POOLS[i];
+
+ dev->priv.ft_pool = ft_pool;
+ return 0;
+}
+
+void mlx5_ft_pool_destroy(struct mlx5_core_dev *dev)
+{
+ kfree(dev->priv.ft_pool);
+}
+
+int
+mlx5_ft_pool_get_avail_sz(struct mlx5_core_dev *dev, enum fs_flow_table_type table_type,
+ int desired_size)
+{
+ u32 max_ft_size = 1 << MLX5_CAP_FLOWTABLE_TYPE(dev, log_max_ft_size, table_type);
+ int i, found_i = -1;
+
+ for (i = ARRAY_SIZE(FT_POOLS) - 1; i >= 0; i--) {
+ if (dev->priv.ft_pool->ft_left[i] && FT_POOLS[i] >= desired_size &&
+ FT_POOLS[i] <= max_ft_size) {
+ found_i = i;
+ if (desired_size != POOL_NEXT_SIZE)
+ break;
+ }
+ }
+
+ if (found_i != -1) {
+ --dev->priv.ft_pool->ft_left[found_i];
+ return FT_POOLS[found_i];
+ }
+
+ return 0;
+}
+
+void
+mlx5_ft_pool_put_sz(struct mlx5_core_dev *dev, int sz)
+{
+ int i;
+
+ if (!sz)
+ return;
+
+ for (i = ARRAY_SIZE(FT_POOLS) - 1; i >= 0; i--) {
+ if (sz == FT_POOLS[i]) {
+ ++dev->priv.ft_pool->ft_left[i];
+ return;
+ }
+ }
+
+ WARN_ONCE(1, "Couldn't find size %d in flow table size pool", sz);
+}
--- /dev/null
+/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
+/* Copyright (c) 2021 Mellanox Technologies. */
+
+#ifndef __MLX5_FS_FT_POOL_H__
+#define __MLX5_FS_FT_POOL_H__
+
+#include <linux/mlx5/driver.h>
+#include "fs_core.h"
+
+#define POOL_NEXT_SIZE 0
+
+int mlx5_ft_pool_init(struct mlx5_core_dev *dev);
+void mlx5_ft_pool_destroy(struct mlx5_core_dev *dev);
+
+int
+mlx5_ft_pool_get_avail_sz(struct mlx5_core_dev *dev, enum fs_flow_table_type table_type,
+ int desired_size);
+void
+mlx5_ft_pool_put_sz(struct mlx5_core_dev *dev, int sz);
+
+#endif /* __MLX5_FS_FT_POOL_H__ */
static void mlx5_infer_tx_affinity_mapping(struct lag_tracker *tracker,
u8 *port1, u8 *port2)
{
+ bool p1en;
+ bool p2en;
+
+ p1en = tracker->netdev_state[MLX5_LAG_P1].tx_enabled &&
+ tracker->netdev_state[MLX5_LAG_P1].link_up;
+
+ p2en = tracker->netdev_state[MLX5_LAG_P2].tx_enabled &&
+ tracker->netdev_state[MLX5_LAG_P2].link_up;
+
*port1 = 1;
*port2 = 2;
- if (!tracker->netdev_state[MLX5_LAG_P1].tx_enabled ||
- !tracker->netdev_state[MLX5_LAG_P1].link_up) {
- *port1 = 2;
+ if ((!p1en && !p2en) || (p1en && p2en))
return;
- }
- if (!tracker->netdev_state[MLX5_LAG_P2].tx_enabled ||
- !tracker->netdev_state[MLX5_LAG_P2].link_up)
+ if (p1en)
*port2 = 1;
+ else
+ *port1 = 2;
}
void mlx5_modify_lag(struct mlx5_lag *ldev,
if (!mlx5_lag_is_ready(ldev))
return;
- spin_lock(&lag_lock);
tracker = ldev->tracker;
- spin_unlock(&lag_lock);
do_bond = tracker.is_bonded && mlx5_lag_check_prereq(ldev);
!mlx5_sriov_is_enabled(dev1);
#ifdef CONFIG_MLX5_ESWITCH
- roce_lag &= dev0->priv.eswitch->mode == MLX5_ESWITCH_NONE &&
- dev1->priv.eswitch->mode == MLX5_ESWITCH_NONE;
+ roce_lag = roce_lag &&
+ dev0->priv.eswitch->mode == MLX5_ESWITCH_NONE &&
+ dev1->priv.eswitch->mode == MLX5_ESWITCH_NONE;
#endif
if (roce_lag)
break;
}
- spin_lock(&lag_lock);
ldev->tracker = tracker;
- spin_unlock(&lag_lock);
if (changed)
mlx5_queue_bond_work(ldev, 0);
#include <linux/mlx5/fs.h>
#include "lib/fs_chains.h"
+#include "fs_ft_pool.h"
#include "en/mapping.h"
#include "fs_core.h"
#include "en_tc.h"
#define chains_lock(chains) ((chains)->lock)
#define chains_ht(chains) ((chains)->chains_ht)
#define prios_ht(chains) ((chains)->prios_ht)
-#define ft_pool_left(chains) ((chains)->ft_left)
#define tc_default_ft(chains) ((chains)->tc_default_ft)
#define tc_end_ft(chains) ((chains)->tc_end_ft)
#define ns_to_chains_fs_prio(ns) ((ns) == MLX5_FLOW_NAMESPACE_FDB ? \
FDB_TC_OFFLOAD : MLX5E_TC_PRIO)
-
-/* Firmware currently has 4 pool of 4 sizes that it supports (FT_POOLS),
- * and a virtual memory region of 16M (MLX5_FT_SIZE), this region is duplicated
- * for each flow table pool. We can allocate up to 16M of each pool,
- * and we keep track of how much we used via get_next_avail_sz_from_pool.
- * Firmware doesn't report any of this for now.
- * ESW_POOL is expected to be sorted from large to small and match firmware
- * pools.
- */
-#define FT_SIZE (16 * 1024 * 1024)
-static const unsigned int FT_POOLS[] = { 4 * 1024 * 1024,
- 1 * 1024 * 1024,
- 64 * 1024,
- 128 };
#define FT_TBL_SZ (64 * 1024)
struct mlx5_fs_chains {
enum mlx5_flow_namespace_type ns;
u32 group_num;
u32 flags;
-
- int ft_left[ARRAY_SIZE(FT_POOLS)];
};
struct fs_chain {
tc_end_ft(chains) = ft;
}
-#define POOL_NEXT_SIZE 0
-static int
-mlx5_chains_get_avail_sz_from_pool(struct mlx5_fs_chains *chains,
- int desired_size)
-{
- int i, found_i = -1;
-
- for (i = ARRAY_SIZE(FT_POOLS) - 1; i >= 0; i--) {
- if (ft_pool_left(chains)[i] && FT_POOLS[i] > desired_size) {
- found_i = i;
- if (desired_size != POOL_NEXT_SIZE)
- break;
- }
- }
-
- if (found_i != -1) {
- --ft_pool_left(chains)[found_i];
- return FT_POOLS[found_i];
- }
-
- return 0;
-}
-
-static void
-mlx5_chains_put_sz_to_pool(struct mlx5_fs_chains *chains, int sz)
-{
- int i;
-
- for (i = ARRAY_SIZE(FT_POOLS) - 1; i >= 0; i--) {
- if (sz == FT_POOLS[i]) {
- ++ft_pool_left(chains)[i];
- return;
- }
- }
-
- WARN_ONCE(1, "Couldn't find size %d in flow table size pool", sz);
-}
-
-static void
-mlx5_chains_init_sz_pool(struct mlx5_fs_chains *chains, u32 ft_max)
-{
- int i;
-
- for (i = ARRAY_SIZE(FT_POOLS) - 1; i >= 0; i--)
- ft_pool_left(chains)[i] =
- FT_POOLS[i] <= ft_max ? FT_SIZE / FT_POOLS[i] : 0;
-}
-
static struct mlx5_flow_table *
mlx5_chains_create_table(struct mlx5_fs_chains *chains,
u32 chain, u32 prio, u32 level)
ft_attr.flags |= (MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT |
MLX5_FLOW_TABLE_TUNNEL_EN_DECAP);
- sz = (chain == mlx5_chains_get_nf_ft_chain(chains)) ?
- mlx5_chains_get_avail_sz_from_pool(chains, FT_TBL_SZ) :
- mlx5_chains_get_avail_sz_from_pool(chains, POOL_NEXT_SIZE);
- if (!sz)
- return ERR_PTR(-ENOSPC);
+ sz = (chain == mlx5_chains_get_nf_ft_chain(chains)) ? FT_TBL_SZ : POOL_NEXT_SIZE;
ft_attr.max_fte = sz;
/* We use tc_default_ft(chains) as the table's next_ft till
if (IS_ERR(ft)) {
mlx5_core_warn(chains->dev, "Failed to create chains table err %d (chain: %d, prio: %d, level: %d, size: %d)\n",
(int)PTR_ERR(ft), chain, prio, level, sz);
- mlx5_chains_put_sz_to_pool(chains, sz);
return ft;
}
return ft;
}
-static void
-mlx5_chains_destroy_table(struct mlx5_fs_chains *chains,
- struct mlx5_flow_table *ft)
-{
- mlx5_chains_put_sz_to_pool(chains, ft->max_fte);
- mlx5_destroy_flow_table(ft);
-}
-
static int
create_chain_restore(struct fs_chain *chain)
{
MLX5_SET(set_action_in, modact, field,
mlx5e_tc_attr_to_reg_mappings[chain_to_reg].mfield);
MLX5_SET(set_action_in, modact, offset,
- mlx5e_tc_attr_to_reg_mappings[chain_to_reg].moffset * 8);
+ mlx5e_tc_attr_to_reg_mappings[chain_to_reg].moffset);
MLX5_SET(set_action_in, modact, length,
- mlx5e_tc_attr_to_reg_mappings[chain_to_reg].mlen * 8);
+ mlx5e_tc_attr_to_reg_mappings[chain_to_reg].mlen == 32 ?
+ 0 : mlx5e_tc_attr_to_reg_mappings[chain_to_reg].mlen);
MLX5_SET(set_action_in, modact, data, chain->id);
mod_hdr = mlx5_modify_header_alloc(chains->dev, chains->ns,
1, modact);
err_miss_rule:
mlx5_destroy_flow_group(miss_group);
err_group:
- mlx5_chains_destroy_table(chains, ft);
+ mlx5_destroy_flow_table(ft);
err_create:
err_alloc:
kvfree(prio_s);
prio_params);
mlx5_del_flow_rules(prio->miss_rule);
mlx5_destroy_flow_group(prio->miss_group);
- mlx5_chains_destroy_table(chains, prio->ft);
+ mlx5_destroy_flow_table(prio->ft);
mlx5_chains_put_chain(chain);
kvfree(prio);
}
mlx5_chains_destroy_global_table(struct mlx5_fs_chains *chains,
struct mlx5_flow_table *ft)
{
- mlx5_chains_destroy_table(chains, ft);
+ mlx5_destroy_flow_table(ft);
}
static struct mlx5_fs_chains *
mlx5_chains_get_chain_range(chains_priv),
mlx5_chains_get_prio_range(chains_priv));
- mlx5_chains_init_sz_pool(chains_priv, attr->max_ft_sz);
-
err = rhashtable_init(&chains_ht(chains_priv), &chain_params);
if (err)
goto init_chains_ht_err;
u32 tx_head;
void *buf;
u32 buf_size;
- struct ib_wc wc[MAX_SEND_CQE];
u8 sync_buff[MIN_READ_SYNC];
struct mlx5dr_mr *sync_mr;
spinlock_t lock; /* Protect the data path of the send ring */
static int mlx5_cmd_dr_create_flow_table(struct mlx5_flow_root_namespace *ns,
struct mlx5_flow_table *ft,
- unsigned int log_size,
+ unsigned int size,
struct mlx5_flow_table *next_ft)
{
struct mlx5dr_table *tbl;
if (mlx5_dr_is_fw_table(ft->flags))
return mlx5_fs_cmd_get_fw_cmds()->create_flow_table(ns, ft,
- log_size,
+ size,
next_ft);
flags = ft->flags;
/* turn off encap/decap if not supported for sw-str by fw */
}
}
+ ft->max_fte = INT_MAX;
+
return 0;
}
int idx;
};
+struct mlx5_ft_pool;
struct mlx5_priv {
/* IRQ table valid only for real pci devices PF or VF */
struct mlx5_irq_table *irq_table;
struct mlx5_core_roce roce;
struct mlx5_fc_stats fc_stats;
struct mlx5_rl_table rl_table;
+ struct mlx5_ft_pool *ft_pool;
struct mlx5_bfreg_data bfregs;
struct mlx5_uars_page *uar;
u16 vport_num);
/* Reg C1 usage:
- * Reg C1 = < ESW_TUN_ID(12) | ESW_TUN_OPTS(12) | ESW_ZONE_ID(8) >
+ * Reg C1 = < Reserved(1) | ESW_TUN_ID(12) | ESW_TUN_OPTS(11) | ESW_ZONE_ID(8) >
*
- * Highest 12 bits of reg c1 is the encapsulation tunnel id, next 12 bits is
- * encapsulation tunnel options, and the lowest 8 bits are used for zone id.
+ * Highest bit is reserved for other offloads as marker bit, next 12 bits of reg c1
+ * is the encapsulation tunnel id, next 11 bits is encapsulation tunnel options,
+ * and the lowest 8 bits are used for zone id.
*
* Zone id is used to restore CT flow when packet misses on chain.
*
* on miss and to support inner header rewrite by means of implicit chain 0
* flows.
*/
+#define ESW_RESERVED_BITS 1
#define ESW_ZONE_ID_BITS 8
-#define ESW_TUN_OPTS_BITS 12
+#define ESW_TUN_OPTS_BITS 11
#define ESW_TUN_ID_BITS 12
#define ESW_TUN_OPTS_OFFSET ESW_ZONE_ID_BITS
#define ESW_TUN_OFFSET ESW_TUN_OPTS_OFFSET
#define ESW_ZONE_ID_MASK GENMASK(ESW_ZONE_ID_BITS - 1, 0)
-#define ESW_TUN_OPTS_MASK GENMASK(32 - ESW_TUN_ID_BITS - 1, ESW_TUN_OPTS_OFFSET)
-#define ESW_TUN_MASK GENMASK(31, ESW_TUN_OFFSET)
+#define ESW_TUN_OPTS_MASK GENMASK(31 - ESW_TUN_ID_BITS - ESW_RESERVED_BITS, ESW_TUN_OPTS_OFFSET)
+#define ESW_TUN_MASK GENMASK(31 - ESW_RESERVED_BITS, ESW_TUN_OFFSET)
#define ESW_TUN_ID_SLOW_TABLE_GOTO_VPORT 0 /* 0 is not a valid tunnel id */
-#define ESW_TUN_OPTS_SLOW_TABLE_GOTO_VPORT 0xFFF /* 0xFFF is a reserved mapping */
+/* 0x7FF is a reserved mapping */
+#define ESW_TUN_OPTS_SLOW_TABLE_GOTO_VPORT GENMASK(ESW_TUN_OPTS_BITS - 1, 0)
#define ESW_TUN_SLOW_TABLE_GOTO_VPORT ((ESW_TUN_ID_SLOW_TABLE_GOTO_VPORT << ESW_TUN_OPTS_BITS) | \
ESW_TUN_OPTS_SLOW_TABLE_GOTO_VPORT)
#define ESW_TUN_SLOW_TABLE_GOTO_VPORT_MARK ESW_TUN_OPTS_MASK
projects / linux-block.git / commitdiff