[linux-block.git] / drivers / net / ethernet / mellanox / mlx5 / core / en_accel / ipsec_offload.c

// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
/* Copyright (c) 2017, Mellanox Technologies inc. All rights reserved. */

#include "mlx5_core.h"
#include "en.h"
#include "ipsec.h"
#include "lib/mlx5.h"

enum {
	MLX5_IPSEC_ASO_REMOVE_FLOW_PKT_CNT_OFFSET,
};

u32 mlx5_ipsec_device_caps(struct mlx5_core_dev *mdev)
{
	u32 caps = 0;

	if (!MLX5_CAP_GEN(mdev, ipsec_offload))
		return 0;

	if (!MLX5_CAP_GEN(mdev, log_max_dek))
		return 0;

	if (!(MLX5_CAP_GEN_64(mdev, general_obj_types) &
	    MLX5_HCA_CAP_GENERAL_OBJECT_TYPES_IPSEC))
		return 0;

	if (!MLX5_CAP_FLOWTABLE_NIC_TX(mdev, ipsec_encrypt) ||
	    !MLX5_CAP_FLOWTABLE_NIC_RX(mdev, ipsec_decrypt))
		return 0;

	if (!MLX5_CAP_IPSEC(mdev, ipsec_crypto_esp_aes_gcm_128_encrypt) ||
	    !MLX5_CAP_IPSEC(mdev, ipsec_crypto_esp_aes_gcm_128_decrypt))
		return 0;

	if (MLX5_CAP_IPSEC(mdev, ipsec_crypto_offload) &&
	    MLX5_CAP_ETH(mdev, insert_trailer) && MLX5_CAP_ETH(mdev, swp))
		caps |= MLX5_IPSEC_CAP_CRYPTO;

	if (MLX5_CAP_IPSEC(mdev, ipsec_full_offload) &&
	    MLX5_CAP_FLOWTABLE_NIC_TX(mdev, reformat_add_esp_trasport) &&
	    MLX5_CAP_FLOWTABLE_NIC_RX(mdev, reformat_del_esp_trasport) &&
	    MLX5_CAP_FLOWTABLE_NIC_RX(mdev, decap))
		caps |= MLX5_IPSEC_CAP_PACKET_OFFLOAD;

	if (!caps)
		return 0;

	if (MLX5_CAP_IPSEC(mdev, ipsec_esn))
		caps |= MLX5_IPSEC_CAP_ESN;

	/* We can accommodate up to 2^24 different IPsec objects
	 * because we use up to 24 bit in flow table metadata
	 * to hold the IPsec Object unique handle.
	 */
	WARN_ON_ONCE(MLX5_CAP_IPSEC(mdev, log_max_ipsec_offload) > 24);
	return caps;
}
EXPORT_SYMBOL_GPL(mlx5_ipsec_device_caps);

static void mlx5e_ipsec_packet_setup(void *obj, u32 pdn,
				     struct mlx5_accel_esp_xfrm_attrs *attrs)
{
	void *aso_ctx;

	aso_ctx = MLX5_ADDR_OF(ipsec_obj, obj, ipsec_aso);
	if (attrs->esn_trigger) {
		MLX5_SET(ipsec_aso, aso_ctx, esn_event_arm, 1);

		if (attrs->dir == XFRM_DEV_OFFLOAD_IN) {
			MLX5_SET(ipsec_aso, aso_ctx, window_sz,
				 attrs->replay_window / 64);
			MLX5_SET(ipsec_aso, aso_ctx, mode,
				 MLX5_IPSEC_ASO_REPLAY_PROTECTION);
			}
	}

	/* ASO context */
	MLX5_SET(ipsec_obj, obj, ipsec_aso_access_pd, pdn);
	MLX5_SET(ipsec_obj, obj, full_offload, 1);
	MLX5_SET(ipsec_aso, aso_ctx, valid, 1);
	/* MLX5_IPSEC_ASO_REG_C_4_5 is type C register that is used
	 * in flow steering to perform matching against. Please be
	 * aware that this register was chosen arbitrary and can't
	 * be used in other places as long as IPsec packet offload
	 * active.
	 */
	MLX5_SET(ipsec_obj, obj, aso_return_reg, MLX5_IPSEC_ASO_REG_C_4_5);
	if (attrs->dir == XFRM_DEV_OFFLOAD_OUT)
		MLX5_SET(ipsec_aso, aso_ctx, mode, MLX5_IPSEC_ASO_INC_SN);

	if (attrs->hard_packet_limit != XFRM_INF) {
		MLX5_SET(ipsec_aso, aso_ctx, remove_flow_pkt_cnt,
			 lower_32_bits(attrs->hard_packet_limit));
		MLX5_SET(ipsec_aso, aso_ctx, hard_lft_arm, 1);
		MLX5_SET(ipsec_aso, aso_ctx, remove_flow_enable, 1);
	}

	if (attrs->soft_packet_limit != XFRM_INF) {
		MLX5_SET(ipsec_aso, aso_ctx, remove_flow_soft_lft,
			 lower_32_bits(attrs->soft_packet_limit));

		MLX5_SET(ipsec_aso, aso_ctx, soft_lft_arm, 1);
	}
}

static int mlx5_create_ipsec_obj(struct mlx5e_ipsec_sa_entry *sa_entry)
{
	struct mlx5_accel_esp_xfrm_attrs *attrs = &sa_entry->attrs;
	struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry);
	struct aes_gcm_keymat *aes_gcm = &attrs->aes_gcm;
	u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
	u32 in[MLX5_ST_SZ_DW(create_ipsec_obj_in)] = {};
	void *obj, *salt_p, *salt_iv_p;
	struct mlx5e_hw_objs *res;
	int err;

	obj = MLX5_ADDR_OF(create_ipsec_obj_in, in, ipsec_object);

	/* salt and seq_iv */
	salt_p = MLX5_ADDR_OF(ipsec_obj, obj, salt);
	memcpy(salt_p, &aes_gcm->salt, sizeof(aes_gcm->salt));

	MLX5_SET(ipsec_obj, obj, icv_length, MLX5_IPSEC_OBJECT_ICV_LEN_16B);
	salt_iv_p = MLX5_ADDR_OF(ipsec_obj, obj, implicit_iv);
	memcpy(salt_iv_p, &aes_gcm->seq_iv, sizeof(aes_gcm->seq_iv));
	/* esn */
	if (attrs->esn_trigger) {
		MLX5_SET(ipsec_obj, obj, esn_en, 1);
		MLX5_SET(ipsec_obj, obj, esn_msb, attrs->esn);
		MLX5_SET(ipsec_obj, obj, esn_overlap, attrs->esn_overlap);
	}

	MLX5_SET(ipsec_obj, obj, dekn, sa_entry->enc_key_id);

	/* general object fields set */
	MLX5_SET(general_obj_in_cmd_hdr, in, opcode,
		 MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
	MLX5_SET(general_obj_in_cmd_hdr, in, obj_type,
		 MLX5_GENERAL_OBJECT_TYPES_IPSEC);

	res = &mdev->mlx5e_res.hw_objs;
	if (attrs->type == XFRM_DEV_OFFLOAD_PACKET)
		mlx5e_ipsec_packet_setup(obj, res->pdn, attrs);

	err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
	if (!err)
		sa_entry->ipsec_obj_id =
			MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);

	return err;
}

static void mlx5_destroy_ipsec_obj(struct mlx5e_ipsec_sa_entry *sa_entry)
{
	struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry);
	u32 in[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)] = {};
	u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];

	MLX5_SET(general_obj_in_cmd_hdr, in, opcode,
		 MLX5_CMD_OP_DESTROY_GENERAL_OBJECT);
	MLX5_SET(general_obj_in_cmd_hdr, in, obj_type,
		 MLX5_GENERAL_OBJECT_TYPES_IPSEC);
	MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, sa_entry->ipsec_obj_id);

	mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
}

int mlx5_ipsec_create_sa_ctx(struct mlx5e_ipsec_sa_entry *sa_entry)
{
	struct aes_gcm_keymat *aes_gcm = &sa_entry->attrs.aes_gcm;
	struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry);
	int err;

	/* key */
	err = mlx5_create_encryption_key(mdev, aes_gcm->aes_key,
					 aes_gcm->key_len / BITS_PER_BYTE,
					 MLX5_ACCEL_OBJ_IPSEC_KEY,
					 &sa_entry->enc_key_id);
	if (err) {
		mlx5_core_dbg(mdev, "Failed to create encryption key (err = %d)\n", err);
		return err;
	}

	err = mlx5_create_ipsec_obj(sa_entry);
	if (err) {
		mlx5_core_dbg(mdev, "Failed to create IPsec object (err = %d)\n", err);
		goto err_enc_key;
	}

	return 0;

err_enc_key:
	mlx5_destroy_encryption_key(mdev, sa_entry->enc_key_id);
	return err;
}

void mlx5_ipsec_free_sa_ctx(struct mlx5e_ipsec_sa_entry *sa_entry)
{
	struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry);

	mlx5_destroy_ipsec_obj(sa_entry);
	mlx5_destroy_encryption_key(mdev, sa_entry->enc_key_id);
}

static int mlx5_modify_ipsec_obj(struct mlx5e_ipsec_sa_entry *sa_entry,
				 const struct mlx5_accel_esp_xfrm_attrs *attrs)
{
	struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry);
	u32 in[MLX5_ST_SZ_DW(modify_ipsec_obj_in)] = {};
	u32 out[MLX5_ST_SZ_DW(query_ipsec_obj_out)];
	u64 modify_field_select = 0;
	u64 general_obj_types;
	void *obj;
	int err;

	if (!attrs->esn_trigger)
		return 0;

	general_obj_types = MLX5_CAP_GEN_64(mdev, general_obj_types);
	if (!(general_obj_types & MLX5_HCA_CAP_GENERAL_OBJECT_TYPES_IPSEC))
		return -EINVAL;

	/* general object fields set */
	MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_QUERY_GENERAL_OBJECT);
	MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, MLX5_GENERAL_OBJECT_TYPES_IPSEC);
	MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, sa_entry->ipsec_obj_id);
	err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
	if (err) {
		mlx5_core_err(mdev, "Query IPsec object failed (Object id %d), err = %d\n",
			      sa_entry->ipsec_obj_id, err);
		return err;
	}

	obj = MLX5_ADDR_OF(query_ipsec_obj_out, out, ipsec_object);
	modify_field_select = MLX5_GET64(ipsec_obj, obj, modify_field_select);

	/* esn */
	if (!(modify_field_select & MLX5_MODIFY_IPSEC_BITMASK_ESN_OVERLAP) ||
	    !(modify_field_select & MLX5_MODIFY_IPSEC_BITMASK_ESN_MSB))
		return -EOPNOTSUPP;

	obj = MLX5_ADDR_OF(modify_ipsec_obj_in, in, ipsec_object);
	MLX5_SET64(ipsec_obj, obj, modify_field_select,
		   MLX5_MODIFY_IPSEC_BITMASK_ESN_OVERLAP |
			   MLX5_MODIFY_IPSEC_BITMASK_ESN_MSB);
	MLX5_SET(ipsec_obj, obj, esn_msb, attrs->esn);
	MLX5_SET(ipsec_obj, obj, esn_overlap, attrs->esn_overlap);

	/* general object fields set */
	MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_MODIFY_GENERAL_OBJECT);

	return mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
}

void mlx5_accel_esp_modify_xfrm(struct mlx5e_ipsec_sa_entry *sa_entry,
				const struct mlx5_accel_esp_xfrm_attrs *attrs)
{
	int err;

	err = mlx5_modify_ipsec_obj(sa_entry, attrs);
	if (err)
		return;

	memcpy(&sa_entry->attrs, attrs, sizeof(sa_entry->attrs));
}

static void
mlx5e_ipsec_aso_update_esn(struct mlx5e_ipsec_sa_entry *sa_entry,
			   const struct mlx5_accel_esp_xfrm_attrs *attrs)
{
	struct mlx5_wqe_aso_ctrl_seg data = {};

	data.data_mask_mode = MLX5_ASO_DATA_MASK_MODE_BITWISE_64BIT << 6;
	data.condition_1_0_operand = MLX5_ASO_ALWAYS_TRUE | MLX5_ASO_ALWAYS_TRUE
								    << 4;
	data.data_offset_condition_operand = MLX5_IPSEC_ASO_REMOVE_FLOW_PKT_CNT_OFFSET;
	data.bitwise_data = cpu_to_be64(BIT_ULL(54));
	data.data_mask = data.bitwise_data;

	mlx5e_ipsec_aso_query(sa_entry, &data);
}

static void mlx5e_ipsec_update_esn_state(struct mlx5e_ipsec_sa_entry *sa_entry,
					 u32 mode_param)
{
	struct mlx5_accel_esp_xfrm_attrs attrs = {};

	if (mode_param < MLX5E_IPSEC_ESN_SCOPE_MID) {
		sa_entry->esn_state.esn++;
		sa_entry->esn_state.overlap = 0;
	} else {
		sa_entry->esn_state.overlap = 1;
	}

	mlx5e_ipsec_build_accel_xfrm_attrs(sa_entry, &attrs);
	mlx5_accel_esp_modify_xfrm(sa_entry, &attrs);
	mlx5e_ipsec_aso_update_esn(sa_entry, &attrs);
}

static void mlx5e_ipsec_handle_event(struct work_struct *_work)
{
	struct mlx5e_ipsec_work *work =
		container_of(_work, struct mlx5e_ipsec_work, work);
	struct mlx5_accel_esp_xfrm_attrs *attrs;
	struct mlx5e_ipsec_sa_entry *sa_entry;
	struct mlx5e_ipsec_aso *aso;
	struct mlx5e_ipsec *ipsec;
	int ret;

	sa_entry = xa_load(&work->ipsec->sadb, work->id);
	if (!sa_entry)
		goto out;

	ipsec = sa_entry->ipsec;
	aso = ipsec->aso;
	attrs = &sa_entry->attrs;

	spin_lock(&sa_entry->x->lock);
	ret = mlx5e_ipsec_aso_query(sa_entry, NULL);
	if (ret)
		goto unlock;

	if (attrs->esn_trigger &&
	    !MLX5_GET(ipsec_aso, aso->ctx, esn_event_arm)) {
		u32 mode_param = MLX5_GET(ipsec_aso, aso->ctx, mode_parameter);

		mlx5e_ipsec_update_esn_state(sa_entry, mode_param);
	}

	if (attrs->soft_packet_limit != XFRM_INF)
		if (!MLX5_GET(ipsec_aso, aso->ctx, soft_lft_arm) ||
		    !MLX5_GET(ipsec_aso, aso->ctx, hard_lft_arm) ||
		    !MLX5_GET(ipsec_aso, aso->ctx, remove_flow_enable))
			xfrm_state_check_expire(sa_entry->x);

unlock:
	spin_unlock(&sa_entry->x->lock);
out:
	kfree(work);
}

static int mlx5e_ipsec_event(struct notifier_block *nb, unsigned long event,
			     void *data)
{
	struct mlx5e_ipsec *ipsec = container_of(nb, struct mlx5e_ipsec, nb);
	struct mlx5_eqe_obj_change *object;
	struct mlx5e_ipsec_work *work;
	struct mlx5_eqe *eqe = data;
	u16 type;

	if (event != MLX5_EVENT_TYPE_OBJECT_CHANGE)
		return NOTIFY_DONE;

	object = &eqe->data.obj_change;
	type = be16_to_cpu(object->obj_type);

	if (type != MLX5_GENERAL_OBJECT_TYPES_IPSEC)
		return NOTIFY_DONE;

	work = kmalloc(sizeof(*work), GFP_ATOMIC);
	if (!work)
		return NOTIFY_DONE;

	INIT_WORK(&work->work, mlx5e_ipsec_handle_event);
	work->ipsec = ipsec;
	work->id = be32_to_cpu(object->obj_id);

	queue_work(ipsec->wq, &work->work);
	return NOTIFY_OK;
}

int mlx5e_ipsec_aso_init(struct mlx5e_ipsec *ipsec)
{
	struct mlx5_core_dev *mdev = ipsec->mdev;
	struct mlx5e_ipsec_aso *aso;
	struct mlx5e_hw_objs *res;
	struct device *pdev;
	int err;

	aso = kzalloc(sizeof(*ipsec->aso), GFP_KERNEL);
	if (!aso)
		return -ENOMEM;

	res = &mdev->mlx5e_res.hw_objs;

	pdev = mlx5_core_dma_dev(mdev);
	aso->dma_addr = dma_map_single(pdev, aso->ctx, sizeof(aso->ctx),
				       DMA_BIDIRECTIONAL);
	err = dma_mapping_error(pdev, aso->dma_addr);
	if (err)
		goto err_dma;

	aso->aso = mlx5_aso_create(mdev, res->pdn);
	if (IS_ERR(aso->aso)) {
		err = PTR_ERR(aso->aso);
		goto err_aso_create;
	}

	spin_lock_init(&aso->lock);
	ipsec->nb.notifier_call = mlx5e_ipsec_event;
	mlx5_notifier_register(mdev, &ipsec->nb);

	ipsec->aso = aso;
	return 0;

err_aso_create:
	dma_unmap_single(pdev, aso->dma_addr, sizeof(aso->ctx),
			 DMA_BIDIRECTIONAL);
err_dma:
	kfree(aso);
	return err;
}

void mlx5e_ipsec_aso_cleanup(struct mlx5e_ipsec *ipsec)
{
	struct mlx5_core_dev *mdev = ipsec->mdev;
	struct mlx5e_ipsec_aso *aso;
	struct device *pdev;

	aso = ipsec->aso;
	pdev = mlx5_core_dma_dev(mdev);

	mlx5_notifier_unregister(mdev, &ipsec->nb);
	mlx5_aso_destroy(aso->aso);
	dma_unmap_single(pdev, aso->dma_addr, sizeof(aso->ctx),
			 DMA_BIDIRECTIONAL);
	kfree(aso);
}

static void mlx5e_ipsec_aso_copy(struct mlx5_wqe_aso_ctrl_seg *ctrl,
				 struct mlx5_wqe_aso_ctrl_seg *data)
{
	if (!data)
		return;

	ctrl->data_mask_mode = data->data_mask_mode;
	ctrl->condition_1_0_operand = data->condition_1_0_operand;
	ctrl->condition_1_0_offset = data->condition_1_0_offset;
	ctrl->data_offset_condition_operand = data->data_offset_condition_operand;
	ctrl->condition_0_data = data->condition_0_data;
	ctrl->condition_0_mask = data->condition_0_mask;
	ctrl->condition_1_data = data->condition_1_data;
	ctrl->condition_1_mask = data->condition_1_mask;
	ctrl->bitwise_data = data->bitwise_data;
	ctrl->data_mask = data->data_mask;
}

int mlx5e_ipsec_aso_query(struct mlx5e_ipsec_sa_entry *sa_entry,
			  struct mlx5_wqe_aso_ctrl_seg *data)
{
	struct mlx5e_ipsec *ipsec = sa_entry->ipsec;
	struct mlx5e_ipsec_aso *aso = ipsec->aso;
	struct mlx5_core_dev *mdev = ipsec->mdev;
	struct mlx5_wqe_aso_ctrl_seg *ctrl;
	struct mlx5e_hw_objs *res;
	struct mlx5_aso_wqe *wqe;
	u8 ds_cnt;
	int ret;

	lockdep_assert_held(&sa_entry->x->lock);
	res = &mdev->mlx5e_res.hw_objs;

	spin_lock_bh(&aso->lock);
	memset(aso->ctx, 0, sizeof(aso->ctx));
	wqe = mlx5_aso_get_wqe(aso->aso);
	ds_cnt = DIV_ROUND_UP(sizeof(*wqe), MLX5_SEND_WQE_DS);
	mlx5_aso_build_wqe(aso->aso, ds_cnt, wqe, sa_entry->ipsec_obj_id,
			   MLX5_ACCESS_ASO_OPC_MOD_IPSEC);

	ctrl = &wqe->aso_ctrl;
	ctrl->va_l =
		cpu_to_be32(lower_32_bits(aso->dma_addr) | ASO_CTRL_READ_EN);
	ctrl->va_h = cpu_to_be32(upper_32_bits(aso->dma_addr));
	ctrl->l_key = cpu_to_be32(res->mkey);
	mlx5e_ipsec_aso_copy(ctrl, data);

	mlx5_aso_post_wqe(aso->aso, false, &wqe->ctrl);
	ret = mlx5_aso_poll_cq(aso->aso, false);
	spin_unlock_bh(&aso->lock);
	return ret;
}

void mlx5e_ipsec_aso_update_curlft(struct mlx5e_ipsec_sa_entry *sa_entry,
				   u64 *packets)
{
	struct mlx5e_ipsec *ipsec = sa_entry->ipsec;
	struct mlx5e_ipsec_aso *aso = ipsec->aso;
	u64 hard_cnt;

	hard_cnt = MLX5_GET(ipsec_aso, aso->ctx, remove_flow_pkt_cnt);
	/* HW decresases the limit till it reaches zero to fire an avent.
	 * We need to fix the calculations, so the returned count is a total
	 * number of passed packets and not how much left.
	 */
	*packets = sa_entry->attrs.hard_packet_limit - hard_cnt;
}
Commit	Line	Data
5ec4f0ce	1	// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
7e4e8491	2	/* Copyright (c) 2017, Mellanox Technologies inc. All rights reserved. */
2d64663c RS	3
2d64663c RS	4	#include "mlx5_core.h"
8518d05b	5	#include "en.h"
c6e3b421	6	#include "ipsec.h"
2d64663c RS	7	#include "lib/mlx5.h"
2d64663c RS	8
cee137a6 LR	9	enum {
	10	MLX5_IPSEC_ASO_REMOVE_FLOW_PKT_CNT_OFFSET,
	11	};
	12
2451da08	13	u32 mlx5_ipsec_device_caps(struct mlx5_core_dev *mdev)
2d64663c	14	{
effbe267	15	u32 caps = 0;
2451da08 LR	16
	17	if (!MLX5_CAP_GEN(mdev, ipsec_offload))
	18	return 0;
	19
	20	if (!MLX5_CAP_GEN(mdev, log_max_dek))
	21	return 0;
	22
	23	if (!(MLX5_CAP_GEN_64(mdev, general_obj_types) &
	24	MLX5_HCA_CAP_GENERAL_OBJECT_TYPES_IPSEC))
	25	return 0;
2d64663c	26
5e466345 HN	27	if (!MLX5_CAP_FLOWTABLE_NIC_TX(mdev, ipsec_encrypt) \|\|
	28	!MLX5_CAP_FLOWTABLE_NIC_RX(mdev, ipsec_decrypt))
	29	return 0;
	30
effbe267 LR	31	if (!MLX5_CAP_IPSEC(mdev, ipsec_crypto_esp_aes_gcm_128_encrypt) \|\|
	32	!MLX5_CAP_IPSEC(mdev, ipsec_crypto_esp_aes_gcm_128_decrypt))
	33	return 0;
2451da08	34
effbe267 LR	35	if (MLX5_CAP_IPSEC(mdev, ipsec_crypto_offload) &&
	36	MLX5_CAP_ETH(mdev, insert_trailer) && MLX5_CAP_ETH(mdev, swp))
	37	caps \|= MLX5_IPSEC_CAP_CRYPTO;
	38
59592cfd LR	39	if (MLX5_CAP_IPSEC(mdev, ipsec_full_offload) &&
	40	MLX5_CAP_FLOWTABLE_NIC_TX(mdev, reformat_add_esp_trasport) &&
	41	MLX5_CAP_FLOWTABLE_NIC_RX(mdev, reformat_del_esp_trasport) &&
	42	MLX5_CAP_FLOWTABLE_NIC_RX(mdev, decap))
	43	caps \|= MLX5_IPSEC_CAP_PACKET_OFFLOAD;
	44
effbe267 LR	45	if (!caps)
effbe267 LR	46	return 0;
2d64663c	47
2984287c	48	if (MLX5_CAP_IPSEC(mdev, ipsec_esn))
effbe267	49	caps \|= MLX5_IPSEC_CAP_ESN;
2d64663c RS	50
	51	/* We can accommodate up to 2^24 different IPsec objects
	52	* because we use up to 24 bit in flow table metadata
	53	* to hold the IPsec Object unique handle.
	54	*/
	55	WARN_ON_ONCE(MLX5_CAP_IPSEC(mdev, log_max_ipsec_offload) > 24);
	56	return caps;
	57	}
2451da08	58	EXPORT_SYMBOL_GPL(mlx5_ipsec_device_caps);
2d64663c	59
8d15f364 LR	60	static void mlx5e_ipsec_packet_setup(void *obj, u32 pdn,
	61	struct mlx5_accel_esp_xfrm_attrs *attrs)
	62	{
	63	void *aso_ctx;
	64
	65	aso_ctx = MLX5_ADDR_OF(ipsec_obj, obj, ipsec_aso);
	66	if (attrs->esn_trigger) {
	67	MLX5_SET(ipsec_aso, aso_ctx, esn_event_arm, 1);
	68
	69	if (attrs->dir == XFRM_DEV_OFFLOAD_IN) {
	70	MLX5_SET(ipsec_aso, aso_ctx, window_sz,
	71	attrs->replay_window / 64);
	72	MLX5_SET(ipsec_aso, aso_ctx, mode,
	73	MLX5_IPSEC_ASO_REPLAY_PROTECTION);
	74	}
	75	}
	76
	77	/* ASO context */
	78	MLX5_SET(ipsec_obj, obj, ipsec_aso_access_pd, pdn);
	79	MLX5_SET(ipsec_obj, obj, full_offload, 1);
	80	MLX5_SET(ipsec_aso, aso_ctx, valid, 1);
	81	/* MLX5_IPSEC_ASO_REG_C_4_5 is type C register that is used
	82	* in flow steering to perform matching against. Please be
	83	* aware that this register was chosen arbitrary and can't
	84	* be used in other places as long as IPsec packet offload
	85	* active.
	86	*/
	87	MLX5_SET(ipsec_obj, obj, aso_return_reg, MLX5_IPSEC_ASO_REG_C_4_5);
	88	if (attrs->dir == XFRM_DEV_OFFLOAD_OUT)
	89	MLX5_SET(ipsec_aso, aso_ctx, mode, MLX5_IPSEC_ASO_INC_SN);
1ed78fc0 LR	90
	91	if (attrs->hard_packet_limit != XFRM_INF) {
	92	MLX5_SET(ipsec_aso, aso_ctx, remove_flow_pkt_cnt,
	93	lower_32_bits(attrs->hard_packet_limit));
	94	MLX5_SET(ipsec_aso, aso_ctx, hard_lft_arm, 1);
	95	MLX5_SET(ipsec_aso, aso_ctx, remove_flow_enable, 1);
	96	}
	97
	98	if (attrs->soft_packet_limit != XFRM_INF) {
	99	MLX5_SET(ipsec_aso, aso_ctx, remove_flow_soft_lft,
	100	lower_32_bits(attrs->soft_packet_limit));
	101
	102	MLX5_SET(ipsec_aso, aso_ctx, soft_lft_arm, 1);
	103	}
8d15f364 LR	104	}
8d15f364 LR	105
b73e6728	106	static int mlx5_create_ipsec_obj(struct mlx5e_ipsec_sa_entry *sa_entry)
2d64663c	107	{
b73e6728 LR	108	struct mlx5_accel_esp_xfrm_attrs *attrs = &sa_entry->attrs;
b73e6728 LR	109	struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry);
6cd2126a	110	struct aes_gcm_keymat *aes_gcm = &attrs->aes_gcm;
2d64663c RS	111	u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
	112	u32 in[MLX5_ST_SZ_DW(create_ipsec_obj_in)] = {};
	113	void obj, salt_p, *salt_iv_p;
8d15f364	114	struct mlx5e_hw_objs *res;
2d64663c RS	115	int err;
	116
	117	obj = MLX5_ADDR_OF(create_ipsec_obj_in, in, ipsec_object);
	118
	119	/* salt and seq_iv */
	120	salt_p = MLX5_ADDR_OF(ipsec_obj, obj, salt);
	121	memcpy(salt_p, &aes_gcm->salt, sizeof(aes_gcm->salt));
	122
1c4a59b9	123	MLX5_SET(ipsec_obj, obj, icv_length, MLX5_IPSEC_OBJECT_ICV_LEN_16B);
2d64663c RS	124	salt_iv_p = MLX5_ADDR_OF(ipsec_obj, obj, implicit_iv);
	125	memcpy(salt_iv_p, &aes_gcm->seq_iv, sizeof(aes_gcm->seq_iv));
	126	/* esn */
e3840530	127	if (attrs->esn_trigger) {
2d64663c	128	MLX5_SET(ipsec_obj, obj, esn_en, 1);
b73e6728	129	MLX5_SET(ipsec_obj, obj, esn_msb, attrs->esn);
e3840530	130	MLX5_SET(ipsec_obj, obj, esn_overlap, attrs->esn_overlap);
2d64663c RS	131	}
2d64663c RS	132
b73e6728	133	MLX5_SET(ipsec_obj, obj, dekn, sa_entry->enc_key_id);
2d64663c RS	134
	135	/* general object fields set */
	136	MLX5_SET(general_obj_in_cmd_hdr, in, opcode,
	137	MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
	138	MLX5_SET(general_obj_in_cmd_hdr, in, obj_type,
	139	MLX5_GENERAL_OBJECT_TYPES_IPSEC);
	140
8d15f364 LR	141	res = &mdev->mlx5e_res.hw_objs;
	142	if (attrs->type == XFRM_DEV_OFFLOAD_PACKET)
	143	mlx5e_ipsec_packet_setup(obj, res->pdn, attrs);
	144
2d64663c RS	145	err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
2d64663c RS	146	if (!err)
b73e6728 LR	147	sa_entry->ipsec_obj_id =
b73e6728 LR	148	MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
2d64663c RS	149
	150	return err;
	151	}
	152
b73e6728	153	static void mlx5_destroy_ipsec_obj(struct mlx5e_ipsec_sa_entry *sa_entry)
2d64663c	154	{
b73e6728	155	struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry);
2d64663c RS	156	u32 in[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)] = {};
	157	u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
	158
	159	MLX5_SET(general_obj_in_cmd_hdr, in, opcode,
	160	MLX5_CMD_OP_DESTROY_GENERAL_OBJECT);
	161	MLX5_SET(general_obj_in_cmd_hdr, in, obj_type,
	162	MLX5_GENERAL_OBJECT_TYPES_IPSEC);
b73e6728	163	MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, sa_entry->ipsec_obj_id);
2d64663c RS	164
	165	mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
	166	}
	167
b73e6728	168	int mlx5_ipsec_create_sa_ctx(struct mlx5e_ipsec_sa_entry *sa_entry)
2d64663c	169	{
6cd2126a	170	struct aes_gcm_keymat *aes_gcm = &sa_entry->attrs.aes_gcm;
b73e6728	171	struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry);
2d64663c RS	172	int err;
2d64663c RS	173
2d64663c RS	174	/* key */
	175	err = mlx5_create_encryption_key(mdev, aes_gcm->aes_key,
	176	aes_gcm->key_len / BITS_PER_BYTE,
	177	MLX5_ACCEL_OBJ_IPSEC_KEY,
b73e6728	178	&sa_entry->enc_key_id);
2d64663c RS	179	if (err) {
2d64663c RS	180	mlx5_core_dbg(mdev, "Failed to create encryption key (err = %d)\n", err);
b73e6728	181	return err;
2d64663c RS	182	}
2d64663c RS	183
b73e6728	184	err = mlx5_create_ipsec_obj(sa_entry);
2d64663c RS	185	if (err) {
	186	mlx5_core_dbg(mdev, "Failed to create IPsec object (err = %d)\n", err);
	187	goto err_enc_key;
	188	}
	189
b73e6728	190	return 0;
2d64663c RS	191
2d64663c RS	192	err_enc_key:
b73e6728 LR	193	mlx5_destroy_encryption_key(mdev, sa_entry->enc_key_id);
b73e6728 LR	194	return err;
2d64663c RS	195	}
2d64663c RS	196
b73e6728	197	void mlx5_ipsec_free_sa_ctx(struct mlx5e_ipsec_sa_entry *sa_entry)
2d64663c	198	{
b73e6728	199	struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry);
2d64663c	200
b73e6728 LR	201	mlx5_destroy_ipsec_obj(sa_entry);
b73e6728 LR	202	mlx5_destroy_encryption_key(mdev, sa_entry->enc_key_id);
2d64663c RS	203	}
2d64663c RS	204
b73e6728 LR	205	static int mlx5_modify_ipsec_obj(struct mlx5e_ipsec_sa_entry *sa_entry,
b73e6728 LR	206	const struct mlx5_accel_esp_xfrm_attrs *attrs)
7ed92f97	207	{
b73e6728	208	struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry);
7ed92f97 RS	209	u32 in[MLX5_ST_SZ_DW(modify_ipsec_obj_in)] = {};
	210	u32 out[MLX5_ST_SZ_DW(query_ipsec_obj_out)];
	211	u64 modify_field_select = 0;
	212	u64 general_obj_types;
	213	void *obj;
	214	int err;
	215
e3840530	216	if (!attrs->esn_trigger)
7ed92f97 RS	217	return 0;
	218
	219	general_obj_types = MLX5_CAP_GEN_64(mdev, general_obj_types);
	220	if (!(general_obj_types & MLX5_HCA_CAP_GENERAL_OBJECT_TYPES_IPSEC))
	221	return -EINVAL;
	222
	223	/* general object fields set */
	224	MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_QUERY_GENERAL_OBJECT);
	225	MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, MLX5_GENERAL_OBJECT_TYPES_IPSEC);
b73e6728	226	MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, sa_entry->ipsec_obj_id);
7ed92f97 RS	227	err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
	228	if (err) {
	229	mlx5_core_err(mdev, "Query IPsec object failed (Object id %d), err = %d\n",
b73e6728	230	sa_entry->ipsec_obj_id, err);
7ed92f97 RS	231	return err;
	232	}
	233
	234	obj = MLX5_ADDR_OF(query_ipsec_obj_out, out, ipsec_object);
	235	modify_field_select = MLX5_GET64(ipsec_obj, obj, modify_field_select);
	236
	237	/* esn */
	238	if (!(modify_field_select & MLX5_MODIFY_IPSEC_BITMASK_ESN_OVERLAP) \|\|
	239	!(modify_field_select & MLX5_MODIFY_IPSEC_BITMASK_ESN_MSB))
	240	return -EOPNOTSUPP;
	241
	242	obj = MLX5_ADDR_OF(modify_ipsec_obj_in, in, ipsec_object);
656d3389 LR	243	MLX5_SET64(ipsec_obj, obj, modify_field_select,
	244	MLX5_MODIFY_IPSEC_BITMASK_ESN_OVERLAP \|
	245	MLX5_MODIFY_IPSEC_BITMASK_ESN_MSB);
b73e6728	246	MLX5_SET(ipsec_obj, obj, esn_msb, attrs->esn);
e3840530	247	MLX5_SET(ipsec_obj, obj, esn_overlap, attrs->esn_overlap);
7ed92f97 RS	248
	249	/* general object fields set */
	250	MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_MODIFY_GENERAL_OBJECT);
	251
	252	return mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out));
	253	}
	254
b73e6728	255	void mlx5_accel_esp_modify_xfrm(struct mlx5e_ipsec_sa_entry *sa_entry,
a534e24d	256	const struct mlx5_accel_esp_xfrm_attrs *attrs)
7ed92f97	257	{
a534e24d	258	int err;
7ed92f97	259
b73e6728	260	err = mlx5_modify_ipsec_obj(sa_entry, attrs);
a534e24d LR	261	if (err)
	262	return;
	263
b73e6728	264	memcpy(&sa_entry->attrs, attrs, sizeof(sa_entry->attrs));
7e4e8491	265	}
8518d05b	266
cee137a6 LR	267	static void
	268	mlx5e_ipsec_aso_update_esn(struct mlx5e_ipsec_sa_entry *sa_entry,
	269	const struct mlx5_accel_esp_xfrm_attrs *attrs)
	270	{
	271	struct mlx5_wqe_aso_ctrl_seg data = {};
	272
	273	data.data_mask_mode = MLX5_ASO_DATA_MASK_MODE_BITWISE_64BIT << 6;
	274	data.condition_1_0_operand = MLX5_ASO_ALWAYS_TRUE \| MLX5_ASO_ALWAYS_TRUE
	275	<< 4;
	276	data.data_offset_condition_operand = MLX5_IPSEC_ASO_REMOVE_FLOW_PKT_CNT_OFFSET;
	277	data.bitwise_data = cpu_to_be64(BIT_ULL(54));
	278	data.data_mask = data.bitwise_data;
	279
	280	mlx5e_ipsec_aso_query(sa_entry, &data);
	281	}
	282
	283	static void mlx5e_ipsec_update_esn_state(struct mlx5e_ipsec_sa_entry *sa_entry,
	284	u32 mode_param)
	285	{
	286	struct mlx5_accel_esp_xfrm_attrs attrs = {};
	287
	288	if (mode_param < MLX5E_IPSEC_ESN_SCOPE_MID) {
	289	sa_entry->esn_state.esn++;
	290	sa_entry->esn_state.overlap = 0;
	291	} else {
	292	sa_entry->esn_state.overlap = 1;
	293	}
	294
	295	mlx5e_ipsec_build_accel_xfrm_attrs(sa_entry, &attrs);
	296	mlx5_accel_esp_modify_xfrm(sa_entry, &attrs);
	297	mlx5e_ipsec_aso_update_esn(sa_entry, &attrs);
	298	}
	299
8c582ddf LR	300	static void mlx5e_ipsec_handle_event(struct work_struct *_work)
	301	{
	302	struct mlx5e_ipsec_work *work =
	303	container_of(_work, struct mlx5e_ipsec_work, work);
	304	struct mlx5_accel_esp_xfrm_attrs *attrs;
	305	struct mlx5e_ipsec_sa_entry *sa_entry;
	306	struct mlx5e_ipsec_aso *aso;
	307	struct mlx5e_ipsec *ipsec;
	308	int ret;
	309
	310	sa_entry = xa_load(&work->ipsec->sadb, work->id);
	311	if (!sa_entry)
	312	goto out;
	313
	314	ipsec = sa_entry->ipsec;
	315	aso = ipsec->aso;
	316	attrs = &sa_entry->attrs;
	317
	318	spin_lock(&sa_entry->x->lock);
	319	ret = mlx5e_ipsec_aso_query(sa_entry, NULL);
	320	if (ret)
	321	goto unlock;
	322
cee137a6 LR	323	if (attrs->esn_trigger &&
	324	!MLX5_GET(ipsec_aso, aso->ctx, esn_event_arm)) {
	325	u32 mode_param = MLX5_GET(ipsec_aso, aso->ctx, mode_parameter);
	326
	327	mlx5e_ipsec_update_esn_state(sa_entry, mode_param);
	328	}
	329
8c582ddf LR	330	if (attrs->soft_packet_limit != XFRM_INF)
	331	if (!MLX5_GET(ipsec_aso, aso->ctx, soft_lft_arm) \|\|
	332	!MLX5_GET(ipsec_aso, aso->ctx, hard_lft_arm) \|\|
	333	!MLX5_GET(ipsec_aso, aso->ctx, remove_flow_enable))
	334	xfrm_state_check_expire(sa_entry->x);
8c582ddf LR	335
	336	unlock:
	337	spin_unlock(&sa_entry->x->lock);
	338	out:
	339	kfree(work);
	340	}
	341
	342	static int mlx5e_ipsec_event(struct notifier_block *nb, unsigned long event,
	343	void *data)
	344	{
	345	struct mlx5e_ipsec *ipsec = container_of(nb, struct mlx5e_ipsec, nb);
	346	struct mlx5_eqe_obj_change *object;
	347	struct mlx5e_ipsec_work *work;
	348	struct mlx5_eqe *eqe = data;
	349	u16 type;
	350
	351	if (event != MLX5_EVENT_TYPE_OBJECT_CHANGE)
	352	return NOTIFY_DONE;
	353
	354	object = &eqe->data.obj_change;
	355	type = be16_to_cpu(object->obj_type);
	356
	357	if (type != MLX5_GENERAL_OBJECT_TYPES_IPSEC)
	358	return NOTIFY_DONE;
	359
	360	work = kmalloc(sizeof(*work), GFP_ATOMIC);
	361	if (!work)
	362	return NOTIFY_DONE;
	363
	364	INIT_WORK(&work->work, mlx5e_ipsec_handle_event);
	365	work->ipsec = ipsec;
	366	work->id = be32_to_cpu(object->obj_id);
	367
	368	queue_work(ipsec->wq, &work->work);
	369	return NOTIFY_OK;
	370	}
	371
8518d05b LR	372	int mlx5e_ipsec_aso_init(struct mlx5e_ipsec *ipsec)
	373	{
	374	struct mlx5_core_dev *mdev = ipsec->mdev;
	375	struct mlx5e_ipsec_aso *aso;
	376	struct mlx5e_hw_objs *res;
	377	struct device *pdev;
	378	int err;
	379
	380	aso = kzalloc(sizeof(*ipsec->aso), GFP_KERNEL);
	381	if (!aso)
	382	return -ENOMEM;
	383
	384	res = &mdev->mlx5e_res.hw_objs;
	385
	386	pdev = mlx5_core_dma_dev(mdev);
	387	aso->dma_addr = dma_map_single(pdev, aso->ctx, sizeof(aso->ctx),
	388	DMA_BIDIRECTIONAL);
	389	err = dma_mapping_error(pdev, aso->dma_addr);
	390	if (err)
	391	goto err_dma;
	392
	393	aso->aso = mlx5_aso_create(mdev, res->pdn);
	394	if (IS_ERR(aso->aso)) {
	395	err = PTR_ERR(aso->aso);
	396	goto err_aso_create;
	397	}
	398
e4d38c45	399	spin_lock_init(&aso->lock);
8c582ddf LR	400	ipsec->nb.notifier_call = mlx5e_ipsec_event;
	401	mlx5_notifier_register(mdev, &ipsec->nb);
	402
8518d05b LR	403	ipsec->aso = aso;
	404	return 0;
	405
	406	err_aso_create:
	407	dma_unmap_single(pdev, aso->dma_addr, sizeof(aso->ctx),
	408	DMA_BIDIRECTIONAL);
	409	err_dma:
	410	kfree(aso);
	411	return err;
	412	}
	413
	414	void mlx5e_ipsec_aso_cleanup(struct mlx5e_ipsec *ipsec)
	415	{
	416	struct mlx5_core_dev *mdev = ipsec->mdev;
	417	struct mlx5e_ipsec_aso *aso;
	418	struct device *pdev;
	419
	420	aso = ipsec->aso;
	421	pdev = mlx5_core_dma_dev(mdev);
	422
8c582ddf	423	mlx5_notifier_unregister(mdev, &ipsec->nb);
8518d05b LR	424	mlx5_aso_destroy(aso->aso);
	425	dma_unmap_single(pdev, aso->dma_addr, sizeof(aso->ctx),
	426	DMA_BIDIRECTIONAL);
	427	kfree(aso);
	428	}
1ed78fc0	429
8c582ddf LR	430	static void mlx5e_ipsec_aso_copy(struct mlx5_wqe_aso_ctrl_seg *ctrl,
	431	struct mlx5_wqe_aso_ctrl_seg *data)
	432	{
	433	if (!data)
	434	return;
	435
	436	ctrl->data_mask_mode = data->data_mask_mode;
	437	ctrl->condition_1_0_operand = data->condition_1_0_operand;
	438	ctrl->condition_1_0_offset = data->condition_1_0_offset;
	439	ctrl->data_offset_condition_operand = data->data_offset_condition_operand;
	440	ctrl->condition_0_data = data->condition_0_data;
	441	ctrl->condition_0_mask = data->condition_0_mask;
	442	ctrl->condition_1_data = data->condition_1_data;
	443	ctrl->condition_1_mask = data->condition_1_mask;
	444	ctrl->bitwise_data = data->bitwise_data;
	445	ctrl->data_mask = data->data_mask;
	446	}
	447
	448	int mlx5e_ipsec_aso_query(struct mlx5e_ipsec_sa_entry *sa_entry,
	449	struct mlx5_wqe_aso_ctrl_seg *data)
1ed78fc0 LR	450	{
	451	struct mlx5e_ipsec *ipsec = sa_entry->ipsec;
	452	struct mlx5e_ipsec_aso *aso = ipsec->aso;
	453	struct mlx5_core_dev *mdev = ipsec->mdev;
	454	struct mlx5_wqe_aso_ctrl_seg *ctrl;
	455	struct mlx5e_hw_objs *res;
	456	struct mlx5_aso_wqe *wqe;
	457	u8 ds_cnt;
e4d38c45	458	int ret;
1ed78fc0	459
8c582ddf	460	lockdep_assert_held(&sa_entry->x->lock);
1ed78fc0 LR	461	res = &mdev->mlx5e_res.hw_objs;
1ed78fc0 LR	462
e4d38c45	463	spin_lock_bh(&aso->lock);
1ed78fc0 LR	464	memset(aso->ctx, 0, sizeof(aso->ctx));
	465	wqe = mlx5_aso_get_wqe(aso->aso);
	466	ds_cnt = DIV_ROUND_UP(sizeof(*wqe), MLX5_SEND_WQE_DS);
	467	mlx5_aso_build_wqe(aso->aso, ds_cnt, wqe, sa_entry->ipsec_obj_id,
	468	MLX5_ACCESS_ASO_OPC_MOD_IPSEC);
	469
	470	ctrl = &wqe->aso_ctrl;
	471	ctrl->va_l =
	472	cpu_to_be32(lower_32_bits(aso->dma_addr) \| ASO_CTRL_READ_EN);
	473	ctrl->va_h = cpu_to_be32(upper_32_bits(aso->dma_addr));
	474	ctrl->l_key = cpu_to_be32(res->mkey);
8c582ddf	475	mlx5e_ipsec_aso_copy(ctrl, data);
1ed78fc0 LR	476
1ed78fc0 LR	477	mlx5_aso_post_wqe(aso->aso, false, &wqe->ctrl);
e4d38c45 LR	478	ret = mlx5_aso_poll_cq(aso->aso, false);
	479	spin_unlock_bh(&aso->lock);
	480	return ret;
1ed78fc0 LR	481	}
	482
	483	void mlx5e_ipsec_aso_update_curlft(struct mlx5e_ipsec_sa_entry *sa_entry,
	484	u64 *packets)
	485	{
	486	struct mlx5e_ipsec *ipsec = sa_entry->ipsec;
	487	struct mlx5e_ipsec_aso *aso = ipsec->aso;
	488	u64 hard_cnt;
	489
	490	hard_cnt = MLX5_GET(ipsec_aso, aso->ctx, remove_flow_pkt_cnt);
	491	/* HW decresases the limit till it reaches zero to fire an avent.
	492	* We need to fix the calculations, so the returned count is a total
	493	* number of passed packets and not how much left.
	494	*/
	495	*packets = sa_entry->attrs.hard_packet_limit - hard_cnt;
	496	}