[linux-block.git] / drivers / net / ethernet / mellanox / mlx5 / core / en_accel / ipsec.c

/*
 * Copyright (c) 2017 Mellanox Technologies. All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 */

#include <crypto/internal/geniv.h>
#include <crypto/aead.h>
#include <linux/inetdevice.h>
#include <linux/netdevice.h>

#include "en.h"
#include "ipsec.h"
#include "ipsec_rxtx.h"

static struct mlx5e_ipsec_sa_entry *to_ipsec_sa_entry(struct xfrm_state *x)
{
	return (struct mlx5e_ipsec_sa_entry *)x->xso.offload_handle;
}

struct xfrm_state *mlx5e_ipsec_sadb_rx_lookup(struct mlx5e_ipsec *ipsec,
					      unsigned int handle)
{
	struct mlx5e_ipsec_sa_entry *sa_entry;
	struct xfrm_state *ret = NULL;

	rcu_read_lock();
	hash_for_each_possible_rcu(ipsec->sadb_rx, sa_entry, hlist, handle)
		if (sa_entry->handle == handle) {
			ret = sa_entry->x;
			xfrm_state_hold(ret);
			break;
		}
	rcu_read_unlock();

	return ret;
}

static int mlx5e_ipsec_sadb_rx_add(struct mlx5e_ipsec_sa_entry *sa_entry)
{
	unsigned int handle = sa_entry->ipsec_obj_id;
	struct mlx5e_ipsec *ipsec = sa_entry->ipsec;
	struct mlx5e_ipsec_sa_entry *_sa_entry;
	unsigned long flags;

	rcu_read_lock();
	hash_for_each_possible_rcu(ipsec->sadb_rx, _sa_entry, hlist, handle)
		if (_sa_entry->handle == handle) {
			rcu_read_unlock();
			return  -EEXIST;
		}
	rcu_read_unlock();

	spin_lock_irqsave(&ipsec->sadb_rx_lock, flags);
	sa_entry->handle = handle;
	hash_add_rcu(ipsec->sadb_rx, &sa_entry->hlist, sa_entry->handle);
	spin_unlock_irqrestore(&ipsec->sadb_rx_lock, flags);

	return 0;
}

static void mlx5e_ipsec_sadb_rx_del(struct mlx5e_ipsec_sa_entry *sa_entry)
{
	struct mlx5e_ipsec *ipsec = sa_entry->ipsec;
	unsigned long flags;

	spin_lock_irqsave(&ipsec->sadb_rx_lock, flags);
	hash_del_rcu(&sa_entry->hlist);
	spin_unlock_irqrestore(&ipsec->sadb_rx_lock, flags);
}

static bool mlx5e_ipsec_update_esn_state(struct mlx5e_ipsec_sa_entry *sa_entry)
{
	struct xfrm_replay_state_esn *replay_esn;
	u32 seq_bottom = 0;
	u8 overlap;
	u32 *esn;

	if (!(sa_entry->x->props.flags & XFRM_STATE_ESN)) {
		sa_entry->esn_state.trigger = 0;
		return false;
	}

	replay_esn = sa_entry->x->replay_esn;
	if (replay_esn->seq >= replay_esn->replay_window)
		seq_bottom = replay_esn->seq - replay_esn->replay_window + 1;

	overlap = sa_entry->esn_state.overlap;

	sa_entry->esn_state.esn = xfrm_replay_seqhi(sa_entry->x,
						    htonl(seq_bottom));
	esn = &sa_entry->esn_state.esn;

	sa_entry->esn_state.trigger = 1;
	if (unlikely(overlap && seq_bottom < MLX5E_IPSEC_ESN_SCOPE_MID)) {
		++(*esn);
		sa_entry->esn_state.overlap = 0;
		return true;
	} else if (unlikely(!overlap &&
			    (seq_bottom >= MLX5E_IPSEC_ESN_SCOPE_MID))) {
		sa_entry->esn_state.overlap = 1;
		return true;
	}

	return false;
}

static void
mlx5e_ipsec_build_accel_xfrm_attrs(struct mlx5e_ipsec_sa_entry *sa_entry,
				   struct mlx5_accel_esp_xfrm_attrs *attrs)
{
	struct xfrm_state *x = sa_entry->x;
	struct aes_gcm_keymat *aes_gcm = &attrs->aes_gcm;
	struct aead_geniv_ctx *geniv_ctx;
	struct crypto_aead *aead;
	unsigned int crypto_data_len, key_len;
	int ivsize;

	memset(attrs, 0, sizeof(*attrs));

	/* key */
	crypto_data_len = (x->aead->alg_key_len + 7) / 8;
	key_len = crypto_data_len - 4; /* 4 bytes salt at end */

	memcpy(aes_gcm->aes_key, x->aead->alg_key, key_len);
	aes_gcm->key_len = key_len * 8;

	/* salt and seq_iv */
	aead = x->data;
	geniv_ctx = crypto_aead_ctx(aead);
	ivsize = crypto_aead_ivsize(aead);
	memcpy(&aes_gcm->seq_iv, &geniv_ctx->salt, ivsize);
	memcpy(&aes_gcm->salt, x->aead->alg_key + key_len,
	       sizeof(aes_gcm->salt));

	/* iv len */
	aes_gcm->icv_len = x->aead->alg_icv_len;

	/* esn */
	if (sa_entry->esn_state.trigger) {
		attrs->flags |= MLX5_ACCEL_ESP_FLAGS_ESN_TRIGGERED;
		attrs->esn = sa_entry->esn_state.esn;
		if (sa_entry->esn_state.overlap)
			attrs->flags |= MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP;
	}

	/* action */
	attrs->action = (x->xso.dir == XFRM_DEV_OFFLOAD_OUT) ?
				MLX5_ACCEL_ESP_ACTION_ENCRYPT :
				      MLX5_ACCEL_ESP_ACTION_DECRYPT;
	/* flags */
	attrs->flags |= (x->props.mode == XFRM_MODE_TRANSPORT) ?
			MLX5_ACCEL_ESP_FLAGS_TRANSPORT :
			MLX5_ACCEL_ESP_FLAGS_TUNNEL;

	/* spi */
	attrs->spi = be32_to_cpu(x->id.spi);

	/* source , destination ips */
	memcpy(&attrs->saddr, x->props.saddr.a6, sizeof(attrs->saddr));
	memcpy(&attrs->daddr, x->id.daddr.a6, sizeof(attrs->daddr));
	attrs->is_ipv6 = (x->props.family != AF_INET);
}

static inline int mlx5e_xfrm_validate_state(struct xfrm_state *x)
{
	struct net_device *netdev = x->xso.real_dev;
	struct mlx5e_priv *priv;

	priv = netdev_priv(netdev);

	if (x->props.aalgo != SADB_AALG_NONE) {
		netdev_info(netdev, "Cannot offload authenticated xfrm states\n");
		return -EINVAL;
	}
	if (x->props.ealgo != SADB_X_EALG_AES_GCM_ICV16) {
		netdev_info(netdev, "Only AES-GCM-ICV16 xfrm state may be offloaded\n");
		return -EINVAL;
	}
	if (x->props.calgo != SADB_X_CALG_NONE) {
		netdev_info(netdev, "Cannot offload compressed xfrm states\n");
		return -EINVAL;
	}
	if (x->props.flags & XFRM_STATE_ESN &&
	    !(mlx5_ipsec_device_caps(priv->mdev) & MLX5_IPSEC_CAP_ESN)) {
		netdev_info(netdev, "Cannot offload ESN xfrm states\n");
		return -EINVAL;
	}
	if (x->props.family != AF_INET &&
	    x->props.family != AF_INET6) {
		netdev_info(netdev, "Only IPv4/6 xfrm states may be offloaded\n");
		return -EINVAL;
	}
	if (x->props.mode != XFRM_MODE_TRANSPORT &&
	    x->props.mode != XFRM_MODE_TUNNEL) {
		dev_info(&netdev->dev, "Only transport and tunnel xfrm states may be offloaded\n");
		return -EINVAL;
	}
	if (x->id.proto != IPPROTO_ESP) {
		netdev_info(netdev, "Only ESP xfrm state may be offloaded\n");
		return -EINVAL;
	}
	if (x->encap) {
		netdev_info(netdev, "Encapsulated xfrm state may not be offloaded\n");
		return -EINVAL;
	}
	if (!x->aead) {
		netdev_info(netdev, "Cannot offload xfrm states without aead\n");
		return -EINVAL;
	}
	if (x->aead->alg_icv_len != 128) {
		netdev_info(netdev, "Cannot offload xfrm states with AEAD ICV length other than 128bit\n");
		return -EINVAL;
	}
	if ((x->aead->alg_key_len != 128 + 32) &&
	    (x->aead->alg_key_len != 256 + 32)) {
		netdev_info(netdev, "Cannot offload xfrm states with AEAD key length other than 128/256 bit\n");
		return -EINVAL;
	}
	if (x->tfcpad) {
		netdev_info(netdev, "Cannot offload xfrm states with tfc padding\n");
		return -EINVAL;
	}
	if (!x->geniv) {
		netdev_info(netdev, "Cannot offload xfrm states without geniv\n");
		return -EINVAL;
	}
	if (strcmp(x->geniv, "seqiv")) {
		netdev_info(netdev, "Cannot offload xfrm states with geniv other than seqiv\n");
		return -EINVAL;
	}
	return 0;
}

static void _update_xfrm_state(struct work_struct *work)
{
	struct mlx5e_ipsec_modify_state_work *modify_work =
		container_of(work, struct mlx5e_ipsec_modify_state_work, work);
	struct mlx5e_ipsec_sa_entry *sa_entry = container_of(
		modify_work, struct mlx5e_ipsec_sa_entry, modify_work);

	mlx5_accel_esp_modify_xfrm(sa_entry, &modify_work->attrs);
}

static int mlx5e_xfrm_add_state(struct xfrm_state *x)
{
	struct mlx5e_ipsec_sa_entry *sa_entry = NULL;
	struct net_device *netdev = x->xso.real_dev;
	struct mlx5e_priv *priv;
	int err;

	priv = netdev_priv(netdev);
	if (!priv->ipsec)
		return -EOPNOTSUPP;

	err = mlx5e_xfrm_validate_state(x);
	if (err)
		return err;

	sa_entry = kzalloc(sizeof(*sa_entry), GFP_KERNEL);
	if (!sa_entry) {
		err = -ENOMEM;
		goto out;
	}

	sa_entry->x = x;
	sa_entry->ipsec = priv->ipsec;

	/* check esn */
	mlx5e_ipsec_update_esn_state(sa_entry);

	mlx5e_ipsec_build_accel_xfrm_attrs(sa_entry, &sa_entry->attrs);
	/* create hw context */
	err = mlx5_ipsec_create_sa_ctx(sa_entry);
	if (err)
		goto err_xfrm;

	err = mlx5e_accel_ipsec_fs_add_rule(priv, sa_entry);
	if (err)
		goto err_hw_ctx;

	if (x->xso.dir == XFRM_DEV_OFFLOAD_IN) {
		err = mlx5e_ipsec_sadb_rx_add(sa_entry);
		if (err)
			goto err_add_rule;
	} else {
		sa_entry->set_iv_op = (x->props.flags & XFRM_STATE_ESN) ?
				mlx5e_ipsec_set_iv_esn : mlx5e_ipsec_set_iv;
	}

	INIT_WORK(&sa_entry->modify_work.work, _update_xfrm_state);
	x->xso.offload_handle = (unsigned long)sa_entry;
	goto out;

err_add_rule:
	mlx5e_accel_ipsec_fs_del_rule(priv, sa_entry);
err_hw_ctx:
	mlx5_ipsec_free_sa_ctx(sa_entry);
err_xfrm:
	kfree(sa_entry);
out:
	return err;
}

static void mlx5e_xfrm_del_state(struct xfrm_state *x)
{
	struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(x);

	if (x->xso.dir == XFRM_DEV_OFFLOAD_IN)
		mlx5e_ipsec_sadb_rx_del(sa_entry);
}

static void mlx5e_xfrm_free_state(struct xfrm_state *x)
{
	struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(x);
	struct mlx5e_priv *priv = netdev_priv(x->xso.dev);

	cancel_work_sync(&sa_entry->modify_work.work);
	mlx5e_accel_ipsec_fs_del_rule(priv, sa_entry);
	mlx5_ipsec_free_sa_ctx(sa_entry);
	kfree(sa_entry);
}

int mlx5e_ipsec_init(struct mlx5e_priv *priv)
{
	struct mlx5e_ipsec *ipsec;
	int ret;

	if (!mlx5_ipsec_device_caps(priv->mdev)) {
		netdev_dbg(priv->netdev, "Not an IPSec offload device\n");
		return 0;
	}

	ipsec = kzalloc(sizeof(*ipsec), GFP_KERNEL);
	if (!ipsec)
		return -ENOMEM;

	hash_init(ipsec->sadb_rx);
	spin_lock_init(&ipsec->sadb_rx_lock);
	ipsec->mdev = priv->mdev;
	ipsec->wq = alloc_ordered_workqueue("mlx5e_ipsec: %s", 0,
					    priv->netdev->name);
	if (!ipsec->wq) {
		ret = -ENOMEM;
		goto err_wq;
	}

	ret = mlx5e_accel_ipsec_fs_init(ipsec);
	if (ret)
		goto err_fs_init;

	priv->ipsec = ipsec;
	netdev_dbg(priv->netdev, "IPSec attached to netdevice\n");
	return 0;

err_fs_init:
	destroy_workqueue(ipsec->wq);
err_wq:
	kfree(ipsec);
	return (ret != -EOPNOTSUPP) ? ret : 0;
}

void mlx5e_ipsec_cleanup(struct mlx5e_priv *priv)
{
	struct mlx5e_ipsec *ipsec = priv->ipsec;

	if (!ipsec)
		return;

	mlx5e_accel_ipsec_fs_cleanup(ipsec);
	destroy_workqueue(ipsec->wq);
	kfree(ipsec);
	priv->ipsec = NULL;
}

static bool mlx5e_ipsec_offload_ok(struct sk_buff *skb, struct xfrm_state *x)
{
	if (x->props.family == AF_INET) {
		/* Offload with IPv4 options is not supported yet */
		if (ip_hdr(skb)->ihl > 5)
			return false;
	} else {
		/* Offload with IPv6 extension headers is not support yet */
		if (ipv6_ext_hdr(ipv6_hdr(skb)->nexthdr))
			return false;
	}

	return true;
}

static void mlx5e_xfrm_advance_esn_state(struct xfrm_state *x)
{
	struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(x);
	struct mlx5e_ipsec_modify_state_work *modify_work =
		&sa_entry->modify_work;
	bool need_update;

	need_update = mlx5e_ipsec_update_esn_state(sa_entry);
	if (!need_update)
		return;

	mlx5e_ipsec_build_accel_xfrm_attrs(sa_entry, &modify_work->attrs);
	queue_work(sa_entry->ipsec->wq, &modify_work->work);
}

static const struct xfrmdev_ops mlx5e_ipsec_xfrmdev_ops = {
	.xdo_dev_state_add	= mlx5e_xfrm_add_state,
	.xdo_dev_state_delete	= mlx5e_xfrm_del_state,
	.xdo_dev_state_free	= mlx5e_xfrm_free_state,
	.xdo_dev_offload_ok	= mlx5e_ipsec_offload_ok,
	.xdo_dev_state_advance_esn = mlx5e_xfrm_advance_esn_state,
};

void mlx5e_ipsec_build_netdev(struct mlx5e_priv *priv)
{
	struct mlx5_core_dev *mdev = priv->mdev;
	struct net_device *netdev = priv->netdev;

	if (!mlx5_ipsec_device_caps(mdev))
		return;

	mlx5_core_info(mdev, "mlx5e: IPSec ESP acceleration enabled\n");
	netdev->xfrmdev_ops = &mlx5e_ipsec_xfrmdev_ops;
	netdev->features |= NETIF_F_HW_ESP;
	netdev->hw_enc_features |= NETIF_F_HW_ESP;

	if (!MLX5_CAP_ETH(mdev, swp_csum)) {
		mlx5_core_dbg(mdev, "mlx5e: SWP checksum not supported\n");
		return;
	}

	netdev->features |= NETIF_F_HW_ESP_TX_CSUM;
	netdev->hw_enc_features |= NETIF_F_HW_ESP_TX_CSUM;

	if (!MLX5_CAP_ETH(mdev, swp_lso)) {
		mlx5_core_dbg(mdev, "mlx5e: ESP LSO not supported\n");
		return;
	}

	netdev->gso_partial_features |= NETIF_F_GSO_ESP;
	mlx5_core_dbg(mdev, "mlx5e: ESP GSO capability turned on\n");
	netdev->features |= NETIF_F_GSO_ESP;
	netdev->hw_features |= NETIF_F_GSO_ESP;
	netdev->hw_enc_features |= NETIF_F_GSO_ESP;
}
Commit	Line	Data
547eede0 IT	1	/*
	2	* Copyright (c) 2017 Mellanox Technologies. All rights reserved.
	3	*
	4	* This software is available to you under a choice of one of two
	5	* licenses. You may choose to be licensed under the terms of the GNU
	6	* General Public License (GPL) Version 2, available from the file
	7	* COPYING in the main directory of this source tree, or the
	8	* OpenIB.org BSD license below:
	9	*
	10	* Redistribution and use in source and binary forms, with or
	11	* without modification, are permitted provided that the following
	12	* conditions are met:
	13	*
	14	* - Redistributions of source code must retain the above
	15	* copyright notice, this list of conditions and the following
	16	* disclaimer.
	17	*
	18	* - Redistributions in binary form must reproduce the above
	19	* copyright notice, this list of conditions and the following
	20	* disclaimer in the documentation and/or other materials
	21	* provided with the distribution.
	22	*
	23	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	24	* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	25	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
	26	* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
	27	* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
	28	* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
	29	* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	30	* SOFTWARE.
	31	*
	32	*/
	33
	34	#include <crypto/internal/geniv.h>
	35	#include <crypto/aead.h>
	36	#include <linux/inetdevice.h>
	37	#include <linux/netdevice.h>
547eede0 IT	38
547eede0 IT	39	#include "en.h"
c6e3b421 LR	40	#include "ipsec.h"
c6e3b421 LR	41	#include "ipsec_rxtx.h"
547eede0	42
75ef3f55 AY	43	static struct mlx5e_ipsec_sa_entry to_ipsec_sa_entry(struct xfrm_state x)
75ef3f55 AY	44	{
021a429b	45	return (struct mlx5e_ipsec_sa_entry *)x->xso.offload_handle;
75ef3f55 AY	46	}
75ef3f55 AY	47
899a59d3 IT	48	struct xfrm_state mlx5e_ipsec_sadb_rx_lookup(struct mlx5e_ipsec ipsec,
	49	unsigned int handle)
	50	{
	51	struct mlx5e_ipsec_sa_entry *sa_entry;
	52	struct xfrm_state *ret = NULL;
	53
	54	rcu_read_lock();
	55	hash_for_each_possible_rcu(ipsec->sadb_rx, sa_entry, hlist, handle)
	56	if (sa_entry->handle == handle) {
	57	ret = sa_entry->x;
	58	xfrm_state_hold(ret);
	59	break;
	60	}
	61	rcu_read_unlock();
	62
	63	return ret;
	64	}
	65
b73e6728	66	static int mlx5e_ipsec_sadb_rx_add(struct mlx5e_ipsec_sa_entry *sa_entry)
547eede0	67	{
b73e6728	68	unsigned int handle = sa_entry->ipsec_obj_id;
547eede0	69	struct mlx5e_ipsec *ipsec = sa_entry->ipsec;
7dfee4b1	70	struct mlx5e_ipsec_sa_entry *_sa_entry;
547eede0	71	unsigned long flags;
547eede0	72
7dfee4b1 RS	73	rcu_read_lock();
	74	hash_for_each_possible_rcu(ipsec->sadb_rx, _sa_entry, hlist, handle)
	75	if (_sa_entry->handle == handle) {
	76	rcu_read_unlock();
	77	return -EEXIST;
	78	}
	79	rcu_read_unlock();
547eede0	80
dc7debec	81	spin_lock_irqsave(&ipsec->sadb_rx_lock, flags);
7dfee4b1	82	sa_entry->handle = handle;
547eede0	83	hash_add_rcu(ipsec->sadb_rx, &sa_entry->hlist, sa_entry->handle);
547eede0	84	spin_unlock_irqrestore(&ipsec->sadb_rx_lock, flags);
dc7debec AY	85
dc7debec AY	86	return 0;
547eede0 IT	87	}
	88
	89	static void mlx5e_ipsec_sadb_rx_del(struct mlx5e_ipsec_sa_entry *sa_entry)
	90	{
	91	struct mlx5e_ipsec *ipsec = sa_entry->ipsec;
	92	unsigned long flags;
	93
	94	spin_lock_irqsave(&ipsec->sadb_rx_lock, flags);
	95	hash_del_rcu(&sa_entry->hlist);
	96	spin_unlock_irqrestore(&ipsec->sadb_rx_lock, flags);
	97	}
	98
cb010083 AY	99	static bool mlx5e_ipsec_update_esn_state(struct mlx5e_ipsec_sa_entry *sa_entry)
	100	{
	101	struct xfrm_replay_state_esn *replay_esn;
2d64663c	102	u32 seq_bottom = 0;
cb010083 AY	103	u8 overlap;
	104	u32 *esn;
	105
	106	if (!(sa_entry->x->props.flags & XFRM_STATE_ESN)) {
	107	sa_entry->esn_state.trigger = 0;
	108	return false;
	109	}
	110
	111	replay_esn = sa_entry->x->replay_esn;
2d64663c RS	112	if (replay_esn->seq >= replay_esn->replay_window)
	113	seq_bottom = replay_esn->seq - replay_esn->replay_window + 1;
	114
cb010083 AY	115	overlap = sa_entry->esn_state.overlap;
	116
	117	sa_entry->esn_state.esn = xfrm_replay_seqhi(sa_entry->x,
	118	htonl(seq_bottom));
	119	esn = &sa_entry->esn_state.esn;
	120
	121	sa_entry->esn_state.trigger = 1;
	122	if (unlikely(overlap && seq_bottom < MLX5E_IPSEC_ESN_SCOPE_MID)) {
	123	++(*esn);
	124	sa_entry->esn_state.overlap = 0;
	125	return true;
	126	} else if (unlikely(!overlap &&
	127	(seq_bottom >= MLX5E_IPSEC_ESN_SCOPE_MID))) {
	128	sa_entry->esn_state.overlap = 1;
	129	return true;
	130	}
	131
	132	return false;
	133	}
	134
d6c4f029 AY	135	static void
	136	mlx5e_ipsec_build_accel_xfrm_attrs(struct mlx5e_ipsec_sa_entry *sa_entry,
	137	struct mlx5_accel_esp_xfrm_attrs *attrs)
547eede0 IT	138	{
547eede0 IT	139	struct xfrm_state *x = sa_entry->x;
6cd2126a	140	struct aes_gcm_keymat *aes_gcm = &attrs->aes_gcm;
547eede0	141	struct aead_geniv_ctx *geniv_ctx;
547eede0	142	struct crypto_aead *aead;
d6c4f029	143	unsigned int crypto_data_len, key_len;
547eede0 IT	144	int ivsize;
547eede0 IT	145
d6c4f029	146	memset(attrs, 0, sizeof(*attrs));
547eede0	147
d6c4f029	148	/* key */
65802f48 AY	149	crypto_data_len = (x->aead->alg_key_len + 7) / 8;
65802f48 AY	150	key_len = crypto_data_len - 4; /* 4 bytes salt at end */
d6c4f029 AY	151
	152	memcpy(aes_gcm->aes_key, x->aead->alg_key, key_len);
	153	aes_gcm->key_len = key_len * 8;
	154
	155	/* salt and seq_iv */
65802f48 AY	156	aead = x->data;
	157	geniv_ctx = crypto_aead_ctx(aead);
	158	ivsize = crypto_aead_ivsize(aead);
d6c4f029 AY	159	memcpy(&aes_gcm->seq_iv, &geniv_ctx->salt, ivsize);
	160	memcpy(&aes_gcm->salt, x->aead->alg_key + key_len,
	161	sizeof(aes_gcm->salt));
	162
	163	/* iv len */
	164	aes_gcm->icv_len = x->aead->alg_icv_len;
	165
cb010083 AY	166	/* esn */
	167	if (sa_entry->esn_state.trigger) {
	168	attrs->flags \|= MLX5_ACCEL_ESP_FLAGS_ESN_TRIGGERED;
	169	attrs->esn = sa_entry->esn_state.esn;
	170	if (sa_entry->esn_state.overlap)
	171	attrs->flags \|= MLX5_ACCEL_ESP_FLAGS_ESN_STATE_OVERLAP;
	172	}
	173
d6c4f029	174	/* action */
3ef535ec LR	175	attrs->action = (x->xso.dir == XFRM_DEV_OFFLOAD_OUT) ?
	176	MLX5_ACCEL_ESP_ACTION_ENCRYPT :
	177	MLX5_ACCEL_ESP_ACTION_DECRYPT;
d6c4f029 AY	178	/* flags */
	179	attrs->flags \|= (x->props.mode == XFRM_MODE_TRANSPORT) ?
	180	MLX5_ACCEL_ESP_FLAGS_TRANSPORT :
	181	MLX5_ACCEL_ESP_FLAGS_TUNNEL;
1dbd51d0 RS	182
1dbd51d0 RS	183	/* spi */
6cd2126a	184	attrs->spi = be32_to_cpu(x->id.spi);
1dbd51d0 RS	185
	186	/* source , destination ips */
	187	memcpy(&attrs->saddr, x->props.saddr.a6, sizeof(attrs->saddr));
	188	memcpy(&attrs->daddr, x->id.daddr.a6, sizeof(attrs->daddr));
	189	attrs->is_ipv6 = (x->props.family != AF_INET);
547eede0 IT	190	}
	191
	192	static inline int mlx5e_xfrm_validate_state(struct xfrm_state *x)
	193	{
bdfd2d1f	194	struct net_device *netdev = x->xso.real_dev;
547eede0 IT	195	struct mlx5e_priv *priv;
	196
	197	priv = netdev_priv(netdev);
	198
	199	if (x->props.aalgo != SADB_AALG_NONE) {
	200	netdev_info(netdev, "Cannot offload authenticated xfrm states\n");
	201	return -EINVAL;
	202	}
	203	if (x->props.ealgo != SADB_X_EALG_AES_GCM_ICV16) {
	204	netdev_info(netdev, "Only AES-GCM-ICV16 xfrm state may be offloaded\n");
	205	return -EINVAL;
	206	}
	207	if (x->props.calgo != SADB_X_CALG_NONE) {
	208	netdev_info(netdev, "Cannot offload compressed xfrm states\n");
	209	return -EINVAL;
	210	}
cb010083	211	if (x->props.flags & XFRM_STATE_ESN &&
effbe267	212	!(mlx5_ipsec_device_caps(priv->mdev) & MLX5_IPSEC_CAP_ESN)) {
547eede0 IT	213	netdev_info(netdev, "Cannot offload ESN xfrm states\n");
	214	return -EINVAL;
	215	}
	216	if (x->props.family != AF_INET &&
	217	x->props.family != AF_INET6) {
	218	netdev_info(netdev, "Only IPv4/6 xfrm states may be offloaded\n");
	219	return -EINVAL;
	220	}
	221	if (x->props.mode != XFRM_MODE_TRANSPORT &&
	222	x->props.mode != XFRM_MODE_TUNNEL) {
	223	dev_info(&netdev->dev, "Only transport and tunnel xfrm states may be offloaded\n");
	224	return -EINVAL;
	225	}
	226	if (x->id.proto != IPPROTO_ESP) {
	227	netdev_info(netdev, "Only ESP xfrm state may be offloaded\n");
	228	return -EINVAL;
	229	}
	230	if (x->encap) {
	231	netdev_info(netdev, "Encapsulated xfrm state may not be offloaded\n");
	232	return -EINVAL;
	233	}
	234	if (!x->aead) {
	235	netdev_info(netdev, "Cannot offload xfrm states without aead\n");
	236	return -EINVAL;
	237	}
	238	if (x->aead->alg_icv_len != 128) {
	239	netdev_info(netdev, "Cannot offload xfrm states with AEAD ICV length other than 128bit\n");
	240	return -EINVAL;
	241	}
	242	if ((x->aead->alg_key_len != 128 + 32) &&
	243	(x->aead->alg_key_len != 256 + 32)) {
	244	netdev_info(netdev, "Cannot offload xfrm states with AEAD key length other than 128/256 bit\n");
	245	return -EINVAL;
	246	}
	247	if (x->tfcpad) {
	248	netdev_info(netdev, "Cannot offload xfrm states with tfc padding\n");
	249	return -EINVAL;
	250	}
	251	if (!x->geniv) {
	252	netdev_info(netdev, "Cannot offload xfrm states without geniv\n");
	253	return -EINVAL;
	254	}
	255	if (strcmp(x->geniv, "seqiv")) {
	256	netdev_info(netdev, "Cannot offload xfrm states with geniv other than seqiv\n");
	257	return -EINVAL;
	258	}
547eede0 IT	259	return 0;
	260	}
	261
c674df97 LR	262	static void _update_xfrm_state(struct work_struct *work)
	263	{
	264	struct mlx5e_ipsec_modify_state_work *modify_work =
	265	container_of(work, struct mlx5e_ipsec_modify_state_work, work);
	266	struct mlx5e_ipsec_sa_entry *sa_entry = container_of(
	267	modify_work, struct mlx5e_ipsec_sa_entry, modify_work);
	268
b73e6728	269	mlx5_accel_esp_modify_xfrm(sa_entry, &modify_work->attrs);
c674df97 LR	270	}
c674df97 LR	271
547eede0 IT	272	static int mlx5e_xfrm_add_state(struct xfrm_state *x)
	273	{
	274	struct mlx5e_ipsec_sa_entry *sa_entry = NULL;
bdfd2d1f	275	struct net_device *netdev = x->xso.real_dev;
547eede0	276	struct mlx5e_priv *priv;
547eede0 IT	277	int err;
	278
	279	priv = netdev_priv(netdev);
021a429b LR	280	if (!priv->ipsec)
021a429b LR	281	return -EOPNOTSUPP;
547eede0 IT	282
	283	err = mlx5e_xfrm_validate_state(x);
	284	if (err)
	285	return err;
	286
	287	sa_entry = kzalloc(sizeof(*sa_entry), GFP_KERNEL);
	288	if (!sa_entry) {
	289	err = -ENOMEM;
	290	goto out;
	291	}
	292
	293	sa_entry->x = x;
	294	sa_entry->ipsec = priv->ipsec;
	295
cb010083 AY	296	/* check esn */
	297	mlx5e_ipsec_update_esn_state(sa_entry);
	298
b73e6728	299	mlx5e_ipsec_build_accel_xfrm_attrs(sa_entry, &sa_entry->attrs);
d6c4f029	300	/* create hw context */
b73e6728 LR	301	err = mlx5_ipsec_create_sa_ctx(sa_entry);
b73e6728 LR	302	if (err)
d6c4f029	303	goto err_xfrm;
547eede0	304
82f7bdba	305	err = mlx5e_accel_ipsec_fs_add_rule(priv, sa_entry);
5e466345 HN	306	if (err)
	307	goto err_hw_ctx;
	308
3ef535ec	309	if (x->xso.dir == XFRM_DEV_OFFLOAD_IN) {
b73e6728	310	err = mlx5e_ipsec_sadb_rx_add(sa_entry);
7dfee4b1	311	if (err)
5e466345	312	goto err_add_rule;
7dfee4b1 RS	313	} else {
	314	sa_entry->set_iv_op = (x->props.flags & XFRM_STATE_ESN) ?
	315	mlx5e_ipsec_set_iv_esn : mlx5e_ipsec_set_iv;
	316	}
	317
c674df97	318	INIT_WORK(&sa_entry->modify_work.work, _update_xfrm_state);
547eede0 IT	319	x->xso.offload_handle = (unsigned long)sa_entry;
	320	goto out;
	321
5e466345	322	err_add_rule:
82f7bdba	323	mlx5e_accel_ipsec_fs_del_rule(priv, sa_entry);
7dfee4b1	324	err_hw_ctx:
b73e6728	325	mlx5_ipsec_free_sa_ctx(sa_entry);
d6c4f029	326	err_xfrm:
547eede0 IT	327	kfree(sa_entry);
	328	out:
	329	return err;
	330	}
	331
	332	static void mlx5e_xfrm_del_state(struct xfrm_state *x)
	333	{
75ef3f55	334	struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(x);
547eede0	335
3ef535ec	336	if (x->xso.dir == XFRM_DEV_OFFLOAD_IN)
547eede0	337	mlx5e_ipsec_sadb_rx_del(sa_entry);
547eede0 IT	338	}
	339
	340	static void mlx5e_xfrm_free_state(struct xfrm_state *x)
	341	{
75ef3f55	342	struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(x);
5e466345	343	struct mlx5e_priv *priv = netdev_priv(x->xso.dev);
547eede0	344
b73e6728	345	cancel_work_sync(&sa_entry->modify_work.work);
82f7bdba	346	mlx5e_accel_ipsec_fs_del_rule(priv, sa_entry);
b73e6728	347	mlx5_ipsec_free_sa_ctx(sa_entry);
547eede0 IT	348	kfree(sa_entry);
	349	}
	350
	351	int mlx5e_ipsec_init(struct mlx5e_priv *priv)
	352	{
021a429b LR	353	struct mlx5e_ipsec *ipsec;
021a429b LR	354	int ret;
547eede0	355
2451da08	356	if (!mlx5_ipsec_device_caps(priv->mdev)) {
547eede0 IT	357	netdev_dbg(priv->netdev, "Not an IPSec offload device\n");
	358	return 0;
	359	}
	360
	361	ipsec = kzalloc(sizeof(*ipsec), GFP_KERNEL);
	362	if (!ipsec)
	363	return -ENOMEM;
	364
	365	hash_init(ipsec->sadb_rx);
	366	spin_lock_init(&ipsec->sadb_rx_lock);
9af1968e	367	ipsec->mdev = priv->mdev;
cb010083 AY	368	ipsec->wq = alloc_ordered_workqueue("mlx5e_ipsec: %s", 0,
	369	priv->netdev->name);
	370	if (!ipsec->wq) {
021a429b LR	371	ret = -ENOMEM;
021a429b LR	372	goto err_wq;
cb010083	373	}
5e466345	374
021a429b LR	375	ret = mlx5e_accel_ipsec_fs_init(ipsec);
	376	if (ret)
	377	goto err_fs_init;
	378
5589b8f1	379	priv->ipsec = ipsec;
547eede0 IT	380	netdev_dbg(priv->netdev, "IPSec attached to netdevice\n");
547eede0 IT	381	return 0;
021a429b LR	382
	383	err_fs_init:
	384	destroy_workqueue(ipsec->wq);
	385	err_wq:
	386	kfree(ipsec);
	387	return (ret != -EOPNOTSUPP) ? ret : 0;
547eede0 IT	388	}
	389
	390	void mlx5e_ipsec_cleanup(struct mlx5e_priv *priv)
	391	{
	392	struct mlx5e_ipsec *ipsec = priv->ipsec;
	393
	394	if (!ipsec)
	395	return;
	396
301e0be8	397	mlx5e_accel_ipsec_fs_cleanup(ipsec);
cb010083	398	destroy_workqueue(ipsec->wq);
547eede0 IT	399	kfree(ipsec);
	400	priv->ipsec = NULL;
	401	}
	402
2ac9cfe7 IT	403	static bool mlx5e_ipsec_offload_ok(struct sk_buff skb, struct xfrm_state x)
	404	{
	405	if (x->props.family == AF_INET) {
	406	/* Offload with IPv4 options is not supported yet */
	407	if (ip_hdr(skb)->ihl > 5)
	408	return false;
	409	} else {
	410	/* Offload with IPv6 extension headers is not support yet */
	411	if (ipv6_ext_hdr(ipv6_hdr(skb)->nexthdr))
	412	return false;
	413	}
	414
	415	return true;
	416	}
	417
cb010083 AY	418	static void mlx5e_xfrm_advance_esn_state(struct xfrm_state *x)
	419	{
	420	struct mlx5e_ipsec_sa_entry *sa_entry = to_ipsec_sa_entry(x);
c674df97 LR	421	struct mlx5e_ipsec_modify_state_work *modify_work =
c674df97 LR	422	&sa_entry->modify_work;
cb010083 AY	423	bool need_update;
cb010083 AY	424
cb010083 AY	425	need_update = mlx5e_ipsec_update_esn_state(sa_entry);
	426	if (!need_update)
	427	return;
	428
cb010083	429	mlx5e_ipsec_build_accel_xfrm_attrs(sa_entry, &modify_work->attrs);
c674df97	430	queue_work(sa_entry->ipsec->wq, &modify_work->work);
cb010083 AY	431	}
cb010083 AY	432
547eede0 IT	433	static const struct xfrmdev_ops mlx5e_ipsec_xfrmdev_ops = {
	434	.xdo_dev_state_add = mlx5e_xfrm_add_state,
	435	.xdo_dev_state_delete = mlx5e_xfrm_del_state,
	436	.xdo_dev_state_free = mlx5e_xfrm_free_state,
2ac9cfe7	437	.xdo_dev_offload_ok = mlx5e_ipsec_offload_ok,
cb010083	438	.xdo_dev_state_advance_esn = mlx5e_xfrm_advance_esn_state,
547eede0 IT	439	};
	440
	441	void mlx5e_ipsec_build_netdev(struct mlx5e_priv *priv)
	442	{
	443	struct mlx5_core_dev *mdev = priv->mdev;
	444	struct net_device *netdev = priv->netdev;
	445
a8444b0b LR	446	if (!mlx5_ipsec_device_caps(mdev))
	447	return;
	448
547eede0 IT	449	mlx5_core_info(mdev, "mlx5e: IPSec ESP acceleration enabled\n");
	450	netdev->xfrmdev_ops = &mlx5e_ipsec_xfrmdev_ops;
	451	netdev->features \|= NETIF_F_HW_ESP;
	452	netdev->hw_enc_features \|= NETIF_F_HW_ESP;
	453
	454	if (!MLX5_CAP_ETH(mdev, swp_csum)) {
	455	mlx5_core_dbg(mdev, "mlx5e: SWP checksum not supported\n");
	456	return;
	457	}
	458
	459	netdev->features \|= NETIF_F_HW_ESP_TX_CSUM;
	460	netdev->hw_enc_features \|= NETIF_F_HW_ESP_TX_CSUM;
2ac9cfe7	461
effbe267	462	if (!MLX5_CAP_ETH(mdev, swp_lso)) {
2ac9cfe7 IT	463	mlx5_core_dbg(mdev, "mlx5e: ESP LSO not supported\n");
	464	return;
	465	}
	466
5a985aa3	467	netdev->gso_partial_features \|= NETIF_F_GSO_ESP;
2ac9cfe7 IT	468	mlx5_core_dbg(mdev, "mlx5e: ESP GSO capability turned on\n");
	469	netdev->features \|= NETIF_F_GSO_ESP;
	470	netdev->hw_features \|= NETIF_F_GSO_ESP;
	471	netdev->hw_enc_features \|= NETIF_F_GSO_ESP;
547eede0	472	}