[linux-2.6-block.git] / drivers / crypto / chelsio / chcr_ktls.c

// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (C) 2020 Chelsio Communications.  All rights reserved. */

#ifdef CONFIG_CHELSIO_TLS_DEVICE
#include "chcr_ktls.h"

static int chcr_init_tcb_fields(struct chcr_ktls_info *tx_info);
/*
 * chcr_ktls_save_keys: calculate and save crypto keys.
 * @tx_info - driver specific tls info.
 * @crypto_info - tls crypto information.
 * @direction - TX/RX direction.
 * return - SUCCESS/FAILURE.
 */
static int chcr_ktls_save_keys(struct chcr_ktls_info *tx_info,
			       struct tls_crypto_info *crypto_info,
			       enum tls_offload_ctx_dir direction)
{
	int ck_size, key_ctx_size, mac_key_size, keylen, ghash_size, ret;
	unsigned char ghash_h[TLS_CIPHER_AES_GCM_256_TAG_SIZE];
	struct tls12_crypto_info_aes_gcm_128 *info_128_gcm;
	struct ktls_key_ctx *kctx = &tx_info->key_ctx;
	struct crypto_cipher *cipher;
	unsigned char *key, *salt;

	switch (crypto_info->cipher_type) {
	case TLS_CIPHER_AES_GCM_128:
		info_128_gcm =
			(struct tls12_crypto_info_aes_gcm_128 *)crypto_info;
		keylen = TLS_CIPHER_AES_GCM_128_KEY_SIZE;
		ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
		tx_info->salt_size = TLS_CIPHER_AES_GCM_128_SALT_SIZE;
		mac_key_size = CHCR_KEYCTX_MAC_KEY_SIZE_128;
		tx_info->iv_size = TLS_CIPHER_AES_GCM_128_IV_SIZE;
		tx_info->iv = be64_to_cpu(*(__be64 *)info_128_gcm->iv);

		ghash_size = TLS_CIPHER_AES_GCM_128_TAG_SIZE;
		key = info_128_gcm->key;
		salt = info_128_gcm->salt;
		tx_info->record_no = *(u64 *)info_128_gcm->rec_seq;

		break;

	default:
		pr_err("GCM: cipher type 0x%x not supported\n",
		       crypto_info->cipher_type);
		ret = -EINVAL;
		goto out;
	}

	key_ctx_size = CHCR_KTLS_KEY_CTX_LEN +
		       roundup(keylen, 16) + ghash_size;
	/* Calculate the H = CIPH(K, 0 repeated 16 times).
	 * It will go in key context
	 */
	cipher = crypto_alloc_cipher("aes", 0, 0);
	if (IS_ERR(cipher)) {
		ret = -ENOMEM;
		goto out;
	}

	ret = crypto_cipher_setkey(cipher, key, keylen);
	if (ret)
		goto out1;

	memset(ghash_h, 0, ghash_size);
	crypto_cipher_encrypt_one(cipher, ghash_h, ghash_h);

	/* fill the Key context */
	if (direction == TLS_OFFLOAD_CTX_DIR_TX) {
		kctx->ctx_hdr = FILL_KEY_CTX_HDR(ck_size,
						 mac_key_size,
						 key_ctx_size >> 4);
	} else {
		ret = -EINVAL;
		goto out1;
	}

	memcpy(kctx->salt, salt, tx_info->salt_size);
	memcpy(kctx->key, key, keylen);
	memcpy(kctx->key + keylen, ghash_h, ghash_size);
	tx_info->key_ctx_len = key_ctx_size;

out1:
	crypto_free_cipher(cipher);
out:
	return ret;
}

static int chcr_ktls_update_connection_state(struct chcr_ktls_info *tx_info,
					     int new_state)
{
	unsigned long flags;

	/* This function can be called from both rx (interrupt context) and tx
	 * queue contexts.
	 */
	spin_lock_irqsave(&tx_info->lock, flags);
	switch (tx_info->connection_state) {
	case KTLS_CONN_CLOSED:
		tx_info->connection_state = new_state;
		break;

	case KTLS_CONN_ACT_OPEN_REQ:
		/* only go forward if state is greater than current state. */
		if (new_state <= tx_info->connection_state)
			break;
		/* update to the next state and also initialize TCB */
		tx_info->connection_state = new_state;
		/* FALLTHRU */
	case KTLS_CONN_ACT_OPEN_RPL:
		/* if we are stuck in this state, means tcb init might not
		 * received by HW, try sending it again.
		 */
		if (!chcr_init_tcb_fields(tx_info))
			tx_info->connection_state = KTLS_CONN_SET_TCB_REQ;
		break;

	case KTLS_CONN_SET_TCB_REQ:
		/* only go forward if state is greater than current state. */
		if (new_state <= tx_info->connection_state)
			break;
		/* update to the next state and check if l2t_state is valid  */
		tx_info->connection_state = new_state;
		/* FALLTHRU */
	case KTLS_CONN_SET_TCB_RPL:
		/* Check if l2t state is valid, then move to ready state. */
		if (cxgb4_check_l2t_valid(tx_info->l2te))
			tx_info->connection_state = KTLS_CONN_TX_READY;
		break;

	case KTLS_CONN_TX_READY:
		/* nothing to be done here */
		break;

	default:
		pr_err("unknown KTLS connection state\n");
		break;
	}
	spin_unlock_irqrestore(&tx_info->lock, flags);

	return tx_info->connection_state;
}
/*
 * chcr_ktls_act_open_req: creates TCB entry for ipv4 connection.
 * @sk - tcp socket.
 * @tx_info - driver specific tls info.
 * @atid - connection active tid.
 * return - send success/failure.
 */
static int chcr_ktls_act_open_req(struct sock *sk,
				  struct chcr_ktls_info *tx_info,
				  int atid)
{
	struct inet_sock *inet = inet_sk(sk);
	struct cpl_t6_act_open_req *cpl6;
	struct cpl_act_open_req *cpl;
	struct sk_buff *skb;
	unsigned int len;
	int qid_atid;
	u64 options;

	len = sizeof(*cpl6);
	skb = alloc_skb(len, GFP_KERNEL);
	if (unlikely(!skb))
		return -ENOMEM;
	/* mark it a control pkt */
	set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id);

	cpl6 = __skb_put_zero(skb, len);
	cpl = (struct cpl_act_open_req *)cpl6;
	INIT_TP_WR(cpl6, 0);
	qid_atid = TID_QID_V(tx_info->rx_qid) |
		   TID_TID_V(atid);
	OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, qid_atid));
	cpl->local_port = inet->inet_sport;
	cpl->peer_port = inet->inet_dport;
	cpl->local_ip = inet->inet_rcv_saddr;
	cpl->peer_ip = inet->inet_daddr;

	/* fill first 64 bit option field. */
	options = TCAM_BYPASS_F | ULP_MODE_V(ULP_MODE_NONE) | NON_OFFLOAD_F |
		  SMAC_SEL_V(tx_info->smt_idx) | TX_CHAN_V(tx_info->tx_chan);
	cpl->opt0 = cpu_to_be64(options);

	/* next 64 bit option field. */
	options =
		TX_QUEUE_V(tx_info->adap->params.tp.tx_modq[tx_info->tx_chan]);
	cpl->opt2 = htonl(options);

	return cxgb4_l2t_send(tx_info->netdev, skb, tx_info->l2te);
}

/*
 * chcr_setup_connection:  create a TCB entry so that TP will form tcp packets.
 * @sk - tcp socket.
 * @tx_info - driver specific tls info.
 * return: NET_TX_OK/NET_XMIT_DROP
 */
static int chcr_setup_connection(struct sock *sk,
				 struct chcr_ktls_info *tx_info)
{
	struct tid_info *t = &tx_info->adap->tids;
	int atid, ret = 0;

	atid = cxgb4_alloc_atid(t, tx_info);
	if (atid == -1)
		return -EINVAL;

	tx_info->atid = atid;
	tx_info->ip_family = sk->sk_family;

	if (sk->sk_family == AF_INET ||
	    (sk->sk_family == AF_INET6 && !sk->sk_ipv6only &&
	     ipv6_addr_type(&sk->sk_v6_daddr) == IPV6_ADDR_MAPPED)) {
		tx_info->ip_family = AF_INET;
		ret = chcr_ktls_act_open_req(sk, tx_info, atid);
	} else {
		tx_info->ip_family = AF_INET6;
		ret = -EOPNOTSUPP;
	}

	/* if return type is NET_XMIT_CN, msg will be sent but delayed, mark ret
	 * success, if any other return type clear atid and return that failure.
	 */
	if (ret) {
		if (ret == NET_XMIT_CN)
			ret = 0;
		else
			cxgb4_free_atid(t, atid);
		goto out;
	}

	/* update the connection state */
	chcr_ktls_update_connection_state(tx_info, KTLS_CONN_ACT_OPEN_REQ);
out:
	return ret;
}

/*
 * chcr_set_tcb_field: update tcb fields.
 * @tx_info - driver specific tls info.
 * @word - TCB word.
 * @mask - TCB word related mask.
 * @val - TCB word related value.
 * @no_reply - set 1 if not looking for TP response.
 */
static int chcr_set_tcb_field(struct chcr_ktls_info *tx_info, u16 word,
			      u64 mask, u64 val, int no_reply)
{
	struct cpl_set_tcb_field *req;
	struct sk_buff *skb;

	skb = alloc_skb(sizeof(struct cpl_set_tcb_field), GFP_ATOMIC);
	if (!skb)
		return -ENOMEM;

	req = (struct cpl_set_tcb_field *)__skb_put_zero(skb, sizeof(*req));
	INIT_TP_WR_CPL(req, CPL_SET_TCB_FIELD, tx_info->tid);
	req->reply_ctrl = htons(QUEUENO_V(tx_info->rx_qid) |
				NO_REPLY_V(no_reply));
	req->word_cookie = htons(TCB_WORD_V(word));
	req->mask = cpu_to_be64(mask);
	req->val = cpu_to_be64(val);

	set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id);
	return cxgb4_ofld_send(tx_info->netdev, skb);
}

/*
 * chcr_ktls_mark_tcb_close: mark tcb state to CLOSE
 * @tx_info - driver specific tls info.
 * return: NET_TX_OK/NET_XMIT_DROP.
 */
static int chcr_ktls_mark_tcb_close(struct chcr_ktls_info *tx_info)
{
	return chcr_set_tcb_field(tx_info, TCB_T_STATE_W,
				  TCB_T_STATE_V(TCB_T_STATE_M),
				  CHCR_TCB_STATE_CLOSED, 1);
}

/*
 * chcr_ktls_dev_del:  call back for tls_dev_del.
 * Remove the tid and l2t entry and close the connection.
 * it per connection basis.
 * @netdev - net device.
 * @tls_cts - tls context.
 * @direction - TX/RX crypto direction
 */
static void chcr_ktls_dev_del(struct net_device *netdev,
			      struct tls_context *tls_ctx,
			      enum tls_offload_ctx_dir direction)
{
	struct chcr_ktls_ofld_ctx_tx *tx_ctx =
				chcr_get_ktls_tx_context(tls_ctx);
	struct chcr_ktls_info *tx_info = tx_ctx->chcr_info;

	if (!tx_info)
		return;

	spin_lock(&tx_info->lock);
	tx_info->connection_state = KTLS_CONN_CLOSED;
	spin_unlock(&tx_info->lock);

	if (tx_info->l2te)
		cxgb4_l2t_release(tx_info->l2te);

	if (tx_info->tid != -1) {
		/* clear tcb state and then release tid */
		chcr_ktls_mark_tcb_close(tx_info);
		cxgb4_remove_tid(&tx_info->adap->tids, tx_info->tx_chan,
				 tx_info->tid, tx_info->ip_family);
	}
	kvfree(tx_info);
	tx_ctx->chcr_info = NULL;
}

/*
 * chcr_ktls_dev_add:  call back for tls_dev_add.
 * Create a tcb entry for TP. Also add l2t entry for the connection. And
 * generate keys & save those keys locally.
 * @netdev - net device.
 * @tls_cts - tls context.
 * @direction - TX/RX crypto direction
 * return: SUCCESS/FAILURE.
 */
static int chcr_ktls_dev_add(struct net_device *netdev, struct sock *sk,
			     enum tls_offload_ctx_dir direction,
			     struct tls_crypto_info *crypto_info,
			     u32 start_offload_tcp_sn)
{
	struct tls_context *tls_ctx = tls_get_ctx(sk);
	struct chcr_ktls_ofld_ctx_tx *tx_ctx;
	struct chcr_ktls_info *tx_info;
	struct dst_entry *dst;
	struct adapter *adap;
	struct port_info *pi;
	struct neighbour *n;
	u8 daaddr[16];
	int ret = -1;

	tx_ctx = chcr_get_ktls_tx_context(tls_ctx);

	pi = netdev_priv(netdev);
	adap = pi->adapter;
	if (direction == TLS_OFFLOAD_CTX_DIR_RX) {
		pr_err("not expecting for RX direction\n");
		ret = -EINVAL;
		goto out;
	}
	if (tx_ctx->chcr_info) {
		ret = -EINVAL;
		goto out;
	}

	tx_info = kvzalloc(sizeof(*tx_info), GFP_KERNEL);
	if (!tx_info) {
		ret = -ENOMEM;
		goto out;
	}

	spin_lock_init(&tx_info->lock);

	/* clear connection state */
	spin_lock(&tx_info->lock);
	tx_info->connection_state = KTLS_CONN_CLOSED;
	spin_unlock(&tx_info->lock);

	tx_info->sk = sk;
	/* initialize tid and atid to -1, 0 is a also a valid id. */
	tx_info->tid = -1;
	tx_info->atid = -1;

	tx_info->adap = adap;
	tx_info->netdev = netdev;
	tx_info->tx_chan = pi->tx_chan;
	tx_info->smt_idx = pi->smt_idx;
	tx_info->port_id = pi->port_id;

	tx_info->rx_qid = chcr_get_first_rx_qid(adap);
	if (unlikely(tx_info->rx_qid < 0))
		goto out2;

	tx_info->prev_seq = start_offload_tcp_sn;
	tx_info->tcp_start_seq_number = start_offload_tcp_sn;

	/* save crypto keys */
	ret = chcr_ktls_save_keys(tx_info, crypto_info, direction);
	if (ret < 0)
		goto out2;

	/* get peer ip */
	if (sk->sk_family == AF_INET ||
	    (sk->sk_family == AF_INET6 && !sk->sk_ipv6only &&
	     ipv6_addr_type(&sk->sk_v6_daddr) == IPV6_ADDR_MAPPED)) {
		memcpy(daaddr, &sk->sk_daddr, 4);
	} else {
		goto out2;
	}

	/* get the l2t index */
	dst = sk_dst_get(sk);
	if (!dst) {
		pr_err("DST entry not found\n");
		goto out2;
	}
	n = dst_neigh_lookup(dst, daaddr);
	if (!n || !n->dev) {
		pr_err("neighbour not found\n");
		dst_release(dst);
		goto out2;
	}
	tx_info->l2te  = cxgb4_l2t_get(adap->l2t, n, n->dev, 0);

	neigh_release(n);
	dst_release(dst);

	if (!tx_info->l2te) {
		pr_err("l2t entry not found\n");
		goto out2;
	}

	tx_ctx->chcr_info = tx_info;

	/* create a filter and call cxgb4_l2t_send to send the packet out, which
	 * will take care of updating l2t entry in hw if not already done.
	 */
	ret = chcr_setup_connection(sk, tx_info);
	if (ret)
		goto out2;

	return 0;
out2:
	kvfree(tx_info);
out:
	return ret;
}

static const struct tlsdev_ops chcr_ktls_ops = {
	.tls_dev_add = chcr_ktls_dev_add,
	.tls_dev_del = chcr_ktls_dev_del,
};

/*
 * chcr_enable_ktls:  add NETIF_F_HW_TLS_TX flag in all the ports.
 */
void chcr_enable_ktls(struct adapter *adap)
{
	struct net_device *netdev;
	int i;

	for_each_port(adap, i) {
		netdev = adap->port[i];
		netdev->features |= NETIF_F_HW_TLS_TX;
		netdev->hw_features |= NETIF_F_HW_TLS_TX;
		netdev->tlsdev_ops = &chcr_ktls_ops;
	}
}

/*
 * chcr_disable_ktls:  remove NETIF_F_HW_TLS_TX flag from all the ports.
 */
void chcr_disable_ktls(struct adapter *adap)
{
	struct net_device *netdev;
	int i;

	for_each_port(adap, i) {
		netdev = adap->port[i];
		netdev->features &= ~NETIF_F_HW_TLS_TX;
		netdev->hw_features &= ~NETIF_F_HW_TLS_TX;
		netdev->tlsdev_ops = NULL;
	}
}

/*
 * chcr_init_tcb_fields:  Initialize tcb fields to handle TCP seq number
 *			  handling.
 * @tx_info - driver specific tls info.
 * return: NET_TX_OK/NET_XMIT_DROP
 */
static int chcr_init_tcb_fields(struct chcr_ktls_info *tx_info)
{
	int  ret = 0;

	/* set tcb in offload and bypass */
	ret =
	chcr_set_tcb_field(tx_info, TCB_T_FLAGS_W,
			   TCB_T_FLAGS_V(TF_CORE_BYPASS_F | TF_NON_OFFLOAD_F),
			   TCB_T_FLAGS_V(TF_CORE_BYPASS_F), 1);
	if (ret)
		return ret;
	/* reset snd_una and snd_next fields in tcb */
	ret = chcr_set_tcb_field(tx_info, TCB_SND_UNA_RAW_W,
				 TCB_SND_NXT_RAW_V(TCB_SND_NXT_RAW_M) |
				 TCB_SND_UNA_RAW_V(TCB_SND_UNA_RAW_M),
				 0, 1);
	if (ret)
		return ret;

	/* reset send max */
	ret = chcr_set_tcb_field(tx_info, TCB_SND_MAX_RAW_W,
				 TCB_SND_MAX_RAW_V(TCB_SND_MAX_RAW_M),
				 0, 1);
	if (ret)
		return ret;

	/* update l2t index and request for tp reply to confirm tcb is
	 * initialised to handle tx traffic.
	 */
	ret = chcr_set_tcb_field(tx_info, TCB_L2T_IX_W,
				 TCB_L2T_IX_V(TCB_L2T_IX_M),
				 TCB_L2T_IX_V(tx_info->l2te->idx), 0);
	return ret;
}

/*
 * chcr_ktls_cpl_act_open_rpl: connection reply received from TP.
 */
int chcr_ktls_cpl_act_open_rpl(struct adapter *adap, unsigned char *input)
{
	const struct cpl_act_open_rpl *p = (void *)input;
	struct chcr_ktls_info *tx_info = NULL;
	unsigned int atid, tid, status;
	struct tid_info *t;

	tid = GET_TID(p);
	status = AOPEN_STATUS_G(ntohl(p->atid_status));
	atid = TID_TID_G(AOPEN_ATID_G(ntohl(p->atid_status)));

	t = &adap->tids;
	tx_info = lookup_atid(t, atid);

	if (!tx_info || tx_info->atid != atid) {
		pr_err("tx_info or atid is not correct\n");
		return -1;
	}

	if (!status) {
		tx_info->tid = tid;
		cxgb4_insert_tid(t, tx_info, tx_info->tid, tx_info->ip_family);

		cxgb4_free_atid(t, atid);
		tx_info->atid = -1;
		/* update the connection state */
		chcr_ktls_update_connection_state(tx_info,
						  KTLS_CONN_ACT_OPEN_RPL);
	}
	return 0;
}

/*
 * chcr_ktls_cpl_set_tcb_rpl: TCB reply received from TP.
 */
int chcr_ktls_cpl_set_tcb_rpl(struct adapter *adap, unsigned char *input)
{
	const struct cpl_set_tcb_rpl *p = (void *)input;
	struct chcr_ktls_info *tx_info = NULL;
	struct tid_info *t;
	u32 tid, status;

	tid = GET_TID(p);
	status = p->status;

	t = &adap->tids;
	tx_info = lookup_tid(t, tid);
	if (!tx_info || tx_info->tid != tid) {
		pr_err("tx_info or atid is not correct\n");
		return -1;
	}
	/* update the connection state */
	chcr_ktls_update_connection_state(tx_info, KTLS_CONN_SET_TCB_RPL);
	return 0;
}
#endif /* CONFIG_CHELSIO_TLS_DEVICE */
Commit	Line	Data
34aba2c4 RM	1	// SPDX-License-Identifier: GPL-2.0-only
	2	/* Copyright (C) 2020 Chelsio Communications. All rights reserved. */
	3
	4	#ifdef CONFIG_CHELSIO_TLS_DEVICE
	5	#include "chcr_ktls.h"
	6
8a30923e RM	7	static int chcr_init_tcb_fields(struct chcr_ktls_info *tx_info);
	8	/*
	9	* chcr_ktls_save_keys: calculate and save crypto keys.
	10	* @tx_info - driver specific tls info.
	11	* @crypto_info - tls crypto information.
	12	* @direction - TX/RX direction.
	13	* return - SUCCESS/FAILURE.
	14	*/
	15	static int chcr_ktls_save_keys(struct chcr_ktls_info *tx_info,
	16	struct tls_crypto_info *crypto_info,
	17	enum tls_offload_ctx_dir direction)
	18	{
	19	int ck_size, key_ctx_size, mac_key_size, keylen, ghash_size, ret;
	20	unsigned char ghash_h[TLS_CIPHER_AES_GCM_256_TAG_SIZE];
	21	struct tls12_crypto_info_aes_gcm_128 *info_128_gcm;
	22	struct ktls_key_ctx *kctx = &tx_info->key_ctx;
	23	struct crypto_cipher *cipher;
	24	unsigned char key, salt;
	25
	26	switch (crypto_info->cipher_type) {
	27	case TLS_CIPHER_AES_GCM_128:
	28	info_128_gcm =
	29	(struct tls12_crypto_info_aes_gcm_128 *)crypto_info;
	30	keylen = TLS_CIPHER_AES_GCM_128_KEY_SIZE;
	31	ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
	32	tx_info->salt_size = TLS_CIPHER_AES_GCM_128_SALT_SIZE;
	33	mac_key_size = CHCR_KEYCTX_MAC_KEY_SIZE_128;
	34	tx_info->iv_size = TLS_CIPHER_AES_GCM_128_IV_SIZE;
	35	tx_info->iv = be64_to_cpu((__be64 )info_128_gcm->iv);
	36
	37	ghash_size = TLS_CIPHER_AES_GCM_128_TAG_SIZE;
	38	key = info_128_gcm->key;
	39	salt = info_128_gcm->salt;
	40	tx_info->record_no = (u64 )info_128_gcm->rec_seq;
	41
	42	break;
	43
	44	default:
	45	pr_err("GCM: cipher type 0x%x not supported\n",
	46	crypto_info->cipher_type);
	47	ret = -EINVAL;
	48	goto out;
	49	}
	50
	51	key_ctx_size = CHCR_KTLS_KEY_CTX_LEN +
	52	roundup(keylen, 16) + ghash_size;
	53	/* Calculate the H = CIPH(K, 0 repeated 16 times).
	54	* It will go in key context
	55	*/
	56	cipher = crypto_alloc_cipher("aes", 0, 0);
	57	if (IS_ERR(cipher)) {
	58	ret = -ENOMEM;
	59	goto out;
	60	}
	61
	62	ret = crypto_cipher_setkey(cipher, key, keylen);
	63	if (ret)
	64	goto out1;
	65
	66	memset(ghash_h, 0, ghash_size);
	67	crypto_cipher_encrypt_one(cipher, ghash_h, ghash_h);
	68
	69	/* fill the Key context */
	70	if (direction == TLS_OFFLOAD_CTX_DIR_TX) {
71	kctx->ctx_hdr = FILL_KEY_CTX_HDR(ck_size,
72	mac_key_size,
73	key_ctx_size >> 4);
74	} else {
75	ret = -EINVAL;
76	goto out1;
77	}
78
79	memcpy(kctx->salt, salt, tx_info->salt_size);
80	memcpy(kctx->key, key, keylen);
81	memcpy(kctx->key + keylen, ghash_h, ghash_size);
82	tx_info->key_ctx_len = key_ctx_size;
83
84	out1:
85	crypto_free_cipher(cipher);
86	out:
87	return ret;
88	}
89
90	static int chcr_ktls_update_connection_state(struct chcr_ktls_info *tx_info,
91	int new_state)
92	{
93	unsigned long flags;
94
95	/* This function can be called from both rx (interrupt context) and tx
96	* queue contexts.
97	*/
98	spin_lock_irqsave(&tx_info->lock, flags);
99	switch (tx_info->connection_state) {
100	case KTLS_CONN_CLOSED:
101	tx_info->connection_state = new_state;
102	break;
103
104	case KTLS_CONN_ACT_OPEN_REQ:
105	/* only go forward if state is greater than current state. */
106	if (new_state <= tx_info->connection_state)
107	break;
108	/* update to the next state and also initialize TCB */
109	tx_info->connection_state = new_state;
110	/* FALLTHRU */
111	case KTLS_CONN_ACT_OPEN_RPL:
112	/* if we are stuck in this state, means tcb init might not
113	* received by HW, try sending it again.
114	*/
115	if (!chcr_init_tcb_fields(tx_info))
116	tx_info->connection_state = KTLS_CONN_SET_TCB_REQ;
117	break;
118
119	case KTLS_CONN_SET_TCB_REQ:
120	/* only go forward if state is greater than current state. */
121	if (new_state <= tx_info->connection_state)
122	break;
123	/* update to the next state and check if l2t_state is valid */
124	tx_info->connection_state = new_state;
125	/* FALLTHRU */
126	case KTLS_CONN_SET_TCB_RPL:
127	/* Check if l2t state is valid, then move to ready state. */
128	if (cxgb4_check_l2t_valid(tx_info->l2te))
129	tx_info->connection_state = KTLS_CONN_TX_READY;
130	break;
131
132	case KTLS_CONN_TX_READY:
133	/* nothing to be done here */
134	break;
135
136	default:
137	pr_err("unknown KTLS connection state\n");
138	break;
139	}
140	spin_unlock_irqrestore(&tx_info->lock, flags);
141
142	return tx_info->connection_state;
143	}
34aba2c4 RM	144	/*
	145	* chcr_ktls_act_open_req: creates TCB entry for ipv4 connection.
	146	* @sk - tcp socket.
	147	* @tx_info - driver specific tls info.
	148	* @atid - connection active tid.
	149	* return - send success/failure.
	150	*/
	151	static int chcr_ktls_act_open_req(struct sock *sk,
	152	struct chcr_ktls_info *tx_info,
	153	int atid)
	154	{
	155	struct inet_sock *inet = inet_sk(sk);
	156	struct cpl_t6_act_open_req *cpl6;
	157	struct cpl_act_open_req *cpl;
	158	struct sk_buff *skb;
	159	unsigned int len;
	160	int qid_atid;
	161	u64 options;
	162
	163	len = sizeof(*cpl6);
	164	skb = alloc_skb(len, GFP_KERNEL);
	165	if (unlikely(!skb))
	166	return -ENOMEM;
	167	/* mark it a control pkt */
	168	set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id);
	169
	170	cpl6 = __skb_put_zero(skb, len);
	171	cpl = (struct cpl_act_open_req *)cpl6;
	172	INIT_TP_WR(cpl6, 0);
	173	qid_atid = TID_QID_V(tx_info->rx_qid) \|
	174	TID_TID_V(atid);
	175	OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, qid_atid));
	176	cpl->local_port = inet->inet_sport;
	177	cpl->peer_port = inet->inet_dport;
	178	cpl->local_ip = inet->inet_rcv_saddr;
	179	cpl->peer_ip = inet->inet_daddr;
	180
	181	/* fill first 64 bit option field. */
	182	options = TCAM_BYPASS_F \| ULP_MODE_V(ULP_MODE_NONE) \| NON_OFFLOAD_F \|
	183	SMAC_SEL_V(tx_info->smt_idx) \| TX_CHAN_V(tx_info->tx_chan);
	184	cpl->opt0 = cpu_to_be64(options);
	185
	186	/* next 64 bit option field. */
	187	options =
	188	TX_QUEUE_V(tx_info->adap->params.tp.tx_modq[tx_info->tx_chan]);
	189	cpl->opt2 = htonl(options);
	190
	191	return cxgb4_l2t_send(tx_info->netdev, skb, tx_info->l2te);
	192	}
	193
	194	/*
	195	* chcr_setup_connection: create a TCB entry so that TP will form tcp packets.
	196	* @sk - tcp socket.
	197	* @tx_info - driver specific tls info.
	198	* return: NET_TX_OK/NET_XMIT_DROP
	199	*/
	200	static int chcr_setup_connection(struct sock *sk,
	201	struct chcr_ktls_info *tx_info)
	202	{
	203	struct tid_info *t = &tx_info->adap->tids;
	204	int atid, ret = 0;
	205
	206	atid = cxgb4_alloc_atid(t, tx_info);
	207	if (atid == -1)
208	return -EINVAL;
209
210	tx_info->atid = atid;
211	tx_info->ip_family = sk->sk_family;
212
213	if (sk->sk_family == AF_INET \|\|
214	(sk->sk_family == AF_INET6 && !sk->sk_ipv6only &&
215	ipv6_addr_type(&sk->sk_v6_daddr) == IPV6_ADDR_MAPPED)) {
216	tx_info->ip_family = AF_INET;
217	ret = chcr_ktls_act_open_req(sk, tx_info, atid);
218	} else {
219	tx_info->ip_family = AF_INET6;
220	ret = -EOPNOTSUPP;
221	}
222
223	/* if return type is NET_XMIT_CN, msg will be sent but delayed, mark ret
224	* success, if any other return type clear atid and return that failure.
225	*/
226	if (ret) {
227	if (ret == NET_XMIT_CN)
228	ret = 0;
229	else
230	cxgb4_free_atid(t, atid);
8a30923e	231	goto out;
34aba2c4 RM	232	}
34aba2c4 RM	233
8a30923e RM	234	/* update the connection state */
	235	chcr_ktls_update_connection_state(tx_info, KTLS_CONN_ACT_OPEN_REQ);
	236	out:
34aba2c4 RM	237	return ret;
	238	}
	239
	240	/*
	241	* chcr_set_tcb_field: update tcb fields.
	242	* @tx_info - driver specific tls info.
	243	* @word - TCB word.
	244	* @mask - TCB word related mask.
	245	* @val - TCB word related value.
	246	* @no_reply - set 1 if not looking for TP response.
	247	*/
	248	static int chcr_set_tcb_field(struct chcr_ktls_info *tx_info, u16 word,
	249	u64 mask, u64 val, int no_reply)
	250	{
	251	struct cpl_set_tcb_field *req;
	252	struct sk_buff *skb;
	253
	254	skb = alloc_skb(sizeof(struct cpl_set_tcb_field), GFP_ATOMIC);
	255	if (!skb)
	256	return -ENOMEM;
	257
	258	req = (struct cpl_set_tcb_field )__skb_put_zero(skb, sizeof(req));
	259	INIT_TP_WR_CPL(req, CPL_SET_TCB_FIELD, tx_info->tid);
	260	req->reply_ctrl = htons(QUEUENO_V(tx_info->rx_qid) \|
	261	NO_REPLY_V(no_reply));
	262	req->word_cookie = htons(TCB_WORD_V(word));
	263	req->mask = cpu_to_be64(mask);
	264	req->val = cpu_to_be64(val);
	265
	266	set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id);
	267	return cxgb4_ofld_send(tx_info->netdev, skb);
	268	}
	269
	270	/*
	271	* chcr_ktls_mark_tcb_close: mark tcb state to CLOSE
	272	* @tx_info - driver specific tls info.
	273	* return: NET_TX_OK/NET_XMIT_DROP.
	274	*/
	275	static int chcr_ktls_mark_tcb_close(struct chcr_ktls_info *tx_info)
	276	{
	277	return chcr_set_tcb_field(tx_info, TCB_T_STATE_W,
	278	TCB_T_STATE_V(TCB_T_STATE_M),
	279	CHCR_TCB_STATE_CLOSED, 1);
	280	}
	281
	282	/*
	283	* chcr_ktls_dev_del: call back for tls_dev_del.
	284	* Remove the tid and l2t entry and close the connection.
	285	* it per connection basis.
	286	* @netdev - net device.
	287	* @tls_cts - tls context.
	288	* @direction - TX/RX crypto direction
	289	*/
	290	static void chcr_ktls_dev_del(struct net_device *netdev,
	291	struct tls_context *tls_ctx,
	292	enum tls_offload_ctx_dir direction)
	293	{
	294	struct chcr_ktls_ofld_ctx_tx *tx_ctx =
	295	chcr_get_ktls_tx_context(tls_ctx);
	296	struct chcr_ktls_info *tx_info = tx_ctx->chcr_info;
	297
	298	if (!tx_info)
	299	return;
	300
301	spin_lock(&tx_info->lock);
302	tx_info->connection_state = KTLS_CONN_CLOSED;
303	spin_unlock(&tx_info->lock);
304
305	if (tx_info->l2te)
306	cxgb4_l2t_release(tx_info->l2te);
307
308	if (tx_info->tid != -1) {
309	/* clear tcb state and then release tid */
310	chcr_ktls_mark_tcb_close(tx_info);
311	cxgb4_remove_tid(&tx_info->adap->tids, tx_info->tx_chan,
312	tx_info->tid, tx_info->ip_family);
313	}
314	kvfree(tx_info);
315	tx_ctx->chcr_info = NULL;
316	}
317
318	/*
319	* chcr_ktls_dev_add: call back for tls_dev_add.
320	* Create a tcb entry for TP. Also add l2t entry for the connection. And
321	* generate keys & save those keys locally.
322	* @netdev - net device.
323	* @tls_cts - tls context.
324	* @direction - TX/RX crypto direction
325	* return: SUCCESS/FAILURE.
326	*/
327	static int chcr_ktls_dev_add(struct net_device netdev, struct sock sk,
328	enum tls_offload_ctx_dir direction,
329	struct tls_crypto_info *crypto_info,
330	u32 start_offload_tcp_sn)
331	{
332	struct tls_context *tls_ctx = tls_get_ctx(sk);
333	struct chcr_ktls_ofld_ctx_tx *tx_ctx;
334	struct chcr_ktls_info *tx_info;
335	struct dst_entry *dst;
336	struct adapter *adap;
337	struct port_info *pi;
338	struct neighbour *n;
339	u8 daaddr[16];
340	int ret = -1;
341
342	tx_ctx = chcr_get_ktls_tx_context(tls_ctx);
343
344	pi = netdev_priv(netdev);
345	adap = pi->adapter;
346	if (direction == TLS_OFFLOAD_CTX_DIR_RX) {
347	pr_err("not expecting for RX direction\n");
348	ret = -EINVAL;
349	goto out;
350	}
351	if (tx_ctx->chcr_info) {
352	ret = -EINVAL;
353	goto out;
354	}
355
356	tx_info = kvzalloc(sizeof(*tx_info), GFP_KERNEL);
357	if (!tx_info) {
358	ret = -ENOMEM;
359	goto out;
360	}
361
362	spin_lock_init(&tx_info->lock);
363
364	/* clear connection state */
365	spin_lock(&tx_info->lock);
366	tx_info->connection_state = KTLS_CONN_CLOSED;
367	spin_unlock(&tx_info->lock);
368
369	tx_info->sk = sk;
370	/* initialize tid and atid to -1, 0 is a also a valid id. */
371	tx_info->tid = -1;
372	tx_info->atid = -1;
373
374	tx_info->adap = adap;
375	tx_info->netdev = netdev;
376	tx_info->tx_chan = pi->tx_chan;
377	tx_info->smt_idx = pi->smt_idx;
378	tx_info->port_id = pi->port_id;
379
380	tx_info->rx_qid = chcr_get_first_rx_qid(adap);
381	if (unlikely(tx_info->rx_qid < 0))
382	goto out2;
383
384	tx_info->prev_seq = start_offload_tcp_sn;
385	tx_info->tcp_start_seq_number = start_offload_tcp_sn;
386
8a30923e RM	387	/* save crypto keys */
	388	ret = chcr_ktls_save_keys(tx_info, crypto_info, direction);
	389	if (ret < 0)
	390	goto out2;
	391
34aba2c4 RM	392	/* get peer ip */
	393	if (sk->sk_family == AF_INET \|\|
	394	(sk->sk_family == AF_INET6 && !sk->sk_ipv6only &&
	395	ipv6_addr_type(&sk->sk_v6_daddr) == IPV6_ADDR_MAPPED)) {
	396	memcpy(daaddr, &sk->sk_daddr, 4);
	397	} else {
	398	goto out2;
	399	}
	400
	401	/* get the l2t index */
	402	dst = sk_dst_get(sk);
	403	if (!dst) {
	404	pr_err("DST entry not found\n");
	405	goto out2;
	406	}
	407	n = dst_neigh_lookup(dst, daaddr);
	408	if (!n \|\| !n->dev) {
	409	pr_err("neighbour not found\n");
	410	dst_release(dst);
	411	goto out2;
	412	}
	413	tx_info->l2te = cxgb4_l2t_get(adap->l2t, n, n->dev, 0);
	414
	415	neigh_release(n);
	416	dst_release(dst);
	417
	418	if (!tx_info->l2te) {
	419	pr_err("l2t entry not found\n");
	420	goto out2;
	421	}
	422
	423	tx_ctx->chcr_info = tx_info;
	424
	425	/* create a filter and call cxgb4_l2t_send to send the packet out, which
	426	* will take care of updating l2t entry in hw if not already done.
	427	*/
	428	ret = chcr_setup_connection(sk, tx_info);
	429	if (ret)
	430	goto out2;
	431
	432	return 0;
	433	out2:
	434	kvfree(tx_info);
	435	out:
	436	return ret;
	437	}
	438
	439	static const struct tlsdev_ops chcr_ktls_ops = {
	440	.tls_dev_add = chcr_ktls_dev_add,
	441	.tls_dev_del = chcr_ktls_dev_del,
	442	};
	443
	444	/*
	445	* chcr_enable_ktls: add NETIF_F_HW_TLS_TX flag in all the ports.
	446	*/
	447	void chcr_enable_ktls(struct adapter *adap)
	448	{
	449	struct net_device *netdev;
	450	int i;
	451
	452	for_each_port(adap, i) {
	453	netdev = adap->port[i];
	454	netdev->features \|= NETIF_F_HW_TLS_TX;
	455	netdev->hw_features \|= NETIF_F_HW_TLS_TX;
456	netdev->tlsdev_ops = &chcr_ktls_ops;
457	}
458	}
459
460	/*
461	* chcr_disable_ktls: remove NETIF_F_HW_TLS_TX flag from all the ports.
462	*/
463	void chcr_disable_ktls(struct adapter *adap)
464	{
465	struct net_device *netdev;
466	int i;
467
468	for_each_port(adap, i) {
469	netdev = adap->port[i];
470	netdev->features &= ~NETIF_F_HW_TLS_TX;
471	netdev->hw_features &= ~NETIF_F_HW_TLS_TX;
472	netdev->tlsdev_ops = NULL;
473	}
474	}
8a30923e RM	475
	476	/*
	477	* chcr_init_tcb_fields: Initialize tcb fields to handle TCP seq number
	478	* handling.
	479	* @tx_info - driver specific tls info.
	480	* return: NET_TX_OK/NET_XMIT_DROP
	481	*/
	482	static int chcr_init_tcb_fields(struct chcr_ktls_info *tx_info)
	483	{
	484	int ret = 0;
	485
	486	/* set tcb in offload and bypass */
	487	ret =
	488	chcr_set_tcb_field(tx_info, TCB_T_FLAGS_W,
	489	TCB_T_FLAGS_V(TF_CORE_BYPASS_F \| TF_NON_OFFLOAD_F),
	490	TCB_T_FLAGS_V(TF_CORE_BYPASS_F), 1);
	491	if (ret)
	492	return ret;
	493	/* reset snd_una and snd_next fields in tcb */
	494	ret = chcr_set_tcb_field(tx_info, TCB_SND_UNA_RAW_W,
	495	TCB_SND_NXT_RAW_V(TCB_SND_NXT_RAW_M) \|
	496	TCB_SND_UNA_RAW_V(TCB_SND_UNA_RAW_M),
	497	0, 1);
	498	if (ret)
	499	return ret;
	500
	501	/* reset send max */
	502	ret = chcr_set_tcb_field(tx_info, TCB_SND_MAX_RAW_W,
	503	TCB_SND_MAX_RAW_V(TCB_SND_MAX_RAW_M),
	504	0, 1);
	505	if (ret)
	506	return ret;
	507
	508	/* update l2t index and request for tp reply to confirm tcb is
	509	* initialised to handle tx traffic.
	510	*/
	511	ret = chcr_set_tcb_field(tx_info, TCB_L2T_IX_W,
	512	TCB_L2T_IX_V(TCB_L2T_IX_M),
	513	TCB_L2T_IX_V(tx_info->l2te->idx), 0);
	514	return ret;
	515	}
	516
	517	/*
	518	* chcr_ktls_cpl_act_open_rpl: connection reply received from TP.
	519	*/
	520	int chcr_ktls_cpl_act_open_rpl(struct adapter adap, unsigned char input)
	521	{
	522	const struct cpl_act_open_rpl p = (void )input;
	523	struct chcr_ktls_info *tx_info = NULL;
	524	unsigned int atid, tid, status;
	525	struct tid_info *t;
	526
	527	tid = GET_TID(p);
	528	status = AOPEN_STATUS_G(ntohl(p->atid_status));
	529	atid = TID_TID_G(AOPEN_ATID_G(ntohl(p->atid_status)));
	530
	531	t = &adap->tids;
	532	tx_info = lookup_atid(t, atid);
	533
	534	if (!tx_info \|\| tx_info->atid != atid) {
	535	pr_err("tx_info or atid is not correct\n");
	536	return -1;
	537	}
	538
539	if (!status) {
540	tx_info->tid = tid;
541	cxgb4_insert_tid(t, tx_info, tx_info->tid, tx_info->ip_family);
542
543	cxgb4_free_atid(t, atid);
544	tx_info->atid = -1;
545	/* update the connection state */
546	chcr_ktls_update_connection_state(tx_info,
547	KTLS_CONN_ACT_OPEN_RPL);
548	}
549	return 0;
550	}
551
552	/*
553	* chcr_ktls_cpl_set_tcb_rpl: TCB reply received from TP.
554	*/
555	int chcr_ktls_cpl_set_tcb_rpl(struct adapter adap, unsigned char input)
556	{
557	const struct cpl_set_tcb_rpl p = (void )input;
558	struct chcr_ktls_info *tx_info = NULL;
559	struct tid_info *t;
560	u32 tid, status;
561
562	tid = GET_TID(p);
563	status = p->status;
564
565	t = &adap->tids;
566	tx_info = lookup_tid(t, tid);
567	if (!tx_info \|\| tx_info->tid != tid) {
568	pr_err("tx_info or atid is not correct\n");
569	return -1;
570	}
571	/* update the connection state */
572	chcr_ktls_update_connection_state(tx_info, KTLS_CONN_SET_TCB_RPL);
573	return 0;
574	}
34aba2c4	575	#endif /* CONFIG_CHELSIO_TLS_DEVICE */