[linux-2.6-block.git] / net / smc / smc_cdc.h

/* SPDX-License-Identifier: GPL-2.0 */
/*
 * Shared Memory Communications over RDMA (SMC-R) and RoCE
 *
 * Connection Data Control (CDC)
 *
 * Copyright IBM Corp. 2016
 *
 * Author(s):  Ursula Braun <ubraun@linux.vnet.ibm.com>
 */

#ifndef SMC_CDC_H
#define SMC_CDC_H

#include <linux/kernel.h> /* max_t */
#include <linux/atomic.h>
#include <linux/in.h>
#include <linux/compiler.h>

#include "smc.h"
#include "smc_core.h"
#include "smc_wr.h"

#define	SMC_CDC_MSG_TYPE		0xFE

/* in network byte order */
union smc_cdc_cursor {		/* SMC cursor */
	struct {
		__be16	reserved;
		__be16	wrap;
		__be32	count;
	};
#ifdef KERNEL_HAS_ATOMIC64
	atomic64_t	acurs;		/* for atomic processing */
#else
	u64		acurs;		/* for atomic processing */
#endif
} __aligned(8);

/* in network byte order */
struct smc_cdc_msg {
	struct smc_wr_rx_hdr		common; /* .type = 0xFE */
	u8				len;	/* 44 */
	__be16				seqno;
	__be32				token;
	union smc_cdc_cursor		prod;
	union smc_cdc_cursor		cons;	/* piggy backed "ack" */
	struct smc_cdc_producer_flags	prod_flags;
	struct smc_cdc_conn_state_flags	conn_state_flags;
	u8				reserved[18];
};

/* SMC-D cursor format */
union smcd_cdc_cursor {
	struct {
		u16	wrap;
		u32	count;
		struct smc_cdc_producer_flags	prod_flags;
		struct smc_cdc_conn_state_flags	conn_state_flags;
	} __packed;
#ifdef KERNEL_HAS_ATOMIC64
	atomic64_t		acurs;		/* for atomic processing */
#else
	u64			acurs;		/* for atomic processing */
#endif
} __aligned(8);

/* CDC message for SMC-D */
struct smcd_cdc_msg {
	struct smc_wr_rx_hdr common;	/* Type = 0xFE */
	u8 res1[7];
	union smcd_cdc_cursor	prod;
	union smcd_cdc_cursor	cons;
	u8 res3[8];
} __aligned(8);

static inline bool smc_cdc_rxed_any_close(struct smc_connection *conn)
{
	return conn->local_rx_ctrl.conn_state_flags.peer_conn_abort ||
	       conn->local_rx_ctrl.conn_state_flags.peer_conn_closed;
}

static inline bool smc_cdc_rxed_any_close_or_senddone(
	struct smc_connection *conn)
{
	return smc_cdc_rxed_any_close(conn) ||
	       conn->local_rx_ctrl.conn_state_flags.peer_done_writing;
}

static inline void smc_curs_add(int size, union smc_host_cursor *curs,
				int value)
{
	curs->count += value;
	if (curs->count >= size) {
		curs->wrap++;
		curs->count -= size;
	}
}

/* SMC cursors are 8 bytes long and require atomic reading and writing */
static inline u64 smc_curs_read(union smc_host_cursor *curs,
				struct smc_connection *conn)
{
#ifndef KERNEL_HAS_ATOMIC64
	unsigned long flags;
	u64 ret;

	spin_lock_irqsave(&conn->acurs_lock, flags);
	ret = curs->acurs;
	spin_unlock_irqrestore(&conn->acurs_lock, flags);
	return ret;
#else
	return atomic64_read(&curs->acurs);
#endif
}

/* Copy cursor src into tgt */
static inline void smc_curs_copy(union smc_host_cursor *tgt,
				 union smc_host_cursor *src,
				 struct smc_connection *conn)
{
#ifndef KERNEL_HAS_ATOMIC64
	unsigned long flags;

	spin_lock_irqsave(&conn->acurs_lock, flags);
	tgt->acurs = src->acurs;
	spin_unlock_irqrestore(&conn->acurs_lock, flags);
#else
	atomic64_set(&tgt->acurs, atomic64_read(&src->acurs));
#endif
}

static inline void smc_curs_copy_net(union smc_cdc_cursor *tgt,
				     union smc_cdc_cursor *src,
				     struct smc_connection *conn)
{
#ifndef KERNEL_HAS_ATOMIC64
	unsigned long flags;

	spin_lock_irqsave(&conn->acurs_lock, flags);
	tgt->acurs = src->acurs;
	spin_unlock_irqrestore(&conn->acurs_lock, flags);
#else
	atomic64_set(&tgt->acurs, atomic64_read(&src->acurs));
#endif
}

static inline void smcd_curs_copy(union smcd_cdc_cursor *tgt,
				  union smcd_cdc_cursor *src,
				  struct smc_connection *conn)
{
#ifndef KERNEL_HAS_ATOMIC64
	unsigned long flags;

	spin_lock_irqsave(&conn->acurs_lock, flags);
	tgt->acurs = src->acurs;
	spin_unlock_irqrestore(&conn->acurs_lock, flags);
#else
	atomic64_set(&tgt->acurs, atomic64_read(&src->acurs));
#endif
}

/* calculate cursor difference between old and new, where old <= new and
 * difference cannot exceed size
 */
static inline int smc_curs_diff(unsigned int size,
				union smc_host_cursor *old,
				union smc_host_cursor *new)
{
	if (old->wrap != new->wrap)
		return max_t(int, 0,
			     ((size - old->count) + new->count));

	return max_t(int, 0, (new->count - old->count));
}

/* calculate cursor difference between old and new - returns negative
 * value in case old > new
 */
static inline int smc_curs_comp(unsigned int size,
				union smc_host_cursor *old,
				union smc_host_cursor *new)
{
	if (old->wrap > new->wrap ||
	    (old->wrap == new->wrap && old->count > new->count))
		return -smc_curs_diff(size, new, old);
	return smc_curs_diff(size, old, new);
}

/* calculate cursor difference between old and new, where old <= new and
 * difference may exceed size
 */
static inline int smc_curs_diff_large(unsigned int size,
				      union smc_host_cursor *old,
				      union smc_host_cursor *new)
{
	if (old->wrap < new->wrap)
		return min_t(int,
			     (size - old->count) + new->count +
			     (new->wrap - old->wrap - 1) * size,
			     size);

	if (old->wrap > new->wrap) /* wrap has switched from 0xffff to 0x0000 */
		return min_t(int,
			     (size - old->count) + new->count +
			     (new->wrap + 0xffff - old->wrap) * size,
			     size);

	return max_t(int, 0, (new->count - old->count));
}

static inline void smc_host_cursor_to_cdc(union smc_cdc_cursor *peer,
					  union smc_host_cursor *local,
					  union smc_host_cursor *save,
					  struct smc_connection *conn)
{
	smc_curs_copy(save, local, conn);
	peer->count = htonl(save->count);
	peer->wrap = htons(save->wrap);
	/* peer->reserved = htons(0); must be ensured by caller */
}

static inline void smc_host_msg_to_cdc(struct smc_cdc_msg *peer,
				       struct smc_connection *conn,
				       union smc_host_cursor *save)
{
	struct smc_host_cdc_msg *local = &conn->local_tx_ctrl;

	peer->common.type = local->common.type;
	peer->len = local->len;
	peer->seqno = htons(local->seqno);
	peer->token = htonl(local->token);
	smc_host_cursor_to_cdc(&peer->prod, &local->prod, save, conn);
	smc_host_cursor_to_cdc(&peer->cons, &local->cons, save, conn);
	peer->prod_flags = local->prod_flags;
	peer->conn_state_flags = local->conn_state_flags;
}

static inline void smc_cdc_cursor_to_host(union smc_host_cursor *local,
					  union smc_cdc_cursor *peer,
					  struct smc_connection *conn)
{
	union smc_host_cursor temp, old;
	union smc_cdc_cursor net;

	smc_curs_copy(&old, local, conn);
	smc_curs_copy_net(&net, peer, conn);
	temp.count = ntohl(net.count);
	temp.wrap = ntohs(net.wrap);
	if ((old.wrap > temp.wrap) && temp.wrap)
		return;
	if ((old.wrap == temp.wrap) &&
	    (old.count > temp.count))
		return;
	smc_curs_copy(local, &temp, conn);
}

static inline void smcr_cdc_msg_to_host(struct smc_host_cdc_msg *local,
					struct smc_cdc_msg *peer,
					struct smc_connection *conn)
{
	local->common.type = peer->common.type;
	local->len = peer->len;
	local->seqno = ntohs(peer->seqno);
	local->token = ntohl(peer->token);
	smc_cdc_cursor_to_host(&local->prod, &peer->prod, conn);
	smc_cdc_cursor_to_host(&local->cons, &peer->cons, conn);
	local->prod_flags = peer->prod_flags;
	local->conn_state_flags = peer->conn_state_flags;
}

static inline void smcd_cdc_msg_to_host(struct smc_host_cdc_msg *local,
					struct smcd_cdc_msg *peer,
					struct smc_connection *conn)
{
	union smc_host_cursor temp;

	temp.wrap = peer->prod.wrap;
	temp.count = peer->prod.count;
	smc_curs_copy(&local->prod, &temp, conn);

	temp.wrap = peer->cons.wrap;
	temp.count = peer->cons.count;
	smc_curs_copy(&local->cons, &temp, conn);
	local->prod_flags = peer->cons.prod_flags;
	local->conn_state_flags = peer->cons.conn_state_flags;
}

static inline void smc_cdc_msg_to_host(struct smc_host_cdc_msg *local,
				       struct smc_cdc_msg *peer,
				       struct smc_connection *conn)
{
	if (conn->lgr->is_smcd)
		smcd_cdc_msg_to_host(local, (struct smcd_cdc_msg *)peer, conn);
	else
		smcr_cdc_msg_to_host(local, peer, conn);
}

struct smc_cdc_tx_pend {
	struct smc_connection	*conn;		/* socket connection */
	union smc_host_cursor	cursor;		/* tx sndbuf cursor sent */
	union smc_host_cursor	p_cursor;	/* rx RMBE cursor produced */
	u16			ctrl_seq;	/* conn. tx sequence # */
};

int smc_cdc_get_free_slot(struct smc_connection *conn,
			  struct smc_wr_buf **wr_buf,
			  struct smc_rdma_wr **wr_rdma_buf,
			  struct smc_cdc_tx_pend **pend);
void smc_cdc_tx_dismiss_slots(struct smc_connection *conn);
int smc_cdc_msg_send(struct smc_connection *conn, struct smc_wr_buf *wr_buf,
		     struct smc_cdc_tx_pend *pend);
int smc_cdc_get_slot_and_msg_send(struct smc_connection *conn);
int smcd_cdc_msg_send(struct smc_connection *conn);
int smc_cdc_init(void) __init;
void smcd_cdc_rx_init(struct smc_connection *conn);

#endif /* SMC_CDC_H */
Commit	Line	Data
b2441318	1	/* SPDX-License-Identifier: GPL-2.0 */
5f08318f UB	2	/*
	3	* Shared Memory Communications over RDMA (SMC-R) and RoCE
	4	*
	5	* Connection Data Control (CDC)
	6	*
	7	* Copyright IBM Corp. 2016
	8	*
	9	* Author(s): Ursula Braun <ubraun@linux.vnet.ibm.com>
	10	*/
	11
	12	#ifndef SMC_CDC_H
	13	#define SMC_CDC_H
	14
	15	#include <linux/kernel.h> /* max_t */
	16	#include <linux/atomic.h>
	17	#include <linux/in.h>
	18	#include <linux/compiler.h>
	19
	20	#include "smc.h"
	21	#include "smc_core.h"
	22	#include "smc_wr.h"
	23
	24	#define SMC_CDC_MSG_TYPE 0xFE
	25
	26	/* in network byte order */
	27	union smc_cdc_cursor { /* SMC cursor */
	28	struct {
	29	__be16 reserved;
	30	__be16 wrap;
	31	__be32 count;
	32	};
	33	#ifdef KERNEL_HAS_ATOMIC64
	34	atomic64_t acurs; /* for atomic processing */
	35	#else
	36	u64 acurs; /* for atomic processing */
	37	#endif
	38	} __aligned(8);
	39
	40	/* in network byte order */
	41	struct smc_cdc_msg {
	42	struct smc_wr_rx_hdr common; /* .type = 0xFE */
	43	u8 len; /* 44 */
	44	__be16 seqno;
	45	__be32 token;
	46	union smc_cdc_cursor prod;
	47	union smc_cdc_cursor cons; /* piggy backed "ack" */
	48	struct smc_cdc_producer_flags prod_flags;
	49	struct smc_cdc_conn_state_flags conn_state_flags;
	50	u8 reserved[18];
b9a22dd9 UB	51	};
	52
	53	/* SMC-D cursor format */
	54	union smcd_cdc_cursor {
	55	struct {
	56	u16 wrap;
	57	u32 count;
	58	struct smc_cdc_producer_flags prod_flags;
	59	struct smc_cdc_conn_state_flags conn_state_flags;
	60	} __packed;
	61	#ifdef KERNEL_HAS_ATOMIC64
	62	atomic64_t acurs; /* for atomic processing */
	63	#else
	64	u64 acurs; /* for atomic processing */
	65	#endif
	66	} __aligned(8);
5f08318f	67
be244f28 HW	68	/* CDC message for SMC-D */
	69	struct smcd_cdc_msg {
	70	struct smc_wr_rx_hdr common; /* Type = 0xFE */
	71	u8 res1[7];
b9a22dd9 UB	72	union smcd_cdc_cursor prod;
b9a22dd9 UB	73	union smcd_cdc_cursor cons;
be244f28	74	u8 res3[8];
b9a22dd9	75	} __aligned(8);
be244f28	76
5f08318f UB	77	static inline bool smc_cdc_rxed_any_close(struct smc_connection *conn)
	78	{
	79	return conn->local_rx_ctrl.conn_state_flags.peer_conn_abort \|\|
	80	conn->local_rx_ctrl.conn_state_flags.peer_conn_closed;
	81	}
	82
	83	static inline bool smc_cdc_rxed_any_close_or_senddone(
	84	struct smc_connection *conn)
	85	{
	86	return smc_cdc_rxed_any_close(conn) \|\|
	87	conn->local_rx_ctrl.conn_state_flags.peer_done_writing;
	88	}
	89
	90	static inline void smc_curs_add(int size, union smc_host_cursor *curs,
	91	int value)
	92	{
	93	curs->count += value;
	94	if (curs->count >= size) {
	95	curs->wrap++;
	96	curs->count -= size;
	97	}
	98	}
	99
	100	/* SMC cursors are 8 bytes long and require atomic reading and writing */
	101	static inline u64 smc_curs_read(union smc_host_cursor *curs,
	102	struct smc_connection *conn)
	103	{
	104	#ifndef KERNEL_HAS_ATOMIC64
	105	unsigned long flags;
	106	u64 ret;
	107
	108	spin_lock_irqsave(&conn->acurs_lock, flags);
	109	ret = curs->acurs;
	110	spin_unlock_irqrestore(&conn->acurs_lock, flags);
	111	return ret;
	112	#else
	113	return atomic64_read(&curs->acurs);
	114	#endif
	115	}
	116
bac6de7b SR	117	/* Copy cursor src into tgt */
	118	static inline void smc_curs_copy(union smc_host_cursor *tgt,
	119	union smc_host_cursor *src,
	120	struct smc_connection *conn)
5f08318f UB	121	{
	122	#ifndef KERNEL_HAS_ATOMIC64
	123	unsigned long flags;
	124
	125	spin_lock_irqsave(&conn->acurs_lock, flags);
bac6de7b	126	tgt->acurs = src->acurs;
5f08318f UB	127	spin_unlock_irqrestore(&conn->acurs_lock, flags);
5f08318f UB	128	#else
bac6de7b	129	atomic64_set(&tgt->acurs, atomic64_read(&src->acurs));
5f08318f UB	130	#endif
	131	}
	132
bac6de7b SR	133	static inline void smc_curs_copy_net(union smc_cdc_cursor *tgt,
	134	union smc_cdc_cursor *src,
	135	struct smc_connection *conn)
5f08318f UB	136	{
	137	#ifndef KERNEL_HAS_ATOMIC64
	138	unsigned long flags;
	139
	140	spin_lock_irqsave(&conn->acurs_lock, flags);
bac6de7b	141	tgt->acurs = src->acurs;
5f08318f UB	142	spin_unlock_irqrestore(&conn->acurs_lock, flags);
5f08318f UB	143	#else
bac6de7b	144	atomic64_set(&tgt->acurs, atomic64_read(&src->acurs));
5f08318f UB	145	#endif
	146	}
	147
b9a22dd9 UB	148	static inline void smcd_curs_copy(union smcd_cdc_cursor *tgt,
	149	union smcd_cdc_cursor *src,
	150	struct smc_connection *conn)
	151	{
	152	#ifndef KERNEL_HAS_ATOMIC64
	153	unsigned long flags;
	154
	155	spin_lock_irqsave(&conn->acurs_lock, flags);
	156	tgt->acurs = src->acurs;
	157	spin_unlock_irqrestore(&conn->acurs_lock, flags);
	158	#else
	159	atomic64_set(&tgt->acurs, atomic64_read(&src->acurs));
	160	#endif
	161	}
	162
b8649efa UB	163	/* calculate cursor difference between old and new, where old <= new and
	164	* difference cannot exceed size
	165	*/
5f08318f UB	166	static inline int smc_curs_diff(unsigned int size,
	167	union smc_host_cursor *old,
	168	union smc_host_cursor *new)
	169	{
	170	if (old->wrap != new->wrap)
	171	return max_t(int, 0,
	172	((size - old->count) + new->count));
	173
	174	return max_t(int, 0, (new->count - old->count));
	175	}
	176
de8474eb SR	177	/* calculate cursor difference between old and new - returns negative
	178	* value in case old > new
	179	*/
	180	static inline int smc_curs_comp(unsigned int size,
	181	union smc_host_cursor *old,
	182	union smc_host_cursor *new)
	183	{
	184	if (old->wrap > new->wrap \|\|
	185	(old->wrap == new->wrap && old->count > new->count))
	186	return -smc_curs_diff(size, new, old);
	187	return smc_curs_diff(size, old, new);
	188	}
	189
b8649efa UB	190	/* calculate cursor difference between old and new, where old <= new and
	191	* difference may exceed size
	192	*/
	193	static inline int smc_curs_diff_large(unsigned int size,
	194	union smc_host_cursor *old,
	195	union smc_host_cursor *new)
	196	{
	197	if (old->wrap < new->wrap)
	198	return min_t(int,
	199	(size - old->count) + new->count +
	200	(new->wrap - old->wrap - 1) * size,
	201	size);
	202
	203	if (old->wrap > new->wrap) /* wrap has switched from 0xffff to 0x0000 */
	204	return min_t(int,
	205	(size - old->count) + new->count +
	206	(new->wrap + 0xffff - old->wrap) * size,
	207	size);
	208
	209	return max_t(int, 0, (new->count - old->count));
	210	}
	211
5f08318f UB	212	static inline void smc_host_cursor_to_cdc(union smc_cdc_cursor *peer,
5f08318f UB	213	union smc_host_cursor *local,
ccc8ca9b	214	union smc_host_cursor *save,
5f08318f UB	215	struct smc_connection *conn)
5f08318f UB	216	{
ccc8ca9b UB	217	smc_curs_copy(save, local, conn);
	218	peer->count = htonl(save->count);
	219	peer->wrap = htons(save->wrap);
5f08318f UB	220	/* peer->reserved = htons(0); must be ensured by caller */
	221	}
	222
	223	static inline void smc_host_msg_to_cdc(struct smc_cdc_msg *peer,
ccc8ca9b UB	224	struct smc_connection *conn,
ccc8ca9b UB	225	union smc_host_cursor *save)
5f08318f	226	{
ccc8ca9b UB	227	struct smc_host_cdc_msg *local = &conn->local_tx_ctrl;
ccc8ca9b UB	228
5f08318f UB	229	peer->common.type = local->common.type;
	230	peer->len = local->len;
	231	peer->seqno = htons(local->seqno);
	232	peer->token = htonl(local->token);
ccc8ca9b UB	233	smc_host_cursor_to_cdc(&peer->prod, &local->prod, save, conn);
ccc8ca9b UB	234	smc_host_cursor_to_cdc(&peer->cons, &local->cons, save, conn);
5f08318f UB	235	peer->prod_flags = local->prod_flags;
	236	peer->conn_state_flags = local->conn_state_flags;
	237	}
	238
	239	static inline void smc_cdc_cursor_to_host(union smc_host_cursor *local,
	240	union smc_cdc_cursor *peer,
	241	struct smc_connection *conn)
	242	{
	243	union smc_host_cursor temp, old;
	244	union smc_cdc_cursor net;
	245
bac6de7b SR	246	smc_curs_copy(&old, local, conn);
bac6de7b SR	247	smc_curs_copy_net(&net, peer, conn);
5f08318f UB	248	temp.count = ntohl(net.count);
	249	temp.wrap = ntohs(net.wrap);
	250	if ((old.wrap > temp.wrap) && temp.wrap)
	251	return;
	252	if ((old.wrap == temp.wrap) &&
	253	(old.count > temp.count))
	254	return;
bac6de7b	255	smc_curs_copy(local, &temp, conn);
5f08318f UB	256	}
5f08318f UB	257
be244f28 HW	258	static inline void smcr_cdc_msg_to_host(struct smc_host_cdc_msg *local,
	259	struct smc_cdc_msg *peer,
	260	struct smc_connection *conn)
5f08318f UB	261	{
	262	local->common.type = peer->common.type;
	263	local->len = peer->len;
	264	local->seqno = ntohs(peer->seqno);
	265	local->token = ntohl(peer->token);
	266	smc_cdc_cursor_to_host(&local->prod, &peer->prod, conn);
	267	smc_cdc_cursor_to_host(&local->cons, &peer->cons, conn);
	268	local->prod_flags = peer->prod_flags;
	269	local->conn_state_flags = peer->conn_state_flags;
	270	}
	271
be244f28	272	static inline void smcd_cdc_msg_to_host(struct smc_host_cdc_msg *local,
a225d2cd UB	273	struct smcd_cdc_msg *peer,
a225d2cd UB	274	struct smc_connection *conn)
be244f28	275	{
b9a22dd9 UB	276	union smc_host_cursor temp;
	277
	278	temp.wrap = peer->prod.wrap;
	279	temp.count = peer->prod.count;
a225d2cd	280	smc_curs_copy(&local->prod, &temp, conn);
b9a22dd9 UB	281
	282	temp.wrap = peer->cons.wrap;
	283	temp.count = peer->cons.count;
a225d2cd	284	smc_curs_copy(&local->cons, &temp, conn);
b9a22dd9 UB	285	local->prod_flags = peer->cons.prod_flags;
b9a22dd9 UB	286	local->conn_state_flags = peer->cons.conn_state_flags;
be244f28 HW	287	}
	288
	289	static inline void smc_cdc_msg_to_host(struct smc_host_cdc_msg *local,
	290	struct smc_cdc_msg *peer,
	291	struct smc_connection *conn)
	292	{
	293	if (conn->lgr->is_smcd)
a225d2cd	294	smcd_cdc_msg_to_host(local, (struct smcd_cdc_msg *)peer, conn);
be244f28 HW	295	else
	296	smcr_cdc_msg_to_host(local, peer, conn);
	297	}
	298
ad6f317f UB	299	struct smc_cdc_tx_pend {
	300	struct smc_connection conn; / socket connection */
	301	union smc_host_cursor cursor; /* tx sndbuf cursor sent */
	302	union smc_host_cursor p_cursor; /* rx RMBE cursor produced */
	303	u16 ctrl_seq; /* conn. tx sequence # */
	304	};
5f08318f	305
51957bc5 UB	306	int smc_cdc_get_free_slot(struct smc_connection *conn,
51957bc5 UB	307	struct smc_wr_buf **wr_buf,
ad6f317f	308	struct smc_rdma_wr **wr_rdma_buf,
5f08318f UB	309	struct smc_cdc_tx_pend **pend);
	310	void smc_cdc_tx_dismiss_slots(struct smc_connection *conn);
	311	int smc_cdc_msg_send(struct smc_connection conn, struct smc_wr_buf wr_buf,
	312	struct smc_cdc_tx_pend *pend);
	313	int smc_cdc_get_slot_and_msg_send(struct smc_connection *conn);
be244f28	314	int smcd_cdc_msg_send(struct smc_connection *conn);
5f08318f	315	int smc_cdc_init(void) __init;
be244f28	316	void smcd_cdc_rx_init(struct smc_connection *conn);
5f08318f UB	317
5f08318f UB	318	#endif /* SMC_CDC_H */