[linux-block.git] / fs / cifs / smbdirect.h

/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
 *   Copyright (C) 2017, Microsoft Corporation.
 *
 *   Author(s): Long Li <longli@microsoft.com>
 */
#ifndef _SMBDIRECT_H
#define _SMBDIRECT_H

#ifdef CONFIG_CIFS_SMB_DIRECT
#define cifs_rdma_enabled(server)	((server)->rdma)

#include "cifsglob.h"
#include <rdma/ib_verbs.h>
#include <rdma/rdma_cm.h>
#include <linux/mempool.h>

extern int rdma_readwrite_threshold;
extern int smbd_max_frmr_depth;
extern int smbd_keep_alive_interval;
extern int smbd_max_receive_size;
extern int smbd_max_fragmented_recv_size;
extern int smbd_max_send_size;
extern int smbd_send_credit_target;
extern int smbd_receive_credit_max;

enum keep_alive_status {
	KEEP_ALIVE_NONE,
	KEEP_ALIVE_PENDING,
	KEEP_ALIVE_SENT,
};

enum smbd_connection_status {
	SMBD_CREATED,
	SMBD_CONNECTING,
	SMBD_CONNECTED,
	SMBD_NEGOTIATE_FAILED,
	SMBD_DISCONNECTING,
	SMBD_DISCONNECTED,
	SMBD_DESTROYED
};

/*
 * The context for the SMBDirect transport
 * Everything related to the transport is here. It has several logical parts
 * 1. RDMA related structures
 * 2. SMBDirect connection parameters
 * 3. Memory registrations
 * 4. Receive and reassembly queues for data receive path
 * 5. mempools for allocating packets
 */
struct smbd_connection {
	enum smbd_connection_status transport_status;

	/* RDMA related */
	struct rdma_cm_id *id;
	struct ib_qp_init_attr qp_attr;
	struct ib_pd *pd;
	struct ib_cq *send_cq, *recv_cq;
	struct ib_device_attr dev_attr;
	int ri_rc;
	struct completion ri_done;
	wait_queue_head_t conn_wait;
	wait_queue_head_t disconn_wait;

	struct completion negotiate_completion;
	bool negotiate_done;

	struct work_struct disconnect_work;
	struct work_struct post_send_credits_work;

	spinlock_t lock_new_credits_offered;
	int new_credits_offered;

	/* Connection parameters defined in [MS-SMBD] 3.1.1.1 */
	int receive_credit_max;
	int send_credit_target;
	int max_send_size;
	int max_fragmented_recv_size;
	int max_fragmented_send_size;
	int max_receive_size;
	int keep_alive_interval;
	int max_readwrite_size;
	enum keep_alive_status keep_alive_requested;
	int protocol;
	atomic_t send_credits;
	atomic_t receive_credits;
	int receive_credit_target;
	int fragment_reassembly_remaining;

	/* Memory registrations */
	/* Maximum number of RDMA read/write outstanding on this connection */
	int responder_resources;
	/* Maximum number of SGEs in a RDMA write/read */
	int max_frmr_depth;
	/*
	 * If payload is less than or equal to the threshold,
	 * use RDMA send/recv to send upper layer I/O.
	 * If payload is more than the threshold,
	 * use RDMA read/write through memory registration for I/O.
	 */
	int rdma_readwrite_threshold;
	enum ib_mr_type mr_type;
	struct list_head mr_list;
	spinlock_t mr_list_lock;
	/* The number of available MRs ready for memory registration */
	atomic_t mr_ready_count;
	atomic_t mr_used_count;
	wait_queue_head_t wait_mr;
	struct work_struct mr_recovery_work;
	/* Used by transport to wait until all MRs are returned */
	wait_queue_head_t wait_for_mr_cleanup;

	/* Activity accoutning */
	atomic_t send_pending;
	wait_queue_head_t wait_send_pending;
	wait_queue_head_t wait_post_send;

	/* Receive queue */
	struct list_head receive_queue;
	int count_receive_queue;
	spinlock_t receive_queue_lock;

	struct list_head empty_packet_queue;
	int count_empty_packet_queue;
	spinlock_t empty_packet_queue_lock;

	wait_queue_head_t wait_receive_queues;

	/* Reassembly queue */
	struct list_head reassembly_queue;
	spinlock_t reassembly_queue_lock;
	wait_queue_head_t wait_reassembly_queue;

	/* total data length of reassembly queue */
	int reassembly_data_length;
	int reassembly_queue_length;
	/* the offset to first buffer in reassembly queue */
	int first_entry_offset;

	bool send_immediate;

	wait_queue_head_t wait_send_queue;

	/*
	 * Indicate if we have received a full packet on the connection
	 * This is used to identify the first SMBD packet of a assembled
	 * payload (SMB packet) in reassembly queue so we can return a
	 * RFC1002 length to upper layer to indicate the length of the SMB
	 * packet received
	 */
	bool full_packet_received;

	struct workqueue_struct *workqueue;
	struct delayed_work idle_timer_work;

	/* Memory pool for preallocating buffers */
	/* request pool for RDMA send */
	struct kmem_cache *request_cache;
	mempool_t *request_mempool;

	/* response pool for RDMA receive */
	struct kmem_cache *response_cache;
	mempool_t *response_mempool;

	/* for debug purposes */
	unsigned int count_get_receive_buffer;
	unsigned int count_put_receive_buffer;
	unsigned int count_reassembly_queue;
	unsigned int count_enqueue_reassembly_queue;
	unsigned int count_dequeue_reassembly_queue;
	unsigned int count_send_empty;
};

enum smbd_message_type {
	SMBD_NEGOTIATE_RESP,
	SMBD_TRANSFER_DATA,
};

#define SMB_DIRECT_RESPONSE_REQUESTED 0x0001

/* SMBD negotiation request packet [MS-SMBD] 2.2.1 */
struct smbd_negotiate_req {
	__le16 min_version;
	__le16 max_version;
	__le16 reserved;
	__le16 credits_requested;
	__le32 preferred_send_size;
	__le32 max_receive_size;
	__le32 max_fragmented_size;
} __packed;

/* SMBD negotiation response packet [MS-SMBD] 2.2.2 */
struct smbd_negotiate_resp {
	__le16 min_version;
	__le16 max_version;
	__le16 negotiated_version;
	__le16 reserved;
	__le16 credits_requested;
	__le16 credits_granted;
	__le32 status;
	__le32 max_readwrite_size;
	__le32 preferred_send_size;
	__le32 max_receive_size;
	__le32 max_fragmented_size;
} __packed;

/* SMBD data transfer packet with payload [MS-SMBD] 2.2.3 */
struct smbd_data_transfer {
	__le16 credits_requested;
	__le16 credits_granted;
	__le16 flags;
	__le16 reserved;
	__le32 remaining_data_length;
	__le32 data_offset;
	__le32 data_length;
	__le32 padding;
	__u8 buffer[];
} __packed;

/* The packet fields for a registered RDMA buffer */
struct smbd_buffer_descriptor_v1 {
	__le64 offset;
	__le32 token;
	__le32 length;
} __packed;

/* Default maximum number of SGEs in a RDMA send/recv */
#define SMBDIRECT_MAX_SGE	16
/* The context for a SMBD request */
struct smbd_request {
	struct smbd_connection *info;
	struct ib_cqe cqe;

	/* the SGE entries for this packet */
	struct ib_sge sge[SMBDIRECT_MAX_SGE];
	int num_sge;

	/* SMBD packet header follows this structure */
	u8 packet[];
};

/* The context for a SMBD response */
struct smbd_response {
	struct smbd_connection *info;
	struct ib_cqe cqe;
	struct ib_sge sge;

	enum smbd_message_type type;

	/* Link to receive queue or reassembly queue */
	struct list_head list;

	/* Indicate if this is the 1st packet of a payload */
	bool first_segment;

	/* SMBD packet header and payload follows this structure */
	u8 packet[];
};

/* Create a SMBDirect session */
struct smbd_connection *smbd_get_connection(
	struct TCP_Server_Info *server, struct sockaddr *dstaddr);

/* Reconnect SMBDirect session */
int smbd_reconnect(struct TCP_Server_Info *server);
/* Destroy SMBDirect session */
void smbd_destroy(struct TCP_Server_Info *server);

/* Interface for carrying upper layer I/O through send/recv */
int smbd_recv(struct smbd_connection *info, struct msghdr *msg);
int smbd_send(struct TCP_Server_Info *server,
	int num_rqst, struct smb_rqst *rqst);

enum mr_state {
	MR_READY,
	MR_REGISTERED,
	MR_INVALIDATED,
	MR_ERROR
};

struct smbd_mr {
	struct smbd_connection	*conn;
	struct list_head	list;
	enum mr_state		state;
	struct ib_mr		*mr;
	struct scatterlist	*sgl;
	int			sgl_count;
	enum dma_data_direction	dir;
	union {
		struct ib_reg_wr	wr;
		struct ib_send_wr	inv_wr;
	};
	struct ib_cqe		cqe;
	bool			need_invalidate;
	struct completion	invalidate_done;
};

/* Interfaces to register and deregister MR for RDMA read/write */
struct smbd_mr *smbd_register_mr(
	struct smbd_connection *info, struct page *pages[], int num_pages,
	int offset, int tailsz, bool writing, bool need_invalidate);
int smbd_deregister_mr(struct smbd_mr *mr);

#else
#define cifs_rdma_enabled(server)	0
struct smbd_connection {};
static inline void *smbd_get_connection(
	struct TCP_Server_Info *server, struct sockaddr *dstaddr) {return NULL;}
static inline int smbd_reconnect(struct TCP_Server_Info *server) {return -1; }
static inline void smbd_destroy(struct TCP_Server_Info *server) {}
static inline int smbd_recv(struct smbd_connection *info, struct msghdr *msg) {return -1; }
static inline int smbd_send(struct TCP_Server_Info *server, int num_rqst, struct smb_rqst *rqst) {return -1; }
#endif

#endif
Commit	Line	Data
c942fddf	1	/* SPDX-License-Identifier: GPL-2.0-or-later */
03bee01d LL	2	/*
	3	* Copyright (C) 2017, Microsoft Corporation.
	4	*
	5	* Author(s): Long Li <longli@microsoft.com>
03bee01d LL	6	*/
	7	#ifndef _SMBDIRECT_H
	8	#define _SMBDIRECT_H
	9
f198186a LL	10	#ifdef CONFIG_CIFS_SMB_DIRECT
	11	#define cifs_rdma_enabled(server) ((server)->rdma)
	12
	13	#include "cifsglob.h"
	14	#include <rdma/ib_verbs.h>
	15	#include <rdma/rdma_cm.h>
	16	#include <linux/mempool.h>
	17
6b314714 SF	18	extern int rdma_readwrite_threshold;
	19	extern int smbd_max_frmr_depth;
	20	extern int smbd_keep_alive_interval;
	21	extern int smbd_max_receive_size;
	22	extern int smbd_max_fragmented_recv_size;
	23	extern int smbd_max_send_size;
	24	extern int smbd_send_credit_target;
	25	extern int smbd_receive_credit_max;
	26
f198186a LL	27	enum keep_alive_status {
	28	KEEP_ALIVE_NONE,
	29	KEEP_ALIVE_PENDING,
	30	KEEP_ALIVE_SENT,
	31	};
	32
	33	enum smbd_connection_status {
	34	SMBD_CREATED,
	35	SMBD_CONNECTING,
	36	SMBD_CONNECTED,
	37	SMBD_NEGOTIATE_FAILED,
	38	SMBD_DISCONNECTING,
	39	SMBD_DISCONNECTED,
	40	SMBD_DESTROYED
	41	};
	42
	43	/*
	44	* The context for the SMBDirect transport
	45	* Everything related to the transport is here. It has several logical parts
	46	* 1. RDMA related structures
	47	* 2. SMBDirect connection parameters
	48	* 3. Memory registrations
	49	* 4. Receive and reassembly queues for data receive path
	50	* 5. mempools for allocating packets
	51	*/
	52	struct smbd_connection {
	53	enum smbd_connection_status transport_status;
	54
	55	/* RDMA related */
	56	struct rdma_cm_id *id;
	57	struct ib_qp_init_attr qp_attr;
	58	struct ib_pd *pd;
	59	struct ib_cq send_cq, recv_cq;
	60	struct ib_device_attr dev_attr;
	61	int ri_rc;
	62	struct completion ri_done;
	63	wait_queue_head_t conn_wait;
050b8c37	64	wait_queue_head_t disconn_wait;
f198186a LL	65
	66	struct completion negotiate_completion;
	67	bool negotiate_done;
	68
f198186a	69	struct work_struct disconnect_work;
f198186a LL	70	struct work_struct post_send_credits_work;
	71
	72	spinlock_t lock_new_credits_offered;
	73	int new_credits_offered;
	74
	75	/* Connection parameters defined in [MS-SMBD] 3.1.1.1 */
	76	int receive_credit_max;
	77	int send_credit_target;
	78	int max_send_size;
	79	int max_fragmented_recv_size;
	80	int max_fragmented_send_size;
	81	int max_receive_size;
	82	int keep_alive_interval;
	83	int max_readwrite_size;
	84	enum keep_alive_status keep_alive_requested;
	85	int protocol;
	86	atomic_t send_credits;
	87	atomic_t receive_credits;
	88	int receive_credit_target;
	89	int fragment_reassembly_remaining;
	90
c7398583 LL	91	/* Memory registrations */
	92	/* Maximum number of RDMA read/write outstanding on this connection */
	93	int responder_resources;
	94	/* Maximum number of SGEs in a RDMA write/read */
	95	int max_frmr_depth;
	96	/*
	97	* If payload is less than or equal to the threshold,
	98	* use RDMA send/recv to send upper layer I/O.
	99	* If payload is more than the threshold,
	100	* use RDMA read/write through memory registration for I/O.
	101	*/
	102	int rdma_readwrite_threshold;
	103	enum ib_mr_type mr_type;
	104	struct list_head mr_list;
	105	spinlock_t mr_list_lock;
	106	/* The number of available MRs ready for memory registration */
	107	atomic_t mr_ready_count;
	108	atomic_t mr_used_count;
	109	wait_queue_head_t wait_mr;
	110	struct work_struct mr_recovery_work;
	111	/* Used by transport to wait until all MRs are returned */
	112	wait_queue_head_t wait_for_mr_cleanup;
	113
f198186a	114	/* Activity accoutning */
f198186a LL	115	atomic_t send_pending;
f198186a LL	116	wait_queue_head_t wait_send_pending;
3ffbe78a	117	wait_queue_head_t wait_post_send;
f198186a LL	118
	119	/* Receive queue */
	120	struct list_head receive_queue;
	121	int count_receive_queue;
	122	spinlock_t receive_queue_lock;
	123
	124	struct list_head empty_packet_queue;
	125	int count_empty_packet_queue;
	126	spinlock_t empty_packet_queue_lock;
	127
	128	wait_queue_head_t wait_receive_queues;
	129
	130	/* Reassembly queue */
	131	struct list_head reassembly_queue;
	132	spinlock_t reassembly_queue_lock;
	133	wait_queue_head_t wait_reassembly_queue;
	134
	135	/* total data length of reassembly queue */
	136	int reassembly_data_length;
	137	int reassembly_queue_length;
	138	/* the offset to first buffer in reassembly queue */
	139	int first_entry_offset;
	140
	141	bool send_immediate;
	142
	143	wait_queue_head_t wait_send_queue;
	144
	145	/*
	146	* Indicate if we have received a full packet on the connection
	147	* This is used to identify the first SMBD packet of a assembled
	148	* payload (SMB packet) in reassembly queue so we can return a
	149	* RFC1002 length to upper layer to indicate the length of the SMB
	150	* packet received
	151	*/
	152	bool full_packet_received;
	153
	154	struct workqueue_struct *workqueue;
	155	struct delayed_work idle_timer_work;
f198186a LL	156
	157	/* Memory pool for preallocating buffers */
	158	/* request pool for RDMA send */
	159	struct kmem_cache *request_cache;
	160	mempool_t *request_mempool;
	161
	162	/* response pool for RDMA receive */
	163	struct kmem_cache *response_cache;
	164	mempool_t *response_mempool;
	165
	166	/* for debug purposes */
	167	unsigned int count_get_receive_buffer;
	168	unsigned int count_put_receive_buffer;
	169	unsigned int count_reassembly_queue;
	170	unsigned int count_enqueue_reassembly_queue;
	171	unsigned int count_dequeue_reassembly_queue;
	172	unsigned int count_send_empty;
	173	};
	174
	175	enum smbd_message_type {
	176	SMBD_NEGOTIATE_RESP,
	177	SMBD_TRANSFER_DATA,
	178	};
	179
	180	#define SMB_DIRECT_RESPONSE_REQUESTED 0x0001
	181
	182	/* SMBD negotiation request packet [MS-SMBD] 2.2.1 */
	183	struct smbd_negotiate_req {
	184	__le16 min_version;
	185	__le16 max_version;
	186	__le16 reserved;
	187	__le16 credits_requested;
	188	__le32 preferred_send_size;
	189	__le32 max_receive_size;
	190	__le32 max_fragmented_size;
	191	} __packed;
	192
	193	/* SMBD negotiation response packet [MS-SMBD] 2.2.2 */
	194	struct smbd_negotiate_resp {
	195	__le16 min_version;
	196	__le16 max_version;
	197	__le16 negotiated_version;
	198	__le16 reserved;
	199	__le16 credits_requested;
	200	__le16 credits_granted;
	201	__le32 status;
	202	__le32 max_readwrite_size;
	203	__le32 preferred_send_size;
	204	__le32 max_receive_size;
	205	__le32 max_fragmented_size;
	206	} __packed;
	207
	208	/* SMBD data transfer packet with payload [MS-SMBD] 2.2.3 */
	209	struct smbd_data_transfer {
	210	__le16 credits_requested;
	211	__le16 credits_granted;
	212	__le16 flags;
	213	__le16 reserved;
	214	__le32 remaining_data_length;
	215	__le32 data_offset;
	216	__le32 data_length;
	217	__le32 padding;
	218	__u8 buffer[];
	219	} __packed;
220
221	/* The packet fields for a registered RDMA buffer */
222	struct smbd_buffer_descriptor_v1 {
223	__le64 offset;
224	__le32 token;
225	__le32 length;
226	} __packed;
227
03bee01d LL	228	/* Default maximum number of SGEs in a RDMA send/recv */
03bee01d LL	229	#define SMBDIRECT_MAX_SGE 16
f198186a LL	230	/* The context for a SMBD request */
	231	struct smbd_request {
	232	struct smbd_connection *info;
	233	struct ib_cqe cqe;
	234
f198186a LL	235	/* the SGE entries for this packet */
	236	struct ib_sge sge[SMBDIRECT_MAX_SGE];
	237	int num_sge;
	238
	239	/* SMBD packet header follows this structure */
	240	u8 packet[];
	241	};
	242
	243	/* The context for a SMBD response */
	244	struct smbd_response {
	245	struct smbd_connection *info;
	246	struct ib_cqe cqe;
	247	struct ib_sge sge;
	248
	249	enum smbd_message_type type;
	250
	251	/* Link to receive queue or reassembly queue */
	252	struct list_head list;
	253
	254	/* Indicate if this is the 1st packet of a payload */
	255	bool first_segment;
	256
	257	/* SMBD packet header and payload follows this structure */
	258	u8 packet[];
	259	};
	260
399f9539 LL	261	/* Create a SMBDirect session */
	262	struct smbd_connection *smbd_get_connection(
	263	struct TCP_Server_Info server, struct sockaddr dstaddr);
	264
ad57b8e1 LL	265	/* Reconnect SMBDirect session */
ad57b8e1 LL	266	int smbd_reconnect(struct TCP_Server_Info *server);
8ef130f9	267	/* Destroy SMBDirect session */
050b8c37	268	void smbd_destroy(struct TCP_Server_Info *server);
ad57b8e1	269
f64b78fd LL	270	/* Interface for carrying upper layer I/O through send/recv */
f64b78fd LL	271	int smbd_recv(struct smbd_connection info, struct msghdr msg);
4739f232 LL	272	int smbd_send(struct TCP_Server_Info *server,
4739f232 LL	273	int num_rqst, struct smb_rqst *rqst);
f64b78fd	274
c7398583 LL	275	enum mr_state {
	276	MR_READY,
	277	MR_REGISTERED,
	278	MR_INVALIDATED,
	279	MR_ERROR
	280	};
	281
	282	struct smbd_mr {
	283	struct smbd_connection *conn;
	284	struct list_head list;
	285	enum mr_state state;
	286	struct ib_mr *mr;
	287	struct scatterlist *sgl;
	288	int sgl_count;
	289	enum dma_data_direction dir;
	290	union {
	291	struct ib_reg_wr wr;
	292	struct ib_send_wr inv_wr;
	293	};
	294	struct ib_cqe cqe;
	295	bool need_invalidate;
	296	struct completion invalidate_done;
	297	};
	298
	299	/* Interfaces to register and deregister MR for RDMA read/write */
	300	struct smbd_mr *smbd_register_mr(
	301	struct smbd_connection info, struct page pages[], int num_pages,
7cf20bce	302	int offset, int tailsz, bool writing, bool need_invalidate);
c7398583 LL	303	int smbd_deregister_mr(struct smbd_mr *mr);
c7398583 LL	304
f198186a LL	305	#else
	306	#define cifs_rdma_enabled(server) 0
	307	struct smbd_connection {};
399f9539 LL	308	static inline void *smbd_get_connection(
399f9539 LL	309	struct TCP_Server_Info server, struct sockaddr dstaddr) {return NULL;}
ad57b8e1	310	static inline int smbd_reconnect(struct TCP_Server_Info *server) {return -1; }
050b8c37	311	static inline void smbd_destroy(struct TCP_Server_Info *server) {}
f64b78fd	312	static inline int smbd_recv(struct smbd_connection info, struct msghdr msg) {return -1; }
4739f232	313	static inline int smbd_send(struct TCP_Server_Info server, int num_rqst, struct smb_rqst rqst) {return -1; }
f198186a LL	314	#endif
f198186a LL	315
03bee01d	316	#endif