[linux-2.6-block.git] / net / smc / smc_wr.c

// SPDX-License-Identifier: GPL-2.0
/*
 * Shared Memory Communications over RDMA (SMC-R) and RoCE
 *
 * Work Requests exploiting Infiniband API
 *
 * Work requests (WR) of type ib_post_send or ib_post_recv respectively
 * are submitted to either RC SQ or RC RQ respectively
 * (reliably connected send/receive queue)
 * and become work queue entries (WQEs).
 * While an SQ WR/WQE is pending, we track it until transmission completion.
 * Through a send or receive completion queue (CQ) respectively,
 * we get completion queue entries (CQEs) [aka work completions (WCs)].
 * Since the CQ callback is called from IRQ context, we split work by using
 * bottom halves implemented by tasklets.
 *
 * SMC uses this to exchange LLC (link layer control)
 * and CDC (connection data control) messages.
 *
 * Copyright IBM Corp. 2016
 *
 * Author(s):  Steffen Maier <maier@linux.vnet.ibm.com>
 */

#include <linux/atomic.h>
#include <linux/hashtable.h>
#include <linux/wait.h>
#include <rdma/ib_verbs.h>
#include <asm/div64.h>

#include "smc.h"
#include "smc_wr.h"

#define SMC_WR_MAX_POLL_CQE 10	/* max. # of compl. queue elements in 1 poll */

#define SMC_WR_RX_HASH_BITS 4
static DEFINE_HASHTABLE(smc_wr_rx_hash, SMC_WR_RX_HASH_BITS);
static DEFINE_SPINLOCK(smc_wr_rx_hash_lock);

struct smc_wr_tx_pend {	/* control data for a pending send request */
	u64			wr_id;		/* work request id sent */
	smc_wr_tx_handler	handler;
	enum ib_wc_status	wc_status;	/* CQE status */
	struct smc_link		*link;
	u32			idx;
	struct smc_wr_tx_pend_priv priv;
};

/******************************** send queue *********************************/

/*------------------------------- completion --------------------------------*/

static inline int smc_wr_tx_find_pending_index(struct smc_link *link, u64 wr_id)
{
	u32 i;

	for (i = 0; i < link->wr_tx_cnt; i++) {
		if (link->wr_tx_pends[i].wr_id == wr_id)
			return i;
	}
	return link->wr_tx_cnt;
}

static inline void smc_wr_tx_process_cqe(struct ib_wc *wc)
{
	struct smc_wr_tx_pend pnd_snd;
	struct smc_link *link;
	u32 pnd_snd_idx;
	int i;

	link = wc->qp->qp_context;

	if (wc->opcode == IB_WC_REG_MR) {
		if (wc->status)
			link->wr_reg_state = FAILED;
		else
			link->wr_reg_state = CONFIRMED;
		wake_up(&link->wr_reg_wait);
		return;
	}

	pnd_snd_idx = smc_wr_tx_find_pending_index(link, wc->wr_id);
	if (pnd_snd_idx == link->wr_tx_cnt)
		return;
	link->wr_tx_pends[pnd_snd_idx].wc_status = wc->status;
	memcpy(&pnd_snd, &link->wr_tx_pends[pnd_snd_idx], sizeof(pnd_snd));
	/* clear the full struct smc_wr_tx_pend including .priv */
	memset(&link->wr_tx_pends[pnd_snd_idx], 0,
	       sizeof(link->wr_tx_pends[pnd_snd_idx]));
	memset(&link->wr_tx_bufs[pnd_snd_idx], 0,
	       sizeof(link->wr_tx_bufs[pnd_snd_idx]));
	if (!test_and_clear_bit(pnd_snd_idx, link->wr_tx_mask))
		return;
	if (wc->status) {
		for_each_set_bit(i, link->wr_tx_mask, link->wr_tx_cnt) {
			/* clear full struct smc_wr_tx_pend including .priv */
			memset(&link->wr_tx_pends[i], 0,
			       sizeof(link->wr_tx_pends[i]));
			memset(&link->wr_tx_bufs[i], 0,
			       sizeof(link->wr_tx_bufs[i]));
			clear_bit(i, link->wr_tx_mask);
		}
		/* terminate connections of this link group abnormally */
		smc_lgr_terminate(smc_get_lgr(link));
	}
	if (pnd_snd.handler)
		pnd_snd.handler(&pnd_snd.priv, link, wc->status);
	wake_up(&link->wr_tx_wait);
}

static void smc_wr_tx_tasklet_fn(unsigned long data)
{
	struct smc_ib_device *dev = (struct smc_ib_device *)data;
	struct ib_wc wc[SMC_WR_MAX_POLL_CQE];
	int i = 0, rc;
	int polled = 0;

again:
	polled++;
	do {
		memset(&wc, 0, sizeof(wc));
		rc = ib_poll_cq(dev->roce_cq_send, SMC_WR_MAX_POLL_CQE, wc);
		if (polled == 1) {
			ib_req_notify_cq(dev->roce_cq_send,
					 IB_CQ_NEXT_COMP |
					 IB_CQ_REPORT_MISSED_EVENTS);
		}
		if (!rc)
			break;
		for (i = 0; i < rc; i++)
			smc_wr_tx_process_cqe(&wc[i]);
	} while (rc > 0);
	if (polled == 1)
		goto again;
}

void smc_wr_tx_cq_handler(struct ib_cq *ib_cq, void *cq_context)
{
	struct smc_ib_device *dev = (struct smc_ib_device *)cq_context;

	tasklet_schedule(&dev->send_tasklet);
}

/*---------------------------- request submission ---------------------------*/

static inline int smc_wr_tx_get_free_slot_index(struct smc_link *link, u32 *idx)
{
	*idx = link->wr_tx_cnt;
	for_each_clear_bit(*idx, link->wr_tx_mask, link->wr_tx_cnt) {
		if (!test_and_set_bit(*idx, link->wr_tx_mask))
			return 0;
	}
	*idx = link->wr_tx_cnt;
	return -EBUSY;
}

/**
 * smc_wr_tx_get_free_slot() - returns buffer for message assembly,
 *			and sets info for pending transmit tracking
 * @link:		Pointer to smc_link used to later send the message.
 * @handler:		Send completion handler function pointer.
 * @wr_buf:		Out value returns pointer to message buffer.
 * @wr_rdma_buf:	Out value returns pointer to rdma work request.
 * @wr_pend_priv:	Out value returns pointer serving as handler context.
 *
 * Return: 0 on success, or -errno on error.
 */
int smc_wr_tx_get_free_slot(struct smc_link *link,
			    smc_wr_tx_handler handler,
			    struct smc_wr_buf **wr_buf,
			    struct smc_rdma_wr **wr_rdma_buf,
			    struct smc_wr_tx_pend_priv **wr_pend_priv)
{
	struct smc_wr_tx_pend *wr_pend;
	u32 idx = link->wr_tx_cnt;
	struct ib_send_wr *wr_ib;
	u64 wr_id;
	int rc;

	*wr_buf = NULL;
	*wr_pend_priv = NULL;
	if (in_softirq()) {
		rc = smc_wr_tx_get_free_slot_index(link, &idx);
		if (rc)
			return rc;
	} else {
		rc = wait_event_timeout(
			link->wr_tx_wait,
			link->state == SMC_LNK_INACTIVE ||
			(smc_wr_tx_get_free_slot_index(link, &idx) != -EBUSY),
			SMC_WR_TX_WAIT_FREE_SLOT_TIME);
		if (!rc) {
			/* timeout - terminate connections */
			smc_lgr_terminate(smc_get_lgr(link));
			return -EPIPE;
		}
		if (idx == link->wr_tx_cnt)
			return -EPIPE;
	}
	wr_id = smc_wr_tx_get_next_wr_id(link);
	wr_pend = &link->wr_tx_pends[idx];
	wr_pend->wr_id = wr_id;
	wr_pend->handler = handler;
	wr_pend->link = link;
	wr_pend->idx = idx;
	wr_ib = &link->wr_tx_ibs[idx];
	wr_ib->wr_id = wr_id;
	*wr_buf = &link->wr_tx_bufs[idx];
	if (wr_rdma_buf)
		*wr_rdma_buf = &link->wr_tx_rdmas[idx];
	*wr_pend_priv = &wr_pend->priv;
	return 0;
}

int smc_wr_tx_put_slot(struct smc_link *link,
		       struct smc_wr_tx_pend_priv *wr_pend_priv)
{
	struct smc_wr_tx_pend *pend;

	pend = container_of(wr_pend_priv, struct smc_wr_tx_pend, priv);
	if (pend->idx < link->wr_tx_cnt) {
		u32 idx = pend->idx;

		/* clear the full struct smc_wr_tx_pend including .priv */
		memset(&link->wr_tx_pends[idx], 0,
		       sizeof(link->wr_tx_pends[idx]));
		memset(&link->wr_tx_bufs[idx], 0,
		       sizeof(link->wr_tx_bufs[idx]));
		test_and_clear_bit(idx, link->wr_tx_mask);
		return 1;
	}

	return 0;
}

/* Send prepared WR slot via ib_post_send.
 * @priv: pointer to smc_wr_tx_pend_priv identifying prepared message buffer
 */
int smc_wr_tx_send(struct smc_link *link, struct smc_wr_tx_pend_priv *priv)
{
	struct smc_wr_tx_pend *pend;
	int rc;

	ib_req_notify_cq(link->smcibdev->roce_cq_send,
			 IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS);
	pend = container_of(priv, struct smc_wr_tx_pend, priv);
	rc = ib_post_send(link->roce_qp, &link->wr_tx_ibs[pend->idx], NULL);
	if (rc) {
		smc_wr_tx_put_slot(link, priv);
		smc_lgr_terminate(smc_get_lgr(link));
	}
	return rc;
}

/* Register a memory region and wait for result. */
int smc_wr_reg_send(struct smc_link *link, struct ib_mr *mr)
{
	int rc;

	ib_req_notify_cq(link->smcibdev->roce_cq_send,
			 IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS);
	link->wr_reg_state = POSTED;
	link->wr_reg.wr.wr_id = (u64)(uintptr_t)mr;
	link->wr_reg.mr = mr;
	link->wr_reg.key = mr->rkey;
	rc = ib_post_send(link->roce_qp, &link->wr_reg.wr, NULL);
	if (rc)
		return rc;

	rc = wait_event_interruptible_timeout(link->wr_reg_wait,
					      (link->wr_reg_state != POSTED),
					      SMC_WR_REG_MR_WAIT_TIME);
	if (!rc) {
		/* timeout - terminate connections */
		smc_lgr_terminate(smc_get_lgr(link));
		return -EPIPE;
	}
	if (rc == -ERESTARTSYS)
		return -EINTR;
	switch (link->wr_reg_state) {
	case CONFIRMED:
		rc = 0;
		break;
	case FAILED:
		rc = -EIO;
		break;
	case POSTED:
		rc = -EPIPE;
		break;
	}
	return rc;
}

void smc_wr_tx_dismiss_slots(struct smc_link *link, u8 wr_tx_hdr_type,
			     smc_wr_tx_filter filter,
			     smc_wr_tx_dismisser dismisser,
			     unsigned long data)
{
	struct smc_wr_tx_pend_priv *tx_pend;
	struct smc_wr_rx_hdr *wr_tx;
	int i;

	for_each_set_bit(i, link->wr_tx_mask, link->wr_tx_cnt) {
		wr_tx = (struct smc_wr_rx_hdr *)&link->wr_tx_bufs[i];
		if (wr_tx->type != wr_tx_hdr_type)
			continue;
		tx_pend = &link->wr_tx_pends[i].priv;
		if (filter(tx_pend, data))
			dismisser(tx_pend);
	}
}

/****************************** receive queue ********************************/

int smc_wr_rx_register_handler(struct smc_wr_rx_handler *handler)
{
	struct smc_wr_rx_handler *h_iter;
	int rc = 0;

	spin_lock(&smc_wr_rx_hash_lock);
	hash_for_each_possible(smc_wr_rx_hash, h_iter, list, handler->type) {
		if (h_iter->type == handler->type) {
			rc = -EEXIST;
			goto out_unlock;
		}
	}
	hash_add(smc_wr_rx_hash, &handler->list, handler->type);
out_unlock:
	spin_unlock(&smc_wr_rx_hash_lock);
	return rc;
}

/* Demultiplex a received work request based on the message type to its handler.
 * Relies on smc_wr_rx_hash having been completely filled before any IB WRs,
 * and not being modified any more afterwards so we don't need to lock it.
 */
static inline void smc_wr_rx_demultiplex(struct ib_wc *wc)
{
	struct smc_link *link = (struct smc_link *)wc->qp->qp_context;
	struct smc_wr_rx_handler *handler;
	struct smc_wr_rx_hdr *wr_rx;
	u64 temp_wr_id;
	u32 index;

	if (wc->byte_len < sizeof(*wr_rx))
		return; /* short message */
	temp_wr_id = wc->wr_id;
	index = do_div(temp_wr_id, link->wr_rx_cnt);
	wr_rx = (struct smc_wr_rx_hdr *)&link->wr_rx_bufs[index];
	hash_for_each_possible(smc_wr_rx_hash, handler, list, wr_rx->type) {
		if (handler->type == wr_rx->type)
			handler->handler(wc, wr_rx);
	}
}

static inline void smc_wr_rx_process_cqes(struct ib_wc wc[], int num)
{
	struct smc_link *link;
	int i;

	for (i = 0; i < num; i++) {
		link = wc[i].qp->qp_context;
		if (wc[i].status == IB_WC_SUCCESS) {
			link->wr_rx_tstamp = jiffies;
			smc_wr_rx_demultiplex(&wc[i]);
			smc_wr_rx_post(link); /* refill WR RX */
		} else {
			/* handle status errors */
			switch (wc[i].status) {
			case IB_WC_RETRY_EXC_ERR:
			case IB_WC_RNR_RETRY_EXC_ERR:
			case IB_WC_WR_FLUSH_ERR:
				/* terminate connections of this link group
				 * abnormally
				 */
				smc_lgr_terminate(smc_get_lgr(link));
				break;
			default:
				smc_wr_rx_post(link); /* refill WR RX */
				break;
			}
		}
	}
}

static void smc_wr_rx_tasklet_fn(unsigned long data)
{
	struct smc_ib_device *dev = (struct smc_ib_device *)data;
	struct ib_wc wc[SMC_WR_MAX_POLL_CQE];
	int polled = 0;
	int rc;

again:
	polled++;
	do {
		memset(&wc, 0, sizeof(wc));
		rc = ib_poll_cq(dev->roce_cq_recv, SMC_WR_MAX_POLL_CQE, wc);
		if (polled == 1) {
			ib_req_notify_cq(dev->roce_cq_recv,
					 IB_CQ_SOLICITED_MASK
					 | IB_CQ_REPORT_MISSED_EVENTS);
		}
		if (!rc)
			break;
		smc_wr_rx_process_cqes(&wc[0], rc);
	} while (rc > 0);
	if (polled == 1)
		goto again;
}

void smc_wr_rx_cq_handler(struct ib_cq *ib_cq, void *cq_context)
{
	struct smc_ib_device *dev = (struct smc_ib_device *)cq_context;

	tasklet_schedule(&dev->recv_tasklet);
}

int smc_wr_rx_post_init(struct smc_link *link)
{
	u32 i;
	int rc = 0;

	for (i = 0; i < link->wr_rx_cnt; i++)
		rc = smc_wr_rx_post(link);
	return rc;
}

/***************************** init, exit, misc ******************************/

void smc_wr_remember_qp_attr(struct smc_link *lnk)
{
	struct ib_qp_attr *attr = &lnk->qp_attr;
	struct ib_qp_init_attr init_attr;

	memset(attr, 0, sizeof(*attr));
	memset(&init_attr, 0, sizeof(init_attr));
	ib_query_qp(lnk->roce_qp, attr,
		    IB_QP_STATE |
		    IB_QP_CUR_STATE |
		    IB_QP_PKEY_INDEX |
		    IB_QP_PORT |
		    IB_QP_QKEY |
		    IB_QP_AV |
		    IB_QP_PATH_MTU |
		    IB_QP_TIMEOUT |
		    IB_QP_RETRY_CNT |
		    IB_QP_RNR_RETRY |
		    IB_QP_RQ_PSN |
		    IB_QP_ALT_PATH |
		    IB_QP_MIN_RNR_TIMER |
		    IB_QP_SQ_PSN |
		    IB_QP_PATH_MIG_STATE |
		    IB_QP_CAP |
		    IB_QP_DEST_QPN,
		    &init_attr);

	lnk->wr_tx_cnt = min_t(size_t, SMC_WR_BUF_CNT,
			       lnk->qp_attr.cap.max_send_wr);
	lnk->wr_rx_cnt = min_t(size_t, SMC_WR_BUF_CNT * 3,
			       lnk->qp_attr.cap.max_recv_wr);
}

static void smc_wr_init_sge(struct smc_link *lnk)
{
	u32 i;

	for (i = 0; i < lnk->wr_tx_cnt; i++) {
		lnk->wr_tx_sges[i].addr =
			lnk->wr_tx_dma_addr + i * SMC_WR_BUF_SIZE;
		lnk->wr_tx_sges[i].length = SMC_WR_TX_SIZE;
		lnk->wr_tx_sges[i].lkey = lnk->roce_pd->local_dma_lkey;
		lnk->wr_tx_rdma_sges[i].tx_rdma_sge[0].wr_tx_rdma_sge[0].lkey =
			lnk->roce_pd->local_dma_lkey;
		lnk->wr_tx_rdma_sges[i].tx_rdma_sge[0].wr_tx_rdma_sge[1].lkey =
			lnk->roce_pd->local_dma_lkey;
		lnk->wr_tx_rdma_sges[i].tx_rdma_sge[1].wr_tx_rdma_sge[0].lkey =
			lnk->roce_pd->local_dma_lkey;
		lnk->wr_tx_rdma_sges[i].tx_rdma_sge[1].wr_tx_rdma_sge[1].lkey =
			lnk->roce_pd->local_dma_lkey;
		lnk->wr_tx_ibs[i].next = NULL;
		lnk->wr_tx_ibs[i].sg_list = &lnk->wr_tx_sges[i];
		lnk->wr_tx_ibs[i].num_sge = 1;
		lnk->wr_tx_ibs[i].opcode = IB_WR_SEND;
		lnk->wr_tx_ibs[i].send_flags =
			IB_SEND_SIGNALED | IB_SEND_SOLICITED;
		lnk->wr_tx_rdmas[i].wr_tx_rdma[0].wr.opcode = IB_WR_RDMA_WRITE;
		lnk->wr_tx_rdmas[i].wr_tx_rdma[1].wr.opcode = IB_WR_RDMA_WRITE;
		lnk->wr_tx_rdmas[i].wr_tx_rdma[0].wr.sg_list =
			lnk->wr_tx_rdma_sges[i].tx_rdma_sge[0].wr_tx_rdma_sge;
		lnk->wr_tx_rdmas[i].wr_tx_rdma[1].wr.sg_list =
			lnk->wr_tx_rdma_sges[i].tx_rdma_sge[1].wr_tx_rdma_sge;
	}
	for (i = 0; i < lnk->wr_rx_cnt; i++) {
		lnk->wr_rx_sges[i].addr =
			lnk->wr_rx_dma_addr + i * SMC_WR_BUF_SIZE;
		lnk->wr_rx_sges[i].length = SMC_WR_BUF_SIZE;
		lnk->wr_rx_sges[i].lkey = lnk->roce_pd->local_dma_lkey;
		lnk->wr_rx_ibs[i].next = NULL;
		lnk->wr_rx_ibs[i].sg_list = &lnk->wr_rx_sges[i];
		lnk->wr_rx_ibs[i].num_sge = 1;
	}
	lnk->wr_reg.wr.next = NULL;
	lnk->wr_reg.wr.num_sge = 0;
	lnk->wr_reg.wr.send_flags = IB_SEND_SIGNALED;
	lnk->wr_reg.wr.opcode = IB_WR_REG_MR;
	lnk->wr_reg.access = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_WRITE;
}

void smc_wr_free_link(struct smc_link *lnk)
{
	struct ib_device *ibdev;

	memset(lnk->wr_tx_mask, 0,
	       BITS_TO_LONGS(SMC_WR_BUF_CNT) * sizeof(*lnk->wr_tx_mask));

	if (!lnk->smcibdev)
		return;
	ibdev = lnk->smcibdev->ibdev;

	if (lnk->wr_rx_dma_addr) {
		ib_dma_unmap_single(ibdev, lnk->wr_rx_dma_addr,
				    SMC_WR_BUF_SIZE * lnk->wr_rx_cnt,
				    DMA_FROM_DEVICE);
		lnk->wr_rx_dma_addr = 0;
	}
	if (lnk->wr_tx_dma_addr) {
		ib_dma_unmap_single(ibdev, lnk->wr_tx_dma_addr,
				    SMC_WR_BUF_SIZE * lnk->wr_tx_cnt,
				    DMA_TO_DEVICE);
		lnk->wr_tx_dma_addr = 0;
	}
}

void smc_wr_free_link_mem(struct smc_link *lnk)
{
	kfree(lnk->wr_tx_pends);
	lnk->wr_tx_pends = NULL;
	kfree(lnk->wr_tx_mask);
	lnk->wr_tx_mask = NULL;
	kfree(lnk->wr_tx_sges);
	lnk->wr_tx_sges = NULL;
	kfree(lnk->wr_tx_rdma_sges);
	lnk->wr_tx_rdma_sges = NULL;
	kfree(lnk->wr_rx_sges);
	lnk->wr_rx_sges = NULL;
	kfree(lnk->wr_tx_rdmas);
	lnk->wr_tx_rdmas = NULL;
	kfree(lnk->wr_rx_ibs);
	lnk->wr_rx_ibs = NULL;
	kfree(lnk->wr_tx_ibs);
	lnk->wr_tx_ibs = NULL;
	kfree(lnk->wr_tx_bufs);
	lnk->wr_tx_bufs = NULL;
	kfree(lnk->wr_rx_bufs);
	lnk->wr_rx_bufs = NULL;
}

int smc_wr_alloc_link_mem(struct smc_link *link)
{
	/* allocate link related memory */
	link->wr_tx_bufs = kcalloc(SMC_WR_BUF_CNT, SMC_WR_BUF_SIZE, GFP_KERNEL);
	if (!link->wr_tx_bufs)
		goto no_mem;
	link->wr_rx_bufs = kcalloc(SMC_WR_BUF_CNT * 3, SMC_WR_BUF_SIZE,
				   GFP_KERNEL);
	if (!link->wr_rx_bufs)
		goto no_mem_wr_tx_bufs;
	link->wr_tx_ibs = kcalloc(SMC_WR_BUF_CNT, sizeof(link->wr_tx_ibs[0]),
				  GFP_KERNEL);
	if (!link->wr_tx_ibs)
		goto no_mem_wr_rx_bufs;
	link->wr_rx_ibs = kcalloc(SMC_WR_BUF_CNT * 3,
				  sizeof(link->wr_rx_ibs[0]),
				  GFP_KERNEL);
	if (!link->wr_rx_ibs)
		goto no_mem_wr_tx_ibs;
	link->wr_tx_rdmas = kcalloc(SMC_WR_BUF_CNT,
				    sizeof(link->wr_tx_rdmas[0]),
				    GFP_KERNEL);
	if (!link->wr_tx_rdmas)
		goto no_mem_wr_rx_ibs;
	link->wr_tx_rdma_sges = kcalloc(SMC_WR_BUF_CNT,
					sizeof(link->wr_tx_rdma_sges[0]),
					GFP_KERNEL);
	if (!link->wr_tx_rdma_sges)
		goto no_mem_wr_tx_rdmas;
	link->wr_tx_sges = kcalloc(SMC_WR_BUF_CNT, sizeof(link->wr_tx_sges[0]),
				   GFP_KERNEL);
	if (!link->wr_tx_sges)
		goto no_mem_wr_tx_rdma_sges;
	link->wr_rx_sges = kcalloc(SMC_WR_BUF_CNT * 3,
				   sizeof(link->wr_rx_sges[0]),
				   GFP_KERNEL);
	if (!link->wr_rx_sges)
		goto no_mem_wr_tx_sges;
	link->wr_tx_mask = kcalloc(BITS_TO_LONGS(SMC_WR_BUF_CNT),
				   sizeof(*link->wr_tx_mask),
				   GFP_KERNEL);
	if (!link->wr_tx_mask)
		goto no_mem_wr_rx_sges;
	link->wr_tx_pends = kcalloc(SMC_WR_BUF_CNT,
				    sizeof(link->wr_tx_pends[0]),
				    GFP_KERNEL);
	if (!link->wr_tx_pends)
		goto no_mem_wr_tx_mask;
	return 0;

no_mem_wr_tx_mask:
	kfree(link->wr_tx_mask);
no_mem_wr_rx_sges:
	kfree(link->wr_rx_sges);
no_mem_wr_tx_sges:
	kfree(link->wr_tx_sges);
no_mem_wr_tx_rdma_sges:
	kfree(link->wr_tx_rdma_sges);
no_mem_wr_tx_rdmas:
	kfree(link->wr_tx_rdmas);
no_mem_wr_rx_ibs:
	kfree(link->wr_rx_ibs);
no_mem_wr_tx_ibs:
	kfree(link->wr_tx_ibs);
no_mem_wr_rx_bufs:
	kfree(link->wr_rx_bufs);
no_mem_wr_tx_bufs:
	kfree(link->wr_tx_bufs);
no_mem:
	return -ENOMEM;
}

void smc_wr_remove_dev(struct smc_ib_device *smcibdev)
{
	tasklet_kill(&smcibdev->recv_tasklet);
	tasklet_kill(&smcibdev->send_tasklet);
}

void smc_wr_add_dev(struct smc_ib_device *smcibdev)
{
	tasklet_init(&smcibdev->recv_tasklet, smc_wr_rx_tasklet_fn,
		     (unsigned long)smcibdev);
	tasklet_init(&smcibdev->send_tasklet, smc_wr_tx_tasklet_fn,
		     (unsigned long)smcibdev);
}

int smc_wr_create_link(struct smc_link *lnk)
{
	struct ib_device *ibdev = lnk->smcibdev->ibdev;
	int rc = 0;

	smc_wr_tx_set_wr_id(&lnk->wr_tx_id, 0);
	lnk->wr_rx_id = 0;
	lnk->wr_rx_dma_addr = ib_dma_map_single(
		ibdev, lnk->wr_rx_bufs,	SMC_WR_BUF_SIZE * lnk->wr_rx_cnt,
		DMA_FROM_DEVICE);
	if (ib_dma_mapping_error(ibdev, lnk->wr_rx_dma_addr)) {
		lnk->wr_rx_dma_addr = 0;
		rc = -EIO;
		goto out;
	}
	lnk->wr_tx_dma_addr = ib_dma_map_single(
		ibdev, lnk->wr_tx_bufs,	SMC_WR_BUF_SIZE * lnk->wr_tx_cnt,
		DMA_TO_DEVICE);
	if (ib_dma_mapping_error(ibdev, lnk->wr_tx_dma_addr)) {
		rc = -EIO;
		goto dma_unmap;
	}
	smc_wr_init_sge(lnk);
	memset(lnk->wr_tx_mask, 0,
	       BITS_TO_LONGS(SMC_WR_BUF_CNT) * sizeof(*lnk->wr_tx_mask));
	init_waitqueue_head(&lnk->wr_tx_wait);
	init_waitqueue_head(&lnk->wr_reg_wait);
	return rc;

dma_unmap:
	ib_dma_unmap_single(ibdev, lnk->wr_rx_dma_addr,
			    SMC_WR_BUF_SIZE * lnk->wr_rx_cnt,
			    DMA_FROM_DEVICE);
	lnk->wr_rx_dma_addr = 0;
out:
	return rc;
}
Commit	Line	Data
b2441318	1	// SPDX-License-Identifier: GPL-2.0
f38ba179 UB	2	/*
	3	* Shared Memory Communications over RDMA (SMC-R) and RoCE
	4	*
	5	* Work Requests exploiting Infiniband API
	6	*
	7	* Work requests (WR) of type ib_post_send or ib_post_recv respectively
	8	* are submitted to either RC SQ or RC RQ respectively
	9	* (reliably connected send/receive queue)
	10	* and become work queue entries (WQEs).
	11	* While an SQ WR/WQE is pending, we track it until transmission completion.
	12	* Through a send or receive completion queue (CQ) respectively,
	13	* we get completion queue entries (CQEs) [aka work completions (WCs)].
	14	* Since the CQ callback is called from IRQ context, we split work by using
	15	* bottom halves implemented by tasklets.
	16	*
	17	* SMC uses this to exchange LLC (link layer control)
	18	* and CDC (connection data control) messages.
	19	*
	20	* Copyright IBM Corp. 2016
	21	*
	22	* Author(s): Steffen Maier <maier@linux.vnet.ibm.com>
	23	*/
	24
	25	#include <linux/atomic.h>
	26	#include <linux/hashtable.h>
	27	#include <linux/wait.h>
	28	#include <rdma/ib_verbs.h>
	29	#include <asm/div64.h>
	30
	31	#include "smc.h"
	32	#include "smc_wr.h"
	33
	34	#define SMC_WR_MAX_POLL_CQE 10 /* max. # of compl. queue elements in 1 poll */
	35
	36	#define SMC_WR_RX_HASH_BITS 4
	37	static DEFINE_HASHTABLE(smc_wr_rx_hash, SMC_WR_RX_HASH_BITS);
	38	static DEFINE_SPINLOCK(smc_wr_rx_hash_lock);
	39
	40	struct smc_wr_tx_pend { /* control data for a pending send request */
	41	u64 wr_id; /* work request id sent */
	42	smc_wr_tx_handler handler;
	43	enum ib_wc_status wc_status; /* CQE status */
	44	struct smc_link *link;
	45	u32 idx;
	46	struct smc_wr_tx_pend_priv priv;
	47	};
	48
	49	/****************************** send queue *******************************/
	50
	51	/------------------------------- completion --------------------------------/
	52
	53	static inline int smc_wr_tx_find_pending_index(struct smc_link *link, u64 wr_id)
	54	{
	55	u32 i;
	56
	57	for (i = 0; i < link->wr_tx_cnt; i++) {
	58	if (link->wr_tx_pends[i].wr_id == wr_id)
	59	return i;
	60	}
	61	return link->wr_tx_cnt;
	62	}
	63
	64	static inline void smc_wr_tx_process_cqe(struct ib_wc *wc)
	65	{
66	struct smc_wr_tx_pend pnd_snd;
67	struct smc_link *link;
68	u32 pnd_snd_idx;
69	int i;
70
71	link = wc->qp->qp_context;
652a1e41 UB	72
	73	if (wc->opcode == IB_WC_REG_MR) {
	74	if (wc->status)
	75	link->wr_reg_state = FAILED;
	76	else
	77	link->wr_reg_state = CONFIRMED;
	78	wake_up(&link->wr_reg_wait);
	79	return;
	80	}
	81
f38ba179 UB	82	pnd_snd_idx = smc_wr_tx_find_pending_index(link, wc->wr_id);
	83	if (pnd_snd_idx == link->wr_tx_cnt)
	84	return;
	85	link->wr_tx_pends[pnd_snd_idx].wc_status = wc->status;
	86	memcpy(&pnd_snd, &link->wr_tx_pends[pnd_snd_idx], sizeof(pnd_snd));
	87	/* clear the full struct smc_wr_tx_pend including .priv */
	88	memset(&link->wr_tx_pends[pnd_snd_idx], 0,
	89	sizeof(link->wr_tx_pends[pnd_snd_idx]));
	90	memset(&link->wr_tx_bufs[pnd_snd_idx], 0,
	91	sizeof(link->wr_tx_bufs[pnd_snd_idx]));
	92	if (!test_and_clear_bit(pnd_snd_idx, link->wr_tx_mask))
	93	return;
	94	if (wc->status) {
	95	for_each_set_bit(i, link->wr_tx_mask, link->wr_tx_cnt) {
	96	/* clear full struct smc_wr_tx_pend including .priv */
	97	memset(&link->wr_tx_pends[i], 0,
	98	sizeof(link->wr_tx_pends[i]));
	99	memset(&link->wr_tx_bufs[i], 0,
	100	sizeof(link->wr_tx_bufs[i]));
	101	clear_bit(i, link->wr_tx_mask);
	102	}
b38d7324	103	/* terminate connections of this link group abnormally */
00e5fb26	104	smc_lgr_terminate(smc_get_lgr(link));
f38ba179 UB	105	}
	106	if (pnd_snd.handler)
	107	pnd_snd.handler(&pnd_snd.priv, link, wc->status);
	108	wake_up(&link->wr_tx_wait);
	109	}
	110
	111	static void smc_wr_tx_tasklet_fn(unsigned long data)
	112	{
	113	struct smc_ib_device dev = (struct smc_ib_device )data;
	114	struct ib_wc wc[SMC_WR_MAX_POLL_CQE];
	115	int i = 0, rc;
	116	int polled = 0;
	117
	118	again:
	119	polled++;
	120	do {
86e780d3	121	memset(&wc, 0, sizeof(wc));
f38ba179 UB	122	rc = ib_poll_cq(dev->roce_cq_send, SMC_WR_MAX_POLL_CQE, wc);
	123	if (polled == 1) {
	124	ib_req_notify_cq(dev->roce_cq_send,
	125	IB_CQ_NEXT_COMP \|
	126	IB_CQ_REPORT_MISSED_EVENTS);
	127	}
	128	if (!rc)
	129	break;
	130	for (i = 0; i < rc; i++)
	131	smc_wr_tx_process_cqe(&wc[i]);
	132	} while (rc > 0);
	133	if (polled == 1)
	134	goto again;
	135	}
	136
	137	void smc_wr_tx_cq_handler(struct ib_cq ib_cq, void cq_context)
	138	{
	139	struct smc_ib_device dev = (struct smc_ib_device )cq_context;
	140
	141	tasklet_schedule(&dev->send_tasklet);
	142	}
	143
	144	/---------------------------- request submission ---------------------------/
	145
	146	static inline int smc_wr_tx_get_free_slot_index(struct smc_link link, u32 idx)
	147	{
	148	*idx = link->wr_tx_cnt;
	149	for_each_clear_bit(*idx, link->wr_tx_mask, link->wr_tx_cnt) {
	150	if (!test_and_set_bit(*idx, link->wr_tx_mask))
	151	return 0;
	152	}
	153	*idx = link->wr_tx_cnt;
	154	return -EBUSY;
	155	}
	156
	157	/**
	158	* smc_wr_tx_get_free_slot() - returns buffer for message assembly,
	159	* and sets info for pending transmit tracking
	160	* @link: Pointer to smc_link used to later send the message.
	161	* @handler: Send completion handler function pointer.
	162	* @wr_buf: Out value returns pointer to message buffer.
ad6f317f	163	* @wr_rdma_buf: Out value returns pointer to rdma work request.
f38ba179 UB	164	* @wr_pend_priv: Out value returns pointer serving as handler context.
	165	*
	166	* Return: 0 on success, or -errno on error.
	167	*/
	168	int smc_wr_tx_get_free_slot(struct smc_link *link,
	169	smc_wr_tx_handler handler,
	170	struct smc_wr_buf **wr_buf,
ad6f317f	171	struct smc_rdma_wr **wr_rdma_buf,
f38ba179 UB	172	struct smc_wr_tx_pend_priv **wr_pend_priv)
	173	{
	174	struct smc_wr_tx_pend *wr_pend;
1a0a04c7	175	u32 idx = link->wr_tx_cnt;
f38ba179 UB	176	struct ib_send_wr *wr_ib;
f38ba179 UB	177	u64 wr_id;
f38ba179 UB	178	int rc;
	179
	180	*wr_buf = NULL;
	181	*wr_pend_priv = NULL;
	182	if (in_softirq()) {
	183	rc = smc_wr_tx_get_free_slot_index(link, &idx);
	184	if (rc)
	185	return rc;
	186	} else {
86e780d3	187	rc = wait_event_timeout(
f38ba179	188	link->wr_tx_wait,
0d18a0cb	189	link->state == SMC_LNK_INACTIVE \|\|
f38ba179 UB	190	(smc_wr_tx_get_free_slot_index(link, &idx) != -EBUSY),
	191	SMC_WR_TX_WAIT_FREE_SLOT_TIME);
	192	if (!rc) {
b38d7324	193	/* timeout - terminate connections */
0d18a0cb	194	smc_lgr_terminate(smc_get_lgr(link));
f38ba179 UB	195	return -EPIPE;
f38ba179 UB	196	}
f38ba179 UB	197	if (idx == link->wr_tx_cnt)
	198	return -EPIPE;
	199	}
	200	wr_id = smc_wr_tx_get_next_wr_id(link);
	201	wr_pend = &link->wr_tx_pends[idx];
	202	wr_pend->wr_id = wr_id;
	203	wr_pend->handler = handler;
	204	wr_pend->link = link;
	205	wr_pend->idx = idx;
	206	wr_ib = &link->wr_tx_ibs[idx];
	207	wr_ib->wr_id = wr_id;
	208	*wr_buf = &link->wr_tx_bufs[idx];
ad6f317f UB	209	if (wr_rdma_buf)
ad6f317f UB	210	*wr_rdma_buf = &link->wr_tx_rdmas[idx];
f38ba179 UB	211	*wr_pend_priv = &wr_pend->priv;
	212	return 0;
	213	}
	214
	215	int smc_wr_tx_put_slot(struct smc_link *link,
	216	struct smc_wr_tx_pend_priv *wr_pend_priv)
	217	{
	218	struct smc_wr_tx_pend *pend;
	219
	220	pend = container_of(wr_pend_priv, struct smc_wr_tx_pend, priv);
	221	if (pend->idx < link->wr_tx_cnt) {
e438bae4 UB	222	u32 idx = pend->idx;
e438bae4 UB	223
f38ba179	224	/* clear the full struct smc_wr_tx_pend including .priv */
46ad0222 KG	225	memset(&link->wr_tx_pends[idx], 0,
	226	sizeof(link->wr_tx_pends[idx]));
	227	memset(&link->wr_tx_bufs[idx], 0,
	228	sizeof(link->wr_tx_bufs[idx]));
e438bae4	229	test_and_clear_bit(idx, link->wr_tx_mask);
f38ba179 UB	230	return 1;
	231	}
	232
	233	return 0;
	234	}
	235
	236	/* Send prepared WR slot via ib_post_send.
	237	* @priv: pointer to smc_wr_tx_pend_priv identifying prepared message buffer
	238	*/
	239	int smc_wr_tx_send(struct smc_link link, struct smc_wr_tx_pend_priv priv)
	240	{
f38ba179 UB	241	struct smc_wr_tx_pend *pend;
	242	int rc;
	243
	244	ib_req_notify_cq(link->smcibdev->roce_cq_send,
8301fa44	245	IB_CQ_NEXT_COMP \| IB_CQ_REPORT_MISSED_EVENTS);
f38ba179	246	pend = container_of(priv, struct smc_wr_tx_pend, priv);
2e3bbe46	247	rc = ib_post_send(link->roce_qp, &link->wr_tx_ibs[pend->idx], NULL);
b4772b3a	248	if (rc) {
f38ba179	249	smc_wr_tx_put_slot(link, priv);
00e5fb26	250	smc_lgr_terminate(smc_get_lgr(link));
b4772b3a	251	}
f38ba179 UB	252	return rc;
	253	}
	254
652a1e41 UB	255	/* Register a memory region and wait for result. */
	256	int smc_wr_reg_send(struct smc_link link, struct ib_mr mr)
	257	{
652a1e41 UB	258	int rc;
	259
	260	ib_req_notify_cq(link->smcibdev->roce_cq_send,
	261	IB_CQ_NEXT_COMP \| IB_CQ_REPORT_MISSED_EVENTS);
	262	link->wr_reg_state = POSTED;
	263	link->wr_reg.wr.wr_id = (u64)(uintptr_t)mr;
	264	link->wr_reg.mr = mr;
	265	link->wr_reg.key = mr->rkey;
2e3bbe46	266	rc = ib_post_send(link->roce_qp, &link->wr_reg.wr, NULL);
652a1e41 UB	267	if (rc)
	268	return rc;
	269
	270	rc = wait_event_interruptible_timeout(link->wr_reg_wait,
	271	(link->wr_reg_state != POSTED),
	272	SMC_WR_REG_MR_WAIT_TIME);
	273	if (!rc) {
	274	/* timeout - terminate connections */
00e5fb26	275	smc_lgr_terminate(smc_get_lgr(link));
652a1e41 UB	276	return -EPIPE;
	277	}
	278	if (rc == -ERESTARTSYS)
	279	return -EINTR;
	280	switch (link->wr_reg_state) {
	281	case CONFIRMED:
	282	rc = 0;
	283	break;
	284	case FAILED:
	285	rc = -EIO;
	286	break;
	287	case POSTED:
	288	rc = -EPIPE;
	289	break;
	290	}
	291	return rc;
	292	}
	293
86e780d3	294	void smc_wr_tx_dismiss_slots(struct smc_link *link, u8 wr_tx_hdr_type,
5f08318f UB	295	smc_wr_tx_filter filter,
	296	smc_wr_tx_dismisser dismisser,
	297	unsigned long data)
	298	{
	299	struct smc_wr_tx_pend_priv *tx_pend;
86e780d3	300	struct smc_wr_rx_hdr *wr_tx;
5f08318f UB	301	int i;
	302
	303	for_each_set_bit(i, link->wr_tx_mask, link->wr_tx_cnt) {
86e780d3 UB	304	wr_tx = (struct smc_wr_rx_hdr *)&link->wr_tx_bufs[i];
86e780d3 UB	305	if (wr_tx->type != wr_tx_hdr_type)
5f08318f UB	306	continue;
	307	tx_pend = &link->wr_tx_pends[i].priv;
	308	if (filter(tx_pend, data))
	309	dismisser(tx_pend);
	310	}
	311	}
	312
f38ba179 UB	313	/**************************** receive queue ******************************/
	314
	315	int smc_wr_rx_register_handler(struct smc_wr_rx_handler *handler)
	316	{
	317	struct smc_wr_rx_handler *h_iter;
	318	int rc = 0;
	319
	320	spin_lock(&smc_wr_rx_hash_lock);
	321	hash_for_each_possible(smc_wr_rx_hash, h_iter, list, handler->type) {
	322	if (h_iter->type == handler->type) {
	323	rc = -EEXIST;
	324	goto out_unlock;
	325	}
	326	}
	327	hash_add(smc_wr_rx_hash, &handler->list, handler->type);
	328	out_unlock:
	329	spin_unlock(&smc_wr_rx_hash_lock);
	330	return rc;
	331	}
	332
	333	/* Demultiplex a received work request based on the message type to its handler.
	334	* Relies on smc_wr_rx_hash having been completely filled before any IB WRs,
	335	* and not being modified any more afterwards so we don't need to lock it.
	336	*/
	337	static inline void smc_wr_rx_demultiplex(struct ib_wc *wc)
	338	{
	339	struct smc_link link = (struct smc_link )wc->qp->qp_context;
	340	struct smc_wr_rx_handler *handler;
	341	struct smc_wr_rx_hdr *wr_rx;
	342	u64 temp_wr_id;
	343	u32 index;
	344
	345	if (wc->byte_len < sizeof(*wr_rx))
	346	return; /* short message */
	347	temp_wr_id = wc->wr_id;
	348	index = do_div(temp_wr_id, link->wr_rx_cnt);
	349	wr_rx = (struct smc_wr_rx_hdr *)&link->wr_rx_bufs[index];
	350	hash_for_each_possible(smc_wr_rx_hash, handler, list, wr_rx->type) {
	351	if (handler->type == wr_rx->type)
	352	handler->handler(wc, wr_rx);
	353	}
	354	}
	355
	356	static inline void smc_wr_rx_process_cqes(struct ib_wc wc[], int num)
	357	{
	358	struct smc_link *link;
	359	int i;
	360
	361	for (i = 0; i < num; i++) {
	362	link = wc[i].qp->qp_context;
	363	if (wc[i].status == IB_WC_SUCCESS) {
877ae5be	364	link->wr_rx_tstamp = jiffies;
f38ba179 UB	365	smc_wr_rx_demultiplex(&wc[i]);
	366	smc_wr_rx_post(link); /* refill WR RX */
	367	} else {
	368	/* handle status errors */
	369	switch (wc[i].status) {
	370	case IB_WC_RETRY_EXC_ERR:
	371	case IB_WC_RNR_RETRY_EXC_ERR:
	372	case IB_WC_WR_FLUSH_ERR:
b38d7324 UB	373	/* terminate connections of this link group
	374	* abnormally
	375	*/
00e5fb26	376	smc_lgr_terminate(smc_get_lgr(link));
f38ba179 UB	377	break;
	378	default:
	379	smc_wr_rx_post(link); /* refill WR RX */
	380	break;
	381	}
	382	}
	383	}
	384	}
	385
	386	static void smc_wr_rx_tasklet_fn(unsigned long data)
	387	{
	388	struct smc_ib_device dev = (struct smc_ib_device )data;
	389	struct ib_wc wc[SMC_WR_MAX_POLL_CQE];
	390	int polled = 0;
	391	int rc;
	392
	393	again:
	394	polled++;
	395	do {
	396	memset(&wc, 0, sizeof(wc));
	397	rc = ib_poll_cq(dev->roce_cq_recv, SMC_WR_MAX_POLL_CQE, wc);
	398	if (polled == 1) {
	399	ib_req_notify_cq(dev->roce_cq_recv,
	400	IB_CQ_SOLICITED_MASK
	401	\| IB_CQ_REPORT_MISSED_EVENTS);
	402	}
	403	if (!rc)
	404	break;
	405	smc_wr_rx_process_cqes(&wc[0], rc);
	406	} while (rc > 0);
	407	if (polled == 1)
	408	goto again;
	409	}
	410
	411	void smc_wr_rx_cq_handler(struct ib_cq ib_cq, void cq_context)
	412	{
	413	struct smc_ib_device dev = (struct smc_ib_device )cq_context;
	414
	415	tasklet_schedule(&dev->recv_tasklet);
	416	}
	417
	418	int smc_wr_rx_post_init(struct smc_link *link)
	419	{
	420	u32 i;
	421	int rc = 0;
	422
	423	for (i = 0; i < link->wr_rx_cnt; i++)
	424	rc = smc_wr_rx_post(link);
	425	return rc;
	426	}
	427
	428	/*************************** init, exit, misc ****************************/
	429
	430	void smc_wr_remember_qp_attr(struct smc_link *lnk)
	431	{
	432	struct ib_qp_attr *attr = &lnk->qp_attr;
	433	struct ib_qp_init_attr init_attr;
	434
	435	memset(attr, 0, sizeof(*attr));
	436	memset(&init_attr, 0, sizeof(init_attr));
	437	ib_query_qp(lnk->roce_qp, attr,
	438	IB_QP_STATE \|
	439	IB_QP_CUR_STATE \|
	440	IB_QP_PKEY_INDEX \|
441	IB_QP_PORT \|
442	IB_QP_QKEY \|
443	IB_QP_AV \|
444	IB_QP_PATH_MTU \|
445	IB_QP_TIMEOUT \|
446	IB_QP_RETRY_CNT \|
447	IB_QP_RNR_RETRY \|
448	IB_QP_RQ_PSN \|
449	IB_QP_ALT_PATH \|
450	IB_QP_MIN_RNR_TIMER \|
451	IB_QP_SQ_PSN \|
452	IB_QP_PATH_MIG_STATE \|
453	IB_QP_CAP \|
454	IB_QP_DEST_QPN,
455	&init_attr);
456
457	lnk->wr_tx_cnt = min_t(size_t, SMC_WR_BUF_CNT,
458	lnk->qp_attr.cap.max_send_wr);
459	lnk->wr_rx_cnt = min_t(size_t, SMC_WR_BUF_CNT * 3,
460	lnk->qp_attr.cap.max_recv_wr);
461	}
462
463	static void smc_wr_init_sge(struct smc_link *lnk)
464	{
465	u32 i;
466
467	for (i = 0; i < lnk->wr_tx_cnt; i++) {
468	lnk->wr_tx_sges[i].addr =
469	lnk->wr_tx_dma_addr + i * SMC_WR_BUF_SIZE;
470	lnk->wr_tx_sges[i].length = SMC_WR_TX_SIZE;
bd4ad577	471	lnk->wr_tx_sges[i].lkey = lnk->roce_pd->local_dma_lkey;
ad6f317f UB	472	lnk->wr_tx_rdma_sges[i].tx_rdma_sge[0].wr_tx_rdma_sge[0].lkey =
	473	lnk->roce_pd->local_dma_lkey;
	474	lnk->wr_tx_rdma_sges[i].tx_rdma_sge[0].wr_tx_rdma_sge[1].lkey =
	475	lnk->roce_pd->local_dma_lkey;
	476	lnk->wr_tx_rdma_sges[i].tx_rdma_sge[1].wr_tx_rdma_sge[0].lkey =
	477	lnk->roce_pd->local_dma_lkey;
	478	lnk->wr_tx_rdma_sges[i].tx_rdma_sge[1].wr_tx_rdma_sge[1].lkey =
	479	lnk->roce_pd->local_dma_lkey;
f38ba179 UB	480	lnk->wr_tx_ibs[i].next = NULL;
	481	lnk->wr_tx_ibs[i].sg_list = &lnk->wr_tx_sges[i];
	482	lnk->wr_tx_ibs[i].num_sge = 1;
	483	lnk->wr_tx_ibs[i].opcode = IB_WR_SEND;
	484	lnk->wr_tx_ibs[i].send_flags =
2c9c1682	485	IB_SEND_SIGNALED \| IB_SEND_SOLICITED;
ad6f317f UB	486	lnk->wr_tx_rdmas[i].wr_tx_rdma[0].wr.opcode = IB_WR_RDMA_WRITE;
	487	lnk->wr_tx_rdmas[i].wr_tx_rdma[1].wr.opcode = IB_WR_RDMA_WRITE;
	488	lnk->wr_tx_rdmas[i].wr_tx_rdma[0].wr.sg_list =
	489	lnk->wr_tx_rdma_sges[i].tx_rdma_sge[0].wr_tx_rdma_sge;
	490	lnk->wr_tx_rdmas[i].wr_tx_rdma[1].wr.sg_list =
	491	lnk->wr_tx_rdma_sges[i].tx_rdma_sge[1].wr_tx_rdma_sge;
f38ba179 UB	492	}
	493	for (i = 0; i < lnk->wr_rx_cnt; i++) {
	494	lnk->wr_rx_sges[i].addr =
	495	lnk->wr_rx_dma_addr + i * SMC_WR_BUF_SIZE;
	496	lnk->wr_rx_sges[i].length = SMC_WR_BUF_SIZE;
bd4ad577	497	lnk->wr_rx_sges[i].lkey = lnk->roce_pd->local_dma_lkey;
f38ba179 UB	498	lnk->wr_rx_ibs[i].next = NULL;
	499	lnk->wr_rx_ibs[i].sg_list = &lnk->wr_rx_sges[i];
	500	lnk->wr_rx_ibs[i].num_sge = 1;
	501	}
652a1e41 UB	502	lnk->wr_reg.wr.next = NULL;
	503	lnk->wr_reg.wr.num_sge = 0;
	504	lnk->wr_reg.wr.send_flags = IB_SEND_SIGNALED;
	505	lnk->wr_reg.wr.opcode = IB_WR_REG_MR;
	506	lnk->wr_reg.access = IB_ACCESS_LOCAL_WRITE \| IB_ACCESS_REMOTE_WRITE;
f38ba179 UB	507	}
	508
	509	void smc_wr_free_link(struct smc_link *lnk)
	510	{
	511	struct ib_device *ibdev;
	512
	513	memset(lnk->wr_tx_mask, 0,
	514	BITS_TO_LONGS(SMC_WR_BUF_CNT) * sizeof(*lnk->wr_tx_mask));
	515
	516	if (!lnk->smcibdev)
	517	return;
	518	ibdev = lnk->smcibdev->ibdev;
	519
	520	if (lnk->wr_rx_dma_addr) {
	521	ib_dma_unmap_single(ibdev, lnk->wr_rx_dma_addr,
	522	SMC_WR_BUF_SIZE * lnk->wr_rx_cnt,
	523	DMA_FROM_DEVICE);
	524	lnk->wr_rx_dma_addr = 0;
	525	}
	526	if (lnk->wr_tx_dma_addr) {
	527	ib_dma_unmap_single(ibdev, lnk->wr_tx_dma_addr,
	528	SMC_WR_BUF_SIZE * lnk->wr_tx_cnt,
	529	DMA_TO_DEVICE);
	530	lnk->wr_tx_dma_addr = 0;
	531	}
	532	}
	533
	534	void smc_wr_free_link_mem(struct smc_link *lnk)
	535	{
	536	kfree(lnk->wr_tx_pends);
	537	lnk->wr_tx_pends = NULL;
	538	kfree(lnk->wr_tx_mask);
	539	lnk->wr_tx_mask = NULL;
	540	kfree(lnk->wr_tx_sges);
	541	lnk->wr_tx_sges = NULL;
ad6f317f UB	542	kfree(lnk->wr_tx_rdma_sges);
ad6f317f UB	543	lnk->wr_tx_rdma_sges = NULL;
f38ba179 UB	544	kfree(lnk->wr_rx_sges);
f38ba179 UB	545	lnk->wr_rx_sges = NULL;
ad6f317f UB	546	kfree(lnk->wr_tx_rdmas);
ad6f317f UB	547	lnk->wr_tx_rdmas = NULL;
f38ba179 UB	548	kfree(lnk->wr_rx_ibs);
	549	lnk->wr_rx_ibs = NULL;
	550	kfree(lnk->wr_tx_ibs);
	551	lnk->wr_tx_ibs = NULL;
	552	kfree(lnk->wr_tx_bufs);
	553	lnk->wr_tx_bufs = NULL;
	554	kfree(lnk->wr_rx_bufs);
	555	lnk->wr_rx_bufs = NULL;
	556	}
	557
	558	int smc_wr_alloc_link_mem(struct smc_link *link)
	559	{
	560	/* allocate link related memory */
	561	link->wr_tx_bufs = kcalloc(SMC_WR_BUF_CNT, SMC_WR_BUF_SIZE, GFP_KERNEL);
	562	if (!link->wr_tx_bufs)
	563	goto no_mem;
	564	link->wr_rx_bufs = kcalloc(SMC_WR_BUF_CNT * 3, SMC_WR_BUF_SIZE,
	565	GFP_KERNEL);
	566	if (!link->wr_rx_bufs)
	567	goto no_mem_wr_tx_bufs;
	568	link->wr_tx_ibs = kcalloc(SMC_WR_BUF_CNT, sizeof(link->wr_tx_ibs[0]),
	569	GFP_KERNEL);
	570	if (!link->wr_tx_ibs)
	571	goto no_mem_wr_rx_bufs;
	572	link->wr_rx_ibs = kcalloc(SMC_WR_BUF_CNT * 3,
	573	sizeof(link->wr_rx_ibs[0]),
	574	GFP_KERNEL);
	575	if (!link->wr_rx_ibs)
	576	goto no_mem_wr_tx_ibs;
ad6f317f UB	577	link->wr_tx_rdmas = kcalloc(SMC_WR_BUF_CNT,
	578	sizeof(link->wr_tx_rdmas[0]),
	579	GFP_KERNEL);
	580	if (!link->wr_tx_rdmas)
	581	goto no_mem_wr_rx_ibs;
	582	link->wr_tx_rdma_sges = kcalloc(SMC_WR_BUF_CNT,
	583	sizeof(link->wr_tx_rdma_sges[0]),
	584	GFP_KERNEL);
	585	if (!link->wr_tx_rdma_sges)
	586	goto no_mem_wr_tx_rdmas;
f38ba179 UB	587	link->wr_tx_sges = kcalloc(SMC_WR_BUF_CNT, sizeof(link->wr_tx_sges[0]),
	588	GFP_KERNEL);
	589	if (!link->wr_tx_sges)
ad6f317f	590	goto no_mem_wr_tx_rdma_sges;
f38ba179 UB	591	link->wr_rx_sges = kcalloc(SMC_WR_BUF_CNT * 3,
	592	sizeof(link->wr_rx_sges[0]),
	593	GFP_KERNEL);
	594	if (!link->wr_rx_sges)
	595	goto no_mem_wr_tx_sges;
6396bb22 KC	596	link->wr_tx_mask = kcalloc(BITS_TO_LONGS(SMC_WR_BUF_CNT),
	597	sizeof(*link->wr_tx_mask),
	598	GFP_KERNEL);
f38ba179 UB	599	if (!link->wr_tx_mask)
	600	goto no_mem_wr_rx_sges;
	601	link->wr_tx_pends = kcalloc(SMC_WR_BUF_CNT,
	602	sizeof(link->wr_tx_pends[0]),
	603	GFP_KERNEL);
	604	if (!link->wr_tx_pends)
	605	goto no_mem_wr_tx_mask;
	606	return 0;
	607
	608	no_mem_wr_tx_mask:
	609	kfree(link->wr_tx_mask);
	610	no_mem_wr_rx_sges:
	611	kfree(link->wr_rx_sges);
	612	no_mem_wr_tx_sges:
	613	kfree(link->wr_tx_sges);
ad6f317f UB	614	no_mem_wr_tx_rdma_sges:
	615	kfree(link->wr_tx_rdma_sges);
	616	no_mem_wr_tx_rdmas:
	617	kfree(link->wr_tx_rdmas);
f38ba179 UB	618	no_mem_wr_rx_ibs:
	619	kfree(link->wr_rx_ibs);
	620	no_mem_wr_tx_ibs:
	621	kfree(link->wr_tx_ibs);
	622	no_mem_wr_rx_bufs:
	623	kfree(link->wr_rx_bufs);
	624	no_mem_wr_tx_bufs:
	625	kfree(link->wr_tx_bufs);
	626	no_mem:
	627	return -ENOMEM;
	628	}
	629
	630	void smc_wr_remove_dev(struct smc_ib_device *smcibdev)
	631	{
	632	tasklet_kill(&smcibdev->recv_tasklet);
	633	tasklet_kill(&smcibdev->send_tasklet);
	634	}
	635
	636	void smc_wr_add_dev(struct smc_ib_device *smcibdev)
	637	{
	638	tasklet_init(&smcibdev->recv_tasklet, smc_wr_rx_tasklet_fn,
	639	(unsigned long)smcibdev);
	640	tasklet_init(&smcibdev->send_tasklet, smc_wr_tx_tasklet_fn,
	641	(unsigned long)smcibdev);
	642	}
	643
	644	int smc_wr_create_link(struct smc_link *lnk)
	645	{
	646	struct ib_device *ibdev = lnk->smcibdev->ibdev;
	647	int rc = 0;
	648
	649	smc_wr_tx_set_wr_id(&lnk->wr_tx_id, 0);
	650	lnk->wr_rx_id = 0;
	651	lnk->wr_rx_dma_addr = ib_dma_map_single(
	652	ibdev, lnk->wr_rx_bufs, SMC_WR_BUF_SIZE * lnk->wr_rx_cnt,
	653	DMA_FROM_DEVICE);
	654	if (ib_dma_mapping_error(ibdev, lnk->wr_rx_dma_addr)) {
	655	lnk->wr_rx_dma_addr = 0;
	656	rc = -EIO;
	657	goto out;
	658	}
	659	lnk->wr_tx_dma_addr = ib_dma_map_single(
	660	ibdev, lnk->wr_tx_bufs, SMC_WR_BUF_SIZE * lnk->wr_tx_cnt,
	661	DMA_TO_DEVICE);
	662	if (ib_dma_mapping_error(ibdev, lnk->wr_tx_dma_addr)) {
	663	rc = -EIO;
	664	goto dma_unmap;
	665	}
	666	smc_wr_init_sge(lnk);
	667	memset(lnk->wr_tx_mask, 0,
	668	BITS_TO_LONGS(SMC_WR_BUF_CNT) * sizeof(*lnk->wr_tx_mask));
652a1e41 UB	669	init_waitqueue_head(&lnk->wr_tx_wait);
652a1e41 UB	670	init_waitqueue_head(&lnk->wr_reg_wait);
f38ba179 UB	671	return rc;
	672
	673	dma_unmap:
	674	ib_dma_unmap_single(ibdev, lnk->wr_rx_dma_addr,
	675	SMC_WR_BUF_SIZE * lnk->wr_rx_cnt,
	676	DMA_FROM_DEVICE);
	677	lnk->wr_rx_dma_addr = 0;
	678	out:
	679	return rc;
	680	}