[linux-block.git] / drivers / infiniband / hw / amso1100 / c2_rnic.c

/*
 * Copyright (c) 2005 Ammasso, Inc. All rights reserved.
 * Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 *
 */


#include <linux/module.h>
#include <linux/moduleparam.h>
#include <linux/pci.h>
#include <linux/netdevice.h>
#include <linux/etherdevice.h>
#include <linux/delay.h>
#include <linux/ethtool.h>
#include <linux/mii.h>
#include <linux/if_vlan.h>
#include <linux/crc32.h>
#include <linux/in.h>
#include <linux/ip.h>
#include <linux/tcp.h>
#include <linux/init.h>
#include <linux/dma-mapping.h>
#include <linux/mm.h>
#include <linux/inet.h>
#include <linux/vmalloc.h>

#include <linux/route.h>

#include <asm/io.h>
#include <asm/irq.h>
#include <asm/byteorder.h>
#include <rdma/ib_smi.h>
#include "c2.h"
#include "c2_vq.h"

/* Device capabilities */
#define C2_MIN_PAGESIZE  1024

#define C2_MAX_MRS       32768
#define C2_MAX_QPS       16000
#define C2_MAX_WQE_SZ    256
#define C2_MAX_QP_WR     ((128*1024)/C2_MAX_WQE_SZ)
#define C2_MAX_SGES      4
#define C2_MAX_SGE_RD    1
#define C2_MAX_CQS       32768
#define C2_MAX_CQES      4096
#define C2_MAX_PDS       16384

/*
 * Send the adapter INIT message to the amso1100
 */
static int c2_adapter_init(struct c2_dev *c2dev)
{
	struct c2wr_init_req wr;
	int err;

	memset(&wr, 0, sizeof(wr));
	c2_wr_set_id(&wr, CCWR_INIT);
	wr.hdr.context = 0;
	wr.hint_count = cpu_to_be64(c2dev->hint_count_dma);
	wr.q0_host_shared = cpu_to_be64(c2dev->req_vq.shared_dma);
	wr.q1_host_shared = cpu_to_be64(c2dev->rep_vq.shared_dma);
	wr.q1_host_msg_pool = cpu_to_be64(c2dev->rep_vq.host_dma);
	wr.q2_host_shared = cpu_to_be64(c2dev->aeq.shared_dma);
	wr.q2_host_msg_pool = cpu_to_be64(c2dev->aeq.host_dma);

	/* Post the init message */
	err = vq_send_wr(c2dev, (union c2wr *) & wr);

	return err;
}

/*
 * Send the adapter TERM message to the amso1100
 */
static void c2_adapter_term(struct c2_dev *c2dev)
{
	struct c2wr_init_req wr;

	memset(&wr, 0, sizeof(wr));
	c2_wr_set_id(&wr, CCWR_TERM);
	wr.hdr.context = 0;

	/* Post the init message */
	vq_send_wr(c2dev, (union c2wr *) & wr);
	c2dev->init = 0;

	return;
}

/*
 * Query the adapter
 */
static int c2_rnic_query(struct c2_dev *c2dev, struct ib_device_attr *props)
{
	struct c2_vq_req *vq_req;
	struct c2wr_rnic_query_req wr;
	struct c2wr_rnic_query_rep *reply;
	int err;

	vq_req = vq_req_alloc(c2dev);
	if (!vq_req)
		return -ENOMEM;

	c2_wr_set_id(&wr, CCWR_RNIC_QUERY);
	wr.hdr.context = (unsigned long) vq_req;
	wr.rnic_handle = c2dev->adapter_handle;

	vq_req_get(c2dev, vq_req);

	err = vq_send_wr(c2dev, (union c2wr *) &wr);
	if (err) {
		vq_req_put(c2dev, vq_req);
		goto bail1;
	}

	err = vq_wait_for_reply(c2dev, vq_req);
	if (err)
		goto bail1;

	reply =
	    (struct c2wr_rnic_query_rep *) (unsigned long) (vq_req->reply_msg);
	if (!reply)
		err = -ENOMEM;
	else
		err = c2_errno(reply);
	if (err)
		goto bail2;

	props->fw_ver =
		((u64)be32_to_cpu(reply->fw_ver_major) << 32) |
		((be32_to_cpu(reply->fw_ver_minor) & 0xFFFF) << 16) |
		(be32_to_cpu(reply->fw_ver_patch) & 0xFFFF);
	memcpy(&props->sys_image_guid, c2dev->netdev->dev_addr, 6);
	props->max_mr_size         = 0xFFFFFFFF;
	props->page_size_cap       = ~(C2_MIN_PAGESIZE-1);
	props->vendor_id           = be32_to_cpu(reply->vendor_id);
	props->vendor_part_id      = be32_to_cpu(reply->part_number);
	props->hw_ver              = be32_to_cpu(reply->hw_version);
	props->max_qp              = be32_to_cpu(reply->max_qps);
	props->max_qp_wr           = be32_to_cpu(reply->max_qp_depth);
	props->device_cap_flags    = c2dev->device_cap_flags;
	props->max_sge             = C2_MAX_SGES;
	props->max_sge_rd          = C2_MAX_SGE_RD;
	props->max_cq              = be32_to_cpu(reply->max_cqs);
	props->max_cqe             = be32_to_cpu(reply->max_cq_depth);
	props->max_mr              = be32_to_cpu(reply->max_mrs);
	props->max_pd              = be32_to_cpu(reply->max_pds);
	props->max_qp_rd_atom      = be32_to_cpu(reply->max_qp_ird);
	props->max_ee_rd_atom      = 0;
	props->max_res_rd_atom     = be32_to_cpu(reply->max_global_ird);
	props->max_qp_init_rd_atom = be32_to_cpu(reply->max_qp_ord);
	props->max_ee_init_rd_atom = 0;
	props->atomic_cap          = IB_ATOMIC_NONE;
	props->max_ee              = 0;
	props->max_rdd             = 0;
	props->max_mw              = be32_to_cpu(reply->max_mws);
	props->max_raw_ipv6_qp     = 0;
	props->max_raw_ethy_qp     = 0;
	props->max_mcast_grp       = 0;
	props->max_mcast_qp_attach = 0;
	props->max_total_mcast_qp_attach = 0;
	props->max_ah              = 0;
	props->max_fmr             = 0;
	props->max_map_per_fmr     = 0;
	props->max_srq             = 0;
	props->max_srq_wr          = 0;
	props->max_srq_sge         = 0;
	props->max_pkeys           = 0;
	props->local_ca_ack_delay  = 0;

 bail2:
	vq_repbuf_free(c2dev, reply);

 bail1:
	vq_req_free(c2dev, vq_req);
	return err;
}

/*
 * Add an IP address to the RNIC interface
 */
int c2_add_addr(struct c2_dev *c2dev, __be32 inaddr, __be32 inmask)
{
	struct c2_vq_req *vq_req;
	struct c2wr_rnic_setconfig_req *wr;
	struct c2wr_rnic_setconfig_rep *reply;
	struct c2_netaddr netaddr;
	int err, len;

	vq_req = vq_req_alloc(c2dev);
	if (!vq_req)
		return -ENOMEM;

	len = sizeof(struct c2_netaddr);
	wr = kmalloc(c2dev->req_vq.msg_size, GFP_KERNEL);
	if (!wr) {
		err = -ENOMEM;
		goto bail0;
	}

	c2_wr_set_id(wr, CCWR_RNIC_SETCONFIG);
	wr->hdr.context = (unsigned long) vq_req;
	wr->rnic_handle = c2dev->adapter_handle;
	wr->option = cpu_to_be32(C2_CFG_ADD_ADDR);

	netaddr.ip_addr = inaddr;
	netaddr.netmask = inmask;
	netaddr.mtu = 0;

	memcpy(wr->data, &netaddr, len);

	vq_req_get(c2dev, vq_req);

	err = vq_send_wr(c2dev, (union c2wr *) wr);
	if (err) {
		vq_req_put(c2dev, vq_req);
		goto bail1;
	}

	err = vq_wait_for_reply(c2dev, vq_req);
	if (err)
		goto bail1;

	reply =
	    (struct c2wr_rnic_setconfig_rep *) (unsigned long) (vq_req->reply_msg);
	if (!reply) {
		err = -ENOMEM;
		goto bail1;
	}

	err = c2_errno(reply);
	vq_repbuf_free(c2dev, reply);

      bail1:
	kfree(wr);
      bail0:
	vq_req_free(c2dev, vq_req);
	return err;
}

/*
 * Delete an IP address from the RNIC interface
 */
int c2_del_addr(struct c2_dev *c2dev, __be32 inaddr, __be32 inmask)
{
	struct c2_vq_req *vq_req;
	struct c2wr_rnic_setconfig_req *wr;
	struct c2wr_rnic_setconfig_rep *reply;
	struct c2_netaddr netaddr;
	int err, len;

	vq_req = vq_req_alloc(c2dev);
	if (!vq_req)
		return -ENOMEM;

	len = sizeof(struct c2_netaddr);
	wr = kmalloc(c2dev->req_vq.msg_size, GFP_KERNEL);
	if (!wr) {
		err = -ENOMEM;
		goto bail0;
	}

	c2_wr_set_id(wr, CCWR_RNIC_SETCONFIG);
	wr->hdr.context = (unsigned long) vq_req;
	wr->rnic_handle = c2dev->adapter_handle;
	wr->option = cpu_to_be32(C2_CFG_DEL_ADDR);

	netaddr.ip_addr = inaddr;
	netaddr.netmask = inmask;
	netaddr.mtu = 0;

	memcpy(wr->data, &netaddr, len);

	vq_req_get(c2dev, vq_req);

	err = vq_send_wr(c2dev, (union c2wr *) wr);
	if (err) {
		vq_req_put(c2dev, vq_req);
		goto bail1;
	}

	err = vq_wait_for_reply(c2dev, vq_req);
	if (err)
		goto bail1;

	reply =
	    (struct c2wr_rnic_setconfig_rep *) (unsigned long) (vq_req->reply_msg);
	if (!reply) {
		err = -ENOMEM;
		goto bail1;
	}

	err = c2_errno(reply);
	vq_repbuf_free(c2dev, reply);

      bail1:
	kfree(wr);
      bail0:
	vq_req_free(c2dev, vq_req);
	return err;
}

/*
 * Open a single RNIC instance to use with all
 * low level openib calls
 */
static int c2_rnic_open(struct c2_dev *c2dev)
{
	struct c2_vq_req *vq_req;
	union c2wr wr;
	struct c2wr_rnic_open_rep *reply;
	int err;

	vq_req = vq_req_alloc(c2dev);
	if (vq_req == NULL) {
		return -ENOMEM;
	}

	memset(&wr, 0, sizeof(wr));
	c2_wr_set_id(&wr, CCWR_RNIC_OPEN);
	wr.rnic_open.req.hdr.context = (unsigned long) (vq_req);
	wr.rnic_open.req.flags = cpu_to_be16(RNIC_PRIV_MODE);
	wr.rnic_open.req.port_num = cpu_to_be16(0);
	wr.rnic_open.req.user_context = (unsigned long) c2dev;

	vq_req_get(c2dev, vq_req);

	err = vq_send_wr(c2dev, &wr);
	if (err) {
		vq_req_put(c2dev, vq_req);
		goto bail0;
	}

	err = vq_wait_for_reply(c2dev, vq_req);
	if (err) {
		goto bail0;
	}

	reply = (struct c2wr_rnic_open_rep *) (unsigned long) (vq_req->reply_msg);
	if (!reply) {
		err = -ENOMEM;
		goto bail0;
	}

	if ((err = c2_errno(reply)) != 0) {
		goto bail1;
	}

	c2dev->adapter_handle = reply->rnic_handle;

      bail1:
	vq_repbuf_free(c2dev, reply);
      bail0:
	vq_req_free(c2dev, vq_req);
	return err;
}

/*
 * Close the RNIC instance
 */
static int c2_rnic_close(struct c2_dev *c2dev)
{
	struct c2_vq_req *vq_req;
	union c2wr wr;
	struct c2wr_rnic_close_rep *reply;
	int err;

	vq_req = vq_req_alloc(c2dev);
	if (vq_req == NULL) {
		return -ENOMEM;
	}

	memset(&wr, 0, sizeof(wr));
	c2_wr_set_id(&wr, CCWR_RNIC_CLOSE);
	wr.rnic_close.req.hdr.context = (unsigned long) vq_req;
	wr.rnic_close.req.rnic_handle = c2dev->adapter_handle;

	vq_req_get(c2dev, vq_req);

	err = vq_send_wr(c2dev, &wr);
	if (err) {
		vq_req_put(c2dev, vq_req);
		goto bail0;
	}

	err = vq_wait_for_reply(c2dev, vq_req);
	if (err) {
		goto bail0;
	}

	reply = (struct c2wr_rnic_close_rep *) (unsigned long) (vq_req->reply_msg);
	if (!reply) {
		err = -ENOMEM;
		goto bail0;
	}

	if ((err = c2_errno(reply)) != 0) {
		goto bail1;
	}

	c2dev->adapter_handle = 0;

      bail1:
	vq_repbuf_free(c2dev, reply);
      bail0:
	vq_req_free(c2dev, vq_req);
	return err;
}

/*
 * Called by c2_probe to initialize the RNIC. This principally
 * involves initalizing the various limits and resouce pools that
 * comprise the RNIC instance.
 */
int __devinit c2_rnic_init(struct c2_dev *c2dev)
{
	int err;
	u32 qsize, msgsize;
	void *q1_pages;
	void *q2_pages;
	void __iomem *mmio_regs;

	/* Device capabilities */
	c2dev->device_cap_flags =
	    (IB_DEVICE_RESIZE_MAX_WR |
	     IB_DEVICE_CURR_QP_STATE_MOD |
	     IB_DEVICE_SYS_IMAGE_GUID |
	     IB_DEVICE_ZERO_STAG |
	     IB_DEVICE_MEM_WINDOW |
	     IB_DEVICE_SEND_W_INV);

	/* Allocate the qptr_array */
	c2dev->qptr_array = vmalloc(C2_MAX_CQS * sizeof(void *));
	if (!c2dev->qptr_array) {
		return -ENOMEM;
	}

	/* Inialize the qptr_array */
	memset(c2dev->qptr_array, 0, C2_MAX_CQS * sizeof(void *));
	c2dev->qptr_array[0] = (void *) &c2dev->req_vq;
	c2dev->qptr_array[1] = (void *) &c2dev->rep_vq;
	c2dev->qptr_array[2] = (void *) &c2dev->aeq;

	/* Initialize data structures */
	init_waitqueue_head(&c2dev->req_vq_wo);
	spin_lock_init(&c2dev->vqlock);
	spin_lock_init(&c2dev->lock);

	/* Allocate MQ shared pointer pool for kernel clients. User
	 * mode client pools are hung off the user context
	 */
	err = c2_init_mqsp_pool(c2dev, GFP_KERNEL, &c2dev->kern_mqsp_pool);
	if (err) {
		goto bail0;
	}

	/* Allocate shared pointers for Q0, Q1, and Q2 from
	 * the shared pointer pool.
	 */

	c2dev->hint_count = c2_alloc_mqsp(c2dev, c2dev->kern_mqsp_pool,
					     &c2dev->hint_count_dma,
					     GFP_KERNEL);
	c2dev->req_vq.shared = c2_alloc_mqsp(c2dev, c2dev->kern_mqsp_pool,
					     &c2dev->req_vq.shared_dma,
					     GFP_KERNEL);
	c2dev->rep_vq.shared = c2_alloc_mqsp(c2dev, c2dev->kern_mqsp_pool,
					     &c2dev->rep_vq.shared_dma,
					     GFP_KERNEL);
	c2dev->aeq.shared = c2_alloc_mqsp(c2dev, c2dev->kern_mqsp_pool,
					  &c2dev->aeq.shared_dma, GFP_KERNEL);
	if (!c2dev->hint_count || !c2dev->req_vq.shared ||
	    !c2dev->rep_vq.shared || !c2dev->aeq.shared) {
		err = -ENOMEM;
		goto bail1;
	}

	mmio_regs = c2dev->kva;
	/* Initialize the Verbs Request Queue */
	c2_mq_req_init(&c2dev->req_vq, 0,
		       be32_to_cpu((__force __be32) readl(mmio_regs + C2_REGS_Q0_QSIZE)),
		       be32_to_cpu((__force __be32) readl(mmio_regs + C2_REGS_Q0_MSGSIZE)),
		       mmio_regs +
		       be32_to_cpu((__force __be32) readl(mmio_regs + C2_REGS_Q0_POOLSTART)),
		       mmio_regs +
		       be32_to_cpu((__force __be32) readl(mmio_regs + C2_REGS_Q0_SHARED)),
		       C2_MQ_ADAPTER_TARGET);

	/* Initialize the Verbs Reply Queue */
	qsize = be32_to_cpu((__force __be32) readl(mmio_regs + C2_REGS_Q1_QSIZE));
	msgsize = be32_to_cpu((__force __be32) readl(mmio_regs + C2_REGS_Q1_MSGSIZE));
	q1_pages = dma_alloc_coherent(&c2dev->pcidev->dev, qsize * msgsize,
				      &c2dev->rep_vq.host_dma, GFP_KERNEL);
	if (!q1_pages) {
		err = -ENOMEM;
		goto bail1;
	}
	pci_unmap_addr_set(&c2dev->rep_vq, mapping, c2dev->rep_vq.host_dma);
	pr_debug("%s rep_vq va %p dma %llx\n", __func__, q1_pages,
		 (unsigned long long) c2dev->rep_vq.host_dma);
	c2_mq_rep_init(&c2dev->rep_vq,
		   1,
		   qsize,
		   msgsize,
		   q1_pages,
		   mmio_regs +
		   be32_to_cpu((__force __be32) readl(mmio_regs + C2_REGS_Q1_SHARED)),
		   C2_MQ_HOST_TARGET);

	/* Initialize the Asynchronus Event Queue */
	qsize = be32_to_cpu((__force __be32) readl(mmio_regs + C2_REGS_Q2_QSIZE));
	msgsize = be32_to_cpu((__force __be32) readl(mmio_regs + C2_REGS_Q2_MSGSIZE));
	q2_pages = dma_alloc_coherent(&c2dev->pcidev->dev, qsize * msgsize,
				      &c2dev->aeq.host_dma, GFP_KERNEL);
	if (!q2_pages) {
		err = -ENOMEM;
		goto bail2;
	}
	pci_unmap_addr_set(&c2dev->aeq, mapping, c2dev->aeq.host_dma);
	pr_debug("%s aeq va %p dma %llx\n", __func__, q2_pages,
		 (unsigned long long) c2dev->aeq.host_dma);
	c2_mq_rep_init(&c2dev->aeq,
		       2,
		       qsize,
		       msgsize,
		       q2_pages,
		       mmio_regs +
		       be32_to_cpu((__force __be32) readl(mmio_regs + C2_REGS_Q2_SHARED)),
		       C2_MQ_HOST_TARGET);

	/* Initialize the verbs request allocator */
	err = vq_init(c2dev);
	if (err)
		goto bail3;

	/* Enable interrupts on the adapter */
	writel(0, c2dev->regs + C2_IDIS);

	/* create the WR init message */
	err = c2_adapter_init(c2dev);
	if (err)
		goto bail4;
	c2dev->init++;

	/* open an adapter instance */
	err = c2_rnic_open(c2dev);
	if (err)
		goto bail4;

	/* Initialize cached the adapter limits */
	if (c2_rnic_query(c2dev, &c2dev->props))
		goto bail5;

	/* Initialize the PD pool */
	err = c2_init_pd_table(c2dev);
	if (err)
		goto bail5;

	/* Initialize the QP pool */
	c2_init_qp_table(c2dev);
	return 0;

      bail5:
	c2_rnic_close(c2dev);
      bail4:
	vq_term(c2dev);
      bail3:
	dma_free_coherent(&c2dev->pcidev->dev,
			  c2dev->aeq.q_size * c2dev->aeq.msg_size,
			  q2_pages, pci_unmap_addr(&c2dev->aeq, mapping));
      bail2:
	dma_free_coherent(&c2dev->pcidev->dev,
			  c2dev->rep_vq.q_size * c2dev->rep_vq.msg_size,
			  q1_pages, pci_unmap_addr(&c2dev->rep_vq, mapping));
      bail1:
	c2_free_mqsp_pool(c2dev, c2dev->kern_mqsp_pool);
      bail0:
	vfree(c2dev->qptr_array);

	return err;
}

/*
 * Called by c2_remove to cleanup the RNIC resources.
 */
void __devexit c2_rnic_term(struct c2_dev *c2dev)
{

	/* Close the open adapter instance */
	c2_rnic_close(c2dev);

	/* Send the TERM message to the adapter */
	c2_adapter_term(c2dev);

	/* Disable interrupts on the adapter */
	writel(1, c2dev->regs + C2_IDIS);

	/* Free the QP pool */
	c2_cleanup_qp_table(c2dev);

	/* Free the PD pool */
	c2_cleanup_pd_table(c2dev);

	/* Free the verbs request allocator */
	vq_term(c2dev);

	/* Free the asynchronus event queue */
	dma_free_coherent(&c2dev->pcidev->dev,
			  c2dev->aeq.q_size * c2dev->aeq.msg_size,
			  c2dev->aeq.msg_pool.host,
			  pci_unmap_addr(&c2dev->aeq, mapping));

	/* Free the verbs reply queue */
	dma_free_coherent(&c2dev->pcidev->dev,
			  c2dev->rep_vq.q_size * c2dev->rep_vq.msg_size,
			  c2dev->rep_vq.msg_pool.host,
			  pci_unmap_addr(&c2dev->rep_vq, mapping));

	/* Free the MQ shared pointer pool */
	c2_free_mqsp_pool(c2dev, c2dev->kern_mqsp_pool);

	/* Free the qptr_array */
	vfree(c2dev->qptr_array);

	return;
}
Commit	Line	Data
f94b533d TT	1	/*
	2	* Copyright (c) 2005 Ammasso, Inc. All rights reserved.
	3	* Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved.
	4	*
	5	* This software is available to you under a choice of one of two
	6	* licenses. You may choose to be licensed under the terms of the GNU
	7	* General Public License (GPL) Version 2, available from the file
	8	* COPYING in the main directory of this source tree, or the
	9	* OpenIB.org BSD license below:
	10	*
	11	* Redistribution and use in source and binary forms, with or
	12	* without modification, are permitted provided that the following
	13	* conditions are met:
	14	*
	15	* - Redistributions of source code must retain the above
	16	* copyright notice, this list of conditions and the following
	17	* disclaimer.
	18	*
	19	* - Redistributions in binary form must reproduce the above
	20	* copyright notice, this list of conditions and the following
	21	* disclaimer in the documentation and/or other materials
	22	* provided with the distribution.
	23	*
	24	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	25	* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	26	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
	27	* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
	28	* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
	29	* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
	30	* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	31	* SOFTWARE.
	32	*
	33	*/
	34
	35
	36	#include <linux/module.h>
	37	#include <linux/moduleparam.h>
	38	#include <linux/pci.h>
	39	#include <linux/netdevice.h>
	40	#include <linux/etherdevice.h>
	41	#include <linux/delay.h>
	42	#include <linux/ethtool.h>
	43	#include <linux/mii.h>
	44	#include <linux/if_vlan.h>
	45	#include <linux/crc32.h>
	46	#include <linux/in.h>
	47	#include <linux/ip.h>
	48	#include <linux/tcp.h>
	49	#include <linux/init.h>
	50	#include <linux/dma-mapping.h>
	51	#include <linux/mm.h>
	52	#include <linux/inet.h>
d7b20045	53	#include <linux/vmalloc.h>
f94b533d TT	54
	55	#include <linux/route.h>
	56
	57	#include <asm/io.h>
	58	#include <asm/irq.h>
	59	#include <asm/byteorder.h>
	60	#include <rdma/ib_smi.h>
	61	#include "c2.h"
	62	#include "c2_vq.h"
	63
	64	/* Device capabilities */
	65	#define C2_MIN_PAGESIZE 1024
	66
	67	#define C2_MAX_MRS 32768
	68	#define C2_MAX_QPS 16000
	69	#define C2_MAX_WQE_SZ 256
	70	#define C2_MAX_QP_WR ((128*1024)/C2_MAX_WQE_SZ)
	71	#define C2_MAX_SGES 4
	72	#define C2_MAX_SGE_RD 1
	73	#define C2_MAX_CQS 32768
	74	#define C2_MAX_CQES 4096
	75	#define C2_MAX_PDS 16384
	76
	77	/*
	78	* Send the adapter INIT message to the amso1100
	79	*/
	80	static int c2_adapter_init(struct c2_dev *c2dev)
	81	{
	82	struct c2wr_init_req wr;
	83	int err;
	84
	85	memset(&wr, 0, sizeof(wr));
	86	c2_wr_set_id(&wr, CCWR_INIT);
	87	wr.hdr.context = 0;
	88	wr.hint_count = cpu_to_be64(c2dev->hint_count_dma);
	89	wr.q0_host_shared = cpu_to_be64(c2dev->req_vq.shared_dma);
	90	wr.q1_host_shared = cpu_to_be64(c2dev->rep_vq.shared_dma);
	91	wr.q1_host_msg_pool = cpu_to_be64(c2dev->rep_vq.host_dma);
	92	wr.q2_host_shared = cpu_to_be64(c2dev->aeq.shared_dma);
	93	wr.q2_host_msg_pool = cpu_to_be64(c2dev->aeq.host_dma);
	94
	95	/* Post the init message */
	96	err = vq_send_wr(c2dev, (union c2wr *) & wr);
	97
	98	return err;
	99	}
	100
	101	/*
	102	* Send the adapter TERM message to the amso1100
	103	*/
	104	static void c2_adapter_term(struct c2_dev *c2dev)
	105	{
	106	struct c2wr_init_req wr;
	107
	108	memset(&wr, 0, sizeof(wr));
	109	c2_wr_set_id(&wr, CCWR_TERM);
	110	wr.hdr.context = 0;
	111
	112	/* Post the init message */
	113	vq_send_wr(c2dev, (union c2wr *) & wr);
	114	c2dev->init = 0;
	115
	116	return;
	117	}
118
119	/*
120	* Query the adapter
121	*/
122	static int c2_rnic_query(struct c2_dev c2dev, struct ib_device_attr props)
123	{
124	struct c2_vq_req *vq_req;
125	struct c2wr_rnic_query_req wr;
126	struct c2wr_rnic_query_rep *reply;
127	int err;
128
129	vq_req = vq_req_alloc(c2dev);
130	if (!vq_req)
131	return -ENOMEM;
132
133	c2_wr_set_id(&wr, CCWR_RNIC_QUERY);
134	wr.hdr.context = (unsigned long) vq_req;
135	wr.rnic_handle = c2dev->adapter_handle;
136
137	vq_req_get(c2dev, vq_req);
138
139	err = vq_send_wr(c2dev, (union c2wr *) &wr);
140	if (err) {
141	vq_req_put(c2dev, vq_req);
142	goto bail1;
143	}
144
145	err = vq_wait_for_reply(c2dev, vq_req);
146	if (err)
147	goto bail1;
148
149	reply =
150	(struct c2wr_rnic_query_rep *) (unsigned long) (vq_req->reply_msg);
151	if (!reply)
152	err = -ENOMEM;
fb7711e7 AB	153	else
fb7711e7 AB	154	err = c2_errno(reply);
f94b533d TT	155	if (err)
	156	goto bail2;
	157
	158	props->fw_ver =
	159	((u64)be32_to_cpu(reply->fw_ver_major) << 32) \|
b26c791e JD	160	((be32_to_cpu(reply->fw_ver_minor) & 0xFFFF) << 16) \|
b26c791e JD	161	(be32_to_cpu(reply->fw_ver_patch) & 0xFFFF);
f94b533d TT	162	memcpy(&props->sys_image_guid, c2dev->netdev->dev_addr, 6);
	163	props->max_mr_size = 0xFFFFFFFF;
	164	props->page_size_cap = ~(C2_MIN_PAGESIZE-1);
	165	props->vendor_id = be32_to_cpu(reply->vendor_id);
	166	props->vendor_part_id = be32_to_cpu(reply->part_number);
	167	props->hw_ver = be32_to_cpu(reply->hw_version);
	168	props->max_qp = be32_to_cpu(reply->max_qps);
	169	props->max_qp_wr = be32_to_cpu(reply->max_qp_depth);
	170	props->device_cap_flags = c2dev->device_cap_flags;
	171	props->max_sge = C2_MAX_SGES;
	172	props->max_sge_rd = C2_MAX_SGE_RD;
	173	props->max_cq = be32_to_cpu(reply->max_cqs);
	174	props->max_cqe = be32_to_cpu(reply->max_cq_depth);
	175	props->max_mr = be32_to_cpu(reply->max_mrs);
	176	props->max_pd = be32_to_cpu(reply->max_pds);
	177	props->max_qp_rd_atom = be32_to_cpu(reply->max_qp_ird);
	178	props->max_ee_rd_atom = 0;
	179	props->max_res_rd_atom = be32_to_cpu(reply->max_global_ird);
	180	props->max_qp_init_rd_atom = be32_to_cpu(reply->max_qp_ord);
	181	props->max_ee_init_rd_atom = 0;
	182	props->atomic_cap = IB_ATOMIC_NONE;
	183	props->max_ee = 0;
	184	props->max_rdd = 0;
	185	props->max_mw = be32_to_cpu(reply->max_mws);
	186	props->max_raw_ipv6_qp = 0;
	187	props->max_raw_ethy_qp = 0;
	188	props->max_mcast_grp = 0;
	189	props->max_mcast_qp_attach = 0;
	190	props->max_total_mcast_qp_attach = 0;
	191	props->max_ah = 0;
	192	props->max_fmr = 0;
	193	props->max_map_per_fmr = 0;
	194	props->max_srq = 0;
	195	props->max_srq_wr = 0;
	196	props->max_srq_sge = 0;
	197	props->max_pkeys = 0;
	198	props->local_ca_ack_delay = 0;
	199
	200	bail2:
	201	vq_repbuf_free(c2dev, reply);
	202
	203	bail1:
	204	vq_req_free(c2dev, vq_req);
	205	return err;
	206	}
	207
	208	/*
	209	* Add an IP address to the RNIC interface
	210	*/
dc544bc9	211	int c2_add_addr(struct c2_dev *c2dev, __be32 inaddr, __be32 inmask)
f94b533d TT	212	{
	213	struct c2_vq_req *vq_req;
	214	struct c2wr_rnic_setconfig_req *wr;
	215	struct c2wr_rnic_setconfig_rep *reply;
	216	struct c2_netaddr netaddr;
	217	int err, len;
	218
	219	vq_req = vq_req_alloc(c2dev);
	220	if (!vq_req)
	221	return -ENOMEM;
	222
	223	len = sizeof(struct c2_netaddr);
	224	wr = kmalloc(c2dev->req_vq.msg_size, GFP_KERNEL);
	225	if (!wr) {
	226	err = -ENOMEM;
	227	goto bail0;
	228	}
	229
	230	c2_wr_set_id(wr, CCWR_RNIC_SETCONFIG);
	231	wr->hdr.context = (unsigned long) vq_req;
	232	wr->rnic_handle = c2dev->adapter_handle;
	233	wr->option = cpu_to_be32(C2_CFG_ADD_ADDR);
	234
	235	netaddr.ip_addr = inaddr;
	236	netaddr.netmask = inmask;
	237	netaddr.mtu = 0;
	238
	239	memcpy(wr->data, &netaddr, len);
	240
	241	vq_req_get(c2dev, vq_req);
	242
	243	err = vq_send_wr(c2dev, (union c2wr *) wr);
	244	if (err) {
	245	vq_req_put(c2dev, vq_req);
	246	goto bail1;
	247	}
	248
	249	err = vq_wait_for_reply(c2dev, vq_req);
	250	if (err)
	251	goto bail1;
	252
	253	reply =
	254	(struct c2wr_rnic_setconfig_rep *) (unsigned long) (vq_req->reply_msg);
	255	if (!reply) {
	256	err = -ENOMEM;
	257	goto bail1;
	258	}
	259
	260	err = c2_errno(reply);
	261	vq_repbuf_free(c2dev, reply);
	262
	263	bail1:
	264	kfree(wr);
	265	bail0:
	266	vq_req_free(c2dev, vq_req);
	267	return err;
	268	}
	269
	270	/*
	271	* Delete an IP address from the RNIC interface
	272	*/
dc544bc9	273	int c2_del_addr(struct c2_dev *c2dev, __be32 inaddr, __be32 inmask)
f94b533d TT	274	{
	275	struct c2_vq_req *vq_req;
	276	struct c2wr_rnic_setconfig_req *wr;
	277	struct c2wr_rnic_setconfig_rep *reply;
	278	struct c2_netaddr netaddr;
	279	int err, len;
	280
	281	vq_req = vq_req_alloc(c2dev);
	282	if (!vq_req)
	283	return -ENOMEM;
	284
	285	len = sizeof(struct c2_netaddr);
	286	wr = kmalloc(c2dev->req_vq.msg_size, GFP_KERNEL);
	287	if (!wr) {
	288	err = -ENOMEM;
	289	goto bail0;
	290	}
	291
	292	c2_wr_set_id(wr, CCWR_RNIC_SETCONFIG);
	293	wr->hdr.context = (unsigned long) vq_req;
	294	wr->rnic_handle = c2dev->adapter_handle;
	295	wr->option = cpu_to_be32(C2_CFG_DEL_ADDR);
	296
	297	netaddr.ip_addr = inaddr;
	298	netaddr.netmask = inmask;
	299	netaddr.mtu = 0;
	300
	301	memcpy(wr->data, &netaddr, len);
	302
	303	vq_req_get(c2dev, vq_req);
	304
	305	err = vq_send_wr(c2dev, (union c2wr *) wr);
	306	if (err) {
	307	vq_req_put(c2dev, vq_req);
	308	goto bail1;
	309	}
	310
	311	err = vq_wait_for_reply(c2dev, vq_req);
	312	if (err)
	313	goto bail1;
	314
	315	reply =
	316	(struct c2wr_rnic_setconfig_rep *) (unsigned long) (vq_req->reply_msg);
	317	if (!reply) {
	318	err = -ENOMEM;
	319	goto bail1;
	320	}
	321
	322	err = c2_errno(reply);
	323	vq_repbuf_free(c2dev, reply);
	324
	325	bail1:
	326	kfree(wr);
	327	bail0:
	328	vq_req_free(c2dev, vq_req);
	329	return err;
	330	}
	331
	332	/*
	333	* Open a single RNIC instance to use with all
	334	* low level openib calls
	335	*/
	336	static int c2_rnic_open(struct c2_dev *c2dev)
	337	{
338	struct c2_vq_req *vq_req;
339	union c2wr wr;
340	struct c2wr_rnic_open_rep *reply;
341	int err;
342
343	vq_req = vq_req_alloc(c2dev);
344	if (vq_req == NULL) {
345	return -ENOMEM;
346	}
347
348	memset(&wr, 0, sizeof(wr));
349	c2_wr_set_id(&wr, CCWR_RNIC_OPEN);
350	wr.rnic_open.req.hdr.context = (unsigned long) (vq_req);
351	wr.rnic_open.req.flags = cpu_to_be16(RNIC_PRIV_MODE);
352	wr.rnic_open.req.port_num = cpu_to_be16(0);
353	wr.rnic_open.req.user_context = (unsigned long) c2dev;
354
355	vq_req_get(c2dev, vq_req);
356
357	err = vq_send_wr(c2dev, &wr);
358	if (err) {
359	vq_req_put(c2dev, vq_req);
360	goto bail0;
361	}
362
363	err = vq_wait_for_reply(c2dev, vq_req);
364	if (err) {
365	goto bail0;
366	}
367
368	reply = (struct c2wr_rnic_open_rep *) (unsigned long) (vq_req->reply_msg);
369	if (!reply) {
370	err = -ENOMEM;
371	goto bail0;
372	}
373
374	if ((err = c2_errno(reply)) != 0) {
375	goto bail1;
376	}
377
378	c2dev->adapter_handle = reply->rnic_handle;
379
380	bail1:
381	vq_repbuf_free(c2dev, reply);
382	bail0:
383	vq_req_free(c2dev, vq_req);
384	return err;
385	}
386
387	/*
388	* Close the RNIC instance
389	*/
390	static int c2_rnic_close(struct c2_dev *c2dev)
391	{
392	struct c2_vq_req *vq_req;
393	union c2wr wr;
394	struct c2wr_rnic_close_rep *reply;
395	int err;
396
397	vq_req = vq_req_alloc(c2dev);
398	if (vq_req == NULL) {
399	return -ENOMEM;
400	}
401
402	memset(&wr, 0, sizeof(wr));
403	c2_wr_set_id(&wr, CCWR_RNIC_CLOSE);
404	wr.rnic_close.req.hdr.context = (unsigned long) vq_req;
405	wr.rnic_close.req.rnic_handle = c2dev->adapter_handle;
406
407	vq_req_get(c2dev, vq_req);
408
409	err = vq_send_wr(c2dev, &wr);
410	if (err) {
411	vq_req_put(c2dev, vq_req);
412	goto bail0;
413	}
414
415	err = vq_wait_for_reply(c2dev, vq_req);
416	if (err) {
417	goto bail0;
418	}
419
420	reply = (struct c2wr_rnic_close_rep *) (unsigned long) (vq_req->reply_msg);
421	if (!reply) {
422	err = -ENOMEM;
423	goto bail0;
424	}
425
426	if ((err = c2_errno(reply)) != 0) {
427	goto bail1;
428	}
429
430	c2dev->adapter_handle = 0;
431
432	bail1:
433	vq_repbuf_free(c2dev, reply);
434	bail0:
435	vq_req_free(c2dev, vq_req);
436	return err;
437	}
438
439	/*
440	* Called by c2_probe to initialize the RNIC. This principally
441	* involves initalizing the various limits and resouce pools that
442	* comprise the RNIC instance.
443	*/
29666128	444	int __devinit c2_rnic_init(struct c2_dev *c2dev)
f94b533d TT	445	{
	446	int err;
	447	u32 qsize, msgsize;
	448	void *q1_pages;
	449	void *q2_pages;
	450	void __iomem *mmio_regs;
	451
	452	/* Device capabilities */
	453	c2dev->device_cap_flags =
	454	(IB_DEVICE_RESIZE_MAX_WR \|
	455	IB_DEVICE_CURR_QP_STATE_MOD \|
	456	IB_DEVICE_SYS_IMAGE_GUID \|
	457	IB_DEVICE_ZERO_STAG \|
139b2db7 RD	458	IB_DEVICE_MEM_WINDOW \|
139b2db7 RD	459	IB_DEVICE_SEND_W_INV);
f94b533d TT	460
	461	/* Allocate the qptr_array */
	462	c2dev->qptr_array = vmalloc(C2_MAX_CQS * sizeof(void *));
	463	if (!c2dev->qptr_array) {
	464	return -ENOMEM;
	465	}
	466
	467	/* Inialize the qptr_array */
	468	memset(c2dev->qptr_array, 0, C2_MAX_CQS * sizeof(void *));
	469	c2dev->qptr_array[0] = (void *) &c2dev->req_vq;
	470	c2dev->qptr_array[1] = (void *) &c2dev->rep_vq;
	471	c2dev->qptr_array[2] = (void *) &c2dev->aeq;
	472
	473	/* Initialize data structures */
	474	init_waitqueue_head(&c2dev->req_vq_wo);
	475	spin_lock_init(&c2dev->vqlock);
	476	spin_lock_init(&c2dev->lock);
	477
	478	/* Allocate MQ shared pointer pool for kernel clients. User
	479	* mode client pools are hung off the user context
	480	*/
	481	err = c2_init_mqsp_pool(c2dev, GFP_KERNEL, &c2dev->kern_mqsp_pool);
	482	if (err) {
	483	goto bail0;
	484	}
	485
	486	/* Allocate shared pointers for Q0, Q1, and Q2 from
	487	* the shared pointer pool.
	488	*/
	489
	490	c2dev->hint_count = c2_alloc_mqsp(c2dev, c2dev->kern_mqsp_pool,
	491	&c2dev->hint_count_dma,
	492	GFP_KERNEL);
	493	c2dev->req_vq.shared = c2_alloc_mqsp(c2dev, c2dev->kern_mqsp_pool,
	494	&c2dev->req_vq.shared_dma,
	495	GFP_KERNEL);
	496	c2dev->rep_vq.shared = c2_alloc_mqsp(c2dev, c2dev->kern_mqsp_pool,
	497	&c2dev->rep_vq.shared_dma,
	498	GFP_KERNEL);
	499	c2dev->aeq.shared = c2_alloc_mqsp(c2dev, c2dev->kern_mqsp_pool,
	500	&c2dev->aeq.shared_dma, GFP_KERNEL);
	501	if (!c2dev->hint_count \|\| !c2dev->req_vq.shared \|\|
	502	!c2dev->rep_vq.shared \|\| !c2dev->aeq.shared) {
	503	err = -ENOMEM;
	504	goto bail1;
	505	}
	506
	507	mmio_regs = c2dev->kva;
	508	/* Initialize the Verbs Request Queue */
	509	c2_mq_req_init(&c2dev->req_vq, 0,
dc544bc9 RD	510	be32_to_cpu((__force __be32) readl(mmio_regs + C2_REGS_Q0_QSIZE)),
dc544bc9 RD	511	be32_to_cpu((__force __be32) readl(mmio_regs + C2_REGS_Q0_MSGSIZE)),
f94b533d	512	mmio_regs +
dc544bc9	513	be32_to_cpu((__force __be32) readl(mmio_regs + C2_REGS_Q0_POOLSTART)),
f94b533d	514	mmio_regs +
dc544bc9	515	be32_to_cpu((__force __be32) readl(mmio_regs + C2_REGS_Q0_SHARED)),
f94b533d TT	516	C2_MQ_ADAPTER_TARGET);
	517
	518	/* Initialize the Verbs Reply Queue */
dc544bc9 RD	519	qsize = be32_to_cpu((__force __be32) readl(mmio_regs + C2_REGS_Q1_QSIZE));
dc544bc9 RD	520	msgsize = be32_to_cpu((__force __be32) readl(mmio_regs + C2_REGS_Q1_MSGSIZE));
8de94ce1 SW	521	q1_pages = dma_alloc_coherent(&c2dev->pcidev->dev, qsize * msgsize,
8de94ce1 SW	522	&c2dev->rep_vq.host_dma, GFP_KERNEL);
f94b533d TT	523	if (!q1_pages) {
	524	err = -ENOMEM;
	525	goto bail1;
	526	}
f94b533d	527	pci_unmap_addr_set(&c2dev->rep_vq, mapping, c2dev->rep_vq.host_dma);
33718363	528	pr_debug("%s rep_vq va %p dma %llx\n", __func__, q1_pages,
6edf6023	529	(unsigned long long) c2dev->rep_vq.host_dma);
f94b533d TT	530	c2_mq_rep_init(&c2dev->rep_vq,
	531	1,
	532	qsize,
	533	msgsize,
	534	q1_pages,
	535	mmio_regs +
dc544bc9	536	be32_to_cpu((__force __be32) readl(mmio_regs + C2_REGS_Q1_SHARED)),
f94b533d TT	537	C2_MQ_HOST_TARGET);
	538
	539	/* Initialize the Asynchronus Event Queue */
dc544bc9 RD	540	qsize = be32_to_cpu((__force __be32) readl(mmio_regs + C2_REGS_Q2_QSIZE));
dc544bc9 RD	541	msgsize = be32_to_cpu((__force __be32) readl(mmio_regs + C2_REGS_Q2_MSGSIZE));
8de94ce1 SW	542	q2_pages = dma_alloc_coherent(&c2dev->pcidev->dev, qsize * msgsize,
8de94ce1 SW	543	&c2dev->aeq.host_dma, GFP_KERNEL);
f94b533d TT	544	if (!q2_pages) {
	545	err = -ENOMEM;
	546	goto bail2;
	547	}
f94b533d	548	pci_unmap_addr_set(&c2dev->aeq, mapping, c2dev->aeq.host_dma);
33718363	549	pr_debug("%s aeq va %p dma %llx\n", __func__, q2_pages,
d7b748d6	550	(unsigned long long) c2dev->aeq.host_dma);
f94b533d TT	551	c2_mq_rep_init(&c2dev->aeq,
	552	2,
	553	qsize,
	554	msgsize,
	555	q2_pages,
	556	mmio_regs +
dc544bc9	557	be32_to_cpu((__force __be32) readl(mmio_regs + C2_REGS_Q2_SHARED)),
f94b533d TT	558	C2_MQ_HOST_TARGET);
	559
	560	/* Initialize the verbs request allocator */
	561	err = vq_init(c2dev);
	562	if (err)
	563	goto bail3;
	564
	565	/* Enable interrupts on the adapter */
	566	writel(0, c2dev->regs + C2_IDIS);
	567
	568	/* create the WR init message */
	569	err = c2_adapter_init(c2dev);
	570	if (err)
	571	goto bail4;
	572	c2dev->init++;
	573
	574	/* open an adapter instance */
	575	err = c2_rnic_open(c2dev);
	576	if (err)
	577	goto bail4;
	578
	579	/* Initialize cached the adapter limits */
	580	if (c2_rnic_query(c2dev, &c2dev->props))
	581	goto bail5;
	582
	583	/* Initialize the PD pool */
	584	err = c2_init_pd_table(c2dev);
	585	if (err)
	586	goto bail5;
	587
	588	/* Initialize the QP pool */
	589	c2_init_qp_table(c2dev);
	590	return 0;
	591
	592	bail5:
	593	c2_rnic_close(c2dev);
	594	bail4:
	595	vq_term(c2dev);
	596	bail3:
8de94ce1 SW	597	dma_free_coherent(&c2dev->pcidev->dev,
	598	c2dev->aeq.q_size * c2dev->aeq.msg_size,
	599	q2_pages, pci_unmap_addr(&c2dev->aeq, mapping));
f94b533d	600	bail2:
8de94ce1 SW	601	dma_free_coherent(&c2dev->pcidev->dev,
	602	c2dev->rep_vq.q_size * c2dev->rep_vq.msg_size,
	603	q1_pages, pci_unmap_addr(&c2dev->rep_vq, mapping));
f94b533d TT	604	bail1:
	605	c2_free_mqsp_pool(c2dev, c2dev->kern_mqsp_pool);
	606	bail0:
	607	vfree(c2dev->qptr_array);
	608
	609	return err;
	610	}
	611
	612	/*
	613	* Called by c2_remove to cleanup the RNIC resources.
	614	*/
29666128	615	void __devexit c2_rnic_term(struct c2_dev *c2dev)
f94b533d TT	616	{
	617
	618	/* Close the open adapter instance */
	619	c2_rnic_close(c2dev);
	620
	621	/* Send the TERM message to the adapter */
	622	c2_adapter_term(c2dev);
	623
	624	/* Disable interrupts on the adapter */
	625	writel(1, c2dev->regs + C2_IDIS);
	626
	627	/* Free the QP pool */
	628	c2_cleanup_qp_table(c2dev);
	629
	630	/* Free the PD pool */
	631	c2_cleanup_pd_table(c2dev);
	632
	633	/* Free the verbs request allocator */
	634	vq_term(c2dev);
	635
8de94ce1 SW	636	/* Free the asynchronus event queue */
	637	dma_free_coherent(&c2dev->pcidev->dev,
	638	c2dev->aeq.q_size * c2dev->aeq.msg_size,
	639	c2dev->aeq.msg_pool.host,
	640	pci_unmap_addr(&c2dev->aeq, mapping));
	641
	642	/* Free the verbs reply queue */
	643	dma_free_coherent(&c2dev->pcidev->dev,
	644	c2dev->rep_vq.q_size * c2dev->rep_vq.msg_size,
	645	c2dev->rep_vq.msg_pool.host,
	646	pci_unmap_addr(&c2dev->rep_vq, mapping));
f94b533d TT	647
	648	/* Free the MQ shared pointer pool */
	649	c2_free_mqsp_pool(c2dev, c2dev->kern_mqsp_pool);
	650
	651	/* Free the qptr_array */
	652	vfree(c2dev->qptr_array);
	653
	654	return;
	655	}