[linux-2.6-block.git] / drivers / infiniband / hw / cxgb4 / provider.c

/*
 * Copyright (c) 2009-2010 Chelsio, Inc. All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */
#include <linux/module.h>
#include <linux/moduleparam.h>
#include <linux/device.h>
#include <linux/netdevice.h>
#include <linux/etherdevice.h>
#include <linux/delay.h>
#include <linux/errno.h>
#include <linux/list.h>
#include <linux/spinlock.h>
#include <linux/ethtool.h>
#include <linux/rtnetlink.h>
#include <linux/inetdevice.h>
#include <linux/io.h>

#include <asm/irq.h>
#include <asm/byteorder.h>

#include <rdma/iw_cm.h>
#include <rdma/ib_verbs.h>
#include <rdma/ib_smi.h>
#include <rdma/ib_umem.h>
#include <rdma/ib_user_verbs.h>

#include "iw_cxgb4.h"

static int fastreg_support = 1;
module_param(fastreg_support, int, 0644);
MODULE_PARM_DESC(fastreg_support, "Advertise fastreg support (default=1)");

static void c4iw_dealloc_ucontext(struct ib_ucontext *context)
{
	struct c4iw_ucontext *ucontext = to_c4iw_ucontext(context);
	struct c4iw_dev *rhp;
	struct c4iw_mm_entry *mm, *tmp;

	pr_debug("context %p\n", context);
	rhp = to_c4iw_dev(ucontext->ibucontext.device);

	list_for_each_entry_safe(mm, tmp, &ucontext->mmaps, entry)
		kfree(mm);
	c4iw_release_dev_ucontext(&rhp->rdev, &ucontext->uctx);
}

static int c4iw_alloc_ucontext(struct ib_ucontext *ucontext,
			       struct ib_udata *udata)
{
	struct ib_device *ibdev = ucontext->device;
	struct c4iw_ucontext *context = to_c4iw_ucontext(ucontext);
	struct c4iw_dev *rhp = to_c4iw_dev(ibdev);
	struct c4iw_alloc_ucontext_resp uresp;
	int ret = 0;
	struct c4iw_mm_entry *mm = NULL;

	pr_debug("ibdev %p\n", ibdev);
	c4iw_init_dev_ucontext(&rhp->rdev, &context->uctx);
	INIT_LIST_HEAD(&context->mmaps);
	spin_lock_init(&context->mmap_lock);

	if (udata->outlen < sizeof(uresp) - sizeof(uresp.reserved)) {
		pr_err_once("Warning - downlevel libcxgb4 (non-fatal), device status page disabled\n");
		rhp->rdev.flags |= T4_STATUS_PAGE_DISABLED;
	} else {
		mm = kmalloc(sizeof(*mm), GFP_KERNEL);
		if (!mm) {
			ret = -ENOMEM;
			goto err;
		}

		uresp.status_page_size = PAGE_SIZE;

		spin_lock(&context->mmap_lock);
		uresp.status_page_key = context->key;
		context->key += PAGE_SIZE;
		spin_unlock(&context->mmap_lock);

		ret = ib_copy_to_udata(udata, &uresp,
				       sizeof(uresp) - sizeof(uresp.reserved));
		if (ret)
			goto err_mm;

		mm->key = uresp.status_page_key;
		mm->addr = virt_to_phys(rhp->rdev.status_page);
		mm->len = PAGE_SIZE;
		insert_mmap(context, mm);
	}
	return 0;
err_mm:
	kfree(mm);
err:
	return ret;
}

static int c4iw_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
{
	int len = vma->vm_end - vma->vm_start;
	u32 key = vma->vm_pgoff << PAGE_SHIFT;
	struct c4iw_rdev *rdev;
	int ret = 0;
	struct c4iw_mm_entry *mm;
	struct c4iw_ucontext *ucontext;
	u64 addr;

	pr_debug("pgoff 0x%lx key 0x%x len %d\n", vma->vm_pgoff,
		 key, len);

	if (vma->vm_start & (PAGE_SIZE-1))
		return -EINVAL;

	rdev = &(to_c4iw_dev(context->device)->rdev);
	ucontext = to_c4iw_ucontext(context);

	mm = remove_mmap(ucontext, key, len);
	if (!mm)
		return -EINVAL;
	addr = mm->addr;
	kfree(mm);

	if ((addr >= pci_resource_start(rdev->lldi.pdev, 0)) &&
	    (addr < (pci_resource_start(rdev->lldi.pdev, 0) +
		    pci_resource_len(rdev->lldi.pdev, 0)))) {

		/*
		 * MA_SYNC register...
		 */
		vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
		ret = io_remap_pfn_range(vma, vma->vm_start,
					 addr >> PAGE_SHIFT,
					 len, vma->vm_page_prot);
	} else if ((addr >= pci_resource_start(rdev->lldi.pdev, 2)) &&
		   (addr < (pci_resource_start(rdev->lldi.pdev, 2) +
		    pci_resource_len(rdev->lldi.pdev, 2)))) {

		/*
		 * Map user DB or OCQP memory...
		 */
		if (addr >= rdev->oc_mw_pa)
			vma->vm_page_prot = t4_pgprot_wc(vma->vm_page_prot);
		else {
			if (!is_t4(rdev->lldi.adapter_type))
				vma->vm_page_prot =
					t4_pgprot_wc(vma->vm_page_prot);
			else
				vma->vm_page_prot =
					pgprot_noncached(vma->vm_page_prot);
		}
		ret = io_remap_pfn_range(vma, vma->vm_start,
					 addr >> PAGE_SHIFT,
					 len, vma->vm_page_prot);
	} else {

		/*
		 * Map WQ or CQ contig dma memory...
		 */
		ret = remap_pfn_range(vma, vma->vm_start,
				      addr >> PAGE_SHIFT,
				      len, vma->vm_page_prot);
	}

	return ret;
}

static int c4iw_deallocate_pd(struct ib_pd *pd, struct ib_udata *udata)
{
	struct c4iw_dev *rhp;
	struct c4iw_pd *php;

	php = to_c4iw_pd(pd);
	rhp = php->rhp;
	pr_debug("ibpd %p pdid 0x%x\n", pd, php->pdid);
	c4iw_put_resource(&rhp->rdev.resource.pdid_table, php->pdid);
	mutex_lock(&rhp->rdev.stats.lock);
	rhp->rdev.stats.pd.cur--;
	mutex_unlock(&rhp->rdev.stats.lock);
	return 0;
}

static int c4iw_allocate_pd(struct ib_pd *pd, struct ib_udata *udata)
{
	struct c4iw_pd *php = to_c4iw_pd(pd);
	struct ib_device *ibdev = pd->device;
	u32 pdid;
	struct c4iw_dev *rhp;

	pr_debug("ibdev %p\n", ibdev);
	rhp = (struct c4iw_dev *) ibdev;
	pdid =  c4iw_get_resource(&rhp->rdev.resource.pdid_table);
	if (!pdid)
		return -EINVAL;

	php->pdid = pdid;
	php->rhp = rhp;
	if (udata) {
		struct c4iw_alloc_pd_resp uresp = {.pdid = php->pdid};

		if (ib_copy_to_udata(udata, &uresp, sizeof(uresp))) {
			c4iw_deallocate_pd(&php->ibpd, udata);
			return -EFAULT;
		}
	}
	mutex_lock(&rhp->rdev.stats.lock);
	rhp->rdev.stats.pd.cur++;
	if (rhp->rdev.stats.pd.cur > rhp->rdev.stats.pd.max)
		rhp->rdev.stats.pd.max = rhp->rdev.stats.pd.cur;
	mutex_unlock(&rhp->rdev.stats.lock);
	pr_debug("pdid 0x%0x ptr 0x%p\n", pdid, php);
	return 0;
}

static int c4iw_query_gid(struct ib_device *ibdev, u32 port, int index,
			  union ib_gid *gid)
{
	struct c4iw_dev *dev;

	pr_debug("ibdev %p, port %u, index %d, gid %p\n",
		 ibdev, port, index, gid);
	if (!port)
		return -EINVAL;
	dev = to_c4iw_dev(ibdev);
	memset(&(gid->raw[0]), 0, sizeof(gid->raw));
	memcpy(&(gid->raw[0]), dev->rdev.lldi.ports[port-1]->dev_addr, 6);
	return 0;
}

static int c4iw_query_device(struct ib_device *ibdev, struct ib_device_attr *props,
			     struct ib_udata *uhw)
{

	struct c4iw_dev *dev;

	pr_debug("ibdev %p\n", ibdev);

	if (uhw->inlen || uhw->outlen)
		return -EINVAL;

	dev = to_c4iw_dev(ibdev);
	memcpy(&props->sys_image_guid, dev->rdev.lldi.ports[0]->dev_addr, 6);
	props->hw_ver = CHELSIO_CHIP_RELEASE(dev->rdev.lldi.adapter_type);
	props->fw_ver = dev->rdev.lldi.fw_vers;
	props->device_cap_flags = dev->device_cap_flags;
	props->page_size_cap = T4_PAGESIZE_MASK;
	props->vendor_id = (u32)dev->rdev.lldi.pdev->vendor;
	props->vendor_part_id = (u32)dev->rdev.lldi.pdev->device;
	props->max_mr_size = T4_MAX_MR_SIZE;
	props->max_qp = dev->rdev.lldi.vr->qp.size / 2;
	props->max_srq = dev->rdev.lldi.vr->srq.size;
	props->max_qp_wr = dev->rdev.hw_queue.t4_max_qp_depth;
	props->max_srq_wr = dev->rdev.hw_queue.t4_max_qp_depth;
	props->max_send_sge = min(T4_MAX_SEND_SGE, T4_MAX_WRITE_SGE);
	props->max_recv_sge = T4_MAX_RECV_SGE;
	props->max_srq_sge = T4_MAX_RECV_SGE;
	props->max_sge_rd = 1;
	props->max_res_rd_atom = dev->rdev.lldi.max_ird_adapter;
	props->max_qp_rd_atom = min(dev->rdev.lldi.max_ordird_qp,
				    c4iw_max_read_depth);
	props->max_qp_init_rd_atom = props->max_qp_rd_atom;
	props->max_cq = dev->rdev.lldi.vr->qp.size;
	props->max_cqe = dev->rdev.hw_queue.t4_max_cq_depth;
	props->max_mr = c4iw_num_stags(&dev->rdev);
	props->max_pd = T4_MAX_NUM_PD;
	props->local_ca_ack_delay = 0;
	props->max_fast_reg_page_list_len =
		t4_max_fr_depth(dev->rdev.lldi.ulptx_memwrite_dsgl && use_dsgl);

	return 0;
}

static int c4iw_query_port(struct ib_device *ibdev, u32 port,
			   struct ib_port_attr *props)
{
	int ret = 0;
	pr_debug("ibdev %p\n", ibdev);
	ret = ib_get_eth_speed(ibdev, port, &props->active_speed,
			       &props->active_width);

	props->port_cap_flags =
	    IB_PORT_CM_SUP |
	    IB_PORT_SNMP_TUNNEL_SUP |
	    IB_PORT_REINIT_SUP |
	    IB_PORT_DEVICE_MGMT_SUP |
	    IB_PORT_VENDOR_CLASS_SUP | IB_PORT_BOOT_MGMT_SUP;
	props->gid_tbl_len = 1;
	props->max_msg_sz = -1;

	return ret;
}

static ssize_t hw_rev_show(struct device *dev,
			   struct device_attribute *attr, char *buf)
{
	struct c4iw_dev *c4iw_dev =
			rdma_device_to_drv_device(dev, struct c4iw_dev, ibdev);

	pr_debug("dev 0x%p\n", dev);
	return sysfs_emit(
		buf, "%d\n",
		CHELSIO_CHIP_RELEASE(c4iw_dev->rdev.lldi.adapter_type));
}
static DEVICE_ATTR_RO(hw_rev);

static ssize_t hca_type_show(struct device *dev,
			     struct device_attribute *attr, char *buf)
{
	struct c4iw_dev *c4iw_dev =
			rdma_device_to_drv_device(dev, struct c4iw_dev, ibdev);
	struct ethtool_drvinfo info;
	struct net_device *lldev = c4iw_dev->rdev.lldi.ports[0];

	pr_debug("dev 0x%p\n", dev);
	lldev->ethtool_ops->get_drvinfo(lldev, &info);
	return sysfs_emit(buf, "%s\n", info.driver);
}
static DEVICE_ATTR_RO(hca_type);

static ssize_t board_id_show(struct device *dev, struct device_attribute *attr,
			     char *buf)
{
	struct c4iw_dev *c4iw_dev =
			rdma_device_to_drv_device(dev, struct c4iw_dev, ibdev);

	pr_debug("dev 0x%p\n", dev);
	return sysfs_emit(buf, "%x.%x\n", c4iw_dev->rdev.lldi.pdev->vendor,
			  c4iw_dev->rdev.lldi.pdev->device);
}
static DEVICE_ATTR_RO(board_id);

enum counters {
	IP4INSEGS,
	IP4OUTSEGS,
	IP4RETRANSSEGS,
	IP4OUTRSTS,
	IP6INSEGS,
	IP6OUTSEGS,
	IP6RETRANSSEGS,
	IP6OUTRSTS,
	NR_COUNTERS
};

static const char * const names[] = {
	[IP4INSEGS] = "ip4InSegs",
	[IP4OUTSEGS] = "ip4OutSegs",
	[IP4RETRANSSEGS] = "ip4RetransSegs",
	[IP4OUTRSTS] = "ip4OutRsts",
	[IP6INSEGS] = "ip6InSegs",
	[IP6OUTSEGS] = "ip6OutSegs",
	[IP6RETRANSSEGS] = "ip6RetransSegs",
	[IP6OUTRSTS] = "ip6OutRsts"
};

static struct rdma_hw_stats *c4iw_alloc_device_stats(struct ib_device *ibdev)
{
	BUILD_BUG_ON(ARRAY_SIZE(names) != NR_COUNTERS);

	/* FIXME: these look like port stats */
	return rdma_alloc_hw_stats_struct(names, NR_COUNTERS,
					  RDMA_HW_STATS_DEFAULT_LIFESPAN);
}

static int c4iw_get_mib(struct ib_device *ibdev,
			struct rdma_hw_stats *stats,
			u32 port, int index)
{
	struct tp_tcp_stats v4, v6;
	struct c4iw_dev *c4iw_dev = to_c4iw_dev(ibdev);

	cxgb4_get_tcp_stats(c4iw_dev->rdev.lldi.pdev, &v4, &v6);
	stats->value[IP4INSEGS] = v4.tcp_in_segs;
	stats->value[IP4OUTSEGS] = v4.tcp_out_segs;
	stats->value[IP4RETRANSSEGS] = v4.tcp_retrans_segs;
	stats->value[IP4OUTRSTS] = v4.tcp_out_rsts;
	stats->value[IP6INSEGS] = v6.tcp_in_segs;
	stats->value[IP6OUTSEGS] = v6.tcp_out_segs;
	stats->value[IP6RETRANSSEGS] = v6.tcp_retrans_segs;
	stats->value[IP6OUTRSTS] = v6.tcp_out_rsts;

	return stats->num_counters;
}

static struct attribute *c4iw_class_attributes[] = {
	&dev_attr_hw_rev.attr,
	&dev_attr_hca_type.attr,
	&dev_attr_board_id.attr,
	NULL
};

static const struct attribute_group c4iw_attr_group = {
	.attrs = c4iw_class_attributes,
};

static int c4iw_port_immutable(struct ib_device *ibdev, u32 port_num,
			       struct ib_port_immutable *immutable)
{
	struct ib_port_attr attr;
	int err;

	immutable->core_cap_flags = RDMA_CORE_PORT_IWARP;

	err = ib_query_port(ibdev, port_num, &attr);
	if (err)
		return err;

	immutable->gid_tbl_len = attr.gid_tbl_len;

	return 0;
}

static void get_dev_fw_str(struct ib_device *dev, char *str)
{
	struct c4iw_dev *c4iw_dev = container_of(dev, struct c4iw_dev,
						 ibdev);
	pr_debug("dev 0x%p\n", dev);

	snprintf(str, IB_FW_VERSION_NAME_MAX, "%u.%u.%u.%u",
		 FW_HDR_FW_VER_MAJOR_G(c4iw_dev->rdev.lldi.fw_vers),
		 FW_HDR_FW_VER_MINOR_G(c4iw_dev->rdev.lldi.fw_vers),
		 FW_HDR_FW_VER_MICRO_G(c4iw_dev->rdev.lldi.fw_vers),
		 FW_HDR_FW_VER_BUILD_G(c4iw_dev->rdev.lldi.fw_vers));
}

static const struct ib_device_ops c4iw_dev_ops = {
	.owner = THIS_MODULE,
	.driver_id = RDMA_DRIVER_CXGB4,
	.uverbs_abi_ver = C4IW_UVERBS_ABI_VERSION,

	.alloc_hw_device_stats = c4iw_alloc_device_stats,
	.alloc_mr = c4iw_alloc_mr,
	.alloc_pd = c4iw_allocate_pd,
	.alloc_ucontext = c4iw_alloc_ucontext,
	.create_cq = c4iw_create_cq,
	.create_qp = c4iw_create_qp,
	.create_srq = c4iw_create_srq,
	.dealloc_pd = c4iw_deallocate_pd,
	.dealloc_ucontext = c4iw_dealloc_ucontext,
	.dereg_mr = c4iw_dereg_mr,
	.destroy_cq = c4iw_destroy_cq,
	.destroy_qp = c4iw_destroy_qp,
	.destroy_srq = c4iw_destroy_srq,
	.device_group = &c4iw_attr_group,
	.fill_res_cq_entry = c4iw_fill_res_cq_entry,
	.fill_res_cm_id_entry = c4iw_fill_res_cm_id_entry,
	.fill_res_mr_entry = c4iw_fill_res_mr_entry,
	.get_dev_fw_str = get_dev_fw_str,
	.get_dma_mr = c4iw_get_dma_mr,
	.get_hw_stats = c4iw_get_mib,
	.get_port_immutable = c4iw_port_immutable,
	.iw_accept = c4iw_accept_cr,
	.iw_add_ref = c4iw_qp_add_ref,
	.iw_connect = c4iw_connect,
	.iw_create_listen = c4iw_create_listen,
	.iw_destroy_listen = c4iw_destroy_listen,
	.iw_get_qp = c4iw_get_qp,
	.iw_reject = c4iw_reject_cr,
	.iw_rem_ref = c4iw_qp_rem_ref,
	.map_mr_sg = c4iw_map_mr_sg,
	.mmap = c4iw_mmap,
	.modify_qp = c4iw_ib_modify_qp,
	.modify_srq = c4iw_modify_srq,
	.poll_cq = c4iw_poll_cq,
	.post_recv = c4iw_post_receive,
	.post_send = c4iw_post_send,
	.post_srq_recv = c4iw_post_srq_recv,
	.query_device = c4iw_query_device,
	.query_gid = c4iw_query_gid,
	.query_port = c4iw_query_port,
	.query_qp = c4iw_ib_query_qp,
	.reg_user_mr = c4iw_reg_user_mr,
	.req_notify_cq = c4iw_arm_cq,

	INIT_RDMA_OBJ_SIZE(ib_cq, c4iw_cq, ibcq),
	INIT_RDMA_OBJ_SIZE(ib_mw, c4iw_mw, ibmw),
	INIT_RDMA_OBJ_SIZE(ib_pd, c4iw_pd, ibpd),
	INIT_RDMA_OBJ_SIZE(ib_srq, c4iw_srq, ibsrq),
	INIT_RDMA_OBJ_SIZE(ib_ucontext, c4iw_ucontext, ibucontext),
};

static int set_netdevs(struct ib_device *ib_dev, struct c4iw_rdev *rdev)
{
	int ret;
	int i;

	for (i = 0; i < rdev->lldi.nports; i++) {
		ret = ib_device_set_netdev(ib_dev, rdev->lldi.ports[i],
					   i + 1);
		if (ret)
			return ret;
	}
	return 0;
}

void c4iw_register_device(struct work_struct *work)
{
	int ret;
	struct uld_ctx *ctx = container_of(work, struct uld_ctx, reg_work);
	struct c4iw_dev *dev = ctx->dev;

	pr_debug("c4iw_dev %p\n", dev);
	memset(&dev->ibdev.node_guid, 0, sizeof(dev->ibdev.node_guid));
	memcpy(&dev->ibdev.node_guid, dev->rdev.lldi.ports[0]->dev_addr, 6);
	dev->device_cap_flags = IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_WINDOW;
	if (fastreg_support)
		dev->device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS;
	dev->ibdev.local_dma_lkey = 0;
	dev->ibdev.node_type = RDMA_NODE_RNIC;
	BUILD_BUG_ON(sizeof(C4IW_NODE_DESC) > IB_DEVICE_NODE_DESC_MAX);
	memcpy(dev->ibdev.node_desc, C4IW_NODE_DESC, sizeof(C4IW_NODE_DESC));
	dev->ibdev.phys_port_cnt = dev->rdev.lldi.nports;
	dev->ibdev.num_comp_vectors =  dev->rdev.lldi.nciq;
	dev->ibdev.dev.parent = &dev->rdev.lldi.pdev->dev;

	memcpy(dev->ibdev.iw_ifname, dev->rdev.lldi.ports[0]->name,
	       sizeof(dev->ibdev.iw_ifname));

	ib_set_device_ops(&dev->ibdev, &c4iw_dev_ops);
	ret = set_netdevs(&dev->ibdev, &dev->rdev);
	if (ret)
		goto err_dealloc_ctx;
	dma_set_max_seg_size(&dev->rdev.lldi.pdev->dev, UINT_MAX);
	ret = ib_register_device(&dev->ibdev, "cxgb4_%d",
				 &dev->rdev.lldi.pdev->dev);
	if (ret)
		goto err_dealloc_ctx;
	return;

err_dealloc_ctx:
	pr_err("%s - Failed registering iwarp device: %d\n",
	       pci_name(ctx->lldi.pdev), ret);
	c4iw_dealloc(ctx);
	return;
}

void c4iw_unregister_device(struct c4iw_dev *dev)
{
	pr_debug("c4iw_dev %p\n", dev);
	ib_unregister_device(&dev->ibdev);
	return;
}
Commit	Line	Data
cfdda9d7 SW	1	/*
	2	* Copyright (c) 2009-2010 Chelsio, Inc. All rights reserved.
	3	*
	4	* This software is available to you under a choice of one of two
	5	* licenses. You may choose to be licensed under the terms of the GNU
	6	* General Public License (GPL) Version 2, available from the file
	7	* COPYING in the main directory of this source tree, or the
	8	* OpenIB.org BSD license below:
	9	*
	10	* Redistribution and use in source and binary forms, with or
	11	* without modification, are permitted provided that the following
	12	* conditions are met:
	13	*
	14	* - Redistributions of source code must retain the above
	15	* copyright notice, this list of conditions and the following
	16	* disclaimer.
	17	*
	18	* - Redistributions in binary form must reproduce the above
	19	* copyright notice, this list of conditions and the following
	20	* disclaimer in the documentation and/or other materials
	21	* provided with the distribution.
	22	*
	23	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	24	* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	25	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
	26	* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
	27	* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
	28	* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
	29	* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	30	* SOFTWARE.
	31	*/
	32	#include <linux/module.h>
	33	#include <linux/moduleparam.h>
	34	#include <linux/device.h>
	35	#include <linux/netdevice.h>
	36	#include <linux/etherdevice.h>
	37	#include <linux/delay.h>
	38	#include <linux/errno.h>
	39	#include <linux/list.h>
	40	#include <linux/spinlock.h>
	41	#include <linux/ethtool.h>
	42	#include <linux/rtnetlink.h>
	43	#include <linux/inetdevice.h>
	44	#include <linux/io.h>
	45
	46	#include <asm/irq.h>
	47	#include <asm/byteorder.h>
	48
	49	#include <rdma/iw_cm.h>
	50	#include <rdma/ib_verbs.h>
	51	#include <rdma/ib_smi.h>
	52	#include <rdma/ib_umem.h>
	53	#include <rdma/ib_user_verbs.h>
	54
	55	#include "iw_cxgb4.h"
	56
40dbf6ee	57	static int fastreg_support = 1;
cfdda9d7	58	module_param(fastreg_support, int, 0644);
40dbf6ee	59	MODULE_PARM_DESC(fastreg_support, "Advertise fastreg support (default=1)");
cfdda9d7	60
a2a074ef	61	static void c4iw_dealloc_ucontext(struct ib_ucontext *context)
cfdda9d7	62	{
cfe876d8	63	struct c4iw_ucontext *ucontext = to_c4iw_ucontext(context);
c12a67fe	64	struct c4iw_dev *rhp;
cfdda9d7 SW	65	struct c4iw_mm_entry mm, tmp;
cfdda9d7 SW	66
cfe876d8	67	pr_debug("context %p\n", context);
c12a67fe SW	68	rhp = to_c4iw_dev(ucontext->ibucontext.device);
c12a67fe SW	69
cfdda9d7 SW	70	list_for_each_entry_safe(mm, tmp, &ucontext->mmaps, entry)
	71	kfree(mm);
	72	c4iw_release_dev_ucontext(&rhp->rdev, &ucontext->uctx);
cfdda9d7 SW	73	}
cfdda9d7 SW	74
a2a074ef LR	75	static int c4iw_alloc_ucontext(struct ib_ucontext *ucontext,
a2a074ef LR	76	struct ib_udata *udata)
cfdda9d7	77	{
a2a074ef LR	78	struct ib_device *ibdev = ucontext->device;
a2a074ef LR	79	struct c4iw_ucontext *context = to_c4iw_ucontext(ucontext);
cfdda9d7	80	struct c4iw_dev *rhp = to_c4iw_dev(ibdev);
05eb2389 SW	81	struct c4iw_alloc_ucontext_resp uresp;
	82	int ret = 0;
	83	struct c4iw_mm_entry *mm = NULL;
cfdda9d7	84
548ddb19	85	pr_debug("ibdev %p\n", ibdev);
cfdda9d7 SW	86	c4iw_init_dev_ucontext(&rhp->rdev, &context->uctx);
	87	INIT_LIST_HEAD(&context->mmaps);
	88	spin_lock_init(&context->mmap_lock);
05eb2389	89
b7dfa889	90	if (udata->outlen < sizeof(uresp) - sizeof(uresp.reserved)) {
700456bd	91	pr_err_once("Warning - downlevel libcxgb4 (non-fatal), device status page disabled\n");
05eb2389 SW	92	rhp->rdev.flags \|= T4_STATUS_PAGE_DISABLED;
	93	} else {
	94	mm = kmalloc(sizeof(*mm), GFP_KERNEL);
bfd2793c YD	95	if (!mm) {
bfd2793c YD	96	ret = -ENOMEM;
a2a074ef	97	goto err;
bfd2793c	98	}
05eb2389 SW	99
	100	uresp.status_page_size = PAGE_SIZE;
	101
	102	spin_lock(&context->mmap_lock);
	103	uresp.status_page_key = context->key;
	104	context->key += PAGE_SIZE;
	105	spin_unlock(&context->mmap_lock);
	106
b7dfa889 YD	107	ret = ib_copy_to_udata(udata, &uresp,
b7dfa889 YD	108	sizeof(uresp) - sizeof(uresp.reserved));
05eb2389 SW	109	if (ret)
	110	goto err_mm;
	111
	112	mm->key = uresp.status_page_key;
	113	mm->addr = virt_to_phys(rhp->rdev.status_page);
	114	mm->len = PAGE_SIZE;
	115	insert_mmap(context, mm);
	116	}
a2a074ef	117	return 0;
05eb2389 SW	118	err_mm:
05eb2389 SW	119	kfree(mm);
05eb2389	120	err:
a2a074ef	121	return ret;
cfdda9d7 SW	122	}
	123
	124	static int c4iw_mmap(struct ib_ucontext context, struct vm_area_struct vma)
	125	{
	126	int len = vma->vm_end - vma->vm_start;
	127	u32 key = vma->vm_pgoff << PAGE_SHIFT;
	128	struct c4iw_rdev *rdev;
	129	int ret = 0;
	130	struct c4iw_mm_entry *mm;
	131	struct c4iw_ucontext *ucontext;
	132	u64 addr;
	133
548ddb19	134	pr_debug("pgoff 0x%lx key 0x%x len %d\n", vma->vm_pgoff,
a9a42886	135	key, len);
cfdda9d7 SW	136
	137	if (vma->vm_start & (PAGE_SIZE-1))
	138	return -EINVAL;
	139
	140	rdev = &(to_c4iw_dev(context->device)->rdev);
	141	ucontext = to_c4iw_ucontext(context);
	142
	143	mm = remove_mmap(ucontext, key, len);
	144	if (!mm)
	145	return -EINVAL;
	146	addr = mm->addr;
	147	kfree(mm);
	148
c6d7b267 SW	149	if ((addr >= pci_resource_start(rdev->lldi.pdev, 0)) &&
	150	(addr < (pci_resource_start(rdev->lldi.pdev, 0) +
	151	pci_resource_len(rdev->lldi.pdev, 0)))) {
cfdda9d7 SW	152
cfdda9d7 SW	153	/*
c6d7b267	154	* MA_SYNC register...
cfdda9d7	155	*/
cfdda9d7	156	vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
c6d7b267 SW	157	ret = io_remap_pfn_range(vma, vma->vm_start,
	158	addr >> PAGE_SHIFT,
	159	len, vma->vm_page_prot);
	160	} else if ((addr >= pci_resource_start(rdev->lldi.pdev, 2)) &&
	161	(addr < (pci_resource_start(rdev->lldi.pdev, 2) +
	162	pci_resource_len(rdev->lldi.pdev, 2)))) {
	163
	164	/*
	165	* Map user DB or OCQP memory...
	166	*/
	167	if (addr >= rdev->oc_mw_pa)
	168	vma->vm_page_prot = t4_pgprot_wc(vma->vm_page_prot);
f079af7a	169	else {
963cab50	170	if (!is_t4(rdev->lldi.adapter_type))
f079af7a VP	171	vma->vm_page_prot =
	172	t4_pgprot_wc(vma->vm_page_prot);
	173	else
	174	vma->vm_page_prot =
	175	pgprot_noncached(vma->vm_page_prot);
	176	}
cfdda9d7 SW	177	ret = io_remap_pfn_range(vma, vma->vm_start,
	178	addr >> PAGE_SHIFT,
	179	len, vma->vm_page_prot);
	180	} else {
	181
	182	/*
	183	* Map WQ or CQ contig dma memory...
	184	*/
	185	ret = remap_pfn_range(vma, vma->vm_start,
	186	addr >> PAGE_SHIFT,
	187	len, vma->vm_page_prot);
	188	}
	189
	190	return ret;
	191	}
	192
91a7c58f	193	static int c4iw_deallocate_pd(struct ib_pd pd, struct ib_udata udata)
cfdda9d7 SW	194	{
	195	struct c4iw_dev *rhp;
	196	struct c4iw_pd *php;
	197
	198	php = to_c4iw_pd(pd);
	199	rhp = php->rhp;
548ddb19	200	pr_debug("ibpd %p pdid 0x%x\n", pd, php->pdid);
ec3eead2	201	c4iw_put_resource(&rhp->rdev.resource.pdid_table, php->pdid);
8d81ef34 VP	202	mutex_lock(&rhp->rdev.stats.lock);
	203	rhp->rdev.stats.pd.cur--;
	204	mutex_unlock(&rhp->rdev.stats.lock);
91a7c58f	205	return 0;
cfdda9d7 SW	206	}
cfdda9d7 SW	207
ff23dfa1	208	static int c4iw_allocate_pd(struct ib_pd pd, struct ib_udata udata)
cfdda9d7	209	{
21a428a0 LR	210	struct c4iw_pd *php = to_c4iw_pd(pd);
21a428a0 LR	211	struct ib_device *ibdev = pd->device;
cfdda9d7 SW	212	u32 pdid;
	213	struct c4iw_dev *rhp;
	214
548ddb19	215	pr_debug("ibdev %p\n", ibdev);
cfdda9d7	216	rhp = (struct c4iw_dev *) ibdev;
ec3eead2	217	pdid = c4iw_get_resource(&rhp->rdev.resource.pdid_table);
cfdda9d7	218	if (!pdid)
21a428a0 LR	219	return -EINVAL;
21a428a0 LR	220
cfdda9d7 SW	221	php->pdid = pdid;
cfdda9d7 SW	222	php->rhp = rhp;
ff23dfa1	223	if (udata) {
7f86260b JG	224	struct c4iw_alloc_pd_resp uresp = {.pdid = php->pdid};
	225
	226	if (ib_copy_to_udata(udata, &uresp, sizeof(uresp))) {
c4367a26	227	c4iw_deallocate_pd(&php->ibpd, udata);
21a428a0	228	return -EFAULT;
cfdda9d7 SW	229	}
cfdda9d7 SW	230	}
8d81ef34 VP	231	mutex_lock(&rhp->rdev.stats.lock);
	232	rhp->rdev.stats.pd.cur++;
	233	if (rhp->rdev.stats.pd.cur > rhp->rdev.stats.pd.max)
	234	rhp->rdev.stats.pd.max = rhp->rdev.stats.pd.cur;
	235	mutex_unlock(&rhp->rdev.stats.lock);
548ddb19	236	pr_debug("pdid 0x%0x ptr 0x%p\n", pdid, php);
21a428a0	237	return 0;
cfdda9d7 SW	238	}
cfdda9d7 SW	239
1fb7f897	240	static int c4iw_query_gid(struct ib_device *ibdev, u32 port, int index,
cfdda9d7 SW	241	union ib_gid *gid)
	242	{
	243	struct c4iw_dev *dev;
	244
1fb7f897	245	pr_debug("ibdev %p, port %u, index %d, gid %p\n",
548ddb19	246	ibdev, port, index, gid);
ba97b749 SW	247	if (!port)
ba97b749 SW	248	return -EINVAL;
cfdda9d7	249	dev = to_c4iw_dev(ibdev);
cfdda9d7 SW	250	memset(&(gid->raw[0]), 0, sizeof(gid->raw));
	251	memcpy(&(gid->raw[0]), dev->rdev.lldi.ports[port-1]->dev_addr, 6);
	252	return 0;
	253	}
	254
2528e33e MB	255	static int c4iw_query_device(struct ib_device ibdev, struct ib_device_attr props,
2528e33e MB	256	struct ib_udata *uhw)
cfdda9d7 SW	257	{
	258
	259	struct c4iw_dev *dev;
2528e33e	260
548ddb19	261	pr_debug("ibdev %p\n", ibdev);
cfdda9d7	262
2528e33e MB	263	if (uhw->inlen \|\| uhw->outlen)
	264	return -EINVAL;
	265
cfdda9d7	266	dev = to_c4iw_dev(ibdev);
cfdda9d7	267	memcpy(&props->sys_image_guid, dev->rdev.lldi.ports[0]->dev_addr, 6);
f079af7a	268	props->hw_ver = CHELSIO_CHIP_RELEASE(dev->rdev.lldi.adapter_type);
cfdda9d7 SW	269	props->fw_ver = dev->rdev.lldi.fw_vers;
	270	props->device_cap_flags = dev->device_cap_flags;
	271	props->page_size_cap = T4_PAGESIZE_MASK;
	272	props->vendor_id = (u32)dev->rdev.lldi.pdev->vendor;
	273	props->vendor_part_id = (u32)dev->rdev.lldi.pdev->device;
	274	props->max_mr_size = T4_MAX_MR_SIZE;
66eb19af	275	props->max_qp = dev->rdev.lldi.vr->qp.size / 2;
6a0b6174	276	props->max_srq = dev->rdev.lldi.vr->srq.size;
04e10e21	277	props->max_qp_wr = dev->rdev.hw_queue.t4_max_qp_depth;
6a0b6174	278	props->max_srq_wr = dev->rdev.hw_queue.t4_max_qp_depth;
33023fb8 SW	279	props->max_send_sge = min(T4_MAX_SEND_SGE, T4_MAX_WRITE_SGE);
33023fb8 SW	280	props->max_recv_sge = T4_MAX_RECV_SGE;
6a0b6174	281	props->max_srq_sge = T4_MAX_RECV_SGE;
cfdda9d7	282	props->max_sge_rd = 1;
4c2c5763 HS	283	props->max_res_rd_atom = dev->rdev.lldi.max_ird_adapter;
	284	props->max_qp_rd_atom = min(dev->rdev.lldi.max_ordird_qp,
	285	c4iw_max_read_depth);
	286	props->max_qp_init_rd_atom = props->max_qp_rd_atom;
66eb19af	287	props->max_cq = dev->rdev.lldi.vr->qp.size;
04e10e21	288	props->max_cqe = dev->rdev.hw_queue.t4_max_cq_depth;
cfdda9d7 SW	289	props->max_mr = c4iw_num_stags(&dev->rdev);
	290	props->max_pd = T4_MAX_NUM_PD;
	291	props->local_ca_ack_delay = 0;
ee30f7d5 H	292	props->max_fast_reg_page_list_len =
ee30f7d5 H	293	t4_max_fr_depth(dev->rdev.lldi.ulptx_memwrite_dsgl && use_dsgl);
cfdda9d7 SW	294
	295	return 0;
	296	}
	297
1fb7f897	298	static int c4iw_query_port(struct ib_device *ibdev, u32 port,
cfdda9d7 SW	299	struct ib_port_attr *props)
cfdda9d7 SW	300	{
5212c3fd	301	int ret = 0;
548ddb19	302	pr_debug("ibdev %p\n", ibdev);
5212c3fd PBT	303	ret = ib_get_eth_speed(ibdev, port, &props->active_speed,
5212c3fd PBT	304	&props->active_width);
cfdda9d7	305
cfdda9d7 SW	306	props->port_cap_flags =
	307	IB_PORT_CM_SUP \|
	308	IB_PORT_SNMP_TUNNEL_SUP \|
	309	IB_PORT_REINIT_SUP \|
	310	IB_PORT_DEVICE_MGMT_SUP \|
	311	IB_PORT_VENDOR_CLASS_SUP \| IB_PORT_BOOT_MGMT_SUP;
	312	props->gid_tbl_len = 1;
cfdda9d7 SW	313	props->max_msg_sz = -1;
cfdda9d7 SW	314
5212c3fd	315	return ret;
cfdda9d7 SW	316	}
cfdda9d7 SW	317
508a523f PP	318	static ssize_t hw_rev_show(struct device *dev,
508a523f PP	319	struct device_attribute attr, char buf)
cfdda9d7	320	{
54747231 PP	321	struct c4iw_dev *c4iw_dev =
	322	rdma_device_to_drv_device(dev, struct c4iw_dev, ibdev);
	323
548ddb19	324	pr_debug("dev 0x%p\n", dev);
1c7fd726 JP	325	return sysfs_emit(
	326	buf, "%d\n",
	327	CHELSIO_CHIP_RELEASE(c4iw_dev->rdev.lldi.adapter_type));
cfdda9d7	328	}
508a523f	329	static DEVICE_ATTR_RO(hw_rev);
cfdda9d7	330
508a523f PP	331	static ssize_t hca_type_show(struct device *dev,
508a523f PP	332	struct device_attribute attr, char buf)
cfdda9d7	333	{
54747231 PP	334	struct c4iw_dev *c4iw_dev =
54747231 PP	335	rdma_device_to_drv_device(dev, struct c4iw_dev, ibdev);
cfdda9d7 SW	336	struct ethtool_drvinfo info;
	337	struct net_device *lldev = c4iw_dev->rdev.lldi.ports[0];
	338
548ddb19	339	pr_debug("dev 0x%p\n", dev);
cfdda9d7	340	lldev->ethtool_ops->get_drvinfo(lldev, &info);
1c7fd726	341	return sysfs_emit(buf, "%s\n", info.driver);
cfdda9d7	342	}
508a523f	343	static DEVICE_ATTR_RO(hca_type);
cfdda9d7	344
508a523f PP	345	static ssize_t board_id_show(struct device dev, struct device_attribute attr,
508a523f PP	346	char *buf)
cfdda9d7	347	{
54747231 PP	348	struct c4iw_dev *c4iw_dev =
	349	rdma_device_to_drv_device(dev, struct c4iw_dev, ibdev);
	350
548ddb19	351	pr_debug("dev 0x%p\n", dev);
1c7fd726 JP	352	return sysfs_emit(buf, "%x.%x\n", c4iw_dev->rdev.lldi.pdev->vendor,
1c7fd726 JP	353	c4iw_dev->rdev.lldi.pdev->device);
cfdda9d7	354	}
508a523f	355	static DEVICE_ATTR_RO(board_id);
cfdda9d7	356
b40f4757 CL	357	enum counters {
	358	IP4INSEGS,
	359	IP4OUTSEGS,
	360	IP4RETRANSSEGS,
	361	IP4OUTRSTS,
	362	IP6INSEGS,
	363	IP6OUTSEGS,
	364	IP6RETRANSSEGS,
	365	IP6OUTRSTS,
	366	NR_COUNTERS
	367	};
	368
	369	static const char * const names[] = {
	370	[IP4INSEGS] = "ip4InSegs",
	371	[IP4OUTSEGS] = "ip4OutSegs",
	372	[IP4RETRANSSEGS] = "ip4RetransSegs",
	373	[IP4OUTRSTS] = "ip4OutRsts",
	374	[IP6INSEGS] = "ip6InSegs",
	375	[IP6OUTSEGS] = "ip6OutSegs",
	376	[IP6RETRANSSEGS] = "ip6RetransSegs",
	377	[IP6OUTRSTS] = "ip6OutRsts"
	378	};
	379
4b5f4d3f	380	static struct rdma_hw_stats c4iw_alloc_device_stats(struct ib_device ibdev)
b40f4757 CL	381	{
	382	BUILD_BUG_ON(ARRAY_SIZE(names) != NR_COUNTERS);
	383
4b5f4d3f	384	/* FIXME: these look like port stats */
b40f4757 CL	385	return rdma_alloc_hw_stats_struct(names, NR_COUNTERS,
	386	RDMA_HW_STATS_DEFAULT_LIFESPAN);
	387	}
	388
cfdda9d7	389	static int c4iw_get_mib(struct ib_device *ibdev,
b40f4757	390	struct rdma_hw_stats *stats,
1fb7f897	391	u32 port, int index)
cfdda9d7	392	{
de5dd81b SW	393	struct tp_tcp_stats v4, v6;
	394	struct c4iw_dev *c4iw_dev = to_c4iw_dev(ibdev);
	395
	396	cxgb4_get_tcp_stats(c4iw_dev->rdev.lldi.pdev, &v4, &v6);
b40f4757 CL	397	stats->value[IP4INSEGS] = v4.tcp_in_segs;
	398	stats->value[IP4OUTSEGS] = v4.tcp_out_segs;
	399	stats->value[IP4RETRANSSEGS] = v4.tcp_retrans_segs;
	400	stats->value[IP4OUTRSTS] = v4.tcp_out_rsts;
	401	stats->value[IP6INSEGS] = v6.tcp_in_segs;
	402	stats->value[IP6OUTSEGS] = v6.tcp_out_segs;
	403	stats->value[IP6RETRANSSEGS] = v6.tcp_retrans_segs;
	404	stats->value[IP6OUTRSTS] = v6.tcp_out_rsts;
	405
	406	return stats->num_counters;
cfdda9d7 SW	407	}
cfdda9d7 SW	408
508a523f PP	409	static struct attribute *c4iw_class_attributes[] = {
	410	&dev_attr_hw_rev.attr,
	411	&dev_attr_hca_type.attr,
	412	&dev_attr_board_id.attr,
	413	NULL
	414	};
cfdda9d7	415
508a523f PP	416	static const struct attribute_group c4iw_attr_group = {
508a523f PP	417	.attrs = c4iw_class_attributes,
cfdda9d7 SW	418	};
cfdda9d7 SW	419
1fb7f897	420	static int c4iw_port_immutable(struct ib_device *ibdev, u32 port_num,
7738613e IW	421	struct ib_port_immutable *immutable)
	422	{
	423	struct ib_port_attr attr;
	424	int err;
	425
c4550c63 OG	426	immutable->core_cap_flags = RDMA_CORE_PORT_IWARP;
	427
	428	err = ib_query_port(ibdev, port_num, &attr);
7738613e IW	429	if (err)
	430	return err;
	431
7738613e IW	432	immutable->gid_tbl_len = attr.gid_tbl_len;
	433
	434	return 0;
	435	}
	436
9abb0d1b	437	static void get_dev_fw_str(struct ib_device dev, char str)
ce192243 IW	438	{
	439	struct c4iw_dev *c4iw_dev = container_of(dev, struct c4iw_dev,
	440	ibdev);
548ddb19	441	pr_debug("dev 0x%p\n", dev);
ce192243	442
9abb0d1b	443	snprintf(str, IB_FW_VERSION_NAME_MAX, "%u.%u.%u.%u",
ce192243 IW	444	FW_HDR_FW_VER_MAJOR_G(c4iw_dev->rdev.lldi.fw_vers),
	445	FW_HDR_FW_VER_MINOR_G(c4iw_dev->rdev.lldi.fw_vers),
	446	FW_HDR_FW_VER_MICRO_G(c4iw_dev->rdev.lldi.fw_vers),
	447	FW_HDR_FW_VER_BUILD_G(c4iw_dev->rdev.lldi.fw_vers));
	448	}
	449
dad3b05d	450	static const struct ib_device_ops c4iw_dev_ops = {
7a154142	451	.owner = THIS_MODULE,
b9560a41	452	.driver_id = RDMA_DRIVER_CXGB4,
72c6ec18	453	.uverbs_abi_ver = C4IW_UVERBS_ABI_VERSION,
b9560a41	454
4b5f4d3f	455	.alloc_hw_device_stats = c4iw_alloc_device_stats,
dad3b05d	456	.alloc_mr = c4iw_alloc_mr,
dad3b05d KH	457	.alloc_pd = c4iw_allocate_pd,
	458	.alloc_ucontext = c4iw_alloc_ucontext,
	459	.create_cq = c4iw_create_cq,
	460	.create_qp = c4iw_create_qp,
	461	.create_srq = c4iw_create_srq,
dad3b05d KH	462	.dealloc_pd = c4iw_deallocate_pd,
	463	.dealloc_ucontext = c4iw_dealloc_ucontext,
	464	.dereg_mr = c4iw_dereg_mr,
	465	.destroy_cq = c4iw_destroy_cq,
	466	.destroy_qp = c4iw_destroy_qp,
	467	.destroy_srq = c4iw_destroy_srq,
915e4af5	468	.device_group = &c4iw_attr_group,
9e2a187a	469	.fill_res_cq_entry = c4iw_fill_res_cq_entry,
211cd945	470	.fill_res_cm_id_entry = c4iw_fill_res_cm_id_entry,
f4434529	471	.fill_res_mr_entry = c4iw_fill_res_mr_entry,
dad3b05d KH	472	.get_dev_fw_str = get_dev_fw_str,
	473	.get_dma_mr = c4iw_get_dma_mr,
	474	.get_hw_stats = c4iw_get_mib,
dad3b05d	475	.get_port_immutable = c4iw_port_immutable,
dd05cb82 KH	476	.iw_accept = c4iw_accept_cr,
	477	.iw_add_ref = c4iw_qp_add_ref,
	478	.iw_connect = c4iw_connect,
	479	.iw_create_listen = c4iw_create_listen,
	480	.iw_destroy_listen = c4iw_destroy_listen,
	481	.iw_get_qp = c4iw_get_qp,
	482	.iw_reject = c4iw_reject_cr,
	483	.iw_rem_ref = c4iw_qp_rem_ref,
dad3b05d KH	484	.map_mr_sg = c4iw_map_mr_sg,
	485	.mmap = c4iw_mmap,
	486	.modify_qp = c4iw_ib_modify_qp,
	487	.modify_srq = c4iw_modify_srq,
	488	.poll_cq = c4iw_poll_cq,
	489	.post_recv = c4iw_post_receive,
	490	.post_send = c4iw_post_send,
	491	.post_srq_recv = c4iw_post_srq_recv,
	492	.query_device = c4iw_query_device,
	493	.query_gid = c4iw_query_gid,
dad3b05d KH	494	.query_port = c4iw_query_port,
	495	.query_qp = c4iw_ib_query_qp,
	496	.reg_user_mr = c4iw_reg_user_mr,
	497	.req_notify_cq = c4iw_arm_cq,
d18bb3e1	498
e39afe3d	499	INIT_RDMA_OBJ_SIZE(ib_cq, c4iw_cq, ibcq),
d18bb3e1 LR	500	INIT_RDMA_OBJ_SIZE(ib_mw, c4iw_mw, ibmw),
d18bb3e1 LR	501	INIT_RDMA_OBJ_SIZE(ib_pd, c4iw_pd, ibpd),
68e326de	502	INIT_RDMA_OBJ_SIZE(ib_srq, c4iw_srq, ibsrq),
a2a074ef	503	INIT_RDMA_OBJ_SIZE(ib_ucontext, c4iw_ucontext, ibucontext),
dad3b05d KH	504	};
dad3b05d KH	505
ab7efbe2 SW	506	static int set_netdevs(struct ib_device ib_dev, struct c4iw_rdev rdev)
	507	{
	508	int ret;
	509	int i;
	510
	511	for (i = 0; i < rdev->lldi.nports; i++) {
	512	ret = ib_device_set_netdev(ib_dev, rdev->lldi.ports[i],
	513	i + 1);
	514	if (ret)
	515	return ret;
	516	}
	517	return 0;
	518	}
	519
1c8f1da5	520	void c4iw_register_device(struct work_struct *work)
cfdda9d7 SW	521	{
cfdda9d7 SW	522	int ret;
1c8f1da5 BP	523	struct uld_ctx *ctx = container_of(work, struct uld_ctx, reg_work);
1c8f1da5 BP	524	struct c4iw_dev *dev = ctx->dev;
cfdda9d7	525
548ddb19	526	pr_debug("c4iw_dev %p\n", dev);
cfdda9d7 SW	527	memset(&dev->ibdev.node_guid, 0, sizeof(dev->ibdev.node_guid));
cfdda9d7 SW	528	memcpy(&dev->ibdev.node_guid, dev->rdev.lldi.ports[0]->dev_addr, 6);
cfdda9d7 SW	529	dev->device_cap_flags = IB_DEVICE_LOCAL_DMA_LKEY \| IB_DEVICE_MEM_WINDOW;
	530	if (fastreg_support)
	531	dev->device_cap_flags \|= IB_DEVICE_MEM_MGT_EXTENSIONS;
	532	dev->ibdev.local_dma_lkey = 0;
cfdda9d7	533	dev->ibdev.node_type = RDMA_NODE_RNIC;
bd99fdea	534	BUILD_BUG_ON(sizeof(C4IW_NODE_DESC) > IB_DEVICE_NODE_DESC_MAX);
cfdda9d7 SW	535	memcpy(dev->ibdev.node_desc, C4IW_NODE_DESC, sizeof(C4IW_NODE_DESC));
cfdda9d7 SW	536	dev->ibdev.phys_port_cnt = dev->rdev.lldi.nports;
cf38be6d	537	dev->ibdev.num_comp_vectors = dev->rdev.lldi.nciq;
d08868a1	538	dev->ibdev.dev.parent = &dev->rdev.lldi.pdev->dev;
cfdda9d7	539
dd05cb82 KH	540	memcpy(dev->ibdev.iw_ifname, dev->rdev.lldi.ports[0]->name,
dd05cb82 KH	541	sizeof(dev->ibdev.iw_ifname));
cfdda9d7	542
dad3b05d	543	ib_set_device_ops(&dev->ibdev, &c4iw_dev_ops);
ab7efbe2 SW	544	ret = set_netdevs(&dev->ibdev, &dev->rdev);
ab7efbe2 SW	545	if (ret)
dd05cb82	546	goto err_dealloc_ctx;
e0477b34 JG	547	dma_set_max_seg_size(&dev->rdev.lldi.pdev->dev, UINT_MAX);
	548	ret = ib_register_device(&dev->ibdev, "cxgb4_%d",
	549	&dev->rdev.lldi.pdev->dev);
cfdda9d7	550	if (ret)
dd05cb82	551	goto err_dealloc_ctx;
1c8f1da5	552	return;
508a523f	553
1c8f1da5 BP	554	err_dealloc_ctx:
	555	pr_err("%s - Failed registering iwarp device: %d\n",
	556	pci_name(ctx->lldi.pdev), ret);
	557	c4iw_dealloc(ctx);
	558	return;
cfdda9d7 SW	559	}
	560
	561	void c4iw_unregister_device(struct c4iw_dev *dev)
	562	{
548ddb19	563	pr_debug("c4iw_dev %p\n", dev);
cfdda9d7	564	ib_unregister_device(&dev->ibdev);
cfdda9d7 SW	565	return;
cfdda9d7 SW	566	}