[linux-2.6-block.git] / drivers / staging / rdma / ehca / ehca_cq.c

/*
 *  IBM eServer eHCA Infiniband device driver for Linux on POWER
 *
 *  Completion queue handling
 *
 *  Authors: Waleri Fomin <fomin@de.ibm.com>
 *           Khadija Souissi <souissi@de.ibm.com>
 *           Reinhard Ernst <rernst@de.ibm.com>
 *           Heiko J Schick <schickhj@de.ibm.com>
 *           Hoang-Nam Nguyen <hnguyen@de.ibm.com>
 *
 *
 *  Copyright (c) 2005 IBM Corporation
 *
 *  All rights reserved.
 *
 *  This source code is distributed under a dual license of GPL v2.0 and OpenIB
 *  BSD.
 *
 * OpenIB BSD License
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 * Redistributions of source code must retain the above copyright notice, this
 * list of conditions and the following disclaimer.
 *
 * Redistributions in binary form must reproduce the above copyright notice,
 * this list of conditions and the following disclaimer in the documentation
 * and/or other materials
 * provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
 * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include <linux/slab.h>

#include "ehca_iverbs.h"
#include "ehca_classes.h"
#include "ehca_irq.h"
#include "hcp_if.h"

static struct kmem_cache *cq_cache;

int ehca_cq_assign_qp(struct ehca_cq *cq, struct ehca_qp *qp)
{
	unsigned int qp_num = qp->real_qp_num;
	unsigned int key = qp_num & (QP_HASHTAB_LEN-1);
	unsigned long flags;

	spin_lock_irqsave(&cq->spinlock, flags);
	hlist_add_head(&qp->list_entries, &cq->qp_hashtab[key]);
	spin_unlock_irqrestore(&cq->spinlock, flags);

	ehca_dbg(cq->ib_cq.device, "cq_num=%x real_qp_num=%x",
		 cq->cq_number, qp_num);

	return 0;
}

int ehca_cq_unassign_qp(struct ehca_cq *cq, unsigned int real_qp_num)
{
	int ret = -EINVAL;
	unsigned int key = real_qp_num & (QP_HASHTAB_LEN-1);
	struct hlist_node *iter;
	struct ehca_qp *qp;
	unsigned long flags;

	spin_lock_irqsave(&cq->spinlock, flags);
	hlist_for_each(iter, &cq->qp_hashtab[key]) {
		qp = hlist_entry(iter, struct ehca_qp, list_entries);
		if (qp->real_qp_num == real_qp_num) {
			hlist_del(iter);
			ehca_dbg(cq->ib_cq.device,
				 "removed qp from cq .cq_num=%x real_qp_num=%x",
				 cq->cq_number, real_qp_num);
			ret = 0;
			break;
		}
	}
	spin_unlock_irqrestore(&cq->spinlock, flags);
	if (ret)
		ehca_err(cq->ib_cq.device,
			 "qp not found cq_num=%x real_qp_num=%x",
			 cq->cq_number, real_qp_num);

	return ret;
}

struct ehca_qp *ehca_cq_get_qp(struct ehca_cq *cq, int real_qp_num)
{
	struct ehca_qp *ret = NULL;
	unsigned int key = real_qp_num & (QP_HASHTAB_LEN-1);
	struct hlist_node *iter;
	struct ehca_qp *qp;
	hlist_for_each(iter, &cq->qp_hashtab[key]) {
		qp = hlist_entry(iter, struct ehca_qp, list_entries);
		if (qp->real_qp_num == real_qp_num) {
			ret = qp;
			break;
		}
	}
	return ret;
}

struct ib_cq *ehca_create_cq(struct ib_device *device,
			     const struct ib_cq_init_attr *attr,
			     struct ib_ucontext *context,
			     struct ib_udata *udata)
{
	int cqe = attr->cqe;
	static const u32 additional_cqe = 20;
	struct ib_cq *cq;
	struct ehca_cq *my_cq;
	struct ehca_shca *shca =
		container_of(device, struct ehca_shca, ib_device);
	struct ipz_adapter_handle adapter_handle;
	struct ehca_alloc_cq_parms param; /* h_call's out parameters */
	struct h_galpa gal;
	void *vpage;
	u32 counter;
	u64 rpage, cqx_fec, h_ret;
	int rc, i;
	unsigned long flags;

	if (attr->flags)
		return ERR_PTR(-EINVAL);

	if (cqe >= 0xFFFFFFFF - 64 - additional_cqe)
		return ERR_PTR(-EINVAL);

	if (!atomic_add_unless(&shca->num_cqs, 1, shca->max_num_cqs)) {
		ehca_err(device, "Unable to create CQ, max number of %i "
			"CQs reached.", shca->max_num_cqs);
		ehca_err(device, "To increase the maximum number of CQs "
			"use the number_of_cqs module parameter.\n");
		return ERR_PTR(-ENOSPC);
	}

	my_cq = kmem_cache_zalloc(cq_cache, GFP_KERNEL);
	if (!my_cq) {
		ehca_err(device, "Out of memory for ehca_cq struct device=%p",
			 device);
		atomic_dec(&shca->num_cqs);
		return ERR_PTR(-ENOMEM);
	}

	memset(&param, 0, sizeof(struct ehca_alloc_cq_parms));

	spin_lock_init(&my_cq->spinlock);
	spin_lock_init(&my_cq->cb_lock);
	spin_lock_init(&my_cq->task_lock);
	atomic_set(&my_cq->nr_events, 0);
	init_waitqueue_head(&my_cq->wait_completion);

	cq = &my_cq->ib_cq;

	adapter_handle = shca->ipz_hca_handle;
	param.eq_handle = shca->eq.ipz_eq_handle;

	idr_preload(GFP_KERNEL);
	write_lock_irqsave(&ehca_cq_idr_lock, flags);
	rc = idr_alloc(&ehca_cq_idr, my_cq, 0, 0x2000000, GFP_NOWAIT);
	write_unlock_irqrestore(&ehca_cq_idr_lock, flags);
	idr_preload_end();

	if (rc < 0) {
		cq = ERR_PTR(-ENOMEM);
		ehca_err(device, "Can't allocate new idr entry. device=%p",
			 device);
		goto create_cq_exit1;
	}
	my_cq->token = rc;

	/*
	 * CQs maximum depth is 4GB-64, but we need additional 20 as buffer
	 * for receiving errors CQEs.
	 */
	param.nr_cqe = cqe + additional_cqe;
	h_ret = hipz_h_alloc_resource_cq(adapter_handle, my_cq, &param);

	if (h_ret != H_SUCCESS) {
		ehca_err(device, "hipz_h_alloc_resource_cq() failed "
			 "h_ret=%lli device=%p", h_ret, device);
		cq = ERR_PTR(ehca2ib_return_code(h_ret));
		goto create_cq_exit2;
	}

	rc = ipz_queue_ctor(NULL, &my_cq->ipz_queue, param.act_pages,
				EHCA_PAGESIZE, sizeof(struct ehca_cqe), 0, 0);
	if (!rc) {
		ehca_err(device, "ipz_queue_ctor() failed ipz_rc=%i device=%p",
			 rc, device);
		cq = ERR_PTR(-EINVAL);
		goto create_cq_exit3;
	}

	for (counter = 0; counter < param.act_pages; counter++) {
		vpage = ipz_qpageit_get_inc(&my_cq->ipz_queue);
		if (!vpage) {
			ehca_err(device, "ipz_qpageit_get_inc() "
				 "returns NULL device=%p", device);
			cq = ERR_PTR(-EAGAIN);
			goto create_cq_exit4;
		}
		rpage = __pa(vpage);

		h_ret = hipz_h_register_rpage_cq(adapter_handle,
						 my_cq->ipz_cq_handle,
						 &my_cq->pf,
						 0,
						 0,
						 rpage,
						 1,
						 my_cq->galpas.
						 kernel);

		if (h_ret < H_SUCCESS) {
			ehca_err(device, "hipz_h_register_rpage_cq() failed "
				 "ehca_cq=%p cq_num=%x h_ret=%lli counter=%i "
				 "act_pages=%i", my_cq, my_cq->cq_number,
				 h_ret, counter, param.act_pages);
			cq = ERR_PTR(-EINVAL);
			goto create_cq_exit4;
		}

		if (counter == (param.act_pages - 1)) {
			vpage = ipz_qpageit_get_inc(&my_cq->ipz_queue);
			if ((h_ret != H_SUCCESS) || vpage) {
				ehca_err(device, "Registration of pages not "
					 "complete ehca_cq=%p cq_num=%x "
					 "h_ret=%lli", my_cq, my_cq->cq_number,
					 h_ret);
				cq = ERR_PTR(-EAGAIN);
				goto create_cq_exit4;
			}
		} else {
			if (h_ret != H_PAGE_REGISTERED) {
				ehca_err(device, "Registration of page failed "
					 "ehca_cq=%p cq_num=%x h_ret=%lli "
					 "counter=%i act_pages=%i",
					 my_cq, my_cq->cq_number,
					 h_ret, counter, param.act_pages);
				cq = ERR_PTR(-ENOMEM);
				goto create_cq_exit4;
			}
		}
	}

	ipz_qeit_reset(&my_cq->ipz_queue);

	gal = my_cq->galpas.kernel;
	cqx_fec = hipz_galpa_load(gal, CQTEMM_OFFSET(cqx_fec));
	ehca_dbg(device, "ehca_cq=%p cq_num=%x CQX_FEC=%llx",
		 my_cq, my_cq->cq_number, cqx_fec);

	my_cq->ib_cq.cqe = my_cq->nr_of_entries =
		param.act_nr_of_entries - additional_cqe;
	my_cq->cq_number = (my_cq->ipz_cq_handle.handle) & 0xffff;

	for (i = 0; i < QP_HASHTAB_LEN; i++)
		INIT_HLIST_HEAD(&my_cq->qp_hashtab[i]);

	INIT_LIST_HEAD(&my_cq->sqp_err_list);
	INIT_LIST_HEAD(&my_cq->rqp_err_list);

	if (context) {
		struct ipz_queue *ipz_queue = &my_cq->ipz_queue;
		struct ehca_create_cq_resp resp;
		memset(&resp, 0, sizeof(resp));
		resp.cq_number = my_cq->cq_number;
		resp.token = my_cq->token;
		resp.ipz_queue.qe_size = ipz_queue->qe_size;
		resp.ipz_queue.act_nr_of_sg = ipz_queue->act_nr_of_sg;
		resp.ipz_queue.queue_length = ipz_queue->queue_length;
		resp.ipz_queue.pagesize = ipz_queue->pagesize;
		resp.ipz_queue.toggle_state = ipz_queue->toggle_state;
		resp.fw_handle_ofs = (u32)
			(my_cq->galpas.user.fw_handle & (PAGE_SIZE - 1));
		if (ib_copy_to_udata(udata, &resp, sizeof(resp))) {
			ehca_err(device, "Copy to udata failed.");
			cq = ERR_PTR(-EFAULT);
			goto create_cq_exit4;
		}
	}

	return cq;

create_cq_exit4:
	ipz_queue_dtor(NULL, &my_cq->ipz_queue);

create_cq_exit3:
	h_ret = hipz_h_destroy_cq(adapter_handle, my_cq, 1);
	if (h_ret != H_SUCCESS)
		ehca_err(device, "hipz_h_destroy_cq() failed ehca_cq=%p "
			 "cq_num=%x h_ret=%lli", my_cq, my_cq->cq_number, h_ret);

create_cq_exit2:
	write_lock_irqsave(&ehca_cq_idr_lock, flags);
	idr_remove(&ehca_cq_idr, my_cq->token);
	write_unlock_irqrestore(&ehca_cq_idr_lock, flags);

create_cq_exit1:
	kmem_cache_free(cq_cache, my_cq);

	atomic_dec(&shca->num_cqs);
	return cq;
}

int ehca_destroy_cq(struct ib_cq *cq)
{
	u64 h_ret;
	struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq);
	int cq_num = my_cq->cq_number;
	struct ib_device *device = cq->device;
	struct ehca_shca *shca = container_of(device, struct ehca_shca,
					      ib_device);
	struct ipz_adapter_handle adapter_handle = shca->ipz_hca_handle;
	unsigned long flags;

	if (cq->uobject) {
		if (my_cq->mm_count_galpa || my_cq->mm_count_queue) {
			ehca_err(device, "Resources still referenced in "
				 "user space cq_num=%x", my_cq->cq_number);
			return -EINVAL;
		}
	}

	/*
	 * remove the CQ from the idr first to make sure
	 * no more interrupt tasklets will touch this CQ
	 */
	write_lock_irqsave(&ehca_cq_idr_lock, flags);
	idr_remove(&ehca_cq_idr, my_cq->token);
	write_unlock_irqrestore(&ehca_cq_idr_lock, flags);

	/* now wait until all pending events have completed */
	wait_event(my_cq->wait_completion, !atomic_read(&my_cq->nr_events));

	/* nobody's using our CQ any longer -- we can destroy it */
	h_ret = hipz_h_destroy_cq(adapter_handle, my_cq, 0);
	if (h_ret == H_R_STATE) {
		/* cq in err: read err data and destroy it forcibly */
		ehca_dbg(device, "ehca_cq=%p cq_num=%x resource=%llx in err "
			 "state. Try to delete it forcibly.",
			 my_cq, cq_num, my_cq->ipz_cq_handle.handle);
		ehca_error_data(shca, my_cq, my_cq->ipz_cq_handle.handle);
		h_ret = hipz_h_destroy_cq(adapter_handle, my_cq, 1);
		if (h_ret == H_SUCCESS)
			ehca_dbg(device, "cq_num=%x deleted successfully.",
				 cq_num);
	}
	if (h_ret != H_SUCCESS) {
		ehca_err(device, "hipz_h_destroy_cq() failed h_ret=%lli "
			 "ehca_cq=%p cq_num=%x", h_ret, my_cq, cq_num);
		return ehca2ib_return_code(h_ret);
	}
	ipz_queue_dtor(NULL, &my_cq->ipz_queue);
	kmem_cache_free(cq_cache, my_cq);

	atomic_dec(&shca->num_cqs);
	return 0;
}

int ehca_resize_cq(struct ib_cq *cq, int cqe, struct ib_udata *udata)
{
	/* TODO: proper resize needs to be done */
	ehca_err(cq->device, "not implemented yet");

	return -EFAULT;
}

int ehca_init_cq_cache(void)
{
	cq_cache = kmem_cache_create("ehca_cache_cq",
				     sizeof(struct ehca_cq), 0,
				     SLAB_HWCACHE_ALIGN,
				     NULL);
	if (!cq_cache)
		return -ENOMEM;
	return 0;
}

void ehca_cleanup_cq_cache(void)
{
	kmem_cache_destroy(cq_cache);
}
Commit	Line	Data
fab97220 HS	1	/*
	2	* IBM eServer eHCA Infiniband device driver for Linux on POWER
	3	*
	4	* Completion queue handling
	5	*
	6	* Authors: Waleri Fomin <fomin@de.ibm.com>
	7	* Khadija Souissi <souissi@de.ibm.com>
	8	* Reinhard Ernst <rernst@de.ibm.com>
	9	* Heiko J Schick <schickhj@de.ibm.com>
	10	* Hoang-Nam Nguyen <hnguyen@de.ibm.com>
	11	*
	12	*
	13	* Copyright (c) 2005 IBM Corporation
	14	*
	15	* All rights reserved.
	16	*
	17	* This source code is distributed under a dual license of GPL v2.0 and OpenIB
	18	* BSD.
	19	*
	20	* OpenIB BSD License
	21	*
	22	* Redistribution and use in source and binary forms, with or without
	23	* modification, are permitted provided that the following conditions are met:
	24	*
	25	* Redistributions of source code must retain the above copyright notice, this
	26	* list of conditions and the following disclaimer.
	27	*
	28	* Redistributions in binary form must reproduce the above copyright notice,
	29	* this list of conditions and the following disclaimer in the documentation
	30	* and/or other materials
	31	* provided with the distribution.
	32	*
	33	* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
	34	* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
	35	* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
	36	* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
	37	* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
	38	* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
	39	* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
	40	* BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
	41	* IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
	42	* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
	43	* POSSIBILITY OF SUCH DAMAGE.
	44	*/
	45
5a0e3ad6 TH	46	#include <linux/slab.h>
5a0e3ad6 TH	47
fab97220 HS	48	#include "ehca_iverbs.h"
	49	#include "ehca_classes.h"
	50	#include "ehca_irq.h"
	51	#include "hcp_if.h"
	52
	53	static struct kmem_cache *cq_cache;
	54
	55	int ehca_cq_assign_qp(struct ehca_cq cq, struct ehca_qp qp)
	56	{
	57	unsigned int qp_num = qp->real_qp_num;
	58	unsigned int key = qp_num & (QP_HASHTAB_LEN-1);
9844b71b	59	unsigned long flags;
fab97220	60
9844b71b	61	spin_lock_irqsave(&cq->spinlock, flags);
fab97220	62	hlist_add_head(&qp->list_entries, &cq->qp_hashtab[key]);
9844b71b	63	spin_unlock_irqrestore(&cq->spinlock, flags);
fab97220 HS	64
	65	ehca_dbg(cq->ib_cq.device, "cq_num=%x real_qp_num=%x",
	66	cq->cq_number, qp_num);
	67
	68	return 0;
	69	}
	70
	71	int ehca_cq_unassign_qp(struct ehca_cq *cq, unsigned int real_qp_num)
	72	{
	73	int ret = -EINVAL;
	74	unsigned int key = real_qp_num & (QP_HASHTAB_LEN-1);
	75	struct hlist_node *iter;
	76	struct ehca_qp *qp;
9844b71b	77	unsigned long flags;
fab97220	78
9844b71b	79	spin_lock_irqsave(&cq->spinlock, flags);
fab97220 HS	80	hlist_for_each(iter, &cq->qp_hashtab[key]) {
	81	qp = hlist_entry(iter, struct ehca_qp, list_entries);
	82	if (qp->real_qp_num == real_qp_num) {
	83	hlist_del(iter);
	84	ehca_dbg(cq->ib_cq.device,
	85	"removed qp from cq .cq_num=%x real_qp_num=%x",
	86	cq->cq_number, real_qp_num);
	87	ret = 0;
	88	break;
	89	}
	90	}
9844b71b	91	spin_unlock_irqrestore(&cq->spinlock, flags);
fab97220 HS	92	if (ret)
	93	ehca_err(cq->ib_cq.device,
	94	"qp not found cq_num=%x real_qp_num=%x",
	95	cq->cq_number, real_qp_num);
	96
	97	return ret;
	98	}
	99
2b94397a	100	struct ehca_qp ehca_cq_get_qp(struct ehca_cq cq, int real_qp_num)
fab97220 HS	101	{
	102	struct ehca_qp *ret = NULL;
	103	unsigned int key = real_qp_num & (QP_HASHTAB_LEN-1);
	104	struct hlist_node *iter;
	105	struct ehca_qp *qp;
	106	hlist_for_each(iter, &cq->qp_hashtab[key]) {
	107	qp = hlist_entry(iter, struct ehca_qp, list_entries);
	108	if (qp->real_qp_num == real_qp_num) {
	109	ret = qp;
	110	break;
	111	}
	112	}
	113	return ret;
	114	}
	115
bcf4c1ea MB	116	struct ib_cq ehca_create_cq(struct ib_device device,
bcf4c1ea MB	117	const struct ib_cq_init_attr *attr,
fab97220 HS	118	struct ib_ucontext *context,
	119	struct ib_udata *udata)
	120	{
bcf4c1ea	121	int cqe = attr->cqe;
fab97220 HS	122	static const u32 additional_cqe = 20;
	123	struct ib_cq *cq;
	124	struct ehca_cq *my_cq;
	125	struct ehca_shca *shca =
	126	container_of(device, struct ehca_shca, ib_device);
	127	struct ipz_adapter_handle adapter_handle;
	128	struct ehca_alloc_cq_parms param; /* h_call's out parameters */
	129	struct h_galpa gal;
	130	void *vpage;
	131	u32 counter;
	132	u64 rpage, cqx_fec, h_ret;
43b2ec01	133	int rc, i;
fab97220 HS	134	unsigned long flags;
fab97220 HS	135
bcf4c1ea MB	136	if (attr->flags)
	137	return ERR_PTR(-EINVAL);
	138
fab97220 HS	139	if (cqe >= 0xFFFFFFFF - 64 - additional_cqe)
	140	return ERR_PTR(-EINVAL);
	141
19f42821	142	if (!atomic_add_unless(&shca->num_cqs, 1, shca->max_num_cqs)) {
d227fa72	143	ehca_err(device, "Unable to create CQ, max number of %i "
19f42821	144	"CQs reached.", shca->max_num_cqs);
d227fa72 SR	145	ehca_err(device, "To increase the maximum number of CQs "
	146	"use the number_of_cqs module parameter.\n");
	147	return ERR_PTR(-ENOSPC);
	148	}
	149
c3762229	150	my_cq = kmem_cache_zalloc(cq_cache, GFP_KERNEL);
fab97220 HS	151	if (!my_cq) {
	152	ehca_err(device, "Out of memory for ehca_cq struct device=%p",
	153	device);
d227fa72	154	atomic_dec(&shca->num_cqs);
fab97220 HS	155	return ERR_PTR(-ENOMEM);
	156	}
	157
fab97220 HS	158	memset(&param, 0, sizeof(struct ehca_alloc_cq_parms));
	159
	160	spin_lock_init(&my_cq->spinlock);
	161	spin_lock_init(&my_cq->cb_lock);
	162	spin_lock_init(&my_cq->task_lock);
28db6beb	163	atomic_set(&my_cq->nr_events, 0);
31726798	164	init_waitqueue_head(&my_cq->wait_completion);
fab97220 HS	165
	166	cq = &my_cq->ib_cq;
	167
	168	adapter_handle = shca->ipz_hca_handle;
	169	param.eq_handle = shca->eq.ipz_eq_handle;
	170
cbbbce1d TH	171	idr_preload(GFP_KERNEL);
cbbbce1d TH	172	write_lock_irqsave(&ehca_cq_idr_lock, flags);
43b2ec01	173	rc = idr_alloc(&ehca_cq_idr, my_cq, 0, 0x2000000, GFP_NOWAIT);
cbbbce1d TH	174	write_unlock_irqrestore(&ehca_cq_idr_lock, flags);
cbbbce1d TH	175	idr_preload_end();
fab97220	176
43b2ec01	177	if (rc < 0) {
fab97220 HS	178	cq = ERR_PTR(-ENOMEM);
	179	ehca_err(device, "Can't allocate new idr entry. device=%p",
	180	device);
	181	goto create_cq_exit1;
	182	}
43b2ec01	183	my_cq->token = rc;
fab97220 HS	184
	185	/*
	186	* CQs maximum depth is 4GB-64, but we need additional 20 as buffer
	187	* for receiving errors CQEs.
	188	*/
	189	param.nr_cqe = cqe + additional_cqe;
	190	h_ret = hipz_h_alloc_resource_cq(adapter_handle, my_cq, &param);
	191
	192	if (h_ret != H_SUCCESS) {
	193	ehca_err(device, "hipz_h_alloc_resource_cq() failed "
3750f605	194	"h_ret=%lli device=%p", h_ret, device);
fab97220 HS	195	cq = ERR_PTR(ehca2ib_return_code(h_ret));
	196	goto create_cq_exit2;
	197	}
	198
43b2ec01	199	rc = ipz_queue_ctor(NULL, &my_cq->ipz_queue, param.act_pages,
e2f81daf	200	EHCA_PAGESIZE, sizeof(struct ehca_cqe), 0, 0);
43b2ec01	201	if (!rc) {
e3722192	202	ehca_err(device, "ipz_queue_ctor() failed ipz_rc=%i device=%p",
43b2ec01	203	rc, device);
fab97220 HS	204	cq = ERR_PTR(-EINVAL);
	205	goto create_cq_exit3;
	206	}
	207
	208	for (counter = 0; counter < param.act_pages; counter++) {
	209	vpage = ipz_qpageit_get_inc(&my_cq->ipz_queue);
	210	if (!vpage) {
	211	ehca_err(device, "ipz_qpageit_get_inc() "
	212	"returns NULL device=%p", device);
	213	cq = ERR_PTR(-EAGAIN);
	214	goto create_cq_exit4;
	215	}
b4b8d1e4	216	rpage = __pa(vpage);
fab97220 HS	217
	218	h_ret = hipz_h_register_rpage_cq(adapter_handle,
	219	my_cq->ipz_cq_handle,
	220	&my_cq->pf,
	221	0,
	222	0,
	223	rpage,
	224	1,
	225	my_cq->galpas.
	226	kernel);
	227
	228	if (h_ret < H_SUCCESS) {
	229	ehca_err(device, "hipz_h_register_rpage_cq() failed "
3750f605	230	"ehca_cq=%p cq_num=%x h_ret=%lli counter=%i "
fab97220 HS	231	"act_pages=%i", my_cq, my_cq->cq_number,
	232	h_ret, counter, param.act_pages);
	233	cq = ERR_PTR(-EINVAL);
	234	goto create_cq_exit4;
	235	}
	236
	237	if (counter == (param.act_pages - 1)) {
	238	vpage = ipz_qpageit_get_inc(&my_cq->ipz_queue);
	239	if ((h_ret != H_SUCCESS) \|\| vpage) {
	240	ehca_err(device, "Registration of pages not "
	241	"complete ehca_cq=%p cq_num=%x "
3750f605	242	"h_ret=%lli", my_cq, my_cq->cq_number,
fab97220 HS	243	h_ret);
	244	cq = ERR_PTR(-EAGAIN);
	245	goto create_cq_exit4;
	246	}
	247	} else {
	248	if (h_ret != H_PAGE_REGISTERED) {
	249	ehca_err(device, "Registration of page failed "
3750f605	250	"ehca_cq=%p cq_num=%x h_ret=%lli "
fab97220 HS	251	"counter=%i act_pages=%i",
	252	my_cq, my_cq->cq_number,
	253	h_ret, counter, param.act_pages);
	254	cq = ERR_PTR(-ENOMEM);
	255	goto create_cq_exit4;
	256	}
	257	}
	258	}
	259
	260	ipz_qeit_reset(&my_cq->ipz_queue);
	261
	262	gal = my_cq->galpas.kernel;
	263	cqx_fec = hipz_galpa_load(gal, CQTEMM_OFFSET(cqx_fec));
3750f605	264	ehca_dbg(device, "ehca_cq=%p cq_num=%x CQX_FEC=%llx",
fab97220 HS	265	my_cq, my_cq->cq_number, cqx_fec);
	266
	267	my_cq->ib_cq.cqe = my_cq->nr_of_entries =
	268	param.act_nr_of_entries - additional_cqe;
	269	my_cq->cq_number = (my_cq->ipz_cq_handle.handle) & 0xffff;
	270
	271	for (i = 0; i < QP_HASHTAB_LEN; i++)
	272	INIT_HLIST_HEAD(&my_cq->qp_hashtab[i]);
	273
b9012e0a AS	274	INIT_LIST_HEAD(&my_cq->sqp_err_list);
	275	INIT_LIST_HEAD(&my_cq->rqp_err_list);
	276
fab97220 HS	277	if (context) {
	278	struct ipz_queue *ipz_queue = &my_cq->ipz_queue;
	279	struct ehca_create_cq_resp resp;
fab97220 HS	280	memset(&resp, 0, sizeof(resp));
	281	resp.cq_number = my_cq->cq_number;
	282	resp.token = my_cq->token;
	283	resp.ipz_queue.qe_size = ipz_queue->qe_size;
	284	resp.ipz_queue.act_nr_of_sg = ipz_queue->act_nr_of_sg;
	285	resp.ipz_queue.queue_length = ipz_queue->queue_length;
	286	resp.ipz_queue.pagesize = ipz_queue->pagesize;
	287	resp.ipz_queue.toggle_state = ipz_queue->toggle_state;
e390d3b5 HNN	288	resp.fw_handle_ofs = (u32)
e390d3b5 HNN	289	(my_cq->galpas.user.fw_handle & (PAGE_SIZE - 1));
fab97220 HS	290	if (ib_copy_to_udata(udata, &resp, sizeof(resp))) {
fab97220 HS	291	ehca_err(device, "Copy to udata failed.");
5bdb0f02	292	cq = ERR_PTR(-EFAULT);
4c34bdf5	293	goto create_cq_exit4;
fab97220 HS	294	}
	295	}
	296
	297	return cq;
	298
fab97220	299	create_cq_exit4:
e2f81daf	300	ipz_queue_dtor(NULL, &my_cq->ipz_queue);
fab97220 HS	301
	302	create_cq_exit3:
	303	h_ret = hipz_h_destroy_cq(adapter_handle, my_cq, 1);
	304	if (h_ret != H_SUCCESS)
	305	ehca_err(device, "hipz_h_destroy_cq() failed ehca_cq=%p "
3750f605	306	"cq_num=%x h_ret=%lli", my_cq, my_cq->cq_number, h_ret);
fab97220 HS	307
fab97220 HS	308	create_cq_exit2:
26ed687f	309	write_lock_irqsave(&ehca_cq_idr_lock, flags);
fab97220	310	idr_remove(&ehca_cq_idr, my_cq->token);
26ed687f	311	write_unlock_irqrestore(&ehca_cq_idr_lock, flags);
fab97220 HS	312
	313	create_cq_exit1:
	314	kmem_cache_free(cq_cache, my_cq);
	315
d227fa72	316	atomic_dec(&shca->num_cqs);
fab97220 HS	317	return cq;
	318	}
	319
	320	int ehca_destroy_cq(struct ib_cq *cq)
	321	{
	322	u64 h_ret;
fab97220 HS	323	struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq);
	324	int cq_num = my_cq->cq_number;
	325	struct ib_device *device = cq->device;
	326	struct ehca_shca *shca = container_of(device, struct ehca_shca,
	327	ib_device);
	328	struct ipz_adapter_handle adapter_handle = shca->ipz_hca_handle;
fab97220 HS	329	unsigned long flags;
fab97220 HS	330
4c34bdf5 HNN	331	if (cq->uobject) {
	332	if (my_cq->mm_count_galpa \|\| my_cq->mm_count_queue) {
	333	ehca_err(device, "Resources still referenced in "
	334	"user space cq_num=%x", my_cq->cq_number);
	335	return -EINVAL;
	336	}
4c34bdf5 HNN	337	}
4c34bdf5 HNN	338
28db6beb JF	339	/*
	340	* remove the CQ from the idr first to make sure
	341	* no more interrupt tasklets will touch this CQ
	342	*/
26ed687f	343	write_lock_irqsave(&ehca_cq_idr_lock, flags);
fab97220	344	idr_remove(&ehca_cq_idr, my_cq->token);
26ed687f	345	write_unlock_irqrestore(&ehca_cq_idr_lock, flags);
fab97220	346
28db6beb JF	347	/* now wait until all pending events have completed */
	348	wait_event(my_cq->wait_completion, !atomic_read(&my_cq->nr_events));
	349
	350	/* nobody's using our CQ any longer -- we can destroy it */
fab97220 HS	351	h_ret = hipz_h_destroy_cq(adapter_handle, my_cq, 0);
	352	if (h_ret == H_R_STATE) {
	353	/* cq in err: read err data and destroy it forcibly */
3750f605	354	ehca_dbg(device, "ehca_cq=%p cq_num=%x resource=%llx in err "
fab97220 HS	355	"state. Try to delete it forcibly.",
	356	my_cq, cq_num, my_cq->ipz_cq_handle.handle);
	357	ehca_error_data(shca, my_cq, my_cq->ipz_cq_handle.handle);
	358	h_ret = hipz_h_destroy_cq(adapter_handle, my_cq, 1);
	359	if (h_ret == H_SUCCESS)
	360	ehca_dbg(device, "cq_num=%x deleted successfully.",
	361	cq_num);
	362	}
	363	if (h_ret != H_SUCCESS) {
3750f605	364	ehca_err(device, "hipz_h_destroy_cq() failed h_ret=%lli "
fab97220 HS	365	"ehca_cq=%p cq_num=%x", h_ret, my_cq, cq_num);
	366	return ehca2ib_return_code(h_ret);
	367	}
e2f81daf	368	ipz_queue_dtor(NULL, &my_cq->ipz_queue);
fab97220 HS	369	kmem_cache_free(cq_cache, my_cq);
fab97220 HS	370
d227fa72	371	atomic_dec(&shca->num_cqs);
fab97220 HS	372	return 0;
	373	}
	374
	375	int ehca_resize_cq(struct ib_cq cq, int cqe, struct ib_udata udata)
	376	{
fab97220 HS	377	/* TODO: proper resize needs to be done */
	378	ehca_err(cq->device, "not implemented yet");
	379
	380	return -EFAULT;
	381	}
	382
	383	int ehca_init_cq_cache(void)
	384	{
	385	cq_cache = kmem_cache_create("ehca_cache_cq",
	386	sizeof(struct ehca_cq), 0,
	387	SLAB_HWCACHE_ALIGN,
20c2df83	388	NULL);
fab97220 HS	389	if (!cq_cache)
	390	return -ENOMEM;
	391	return 0;
	392	}
	393
	394	void ehca_cleanup_cq_cache(void)
	395	{
7438370a	396	kmem_cache_destroy(cq_cache);
fab97220	397	}