projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
IB/iser: Remove unused "write-only" variables
[linux-2.6-block.git] / drivers / infiniband / hw / amso1100 / c2_qp.c
CommitLineData
f94b533d
TT		 1/*
2 * Copyright (c) 2004 Topspin Communications. All rights reserved.
3 * Copyright (c) 2005 Cisco Systems. All rights reserved.
4 * Copyright (c) 2005 Mellanox Technologies. All rights reserved.
5 * Copyright (c) 2004 Voltaire, Inc. All rights reserved.
6 * Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved.
7 *
8 * This software is available to you under a choice of one of two
9 * licenses. You may choose to be licensed under the terms of the GNU
10 * General Public License (GPL) Version 2, available from the file
11 * COPYING in the main directory of this source tree, or the
12 * OpenIB.org BSD license below:
13 *
14 * Redistribution and use in source and binary forms, with or
15 * without modification, are permitted provided that the following
16 * conditions are met:
17 *
18 * - Redistributions of source code must retain the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer.
21 *
22 * - Redistributions in binary form must reproduce the above
23 * copyright notice, this list of conditions and the following
24 * disclaimer in the documentation and/or other materials
25 * provided with the distribution.
26 *
27 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
28 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
29 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
30 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
31 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
32 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
33 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34 * SOFTWARE.
35 *
36 */
37
e52e6080
TT		 38#include <linux/delay.h>
39
f94b533d
TT		 40#include "c2.h"
41#include "c2_vq.h"
42#include "c2_status.h"
43
44#define C2_MAX_ORD_PER_QP 128
45#define C2_MAX_IRD_PER_QP 128
46
47#define C2_HINT_MAKE(q_index, hint_count) (((q_index) << 16) | hint_count)
48#define C2_HINT_GET_INDEX(hint) (((hint) & 0x7FFF0000) >> 16)
49#define C2_HINT_GET_COUNT(hint) ((hint) & 0x0000FFFF)
50
51#define NO_SUPPORT -1
52static const u8 c2_opcode[] = {
53 [IB_WR_SEND] = C2_WR_TYPE_SEND,
54 [IB_WR_SEND_WITH_IMM] = NO_SUPPORT,
55 [IB_WR_RDMA_WRITE] = C2_WR_TYPE_RDMA_WRITE,
56 [IB_WR_RDMA_WRITE_WITH_IMM] = NO_SUPPORT,
57 [IB_WR_RDMA_READ] = C2_WR_TYPE_RDMA_READ,
58 [IB_WR_ATOMIC_CMP_AND_SWP] = NO_SUPPORT,
59 [IB_WR_ATOMIC_FETCH_AND_ADD] = NO_SUPPORT,
60};
61
62static int to_c2_state(enum ib_qp_state ib_state)
63{
64 switch (ib_state) {
65 case IB_QPS_RESET:
66 return C2_QP_STATE_IDLE;
67 case IB_QPS_RTS:
68 return C2_QP_STATE_RTS;
69 case IB_QPS_SQD:
70 return C2_QP_STATE_CLOSING;
71 case IB_QPS_SQE:
72 return C2_QP_STATE_CLOSING;
73 case IB_QPS_ERR:
74 return C2_QP_STATE_ERROR;
75 default:
76 return -1;
77 }
78}
79
80static int to_ib_state(enum c2_qp_state c2_state)
81{
82 switch (c2_state) {
83 case C2_QP_STATE_IDLE:
84 return IB_QPS_RESET;
85 case C2_QP_STATE_CONNECTING:
86 return IB_QPS_RTR;
87 case C2_QP_STATE_RTS:
88 return IB_QPS_RTS;
89 case C2_QP_STATE_CLOSING:
90 return IB_QPS_SQD;
91 case C2_QP_STATE_ERROR:
92 return IB_QPS_ERR;
93 case C2_QP_STATE_TERMINATE:
94 return IB_QPS_SQE;
95 default:
96 return -1;
97 }
98}
99
100static const char *to_ib_state_str(int ib_state)
101{
102 static const char *state_str[] = {
103 "IB_QPS_RESET",
104 "IB_QPS_INIT",
105 "IB_QPS_RTR",
106 "IB_QPS_RTS",
107 "IB_QPS_SQD",
108 "IB_QPS_SQE",
109 "IB_QPS_ERR"
110 };
111 if (ib_state < IB_QPS_RESET ||
112 ib_state > IB_QPS_ERR)
113 return "<invalid IB QP state>";
114
115 ib_state -= IB_QPS_RESET;
116 return state_str[ib_state];
117}
118
119void c2_set_qp_state(struct c2_qp *qp, int c2_state)
120{
121 int new_state = to_ib_state(c2_state);
122
123 pr_debug("%s: qp[%p] state modify %s --> %s\n",
124 __FUNCTION__,
125 qp,
126 to_ib_state_str(qp->state),
127 to_ib_state_str(new_state));
128 qp->state = new_state;
129}
130
131#define C2_QP_NO_ATTR_CHANGE 0xFFFFFFFF
132
133int c2_qp_modify(struct c2_dev *c2dev, struct c2_qp *qp,
134 struct ib_qp_attr *attr, int attr_mask)
135{
136 struct c2wr_qp_modify_req wr;
137 struct c2wr_qp_modify_rep *reply;
138 struct c2_vq_req *vq_req;
139 unsigned long flags;
140 u8 next_state;
141 int err;
142
143 pr_debug("%s:%d qp=%p, %s --> %s\n",
144 __FUNCTION__, __LINE__,
145 qp,
146 to_ib_state_str(qp->state),
147 to_ib_state_str(attr->qp_state));
148
149 vq_req = vq_req_alloc(c2dev);
150 if (!vq_req)
151 return -ENOMEM;
152
153 c2_wr_set_id(&wr, CCWR_QP_MODIFY);
154 wr.hdr.context = (unsigned long) vq_req;
155 wr.rnic_handle = c2dev->adapter_handle;
156 wr.qp_handle = qp->adapter_handle;
157 wr.ord = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
158 wr.ird = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
159 wr.sq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
160 wr.rq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
161
162 if (attr_mask & IB_QP_STATE) {
163 /* Ensure the state is valid */
164 if (attr->qp_state < 0 || attr->qp_state > IB_QPS_ERR)
165 return -EINVAL;
166
167 wr.next_qp_state = cpu_to_be32(to_c2_state(attr->qp_state));
168
169 if (attr->qp_state == IB_QPS_ERR) {
170 spin_lock_irqsave(&qp->lock, flags);
171 if (qp->cm_id && qp->state == IB_QPS_RTS) {
172 pr_debug("Generating CLOSE event for QP-->ERR, "
173 "qp=%p, cm_id=%p\n",qp,qp->cm_id);
174 /* Generate an CLOSE event */
175 vq_req->cm_id = qp->cm_id;
176 vq_req->event = IW_CM_EVENT_CLOSE;
177 }
178 spin_unlock_irqrestore(&qp->lock, flags);
179 }
180 next_state = attr->qp_state;
181
182 } else if (attr_mask & IB_QP_CUR_STATE) {
183
184 if (attr->cur_qp_state != IB_QPS_RTR &&
185 attr->cur_qp_state != IB_QPS_RTS &&
186 attr->cur_qp_state != IB_QPS_SQD &&
187 attr->cur_qp_state != IB_QPS_SQE)
188 return -EINVAL;
189 else
190 wr.next_qp_state =
191 cpu_to_be32(to_c2_state(attr->cur_qp_state));
192
193 next_state = attr->cur_qp_state;
194
195 } else {
196 err = 0;
197 goto bail0;
198 }
199
200 /* reference the request struct */
201 vq_req_get(c2dev, vq_req);
202
203 err = vq_send_wr(c2dev, (union c2wr *) & wr);
204 if (err) {
205 vq_req_put(c2dev, vq_req);
206 goto bail0;
207 }
208
209 err = vq_wait_for_reply(c2dev, vq_req);
210 if (err)
211 goto bail0;
212
213 reply = (struct c2wr_qp_modify_rep *) (unsigned long) vq_req->reply_msg;
214 if (!reply) {
215 err = -ENOMEM;
216 goto bail0;
217 }
218
219 err = c2_errno(reply);
220 if (!err)
221 qp->state = next_state;
222#ifdef DEBUG
223 else
224 pr_debug("%s: c2_errno=%d\n", __FUNCTION__, err);
225#endif
226 /*
227 * If we're going to error and generating the event here, then
228 * we need to remove the reference because there will be no
229 * close event generated by the adapter
230 */
231 spin_lock_irqsave(&qp->lock, flags);
232 if (vq_req->event==IW_CM_EVENT_CLOSE && qp->cm_id) {
233 qp->cm_id->rem_ref(qp->cm_id);
234 qp->cm_id = NULL;
235 }
236 spin_unlock_irqrestore(&qp->lock, flags);
237
238 vq_repbuf_free(c2dev, reply);
239 bail0:
240 vq_req_free(c2dev, vq_req);
241
242 pr_debug("%s:%d qp=%p, cur_state=%s\n",
243 __FUNCTION__, __LINE__,
244 qp,
245 to_ib_state_str(qp->state));
246 return err;
247}
248
249int c2_qp_set_read_limits(struct c2_dev *c2dev, struct c2_qp *qp,
250 int ord, int ird)
251{
252 struct c2wr_qp_modify_req wr;
253 struct c2wr_qp_modify_rep *reply;
254 struct c2_vq_req *vq_req;
255 int err;
256
257 vq_req = vq_req_alloc(c2dev);
258 if (!vq_req)
259 return -ENOMEM;
260
261 c2_wr_set_id(&wr, CCWR_QP_MODIFY);
262 wr.hdr.context = (unsigned long) vq_req;
263 wr.rnic_handle = c2dev->adapter_handle;
264 wr.qp_handle = qp->adapter_handle;
265 wr.ord = cpu_to_be32(ord);
266 wr.ird = cpu_to_be32(ird);
267 wr.sq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
268 wr.rq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
269 wr.next_qp_state = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
270
271 /* reference the request struct */
272 vq_req_get(c2dev, vq_req);
273
274 err = vq_send_wr(c2dev, (union c2wr *) & wr);
275 if (err) {
276 vq_req_put(c2dev, vq_req);
277 goto bail0;
278 }
279
280 err = vq_wait_for_reply(c2dev, vq_req);
281 if (err)
282 goto bail0;
283
284 reply = (struct c2wr_qp_modify_rep *) (unsigned long)
285 vq_req->reply_msg;
286 if (!reply) {
287 err = -ENOMEM;
288 goto bail0;
289 }
290
291 err = c2_errno(reply);
292 vq_repbuf_free(c2dev, reply);
293 bail0:
294 vq_req_free(c2dev, vq_req);
295 return err;
296}
297
298static int destroy_qp(struct c2_dev *c2dev, struct c2_qp *qp)
299{
300 struct c2_vq_req *vq_req;
301 struct c2wr_qp_destroy_req wr;
302 struct c2wr_qp_destroy_rep *reply;
303 unsigned long flags;
304 int err;
305
306 /*
307 * Allocate a verb request message
308 */
309 vq_req = vq_req_alloc(c2dev);
310 if (!vq_req) {
311 return -ENOMEM;
312 }
313
314 /*
315 * Initialize the WR
316 */
317 c2_wr_set_id(&wr, CCWR_QP_DESTROY);
318 wr.hdr.context = (unsigned long) vq_req;
319 wr.rnic_handle = c2dev->adapter_handle;
320 wr.qp_handle = qp->adapter_handle;
321
322 /*
323 * reference the request struct. dereferenced in the int handler.
324 */
325 vq_req_get(c2dev, vq_req);
326
327 spin_lock_irqsave(&qp->lock, flags);
328 if (qp->cm_id && qp->state == IB_QPS_RTS) {
329 pr_debug("destroy_qp: generating CLOSE event for QP-->ERR, "
330 "qp=%p, cm_id=%p\n",qp,qp->cm_id);
331 /* Generate an CLOSE event */
332 vq_req->qp = qp;
333 vq_req->cm_id = qp->cm_id;
334 vq_req->event = IW_CM_EVENT_CLOSE;
335 }
336 spin_unlock_irqrestore(&qp->lock, flags);
337
338 /*
339 * Send WR to adapter
340 */
341 err = vq_send_wr(c2dev, (union c2wr *) & wr);
342 if (err) {
343 vq_req_put(c2dev, vq_req);
344 goto bail0;
345 }
346
347 /*
348 * Wait for reply from adapter
349 */
350 err = vq_wait_for_reply(c2dev, vq_req);
351 if (err) {
352 goto bail0;
353 }
354
355 /*
356 * Process reply
357 */
358 reply = (struct c2wr_qp_destroy_rep *) (unsigned long) (vq_req->reply_msg);
359 if (!reply) {
360 err = -ENOMEM;
361 goto bail0;
362 }
363
364 spin_lock_irqsave(&qp->lock, flags);
365 if (qp->cm_id) {
366 qp->cm_id->rem_ref(qp->cm_id);
367 qp->cm_id = NULL;
368 }
369 spin_unlock_irqrestore(&qp->lock, flags);
370
371 vq_repbuf_free(c2dev, reply);
372 bail0:
373 vq_req_free(c2dev, vq_req);
374 return err;
375}
376
377static int c2_alloc_qpn(struct c2_dev *c2dev, struct c2_qp *qp)
378{
379 int ret;
380
381 do {
382 spin_lock_irq(&c2dev->qp_table.lock);
383 ret = idr_get_new_above(&c2dev->qp_table.idr, qp,
384 c2dev->qp_table.last++, &qp->qpn);
385 spin_unlock_irq(&c2dev->qp_table.lock);
386 } while ((ret == -EAGAIN) &&
387 idr_pre_get(&c2dev->qp_table.idr, GFP_KERNEL));
388 return ret;
389}
390
391static void c2_free_qpn(struct c2_dev *c2dev, int qpn)
392{
393 spin_lock_irq(&c2dev->qp_table.lock);
394 idr_remove(&c2dev->qp_table.idr, qpn);
395 spin_unlock_irq(&c2dev->qp_table.lock);
396}
397
398struct c2_qp *c2_find_qpn(struct c2_dev *c2dev, int qpn)
399{
400 unsigned long flags;
401 struct c2_qp *qp;
402
403 spin_lock_irqsave(&c2dev->qp_table.lock, flags);
404 qp = idr_find(&c2dev->qp_table.idr, qpn);
405 spin_unlock_irqrestore(&c2dev->qp_table.lock, flags);
406 return qp;
407}
408
409int c2_alloc_qp(struct c2_dev *c2dev,
410 struct c2_pd *pd,
411 struct ib_qp_init_attr *qp_attrs, struct c2_qp *qp)
412{
413 struct c2wr_qp_create_req wr;
414 struct c2wr_qp_create_rep *reply;
415 struct c2_vq_req *vq_req;
416 struct c2_cq *send_cq = to_c2cq(qp_attrs->send_cq);
417 struct c2_cq *recv_cq = to_c2cq(qp_attrs->recv_cq);
418 unsigned long peer_pa;
419 u32 q_size, msg_size, mmap_size;
420 void __iomem *mmap;
421 int err;
422
423 err = c2_alloc_qpn(c2dev, qp);
424 if (err)
425 return err;
426 qp->ibqp.qp_num = qp->qpn;
427 qp->ibqp.qp_type = IB_QPT_RC;
428
429 /* Allocate the SQ and RQ shared pointers */
430 qp->sq_mq.shared = c2_alloc_mqsp(c2dev, c2dev->kern_mqsp_pool,
431 &qp->sq_mq.shared_dma, GFP_KERNEL);
432 if (!qp->sq_mq.shared) {
433 err = -ENOMEM;
434 goto bail0;
435 }
436
437 qp->rq_mq.shared = c2_alloc_mqsp(c2dev, c2dev->kern_mqsp_pool,
438 &qp->rq_mq.shared_dma, GFP_KERNEL);
439 if (!qp->rq_mq.shared) {
440 err = -ENOMEM;
441 goto bail1;
442 }
443
444 /* Allocate the verbs request */
445 vq_req = vq_req_alloc(c2dev);
446 if (vq_req == NULL) {
447 err = -ENOMEM;
448 goto bail2;
449 }
450
451 /* Initialize the work request */
452 memset(&wr, 0, sizeof(wr));
453 c2_wr_set_id(&wr, CCWR_QP_CREATE);
454 wr.hdr.context = (unsigned long) vq_req;
455 wr.rnic_handle = c2dev->adapter_handle;
456 wr.sq_cq_handle = send_cq->adapter_handle;
457 wr.rq_cq_handle = recv_cq->adapter_handle;
458 wr.sq_depth = cpu_to_be32(qp_attrs->cap.max_send_wr + 1);
459 wr.rq_depth = cpu_to_be32(qp_attrs->cap.max_recv_wr + 1);
460 wr.srq_handle = 0;
461 wr.flags = cpu_to_be32(QP_RDMA_READ | QP_RDMA_WRITE | QP_MW_BIND |
462 QP_ZERO_STAG | QP_RDMA_READ_RESPONSE);
463 wr.send_sgl_depth = cpu_to_be32(qp_attrs->cap.max_send_sge);
464 wr.recv_sgl_depth = cpu_to_be32(qp_attrs->cap.max_recv_sge);
465 wr.rdma_write_sgl_depth = cpu_to_be32(qp_attrs->cap.max_send_sge);
466 wr.shared_sq_ht = cpu_to_be64(qp->sq_mq.shared_dma);
467 wr.shared_rq_ht = cpu_to_be64(qp->rq_mq.shared_dma);
468 wr.ord = cpu_to_be32(C2_MAX_ORD_PER_QP);
469 wr.ird = cpu_to_be32(C2_MAX_IRD_PER_QP);
470 wr.pd_id = pd->pd_id;
471 wr.user_context = (unsigned long) qp;
472
473 vq_req_get(c2dev, vq_req);
474
475 /* Send the WR to the adapter */
476 err = vq_send_wr(c2dev, (union c2wr *) & wr);
477 if (err) {
478 vq_req_put(c2dev, vq_req);
479 goto bail3;
480 }
481
482 /* Wait for the verb reply */
483 err = vq_wait_for_reply(c2dev, vq_req);
484 if (err) {
485 goto bail3;
486 }
487
488 /* Process the reply */
489 reply = (struct c2wr_qp_create_rep *) (unsigned long) (vq_req->reply_msg);
490 if (!reply) {
491 err = -ENOMEM;
492 goto bail3;
493 }
494
495 if ((err = c2_wr_get_result(reply)) != 0) {
496 goto bail4;
497 }
498
499 /* Fill in the kernel QP struct */
500 atomic_set(&qp->refcount, 1);
501 qp->adapter_handle = reply->qp_handle;
502 qp->state = IB_QPS_RESET;
503 qp->send_sgl_depth = qp_attrs->cap.max_send_sge;
504 qp->rdma_write_sgl_depth = qp_attrs->cap.max_send_sge;
505 qp->recv_sgl_depth = qp_attrs->cap.max_recv_sge;
506
507 /* Initialize the SQ MQ */
508 q_size = be32_to_cpu(reply->sq_depth);
509 msg_size = be32_to_cpu(reply->sq_msg_size);
510 peer_pa = c2dev->pa + be32_to_cpu(reply->sq_mq_start);
511 mmap_size = PAGE_ALIGN(sizeof(struct c2_mq_shared) + msg_size * q_size);
512 mmap = ioremap_nocache(peer_pa, mmap_size);
513 if (!mmap) {
514 err = -ENOMEM;
515 goto bail5;
516 }
517
518 c2_mq_req_init(&qp->sq_mq,
519 be32_to_cpu(reply->sq_mq_index),
520 q_size,
521 msg_size,
522 mmap + sizeof(struct c2_mq_shared), /* pool start */
523 mmap, /* peer */
524 C2_MQ_ADAPTER_TARGET);
525
526 /* Initialize the RQ mq */
527 q_size = be32_to_cpu(reply->rq_depth);
528 msg_size = be32_to_cpu(reply->rq_msg_size);
529 peer_pa = c2dev->pa + be32_to_cpu(reply->rq_mq_start);
530 mmap_size = PAGE_ALIGN(sizeof(struct c2_mq_shared) + msg_size * q_size);
531 mmap = ioremap_nocache(peer_pa, mmap_size);
532 if (!mmap) {
533 err = -ENOMEM;
534 goto bail6;
535 }
536
537 c2_mq_req_init(&qp->rq_mq,
538 be32_to_cpu(reply->rq_mq_index),
539 q_size,
540 msg_size,
541 mmap + sizeof(struct c2_mq_shared), /* pool start */
542 mmap, /* peer */
543 C2_MQ_ADAPTER_TARGET);
544
545 vq_repbuf_free(c2dev, reply);
546 vq_req_free(c2dev, vq_req);
547
548 return 0;
549
550 bail6:
551 iounmap(qp->sq_mq.peer);
552 bail5:
553 destroy_qp(c2dev, qp);
554 bail4:
555 vq_repbuf_free(c2dev, reply);
556 bail3:
557 vq_req_free(c2dev, vq_req);
558 bail2:
559 c2_free_mqsp(qp->rq_mq.shared);
560 bail1:
561 c2_free_mqsp(qp->sq_mq.shared);
562 bail0:
563 c2_free_qpn(c2dev, qp->qpn);
564 return err;
565}
566
c9edea29
KK		 567static inline void c2_lock_cqs(struct c2_cq *send_cq, struct c2_cq *recv_cq)
568{
569 if (send_cq == recv_cq)
570 spin_lock_irq(&send_cq->lock);
571 else if (send_cq > recv_cq) {
572 spin_lock_irq(&send_cq->lock);
573 spin_lock_nested(&recv_cq->lock, SINGLE_DEPTH_NESTING);
574 } else {
575 spin_lock_irq(&recv_cq->lock);
576 spin_lock_nested(&send_cq->lock, SINGLE_DEPTH_NESTING);
577 }
578}
579
580static inline void c2_unlock_cqs(struct c2_cq *send_cq, struct c2_cq *recv_cq)
581{
582 if (send_cq == recv_cq)
583 spin_unlock_irq(&send_cq->lock);
584 else if (send_cq > recv_cq) {
585 spin_unlock(&recv_cq->lock);
586 spin_unlock_irq(&send_cq->lock);
587 } else {
588 spin_unlock(&send_cq->lock);
589 spin_unlock_irq(&recv_cq->lock);
590 }
591}
592
f94b533d
TT		 593void c2_free_qp(struct c2_dev *c2dev, struct c2_qp *qp)
594{
595 struct c2_cq *send_cq;
596 struct c2_cq *recv_cq;
597
598 send_cq = to_c2cq(qp->ibqp.send_cq);
599 recv_cq = to_c2cq(qp->ibqp.recv_cq);
600
601 /*
602 * Lock CQs here, so that CQ polling code can do QP lookup
603 * without taking a lock.
604 */
c9edea29 605 c2_lock_cqs(send_cq, recv_cq);
f94b533d 606 c2_free_qpn(c2dev, qp->qpn);
c9edea29 607 c2_unlock_cqs(send_cq, recv_cq);
f94b533d
TT		 608
609 /*
610 * Destory qp in the rnic...
611 */
612 destroy_qp(c2dev, qp);
613
614 /*
615 * Mark any unreaped CQEs as null and void.
616 */
617 c2_cq_clean(c2dev, qp, send_cq->cqn);
618 if (send_cq != recv_cq)
619 c2_cq_clean(c2dev, qp, recv_cq->cqn);
620 /*
621 * Unmap the MQs and return the shared pointers
622 * to the message pool.
623 */
624 iounmap(qp->sq_mq.peer);
625 iounmap(qp->rq_mq.peer);
626 c2_free_mqsp(qp->sq_mq.shared);
627 c2_free_mqsp(qp->rq_mq.shared);
628
629 atomic_dec(&qp->refcount);
630 wait_event(qp->wait, !atomic_read(&qp->refcount));
631}
632
633/*
634 * Function: move_sgl
635 *
636 * Description:
637 * Move an SGL from the user's work request struct into a CCIL Work Request
638 * message, swapping to WR byte order and ensure the total length doesn't
639 * overflow.
640 *
641 * IN:
642 * dst - ptr to CCIL Work Request message SGL memory.
643 * src - ptr to the consumers SGL memory.
644 *
645 * OUT: none
646 *
647 * Return:
648 * CCIL status codes.
649 */
650static int
651move_sgl(struct c2_data_addr * dst, struct ib_sge *src, int count, u32 * p_len,
652 u8 * actual_count)
653{
654 u32 tot = 0; /* running total */
655 u8 acount = 0; /* running total non-0 len sge's */
656
657 while (count > 0) {
658 /*
659 * If the addition of this SGE causes the
660 * total SGL length to exceed 2^32-1, then
661 * fail-n-bail.
662 *
663 * If the current total plus the next element length
664 * wraps, then it will go negative and be less than the
665 * current total...
666 */
667 if ((tot + src->length) < tot) {
668 return -EINVAL;
669 }
670 /*
671 * Bug: 1456 (as well as 1498 & 1643)
672 * Skip over any sge's supplied with len=0
673 */
674 if (src->length) {
675 tot += src->length;
676 dst->stag = cpu_to_be32(src->lkey);
677 dst->to = cpu_to_be64(src->addr);
678 dst->length = cpu_to_be32(src->length);
679 dst++;
680 acount++;
681 }
682 src++;
683 count--;
684 }
685
686 if (acount == 0) {
687 /*
688 * Bug: 1476 (as well as 1498, 1456 and 1643)
689 * Setup the SGL in the WR to make it easier for the RNIC.
690 * This way, the FW doesn't have to deal with special cases.
691 * Setting length=0 should be sufficient.
692 */
693 dst->stag = 0;
694 dst->to = 0;
695 dst->length = 0;
696 }
697
698 *p_len = tot;
699 *actual_count = acount;
700 return 0;
701}
702
703/*
704 * Function: c2_activity (private function)
705 *
706 * Description:
707 * Post an mq index to the host->adapter activity fifo.
708 *
709 * IN:
710 * c2dev - ptr to c2dev structure
711 * mq_index - mq index to post
712 * shared - value most recently written to shared
713 *
714 * OUT:
715 *
716 * Return:
717 * none
718 */
719static inline void c2_activity(struct c2_dev *c2dev, u32 mq_index, u16 shared)
720{
721 /*
722 * First read the register to see if the FIFO is full, and if so,
723 * spin until it's not. This isn't perfect -- there is no
724 * synchronization among the clients of the register, but in
725 * practice it prevents multiple CPU from hammering the bus
726 * with PCI RETRY. Note that when this does happen, the card
727 * cannot get on the bus and the card and system hang in a
728 * deadlock -- thus the need for this code. [TOT]
729 */
e52e6080
TT		 730 while (readl(c2dev->regs + PCI_BAR0_ADAPTER_HINT) & 0x80000000)
731 udelay(10);
f94b533d
TT		 732
733 __raw_writel(C2_HINT_MAKE(mq_index, shared),
734 c2dev->regs + PCI_BAR0_ADAPTER_HINT);
735}
736
737/*
738 * Function: qp_wr_post
739 *
740 * Description:
741 * This in-line function allocates a MQ msg, then moves the host-copy of
742 * the completed WR into msg. Then it posts the message.
743 *
744 * IN:
745 * q - ptr to user MQ.
746 * wr - ptr to host-copy of the WR.
747 * qp - ptr to user qp
748 * size - Number of bytes to post. Assumed to be divisible by 4.
749 *
750 * OUT: none
751 *
752 * Return:
753 * CCIL status codes.
754 */
755static int qp_wr_post(struct c2_mq *q, union c2wr * wr, struct c2_qp *qp, u32 size)
756{
757 union c2wr *msg;
758
759 msg = c2_mq_alloc(q);
760 if (msg == NULL) {
761 return -EINVAL;
762 }
763#ifdef CCMSGMAGIC
764 ((c2wr_hdr_t *) wr)->magic = cpu_to_be32(CCWR_MAGIC);
765#endif
766
767 /*
768 * Since all header fields in the WR are the same as the
769 * CQE, set the following so the adapter need not.
770 */
771 c2_wr_set_result(wr, CCERR_PENDING);
772
773 /*
774 * Copy the wr down to the adapter
775 */
776 memcpy((void *) msg, (void *) wr, size);
777
778 c2_mq_produce(q);
779 return 0;
780}
781
782
783int c2_post_send(struct ib_qp *ibqp, struct ib_send_wr *ib_wr,
784 struct ib_send_wr **bad_wr)
785{
786 struct c2_dev *c2dev = to_c2dev(ibqp->device);
787 struct c2_qp *qp = to_c2qp(ibqp);
788 union c2wr wr;
e52e6080 789 unsigned long lock_flags;
f94b533d
TT		 790 int err = 0;
791
792 u32 flags;
793 u32 tot_len;
794 u8 actual_sge_count;
795 u32 msg_size;
796
797 if (qp->state > IB_QPS_RTS)
798 return -EINVAL;
799
800 while (ib_wr) {
801
802 flags = 0;
803 wr.sqwr.sq_hdr.user_hdr.hdr.context = ib_wr->wr_id;
804 if (ib_wr->send_flags & IB_SEND_SIGNALED) {
805 flags |= SQ_SIGNALED;
806 }
807
808 switch (ib_wr->opcode) {
809 case IB_WR_SEND:
810 if (ib_wr->send_flags & IB_SEND_SOLICITED) {
811 c2_wr_set_id(&wr, C2_WR_TYPE_SEND_SE);
812 msg_size = sizeof(struct c2wr_send_req);
813 } else {
814 c2_wr_set_id(&wr, C2_WR_TYPE_SEND);
815 msg_size = sizeof(struct c2wr_send_req);
816 }
817
818 wr.sqwr.send.remote_stag = 0;
819 msg_size += sizeof(struct c2_data_addr) * ib_wr->num_sge;
820 if (ib_wr->num_sge > qp->send_sgl_depth) {
821 err = -EINVAL;
822 break;
823 }
824 if (ib_wr->send_flags & IB_SEND_FENCE) {
825 flags |= SQ_READ_FENCE;
826 }
827 err = move_sgl((struct c2_data_addr *) & (wr.sqwr.send.data),
828 ib_wr->sg_list,
829 ib_wr->num_sge,
830 &tot_len, &actual_sge_count);
831 wr.sqwr.send.sge_len = cpu_to_be32(tot_len);
832 c2_wr_set_sge_count(&wr, actual_sge_count);
833 break;
834 case IB_WR_RDMA_WRITE:
835 c2_wr_set_id(&wr, C2_WR_TYPE_RDMA_WRITE);
836 msg_size = sizeof(struct c2wr_rdma_write_req) +
837 (sizeof(struct c2_data_addr) * ib_wr->num_sge);
838 if (ib_wr->num_sge > qp->rdma_write_sgl_depth) {
839 err = -EINVAL;
840 break;
841 }
842 if (ib_wr->send_flags & IB_SEND_FENCE) {
843 flags |= SQ_READ_FENCE;
844 }
845 wr.sqwr.rdma_write.remote_stag =
846 cpu_to_be32(ib_wr->wr.rdma.rkey);
847 wr.sqwr.rdma_write.remote_to =
848 cpu_to_be64(ib_wr->wr.rdma.remote_addr);
849 err = move_sgl((struct c2_data_addr *)
850 & (wr.sqwr.rdma_write.data),
851 ib_wr->sg_list,
852 ib_wr->num_sge,
853 &tot_len, &actual_sge_count);
854 wr.sqwr.rdma_write.sge_len = cpu_to_be32(tot_len);
855 c2_wr_set_sge_count(&wr, actual_sge_count);
856 break;
857 case IB_WR_RDMA_READ:
858 c2_wr_set_id(&wr, C2_WR_TYPE_RDMA_READ);
859 msg_size = sizeof(struct c2wr_rdma_read_req);
860
861 /* IWarp only suppots 1 sge for RDMA reads */
862 if (ib_wr->num_sge > 1) {
863 err = -EINVAL;
864 break;
865 }
866
867 /*
868 * Move the local and remote stag/to/len into the WR.
869 */
870 wr.sqwr.rdma_read.local_stag =
871 cpu_to_be32(ib_wr->sg_list->lkey);
872 wr.sqwr.rdma_read.local_to =
873 cpu_to_be64(ib_wr->sg_list->addr);
874 wr.sqwr.rdma_read.remote_stag =
875 cpu_to_be32(ib_wr->wr.rdma.rkey);
876 wr.sqwr.rdma_read.remote_to =
877 cpu_to_be64(ib_wr->wr.rdma.remote_addr);
878 wr.sqwr.rdma_read.length =
879 cpu_to_be32(ib_wr->sg_list->length);
880 break;
881 default:
882 /* error */
883 msg_size = 0;
884 err = -EINVAL;
885 break;
886 }
887
888 /*
889 * If we had an error on the last wr build, then
890 * break out. Possible errors include bogus WR
891 * type, and a bogus SGL length...
892 */
893 if (err) {
894 break;
895 }
896
897 /*
898 * Store flags
899 */
900 c2_wr_set_flags(&wr, flags);
901
902 /*
903 * Post the puppy!
904 */
e52e6080 905 spin_lock_irqsave(&qp->lock, lock_flags);
f94b533d
TT		 906 err = qp_wr_post(&qp->sq_mq, &wr, qp, msg_size);
907 if (err) {
e52e6080 908 spin_unlock_irqrestore(&qp->lock, lock_flags);
f94b533d
TT		 909 break;
910 }
911
912 /*
913 * Enqueue mq index to activity FIFO.
914 */
915 c2_activity(c2dev, qp->sq_mq.index, qp->sq_mq.hint_count);
e52e6080 916 spin_unlock_irqrestore(&qp->lock, lock_flags);
f94b533d
TT		 917
918 ib_wr = ib_wr->next;
919 }
920
921 if (err)
922 *bad_wr = ib_wr;
923 return err;
924}
925
926int c2_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *ib_wr,
927 struct ib_recv_wr **bad_wr)
928{
929 struct c2_dev *c2dev = to_c2dev(ibqp->device);
930 struct c2_qp *qp = to_c2qp(ibqp);
931 union c2wr wr;
e52e6080 932 unsigned long lock_flags;
f94b533d
TT		 933 int err = 0;
934
935 if (qp->state > IB_QPS_RTS)
936 return -EINVAL;
937
938 /*
939 * Try and post each work request
940 */
941 while (ib_wr) {
942 u32 tot_len;
943 u8 actual_sge_count;
944
945 if (ib_wr->num_sge > qp->recv_sgl_depth) {
946 err = -EINVAL;
947 break;
948 }
949
950 /*
951 * Create local host-copy of the WR
952 */
953 wr.rqwr.rq_hdr.user_hdr.hdr.context = ib_wr->wr_id;
954 c2_wr_set_id(&wr, CCWR_RECV);
955 c2_wr_set_flags(&wr, 0);
956
957 /* sge_count is limited to eight bits. */
958 BUG_ON(ib_wr->num_sge >= 256);
959 err = move_sgl((struct c2_data_addr *) & (wr.rqwr.data),
960 ib_wr->sg_list,
961 ib_wr->num_sge, &tot_len, &actual_sge_count);
962 c2_wr_set_sge_count(&wr, actual_sge_count);
963
964 /*
965 * If we had an error on the last wr build, then
966 * break out. Possible errors include bogus WR
967 * type, and a bogus SGL length...
968 */
969 if (err) {
970 break;
971 }
972
e52e6080 973 spin_lock_irqsave(&qp->lock, lock_flags);
f94b533d
TT		 974 err = qp_wr_post(&qp->rq_mq, &wr, qp, qp->rq_mq.msg_size);
975 if (err) {
e52e6080 976 spin_unlock_irqrestore(&qp->lock, lock_flags);
f94b533d
TT		 977 break;
978 }
979
980 /*
981 * Enqueue mq index to activity FIFO
982 */
983 c2_activity(c2dev, qp->rq_mq.index, qp->rq_mq.hint_count);
e52e6080 984 spin_unlock_irqrestore(&qp->lock, lock_flags);
f94b533d
TT		 985
986 ib_wr = ib_wr->next;
987 }
988
989 if (err)
990 *bad_wr = ib_wr;
991 return err;
992}
993
994void __devinit c2_init_qp_table(struct c2_dev *c2dev)
995{
996 spin_lock_init(&c2dev->qp_table.lock);
997 idr_init(&c2dev->qp_table.idr);
998}
999
1000void __devexit c2_cleanup_qp_table(struct c2_dev *c2dev)
1001{
1002 idr_destroy(&c2dev->qp_table.idr);
1003}
Linux 6.x block layer and io_uring tree(s)