Commit | Line | Data |
---|---|---|
f94b533d TT |
1 | /* |
2 | * Copyright (c) 2004 Topspin Communications. All rights reserved. | |
3 | * Copyright (c) 2005 Cisco Systems. All rights reserved. | |
4 | * Copyright (c) 2005 Mellanox Technologies. All rights reserved. | |
5 | * Copyright (c) 2004 Voltaire, Inc. All rights reserved. | |
6 | * Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved. | |
7 | * | |
8 | * This software is available to you under a choice of one of two | |
9 | * licenses. You may choose to be licensed under the terms of the GNU | |
10 | * General Public License (GPL) Version 2, available from the file | |
11 | * COPYING in the main directory of this source tree, or the | |
12 | * OpenIB.org BSD license below: | |
13 | * | |
14 | * Redistribution and use in source and binary forms, with or | |
15 | * without modification, are permitted provided that the following | |
16 | * conditions are met: | |
17 | * | |
18 | * - Redistributions of source code must retain the above | |
19 | * copyright notice, this list of conditions and the following | |
20 | * disclaimer. | |
21 | * | |
22 | * - Redistributions in binary form must reproduce the above | |
23 | * copyright notice, this list of conditions and the following | |
24 | * disclaimer in the documentation and/or other materials | |
25 | * provided with the distribution. | |
26 | * | |
27 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
28 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
29 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
30 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
31 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
32 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
33 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
34 | * SOFTWARE. | |
35 | * | |
36 | */ | |
37 | ||
e52e6080 TT |
38 | #include <linux/delay.h> |
39 | ||
f94b533d TT |
40 | #include "c2.h" |
41 | #include "c2_vq.h" | |
42 | #include "c2_status.h" | |
43 | ||
44 | #define C2_MAX_ORD_PER_QP 128 | |
45 | #define C2_MAX_IRD_PER_QP 128 | |
46 | ||
47 | #define C2_HINT_MAKE(q_index, hint_count) (((q_index) << 16) | hint_count) | |
48 | #define C2_HINT_GET_INDEX(hint) (((hint) & 0x7FFF0000) >> 16) | |
49 | #define C2_HINT_GET_COUNT(hint) ((hint) & 0x0000FFFF) | |
50 | ||
51 | #define NO_SUPPORT -1 | |
52 | static const u8 c2_opcode[] = { | |
53 | [IB_WR_SEND] = C2_WR_TYPE_SEND, | |
54 | [IB_WR_SEND_WITH_IMM] = NO_SUPPORT, | |
55 | [IB_WR_RDMA_WRITE] = C2_WR_TYPE_RDMA_WRITE, | |
56 | [IB_WR_RDMA_WRITE_WITH_IMM] = NO_SUPPORT, | |
57 | [IB_WR_RDMA_READ] = C2_WR_TYPE_RDMA_READ, | |
58 | [IB_WR_ATOMIC_CMP_AND_SWP] = NO_SUPPORT, | |
59 | [IB_WR_ATOMIC_FETCH_AND_ADD] = NO_SUPPORT, | |
60 | }; | |
61 | ||
62 | static int to_c2_state(enum ib_qp_state ib_state) | |
63 | { | |
64 | switch (ib_state) { | |
65 | case IB_QPS_RESET: | |
66 | return C2_QP_STATE_IDLE; | |
67 | case IB_QPS_RTS: | |
68 | return C2_QP_STATE_RTS; | |
69 | case IB_QPS_SQD: | |
70 | return C2_QP_STATE_CLOSING; | |
71 | case IB_QPS_SQE: | |
72 | return C2_QP_STATE_CLOSING; | |
73 | case IB_QPS_ERR: | |
74 | return C2_QP_STATE_ERROR; | |
75 | default: | |
76 | return -1; | |
77 | } | |
78 | } | |
79 | ||
80 | static int to_ib_state(enum c2_qp_state c2_state) | |
81 | { | |
82 | switch (c2_state) { | |
83 | case C2_QP_STATE_IDLE: | |
84 | return IB_QPS_RESET; | |
85 | case C2_QP_STATE_CONNECTING: | |
86 | return IB_QPS_RTR; | |
87 | case C2_QP_STATE_RTS: | |
88 | return IB_QPS_RTS; | |
89 | case C2_QP_STATE_CLOSING: | |
90 | return IB_QPS_SQD; | |
91 | case C2_QP_STATE_ERROR: | |
92 | return IB_QPS_ERR; | |
93 | case C2_QP_STATE_TERMINATE: | |
94 | return IB_QPS_SQE; | |
95 | default: | |
96 | return -1; | |
97 | } | |
98 | } | |
99 | ||
100 | static const char *to_ib_state_str(int ib_state) | |
101 | { | |
102 | static const char *state_str[] = { | |
103 | "IB_QPS_RESET", | |
104 | "IB_QPS_INIT", | |
105 | "IB_QPS_RTR", | |
106 | "IB_QPS_RTS", | |
107 | "IB_QPS_SQD", | |
108 | "IB_QPS_SQE", | |
109 | "IB_QPS_ERR" | |
110 | }; | |
111 | if (ib_state < IB_QPS_RESET || | |
112 | ib_state > IB_QPS_ERR) | |
113 | return "<invalid IB QP state>"; | |
114 | ||
115 | ib_state -= IB_QPS_RESET; | |
116 | return state_str[ib_state]; | |
117 | } | |
118 | ||
119 | void c2_set_qp_state(struct c2_qp *qp, int c2_state) | |
120 | { | |
121 | int new_state = to_ib_state(c2_state); | |
122 | ||
123 | pr_debug("%s: qp[%p] state modify %s --> %s\n", | |
33718363 | 124 | __func__, |
f94b533d TT |
125 | qp, |
126 | to_ib_state_str(qp->state), | |
127 | to_ib_state_str(new_state)); | |
128 | qp->state = new_state; | |
129 | } | |
130 | ||
131 | #define C2_QP_NO_ATTR_CHANGE 0xFFFFFFFF | |
132 | ||
133 | int c2_qp_modify(struct c2_dev *c2dev, struct c2_qp *qp, | |
134 | struct ib_qp_attr *attr, int attr_mask) | |
135 | { | |
136 | struct c2wr_qp_modify_req wr; | |
137 | struct c2wr_qp_modify_rep *reply; | |
138 | struct c2_vq_req *vq_req; | |
139 | unsigned long flags; | |
140 | u8 next_state; | |
141 | int err; | |
142 | ||
143 | pr_debug("%s:%d qp=%p, %s --> %s\n", | |
33718363 | 144 | __func__, __LINE__, |
f94b533d TT |
145 | qp, |
146 | to_ib_state_str(qp->state), | |
147 | to_ib_state_str(attr->qp_state)); | |
148 | ||
149 | vq_req = vq_req_alloc(c2dev); | |
150 | if (!vq_req) | |
151 | return -ENOMEM; | |
152 | ||
153 | c2_wr_set_id(&wr, CCWR_QP_MODIFY); | |
154 | wr.hdr.context = (unsigned long) vq_req; | |
155 | wr.rnic_handle = c2dev->adapter_handle; | |
156 | wr.qp_handle = qp->adapter_handle; | |
157 | wr.ord = cpu_to_be32(C2_QP_NO_ATTR_CHANGE); | |
158 | wr.ird = cpu_to_be32(C2_QP_NO_ATTR_CHANGE); | |
159 | wr.sq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE); | |
160 | wr.rq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE); | |
161 | ||
162 | if (attr_mask & IB_QP_STATE) { | |
163 | /* Ensure the state is valid */ | |
ad1f9791 KK |
164 | if (attr->qp_state < 0 || attr->qp_state > IB_QPS_ERR) { |
165 | err = -EINVAL; | |
166 | goto bail0; | |
167 | } | |
f94b533d TT |
168 | |
169 | wr.next_qp_state = cpu_to_be32(to_c2_state(attr->qp_state)); | |
170 | ||
171 | if (attr->qp_state == IB_QPS_ERR) { | |
172 | spin_lock_irqsave(&qp->lock, flags); | |
173 | if (qp->cm_id && qp->state == IB_QPS_RTS) { | |
174 | pr_debug("Generating CLOSE event for QP-->ERR, " | |
175 | "qp=%p, cm_id=%p\n",qp,qp->cm_id); | |
176 | /* Generate an CLOSE event */ | |
177 | vq_req->cm_id = qp->cm_id; | |
178 | vq_req->event = IW_CM_EVENT_CLOSE; | |
179 | } | |
180 | spin_unlock_irqrestore(&qp->lock, flags); | |
181 | } | |
182 | next_state = attr->qp_state; | |
183 | ||
184 | } else if (attr_mask & IB_QP_CUR_STATE) { | |
185 | ||
186 | if (attr->cur_qp_state != IB_QPS_RTR && | |
187 | attr->cur_qp_state != IB_QPS_RTS && | |
188 | attr->cur_qp_state != IB_QPS_SQD && | |
ad1f9791 KK |
189 | attr->cur_qp_state != IB_QPS_SQE) { |
190 | err = -EINVAL; | |
191 | goto bail0; | |
192 | } else | |
f94b533d TT |
193 | wr.next_qp_state = |
194 | cpu_to_be32(to_c2_state(attr->cur_qp_state)); | |
195 | ||
196 | next_state = attr->cur_qp_state; | |
197 | ||
198 | } else { | |
199 | err = 0; | |
200 | goto bail0; | |
201 | } | |
202 | ||
203 | /* reference the request struct */ | |
204 | vq_req_get(c2dev, vq_req); | |
205 | ||
206 | err = vq_send_wr(c2dev, (union c2wr *) & wr); | |
207 | if (err) { | |
208 | vq_req_put(c2dev, vq_req); | |
209 | goto bail0; | |
210 | } | |
211 | ||
212 | err = vq_wait_for_reply(c2dev, vq_req); | |
213 | if (err) | |
214 | goto bail0; | |
215 | ||
216 | reply = (struct c2wr_qp_modify_rep *) (unsigned long) vq_req->reply_msg; | |
217 | if (!reply) { | |
218 | err = -ENOMEM; | |
219 | goto bail0; | |
220 | } | |
221 | ||
222 | err = c2_errno(reply); | |
223 | if (!err) | |
224 | qp->state = next_state; | |
225 | #ifdef DEBUG | |
226 | else | |
33718363 | 227 | pr_debug("%s: c2_errno=%d\n", __func__, err); |
f94b533d TT |
228 | #endif |
229 | /* | |
230 | * If we're going to error and generating the event here, then | |
231 | * we need to remove the reference because there will be no | |
232 | * close event generated by the adapter | |
233 | */ | |
234 | spin_lock_irqsave(&qp->lock, flags); | |
235 | if (vq_req->event==IW_CM_EVENT_CLOSE && qp->cm_id) { | |
236 | qp->cm_id->rem_ref(qp->cm_id); | |
237 | qp->cm_id = NULL; | |
238 | } | |
239 | spin_unlock_irqrestore(&qp->lock, flags); | |
240 | ||
241 | vq_repbuf_free(c2dev, reply); | |
242 | bail0: | |
243 | vq_req_free(c2dev, vq_req); | |
244 | ||
245 | pr_debug("%s:%d qp=%p, cur_state=%s\n", | |
33718363 | 246 | __func__, __LINE__, |
f94b533d TT |
247 | qp, |
248 | to_ib_state_str(qp->state)); | |
249 | return err; | |
250 | } | |
251 | ||
252 | int c2_qp_set_read_limits(struct c2_dev *c2dev, struct c2_qp *qp, | |
253 | int ord, int ird) | |
254 | { | |
255 | struct c2wr_qp_modify_req wr; | |
256 | struct c2wr_qp_modify_rep *reply; | |
257 | struct c2_vq_req *vq_req; | |
258 | int err; | |
259 | ||
260 | vq_req = vq_req_alloc(c2dev); | |
261 | if (!vq_req) | |
262 | return -ENOMEM; | |
263 | ||
264 | c2_wr_set_id(&wr, CCWR_QP_MODIFY); | |
265 | wr.hdr.context = (unsigned long) vq_req; | |
266 | wr.rnic_handle = c2dev->adapter_handle; | |
267 | wr.qp_handle = qp->adapter_handle; | |
268 | wr.ord = cpu_to_be32(ord); | |
269 | wr.ird = cpu_to_be32(ird); | |
270 | wr.sq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE); | |
271 | wr.rq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE); | |
272 | wr.next_qp_state = cpu_to_be32(C2_QP_NO_ATTR_CHANGE); | |
273 | ||
274 | /* reference the request struct */ | |
275 | vq_req_get(c2dev, vq_req); | |
276 | ||
277 | err = vq_send_wr(c2dev, (union c2wr *) & wr); | |
278 | if (err) { | |
279 | vq_req_put(c2dev, vq_req); | |
280 | goto bail0; | |
281 | } | |
282 | ||
283 | err = vq_wait_for_reply(c2dev, vq_req); | |
284 | if (err) | |
285 | goto bail0; | |
286 | ||
287 | reply = (struct c2wr_qp_modify_rep *) (unsigned long) | |
288 | vq_req->reply_msg; | |
289 | if (!reply) { | |
290 | err = -ENOMEM; | |
291 | goto bail0; | |
292 | } | |
293 | ||
294 | err = c2_errno(reply); | |
295 | vq_repbuf_free(c2dev, reply); | |
296 | bail0: | |
297 | vq_req_free(c2dev, vq_req); | |
298 | return err; | |
299 | } | |
300 | ||
301 | static int destroy_qp(struct c2_dev *c2dev, struct c2_qp *qp) | |
302 | { | |
303 | struct c2_vq_req *vq_req; | |
304 | struct c2wr_qp_destroy_req wr; | |
305 | struct c2wr_qp_destroy_rep *reply; | |
306 | unsigned long flags; | |
307 | int err; | |
308 | ||
309 | /* | |
310 | * Allocate a verb request message | |
311 | */ | |
312 | vq_req = vq_req_alloc(c2dev); | |
313 | if (!vq_req) { | |
314 | return -ENOMEM; | |
315 | } | |
316 | ||
317 | /* | |
318 | * Initialize the WR | |
319 | */ | |
320 | c2_wr_set_id(&wr, CCWR_QP_DESTROY); | |
321 | wr.hdr.context = (unsigned long) vq_req; | |
322 | wr.rnic_handle = c2dev->adapter_handle; | |
323 | wr.qp_handle = qp->adapter_handle; | |
324 | ||
325 | /* | |
326 | * reference the request struct. dereferenced in the int handler. | |
327 | */ | |
328 | vq_req_get(c2dev, vq_req); | |
329 | ||
330 | spin_lock_irqsave(&qp->lock, flags); | |
331 | if (qp->cm_id && qp->state == IB_QPS_RTS) { | |
332 | pr_debug("destroy_qp: generating CLOSE event for QP-->ERR, " | |
333 | "qp=%p, cm_id=%p\n",qp,qp->cm_id); | |
334 | /* Generate an CLOSE event */ | |
335 | vq_req->qp = qp; | |
336 | vq_req->cm_id = qp->cm_id; | |
337 | vq_req->event = IW_CM_EVENT_CLOSE; | |
338 | } | |
339 | spin_unlock_irqrestore(&qp->lock, flags); | |
340 | ||
341 | /* | |
342 | * Send WR to adapter | |
343 | */ | |
344 | err = vq_send_wr(c2dev, (union c2wr *) & wr); | |
345 | if (err) { | |
346 | vq_req_put(c2dev, vq_req); | |
347 | goto bail0; | |
348 | } | |
349 | ||
350 | /* | |
351 | * Wait for reply from adapter | |
352 | */ | |
353 | err = vq_wait_for_reply(c2dev, vq_req); | |
354 | if (err) { | |
355 | goto bail0; | |
356 | } | |
357 | ||
358 | /* | |
359 | * Process reply | |
360 | */ | |
361 | reply = (struct c2wr_qp_destroy_rep *) (unsigned long) (vq_req->reply_msg); | |
362 | if (!reply) { | |
363 | err = -ENOMEM; | |
364 | goto bail0; | |
365 | } | |
366 | ||
367 | spin_lock_irqsave(&qp->lock, flags); | |
368 | if (qp->cm_id) { | |
369 | qp->cm_id->rem_ref(qp->cm_id); | |
370 | qp->cm_id = NULL; | |
371 | } | |
372 | spin_unlock_irqrestore(&qp->lock, flags); | |
373 | ||
374 | vq_repbuf_free(c2dev, reply); | |
375 | bail0: | |
376 | vq_req_free(c2dev, vq_req); | |
377 | return err; | |
378 | } | |
379 | ||
380 | static int c2_alloc_qpn(struct c2_dev *c2dev, struct c2_qp *qp) | |
381 | { | |
382 | int ret; | |
383 | ||
384 | do { | |
385 | spin_lock_irq(&c2dev->qp_table.lock); | |
386 | ret = idr_get_new_above(&c2dev->qp_table.idr, qp, | |
387 | c2dev->qp_table.last++, &qp->qpn); | |
388 | spin_unlock_irq(&c2dev->qp_table.lock); | |
389 | } while ((ret == -EAGAIN) && | |
390 | idr_pre_get(&c2dev->qp_table.idr, GFP_KERNEL)); | |
391 | return ret; | |
392 | } | |
393 | ||
394 | static void c2_free_qpn(struct c2_dev *c2dev, int qpn) | |
395 | { | |
396 | spin_lock_irq(&c2dev->qp_table.lock); | |
397 | idr_remove(&c2dev->qp_table.idr, qpn); | |
398 | spin_unlock_irq(&c2dev->qp_table.lock); | |
399 | } | |
400 | ||
401 | struct c2_qp *c2_find_qpn(struct c2_dev *c2dev, int qpn) | |
402 | { | |
403 | unsigned long flags; | |
404 | struct c2_qp *qp; | |
405 | ||
406 | spin_lock_irqsave(&c2dev->qp_table.lock, flags); | |
407 | qp = idr_find(&c2dev->qp_table.idr, qpn); | |
408 | spin_unlock_irqrestore(&c2dev->qp_table.lock, flags); | |
409 | return qp; | |
410 | } | |
411 | ||
412 | int c2_alloc_qp(struct c2_dev *c2dev, | |
413 | struct c2_pd *pd, | |
414 | struct ib_qp_init_attr *qp_attrs, struct c2_qp *qp) | |
415 | { | |
416 | struct c2wr_qp_create_req wr; | |
417 | struct c2wr_qp_create_rep *reply; | |
418 | struct c2_vq_req *vq_req; | |
419 | struct c2_cq *send_cq = to_c2cq(qp_attrs->send_cq); | |
420 | struct c2_cq *recv_cq = to_c2cq(qp_attrs->recv_cq); | |
421 | unsigned long peer_pa; | |
422 | u32 q_size, msg_size, mmap_size; | |
423 | void __iomem *mmap; | |
424 | int err; | |
425 | ||
426 | err = c2_alloc_qpn(c2dev, qp); | |
427 | if (err) | |
428 | return err; | |
429 | qp->ibqp.qp_num = qp->qpn; | |
430 | qp->ibqp.qp_type = IB_QPT_RC; | |
431 | ||
432 | /* Allocate the SQ and RQ shared pointers */ | |
433 | qp->sq_mq.shared = c2_alloc_mqsp(c2dev, c2dev->kern_mqsp_pool, | |
434 | &qp->sq_mq.shared_dma, GFP_KERNEL); | |
435 | if (!qp->sq_mq.shared) { | |
436 | err = -ENOMEM; | |
437 | goto bail0; | |
438 | } | |
439 | ||
440 | qp->rq_mq.shared = c2_alloc_mqsp(c2dev, c2dev->kern_mqsp_pool, | |
441 | &qp->rq_mq.shared_dma, GFP_KERNEL); | |
442 | if (!qp->rq_mq.shared) { | |
443 | err = -ENOMEM; | |
444 | goto bail1; | |
445 | } | |
446 | ||
447 | /* Allocate the verbs request */ | |
448 | vq_req = vq_req_alloc(c2dev); | |
449 | if (vq_req == NULL) { | |
450 | err = -ENOMEM; | |
451 | goto bail2; | |
452 | } | |
453 | ||
454 | /* Initialize the work request */ | |
455 | memset(&wr, 0, sizeof(wr)); | |
456 | c2_wr_set_id(&wr, CCWR_QP_CREATE); | |
457 | wr.hdr.context = (unsigned long) vq_req; | |
458 | wr.rnic_handle = c2dev->adapter_handle; | |
459 | wr.sq_cq_handle = send_cq->adapter_handle; | |
460 | wr.rq_cq_handle = recv_cq->adapter_handle; | |
461 | wr.sq_depth = cpu_to_be32(qp_attrs->cap.max_send_wr + 1); | |
462 | wr.rq_depth = cpu_to_be32(qp_attrs->cap.max_recv_wr + 1); | |
463 | wr.srq_handle = 0; | |
464 | wr.flags = cpu_to_be32(QP_RDMA_READ | QP_RDMA_WRITE | QP_MW_BIND | | |
465 | QP_ZERO_STAG | QP_RDMA_READ_RESPONSE); | |
466 | wr.send_sgl_depth = cpu_to_be32(qp_attrs->cap.max_send_sge); | |
467 | wr.recv_sgl_depth = cpu_to_be32(qp_attrs->cap.max_recv_sge); | |
468 | wr.rdma_write_sgl_depth = cpu_to_be32(qp_attrs->cap.max_send_sge); | |
469 | wr.shared_sq_ht = cpu_to_be64(qp->sq_mq.shared_dma); | |
470 | wr.shared_rq_ht = cpu_to_be64(qp->rq_mq.shared_dma); | |
471 | wr.ord = cpu_to_be32(C2_MAX_ORD_PER_QP); | |
472 | wr.ird = cpu_to_be32(C2_MAX_IRD_PER_QP); | |
473 | wr.pd_id = pd->pd_id; | |
474 | wr.user_context = (unsigned long) qp; | |
475 | ||
476 | vq_req_get(c2dev, vq_req); | |
477 | ||
478 | /* Send the WR to the adapter */ | |
479 | err = vq_send_wr(c2dev, (union c2wr *) & wr); | |
480 | if (err) { | |
481 | vq_req_put(c2dev, vq_req); | |
482 | goto bail3; | |
483 | } | |
484 | ||
485 | /* Wait for the verb reply */ | |
486 | err = vq_wait_for_reply(c2dev, vq_req); | |
487 | if (err) { | |
488 | goto bail3; | |
489 | } | |
490 | ||
491 | /* Process the reply */ | |
492 | reply = (struct c2wr_qp_create_rep *) (unsigned long) (vq_req->reply_msg); | |
493 | if (!reply) { | |
494 | err = -ENOMEM; | |
495 | goto bail3; | |
496 | } | |
497 | ||
498 | if ((err = c2_wr_get_result(reply)) != 0) { | |
499 | goto bail4; | |
500 | } | |
501 | ||
502 | /* Fill in the kernel QP struct */ | |
503 | atomic_set(&qp->refcount, 1); | |
504 | qp->adapter_handle = reply->qp_handle; | |
505 | qp->state = IB_QPS_RESET; | |
506 | qp->send_sgl_depth = qp_attrs->cap.max_send_sge; | |
507 | qp->rdma_write_sgl_depth = qp_attrs->cap.max_send_sge; | |
508 | qp->recv_sgl_depth = qp_attrs->cap.max_recv_sge; | |
4e8e6ee3 | 509 | init_waitqueue_head(&qp->wait); |
f94b533d TT |
510 | |
511 | /* Initialize the SQ MQ */ | |
512 | q_size = be32_to_cpu(reply->sq_depth); | |
513 | msg_size = be32_to_cpu(reply->sq_msg_size); | |
514 | peer_pa = c2dev->pa + be32_to_cpu(reply->sq_mq_start); | |
515 | mmap_size = PAGE_ALIGN(sizeof(struct c2_mq_shared) + msg_size * q_size); | |
516 | mmap = ioremap_nocache(peer_pa, mmap_size); | |
517 | if (!mmap) { | |
518 | err = -ENOMEM; | |
519 | goto bail5; | |
520 | } | |
521 | ||
522 | c2_mq_req_init(&qp->sq_mq, | |
523 | be32_to_cpu(reply->sq_mq_index), | |
524 | q_size, | |
525 | msg_size, | |
526 | mmap + sizeof(struct c2_mq_shared), /* pool start */ | |
527 | mmap, /* peer */ | |
528 | C2_MQ_ADAPTER_TARGET); | |
529 | ||
530 | /* Initialize the RQ mq */ | |
531 | q_size = be32_to_cpu(reply->rq_depth); | |
532 | msg_size = be32_to_cpu(reply->rq_msg_size); | |
533 | peer_pa = c2dev->pa + be32_to_cpu(reply->rq_mq_start); | |
534 | mmap_size = PAGE_ALIGN(sizeof(struct c2_mq_shared) + msg_size * q_size); | |
535 | mmap = ioremap_nocache(peer_pa, mmap_size); | |
536 | if (!mmap) { | |
537 | err = -ENOMEM; | |
538 | goto bail6; | |
539 | } | |
540 | ||
541 | c2_mq_req_init(&qp->rq_mq, | |
542 | be32_to_cpu(reply->rq_mq_index), | |
543 | q_size, | |
544 | msg_size, | |
545 | mmap + sizeof(struct c2_mq_shared), /* pool start */ | |
546 | mmap, /* peer */ | |
547 | C2_MQ_ADAPTER_TARGET); | |
548 | ||
549 | vq_repbuf_free(c2dev, reply); | |
550 | vq_req_free(c2dev, vq_req); | |
551 | ||
552 | return 0; | |
553 | ||
554 | bail6: | |
555 | iounmap(qp->sq_mq.peer); | |
556 | bail5: | |
557 | destroy_qp(c2dev, qp); | |
558 | bail4: | |
559 | vq_repbuf_free(c2dev, reply); | |
560 | bail3: | |
561 | vq_req_free(c2dev, vq_req); | |
562 | bail2: | |
563 | c2_free_mqsp(qp->rq_mq.shared); | |
564 | bail1: | |
565 | c2_free_mqsp(qp->sq_mq.shared); | |
566 | bail0: | |
567 | c2_free_qpn(c2dev, qp->qpn); | |
568 | return err; | |
569 | } | |
570 | ||
c9edea29 KK |
571 | static inline void c2_lock_cqs(struct c2_cq *send_cq, struct c2_cq *recv_cq) |
572 | { | |
573 | if (send_cq == recv_cq) | |
574 | spin_lock_irq(&send_cq->lock); | |
575 | else if (send_cq > recv_cq) { | |
576 | spin_lock_irq(&send_cq->lock); | |
577 | spin_lock_nested(&recv_cq->lock, SINGLE_DEPTH_NESTING); | |
578 | } else { | |
579 | spin_lock_irq(&recv_cq->lock); | |
580 | spin_lock_nested(&send_cq->lock, SINGLE_DEPTH_NESTING); | |
581 | } | |
582 | } | |
583 | ||
584 | static inline void c2_unlock_cqs(struct c2_cq *send_cq, struct c2_cq *recv_cq) | |
585 | { | |
586 | if (send_cq == recv_cq) | |
587 | spin_unlock_irq(&send_cq->lock); | |
588 | else if (send_cq > recv_cq) { | |
589 | spin_unlock(&recv_cq->lock); | |
590 | spin_unlock_irq(&send_cq->lock); | |
591 | } else { | |
592 | spin_unlock(&send_cq->lock); | |
593 | spin_unlock_irq(&recv_cq->lock); | |
594 | } | |
595 | } | |
596 | ||
f94b533d TT |
597 | void c2_free_qp(struct c2_dev *c2dev, struct c2_qp *qp) |
598 | { | |
599 | struct c2_cq *send_cq; | |
600 | struct c2_cq *recv_cq; | |
601 | ||
602 | send_cq = to_c2cq(qp->ibqp.send_cq); | |
603 | recv_cq = to_c2cq(qp->ibqp.recv_cq); | |
604 | ||
605 | /* | |
606 | * Lock CQs here, so that CQ polling code can do QP lookup | |
607 | * without taking a lock. | |
608 | */ | |
c9edea29 | 609 | c2_lock_cqs(send_cq, recv_cq); |
f94b533d | 610 | c2_free_qpn(c2dev, qp->qpn); |
c9edea29 | 611 | c2_unlock_cqs(send_cq, recv_cq); |
f94b533d TT |
612 | |
613 | /* | |
614 | * Destory qp in the rnic... | |
615 | */ | |
616 | destroy_qp(c2dev, qp); | |
617 | ||
618 | /* | |
619 | * Mark any unreaped CQEs as null and void. | |
620 | */ | |
621 | c2_cq_clean(c2dev, qp, send_cq->cqn); | |
622 | if (send_cq != recv_cq) | |
623 | c2_cq_clean(c2dev, qp, recv_cq->cqn); | |
624 | /* | |
625 | * Unmap the MQs and return the shared pointers | |
626 | * to the message pool. | |
627 | */ | |
628 | iounmap(qp->sq_mq.peer); | |
629 | iounmap(qp->rq_mq.peer); | |
630 | c2_free_mqsp(qp->sq_mq.shared); | |
631 | c2_free_mqsp(qp->rq_mq.shared); | |
632 | ||
633 | atomic_dec(&qp->refcount); | |
634 | wait_event(qp->wait, !atomic_read(&qp->refcount)); | |
635 | } | |
636 | ||
637 | /* | |
638 | * Function: move_sgl | |
639 | * | |
640 | * Description: | |
641 | * Move an SGL from the user's work request struct into a CCIL Work Request | |
642 | * message, swapping to WR byte order and ensure the total length doesn't | |
643 | * overflow. | |
644 | * | |
645 | * IN: | |
646 | * dst - ptr to CCIL Work Request message SGL memory. | |
647 | * src - ptr to the consumers SGL memory. | |
648 | * | |
649 | * OUT: none | |
650 | * | |
651 | * Return: | |
652 | * CCIL status codes. | |
653 | */ | |
654 | static int | |
655 | move_sgl(struct c2_data_addr * dst, struct ib_sge *src, int count, u32 * p_len, | |
656 | u8 * actual_count) | |
657 | { | |
658 | u32 tot = 0; /* running total */ | |
659 | u8 acount = 0; /* running total non-0 len sge's */ | |
660 | ||
661 | while (count > 0) { | |
662 | /* | |
663 | * If the addition of this SGE causes the | |
664 | * total SGL length to exceed 2^32-1, then | |
665 | * fail-n-bail. | |
666 | * | |
667 | * If the current total plus the next element length | |
668 | * wraps, then it will go negative and be less than the | |
669 | * current total... | |
670 | */ | |
671 | if ((tot + src->length) < tot) { | |
672 | return -EINVAL; | |
673 | } | |
674 | /* | |
675 | * Bug: 1456 (as well as 1498 & 1643) | |
676 | * Skip over any sge's supplied with len=0 | |
677 | */ | |
678 | if (src->length) { | |
679 | tot += src->length; | |
680 | dst->stag = cpu_to_be32(src->lkey); | |
681 | dst->to = cpu_to_be64(src->addr); | |
682 | dst->length = cpu_to_be32(src->length); | |
683 | dst++; | |
684 | acount++; | |
685 | } | |
686 | src++; | |
687 | count--; | |
688 | } | |
689 | ||
690 | if (acount == 0) { | |
691 | /* | |
692 | * Bug: 1476 (as well as 1498, 1456 and 1643) | |
693 | * Setup the SGL in the WR to make it easier for the RNIC. | |
694 | * This way, the FW doesn't have to deal with special cases. | |
695 | * Setting length=0 should be sufficient. | |
696 | */ | |
697 | dst->stag = 0; | |
698 | dst->to = 0; | |
699 | dst->length = 0; | |
700 | } | |
701 | ||
702 | *p_len = tot; | |
703 | *actual_count = acount; | |
704 | return 0; | |
705 | } | |
706 | ||
707 | /* | |
708 | * Function: c2_activity (private function) | |
709 | * | |
710 | * Description: | |
711 | * Post an mq index to the host->adapter activity fifo. | |
712 | * | |
713 | * IN: | |
714 | * c2dev - ptr to c2dev structure | |
715 | * mq_index - mq index to post | |
716 | * shared - value most recently written to shared | |
717 | * | |
718 | * OUT: | |
719 | * | |
720 | * Return: | |
721 | * none | |
722 | */ | |
723 | static inline void c2_activity(struct c2_dev *c2dev, u32 mq_index, u16 shared) | |
724 | { | |
725 | /* | |
726 | * First read the register to see if the FIFO is full, and if so, | |
727 | * spin until it's not. This isn't perfect -- there is no | |
728 | * synchronization among the clients of the register, but in | |
729 | * practice it prevents multiple CPU from hammering the bus | |
730 | * with PCI RETRY. Note that when this does happen, the card | |
731 | * cannot get on the bus and the card and system hang in a | |
732 | * deadlock -- thus the need for this code. [TOT] | |
733 | */ | |
e52e6080 TT |
734 | while (readl(c2dev->regs + PCI_BAR0_ADAPTER_HINT) & 0x80000000) |
735 | udelay(10); | |
f94b533d TT |
736 | |
737 | __raw_writel(C2_HINT_MAKE(mq_index, shared), | |
738 | c2dev->regs + PCI_BAR0_ADAPTER_HINT); | |
739 | } | |
740 | ||
741 | /* | |
742 | * Function: qp_wr_post | |
743 | * | |
744 | * Description: | |
745 | * This in-line function allocates a MQ msg, then moves the host-copy of | |
746 | * the completed WR into msg. Then it posts the message. | |
747 | * | |
748 | * IN: | |
749 | * q - ptr to user MQ. | |
750 | * wr - ptr to host-copy of the WR. | |
751 | * qp - ptr to user qp | |
752 | * size - Number of bytes to post. Assumed to be divisible by 4. | |
753 | * | |
754 | * OUT: none | |
755 | * | |
756 | * Return: | |
757 | * CCIL status codes. | |
758 | */ | |
759 | static int qp_wr_post(struct c2_mq *q, union c2wr * wr, struct c2_qp *qp, u32 size) | |
760 | { | |
761 | union c2wr *msg; | |
762 | ||
763 | msg = c2_mq_alloc(q); | |
764 | if (msg == NULL) { | |
765 | return -EINVAL; | |
766 | } | |
767 | #ifdef CCMSGMAGIC | |
768 | ((c2wr_hdr_t *) wr)->magic = cpu_to_be32(CCWR_MAGIC); | |
769 | #endif | |
770 | ||
771 | /* | |
772 | * Since all header fields in the WR are the same as the | |
773 | * CQE, set the following so the adapter need not. | |
774 | */ | |
775 | c2_wr_set_result(wr, CCERR_PENDING); | |
776 | ||
777 | /* | |
778 | * Copy the wr down to the adapter | |
779 | */ | |
780 | memcpy((void *) msg, (void *) wr, size); | |
781 | ||
782 | c2_mq_produce(q); | |
783 | return 0; | |
784 | } | |
785 | ||
786 | ||
787 | int c2_post_send(struct ib_qp *ibqp, struct ib_send_wr *ib_wr, | |
788 | struct ib_send_wr **bad_wr) | |
789 | { | |
790 | struct c2_dev *c2dev = to_c2dev(ibqp->device); | |
791 | struct c2_qp *qp = to_c2qp(ibqp); | |
792 | union c2wr wr; | |
e52e6080 | 793 | unsigned long lock_flags; |
f94b533d TT |
794 | int err = 0; |
795 | ||
796 | u32 flags; | |
797 | u32 tot_len; | |
798 | u8 actual_sge_count; | |
799 | u32 msg_size; | |
800 | ||
c597b024 FZ |
801 | if (qp->state > IB_QPS_RTS) { |
802 | err = -EINVAL; | |
803 | goto out; | |
804 | } | |
f94b533d TT |
805 | |
806 | while (ib_wr) { | |
807 | ||
808 | flags = 0; | |
809 | wr.sqwr.sq_hdr.user_hdr.hdr.context = ib_wr->wr_id; | |
810 | if (ib_wr->send_flags & IB_SEND_SIGNALED) { | |
811 | flags |= SQ_SIGNALED; | |
812 | } | |
813 | ||
814 | switch (ib_wr->opcode) { | |
815 | case IB_WR_SEND: | |
139b2db7 RD |
816 | case IB_WR_SEND_WITH_INV: |
817 | if (ib_wr->opcode == IB_WR_SEND) { | |
818 | if (ib_wr->send_flags & IB_SEND_SOLICITED) | |
819 | c2_wr_set_id(&wr, C2_WR_TYPE_SEND_SE); | |
820 | else | |
821 | c2_wr_set_id(&wr, C2_WR_TYPE_SEND); | |
822 | wr.sqwr.send.remote_stag = 0; | |
f94b533d | 823 | } else { |
139b2db7 RD |
824 | if (ib_wr->send_flags & IB_SEND_SOLICITED) |
825 | c2_wr_set_id(&wr, C2_WR_TYPE_SEND_SE_INV); | |
826 | else | |
827 | c2_wr_set_id(&wr, C2_WR_TYPE_SEND_INV); | |
828 | wr.sqwr.send.remote_stag = | |
829 | cpu_to_be32(ib_wr->ex.invalidate_rkey); | |
f94b533d TT |
830 | } |
831 | ||
139b2db7 RD |
832 | msg_size = sizeof(struct c2wr_send_req) + |
833 | sizeof(struct c2_data_addr) * ib_wr->num_sge; | |
f94b533d TT |
834 | if (ib_wr->num_sge > qp->send_sgl_depth) { |
835 | err = -EINVAL; | |
836 | break; | |
837 | } | |
838 | if (ib_wr->send_flags & IB_SEND_FENCE) { | |
839 | flags |= SQ_READ_FENCE; | |
840 | } | |
841 | err = move_sgl((struct c2_data_addr *) & (wr.sqwr.send.data), | |
842 | ib_wr->sg_list, | |
843 | ib_wr->num_sge, | |
844 | &tot_len, &actual_sge_count); | |
845 | wr.sqwr.send.sge_len = cpu_to_be32(tot_len); | |
846 | c2_wr_set_sge_count(&wr, actual_sge_count); | |
847 | break; | |
848 | case IB_WR_RDMA_WRITE: | |
849 | c2_wr_set_id(&wr, C2_WR_TYPE_RDMA_WRITE); | |
850 | msg_size = sizeof(struct c2wr_rdma_write_req) + | |
851 | (sizeof(struct c2_data_addr) * ib_wr->num_sge); | |
852 | if (ib_wr->num_sge > qp->rdma_write_sgl_depth) { | |
853 | err = -EINVAL; | |
854 | break; | |
855 | } | |
856 | if (ib_wr->send_flags & IB_SEND_FENCE) { | |
857 | flags |= SQ_READ_FENCE; | |
858 | } | |
859 | wr.sqwr.rdma_write.remote_stag = | |
860 | cpu_to_be32(ib_wr->wr.rdma.rkey); | |
861 | wr.sqwr.rdma_write.remote_to = | |
862 | cpu_to_be64(ib_wr->wr.rdma.remote_addr); | |
863 | err = move_sgl((struct c2_data_addr *) | |
864 | & (wr.sqwr.rdma_write.data), | |
865 | ib_wr->sg_list, | |
866 | ib_wr->num_sge, | |
867 | &tot_len, &actual_sge_count); | |
868 | wr.sqwr.rdma_write.sge_len = cpu_to_be32(tot_len); | |
869 | c2_wr_set_sge_count(&wr, actual_sge_count); | |
870 | break; | |
871 | case IB_WR_RDMA_READ: | |
872 | c2_wr_set_id(&wr, C2_WR_TYPE_RDMA_READ); | |
873 | msg_size = sizeof(struct c2wr_rdma_read_req); | |
874 | ||
875 | /* IWarp only suppots 1 sge for RDMA reads */ | |
876 | if (ib_wr->num_sge > 1) { | |
877 | err = -EINVAL; | |
878 | break; | |
879 | } | |
880 | ||
881 | /* | |
882 | * Move the local and remote stag/to/len into the WR. | |
883 | */ | |
884 | wr.sqwr.rdma_read.local_stag = | |
885 | cpu_to_be32(ib_wr->sg_list->lkey); | |
886 | wr.sqwr.rdma_read.local_to = | |
887 | cpu_to_be64(ib_wr->sg_list->addr); | |
888 | wr.sqwr.rdma_read.remote_stag = | |
889 | cpu_to_be32(ib_wr->wr.rdma.rkey); | |
890 | wr.sqwr.rdma_read.remote_to = | |
891 | cpu_to_be64(ib_wr->wr.rdma.remote_addr); | |
892 | wr.sqwr.rdma_read.length = | |
893 | cpu_to_be32(ib_wr->sg_list->length); | |
894 | break; | |
895 | default: | |
896 | /* error */ | |
897 | msg_size = 0; | |
898 | err = -EINVAL; | |
899 | break; | |
900 | } | |
901 | ||
902 | /* | |
903 | * If we had an error on the last wr build, then | |
904 | * break out. Possible errors include bogus WR | |
905 | * type, and a bogus SGL length... | |
906 | */ | |
907 | if (err) { | |
908 | break; | |
909 | } | |
910 | ||
911 | /* | |
912 | * Store flags | |
913 | */ | |
914 | c2_wr_set_flags(&wr, flags); | |
915 | ||
916 | /* | |
917 | * Post the puppy! | |
918 | */ | |
e52e6080 | 919 | spin_lock_irqsave(&qp->lock, lock_flags); |
f94b533d TT |
920 | err = qp_wr_post(&qp->sq_mq, &wr, qp, msg_size); |
921 | if (err) { | |
e52e6080 | 922 | spin_unlock_irqrestore(&qp->lock, lock_flags); |
f94b533d TT |
923 | break; |
924 | } | |
925 | ||
926 | /* | |
927 | * Enqueue mq index to activity FIFO. | |
928 | */ | |
929 | c2_activity(c2dev, qp->sq_mq.index, qp->sq_mq.hint_count); | |
e52e6080 | 930 | spin_unlock_irqrestore(&qp->lock, lock_flags); |
f94b533d TT |
931 | |
932 | ib_wr = ib_wr->next; | |
933 | } | |
934 | ||
c597b024 | 935 | out: |
f94b533d TT |
936 | if (err) |
937 | *bad_wr = ib_wr; | |
938 | return err; | |
939 | } | |
940 | ||
941 | int c2_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *ib_wr, | |
942 | struct ib_recv_wr **bad_wr) | |
943 | { | |
944 | struct c2_dev *c2dev = to_c2dev(ibqp->device); | |
945 | struct c2_qp *qp = to_c2qp(ibqp); | |
946 | union c2wr wr; | |
e52e6080 | 947 | unsigned long lock_flags; |
f94b533d TT |
948 | int err = 0; |
949 | ||
c597b024 FZ |
950 | if (qp->state > IB_QPS_RTS) { |
951 | err = -EINVAL; | |
952 | goto out; | |
953 | } | |
f94b533d TT |
954 | |
955 | /* | |
956 | * Try and post each work request | |
957 | */ | |
958 | while (ib_wr) { | |
959 | u32 tot_len; | |
960 | u8 actual_sge_count; | |
961 | ||
962 | if (ib_wr->num_sge > qp->recv_sgl_depth) { | |
963 | err = -EINVAL; | |
964 | break; | |
965 | } | |
966 | ||
967 | /* | |
968 | * Create local host-copy of the WR | |
969 | */ | |
970 | wr.rqwr.rq_hdr.user_hdr.hdr.context = ib_wr->wr_id; | |
971 | c2_wr_set_id(&wr, CCWR_RECV); | |
972 | c2_wr_set_flags(&wr, 0); | |
973 | ||
974 | /* sge_count is limited to eight bits. */ | |
975 | BUG_ON(ib_wr->num_sge >= 256); | |
976 | err = move_sgl((struct c2_data_addr *) & (wr.rqwr.data), | |
977 | ib_wr->sg_list, | |
978 | ib_wr->num_sge, &tot_len, &actual_sge_count); | |
979 | c2_wr_set_sge_count(&wr, actual_sge_count); | |
980 | ||
981 | /* | |
982 | * If we had an error on the last wr build, then | |
983 | * break out. Possible errors include bogus WR | |
984 | * type, and a bogus SGL length... | |
985 | */ | |
986 | if (err) { | |
987 | break; | |
988 | } | |
989 | ||
e52e6080 | 990 | spin_lock_irqsave(&qp->lock, lock_flags); |
f94b533d TT |
991 | err = qp_wr_post(&qp->rq_mq, &wr, qp, qp->rq_mq.msg_size); |
992 | if (err) { | |
e52e6080 | 993 | spin_unlock_irqrestore(&qp->lock, lock_flags); |
f94b533d TT |
994 | break; |
995 | } | |
996 | ||
997 | /* | |
998 | * Enqueue mq index to activity FIFO | |
999 | */ | |
1000 | c2_activity(c2dev, qp->rq_mq.index, qp->rq_mq.hint_count); | |
e52e6080 | 1001 | spin_unlock_irqrestore(&qp->lock, lock_flags); |
f94b533d TT |
1002 | |
1003 | ib_wr = ib_wr->next; | |
1004 | } | |
1005 | ||
c597b024 | 1006 | out: |
f94b533d TT |
1007 | if (err) |
1008 | *bad_wr = ib_wr; | |
1009 | return err; | |
1010 | } | |
1011 | ||
1012 | void __devinit c2_init_qp_table(struct c2_dev *c2dev) | |
1013 | { | |
1014 | spin_lock_init(&c2dev->qp_table.lock); | |
1015 | idr_init(&c2dev->qp_table.idr); | |
1016 | } | |
1017 | ||
1018 | void __devexit c2_cleanup_qp_table(struct c2_dev *c2dev) | |
1019 | { | |
1020 | idr_destroy(&c2dev->qp_table.idr); | |
1021 | } |