Commit | Line | Data |
---|---|---|
b038ced7 SW |
1 | /* |
2 | * Copyright (c) 2006 Chelsio, Inc. All rights reserved. | |
b038ced7 SW |
3 | * |
4 | * This software is available to you under a choice of one of two | |
5 | * licenses. You may choose to be licensed under the terms of the GNU | |
6 | * General Public License (GPL) Version 2, available from the file | |
7 | * COPYING in the main directory of this source tree, or the | |
8 | * OpenIB.org BSD license below: | |
9 | * | |
10 | * Redistribution and use in source and binary forms, with or | |
11 | * without modification, are permitted provided that the following | |
12 | * conditions are met: | |
13 | * | |
14 | * - Redistributions of source code must retain the above | |
15 | * copyright notice, this list of conditions and the following | |
16 | * disclaimer. | |
17 | * | |
18 | * - Redistributions in binary form must reproduce the above | |
19 | * copyright notice, this list of conditions and the following | |
20 | * disclaimer in the documentation and/or other materials | |
21 | * provided with the distribution. | |
22 | * | |
23 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
24 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
25 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
26 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
27 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
28 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
29 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
30 | * SOFTWARE. | |
31 | */ | |
32 | #include "iwch_provider.h" | |
33 | #include "iwch.h" | |
34 | #include "iwch_cm.h" | |
35 | #include "cxio_hal.h" | |
4ab928f6 | 36 | #include "cxio_resource.h" |
b038ced7 SW |
37 | |
38 | #define NO_SUPPORT -1 | |
39 | ||
4ab928f6 | 40 | static int build_rdma_send(union t3_wr *wqe, struct ib_send_wr *wr, |
2b540355 | 41 | u8 * flit_cnt) |
b038ced7 SW |
42 | { |
43 | int i; | |
44 | u32 plen; | |
45 | ||
46 | switch (wr->opcode) { | |
47 | case IB_WR_SEND: | |
b038ced7 SW |
48 | if (wr->send_flags & IB_SEND_SOLICITED) |
49 | wqe->send.rdmaop = T3_SEND_WITH_SE; | |
50 | else | |
51 | wqe->send.rdmaop = T3_SEND; | |
52 | wqe->send.rem_stag = 0; | |
53 | break; | |
e7e55829 SW |
54 | case IB_WR_SEND_WITH_INV: |
55 | if (wr->send_flags & IB_SEND_SOLICITED) | |
56 | wqe->send.rdmaop = T3_SEND_WITH_SE_INV; | |
57 | else | |
58 | wqe->send.rdmaop = T3_SEND_WITH_INV; | |
59 | wqe->send.rem_stag = cpu_to_be32(wr->ex.invalidate_rkey); | |
b038ced7 | 60 | break; |
b038ced7 | 61 | default: |
e7e55829 | 62 | return -EINVAL; |
b038ced7 SW |
63 | } |
64 | if (wr->num_sge > T3_MAX_SGE) | |
65 | return -EINVAL; | |
66 | wqe->send.reserved[0] = 0; | |
67 | wqe->send.reserved[1] = 0; | |
68 | wqe->send.reserved[2] = 0; | |
e7e55829 SW |
69 | plen = 0; |
70 | for (i = 0; i < wr->num_sge; i++) { | |
71 | if ((plen + wr->sg_list[i].length) < plen) | |
72 | return -EMSGSIZE; | |
73 | ||
74 | plen += wr->sg_list[i].length; | |
75 | wqe->send.sgl[i].stag = cpu_to_be32(wr->sg_list[i].lkey); | |
76 | wqe->send.sgl[i].len = cpu_to_be32(wr->sg_list[i].length); | |
77 | wqe->send.sgl[i].to = cpu_to_be64(wr->sg_list[i].addr); | |
b038ced7 | 78 | } |
e7e55829 SW |
79 | wqe->send.num_sgle = cpu_to_be32(wr->num_sge); |
80 | *flit_cnt = 4 + ((wr->num_sge) << 1); | |
b038ced7 SW |
81 | wqe->send.plen = cpu_to_be32(plen); |
82 | return 0; | |
83 | } | |
84 | ||
4ab928f6 | 85 | static int build_rdma_write(union t3_wr *wqe, struct ib_send_wr *wr, |
2b540355 | 86 | u8 *flit_cnt) |
b038ced7 SW |
87 | { |
88 | int i; | |
89 | u32 plen; | |
90 | if (wr->num_sge > T3_MAX_SGE) | |
91 | return -EINVAL; | |
92 | wqe->write.rdmaop = T3_RDMA_WRITE; | |
93 | wqe->write.reserved[0] = 0; | |
94 | wqe->write.reserved[1] = 0; | |
95 | wqe->write.reserved[2] = 0; | |
96 | wqe->write.stag_sink = cpu_to_be32(wr->wr.rdma.rkey); | |
97 | wqe->write.to_sink = cpu_to_be64(wr->wr.rdma.remote_addr); | |
98 | ||
99 | if (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) { | |
100 | plen = 4; | |
0f39cf3d | 101 | wqe->write.sgl[0].stag = wr->ex.imm_data; |
9c3da099 HH |
102 | wqe->write.sgl[0].len = cpu_to_be32(0); |
103 | wqe->write.num_sgle = cpu_to_be32(0); | |
b038ced7 SW |
104 | *flit_cnt = 6; |
105 | } else { | |
106 | plen = 0; | |
107 | for (i = 0; i < wr->num_sge; i++) { | |
108 | if ((plen + wr->sg_list[i].length) < plen) { | |
109 | return -EMSGSIZE; | |
110 | } | |
111 | plen += wr->sg_list[i].length; | |
112 | wqe->write.sgl[i].stag = | |
113 | cpu_to_be32(wr->sg_list[i].lkey); | |
114 | wqe->write.sgl[i].len = | |
115 | cpu_to_be32(wr->sg_list[i].length); | |
116 | wqe->write.sgl[i].to = | |
117 | cpu_to_be64(wr->sg_list[i].addr); | |
118 | } | |
119 | wqe->write.num_sgle = cpu_to_be32(wr->num_sge); | |
120 | *flit_cnt = 5 + ((wr->num_sge) << 1); | |
121 | } | |
122 | wqe->write.plen = cpu_to_be32(plen); | |
123 | return 0; | |
124 | } | |
125 | ||
4ab928f6 | 126 | static int build_rdma_read(union t3_wr *wqe, struct ib_send_wr *wr, |
2b540355 | 127 | u8 *flit_cnt) |
b038ced7 SW |
128 | { |
129 | if (wr->num_sge > 1) | |
130 | return -EINVAL; | |
131 | wqe->read.rdmaop = T3_READ_REQ; | |
e7e55829 SW |
132 | if (wr->opcode == IB_WR_RDMA_READ_WITH_INV) |
133 | wqe->read.local_inv = 1; | |
134 | else | |
135 | wqe->read.local_inv = 0; | |
b038ced7 SW |
136 | wqe->read.reserved[0] = 0; |
137 | wqe->read.reserved[1] = 0; | |
b038ced7 SW |
138 | wqe->read.rem_stag = cpu_to_be32(wr->wr.rdma.rkey); |
139 | wqe->read.rem_to = cpu_to_be64(wr->wr.rdma.remote_addr); | |
140 | wqe->read.local_stag = cpu_to_be32(wr->sg_list[0].lkey); | |
141 | wqe->read.local_len = cpu_to_be32(wr->sg_list[0].length); | |
142 | wqe->read.local_to = cpu_to_be64(wr->sg_list[0].addr); | |
143 | *flit_cnt = sizeof(struct t3_rdma_read_wr) >> 3; | |
144 | return 0; | |
145 | } | |
146 | ||
4ab928f6 | 147 | static int build_fastreg(union t3_wr *wqe, struct ib_send_wr *wr, |
e7e55829 SW |
148 | u8 *flit_cnt, int *wr_cnt, struct t3_wq *wq) |
149 | { | |
150 | int i; | |
151 | __be64 *p; | |
152 | ||
153 | if (wr->wr.fast_reg.page_list_len > T3_MAX_FASTREG_DEPTH) | |
154 | return -EINVAL; | |
155 | *wr_cnt = 1; | |
156 | wqe->fastreg.stag = cpu_to_be32(wr->wr.fast_reg.rkey); | |
157 | wqe->fastreg.len = cpu_to_be32(wr->wr.fast_reg.length); | |
158 | wqe->fastreg.va_base_hi = cpu_to_be32(wr->wr.fast_reg.iova_start >> 32); | |
159 | wqe->fastreg.va_base_lo_fbo = | |
160 | cpu_to_be32(wr->wr.fast_reg.iova_start & 0xffffffff); | |
161 | wqe->fastreg.page_type_perms = cpu_to_be32( | |
162 | V_FR_PAGE_COUNT(wr->wr.fast_reg.page_list_len) | | |
163 | V_FR_PAGE_SIZE(wr->wr.fast_reg.page_shift-12) | | |
164 | V_FR_TYPE(TPT_VATO) | | |
165 | V_FR_PERMS(iwch_ib_to_tpt_access(wr->wr.fast_reg.access_flags))); | |
166 | p = &wqe->fastreg.pbl_addrs[0]; | |
167 | for (i = 0; i < wr->wr.fast_reg.page_list_len; i++, p++) { | |
168 | ||
169 | /* If we need a 2nd WR, then set it up */ | |
170 | if (i == T3_MAX_FASTREG_FRAG) { | |
171 | *wr_cnt = 2; | |
172 | wqe = (union t3_wr *)(wq->queue + | |
173 | Q_PTR2IDX((wq->wptr+1), wq->size_log2)); | |
174 | build_fw_riwrh((void *)wqe, T3_WR_FASTREG, 0, | |
175 | Q_GENBIT(wq->wptr + 1, wq->size_log2), | |
176 | 0, 1 + wr->wr.fast_reg.page_list_len - T3_MAX_FASTREG_FRAG, | |
177 | T3_EOP); | |
178 | ||
179 | p = &wqe->pbl_frag.pbl_addrs[0]; | |
180 | } | |
181 | *p = cpu_to_be64((u64)wr->wr.fast_reg.page_list->page_list[i]); | |
182 | } | |
183 | *flit_cnt = 5 + wr->wr.fast_reg.page_list_len; | |
184 | if (*flit_cnt > 15) | |
185 | *flit_cnt = 15; | |
186 | return 0; | |
187 | } | |
188 | ||
4ab928f6 | 189 | static int build_inv_stag(union t3_wr *wqe, struct ib_send_wr *wr, |
e7e55829 SW |
190 | u8 *flit_cnt) |
191 | { | |
192 | wqe->local_inv.stag = cpu_to_be32(wr->ex.invalidate_rkey); | |
193 | wqe->local_inv.reserved = 0; | |
194 | *flit_cnt = sizeof(struct t3_local_inv_wr) >> 3; | |
195 | return 0; | |
196 | } | |
197 | ||
2b540355 AB |
198 | static int iwch_sgl2pbl_map(struct iwch_dev *rhp, struct ib_sge *sg_list, |
199 | u32 num_sgle, u32 * pbl_addr, u8 * page_size) | |
b038ced7 SW |
200 | { |
201 | int i; | |
202 | struct iwch_mr *mhp; | |
900f4c16 | 203 | u64 offset; |
b038ced7 SW |
204 | for (i = 0; i < num_sgle; i++) { |
205 | ||
206 | mhp = get_mhp(rhp, (sg_list[i].lkey) >> 8); | |
207 | if (!mhp) { | |
33718363 | 208 | PDBG("%s %d\n", __func__, __LINE__); |
b038ced7 SW |
209 | return -EIO; |
210 | } | |
211 | if (!mhp->attr.state) { | |
33718363 | 212 | PDBG("%s %d\n", __func__, __LINE__); |
b038ced7 SW |
213 | return -EIO; |
214 | } | |
215 | if (mhp->attr.zbva) { | |
33718363 | 216 | PDBG("%s %d\n", __func__, __LINE__); |
b038ced7 SW |
217 | return -EIO; |
218 | } | |
219 | ||
220 | if (sg_list[i].addr < mhp->attr.va_fbo) { | |
33718363 | 221 | PDBG("%s %d\n", __func__, __LINE__); |
b038ced7 SW |
222 | return -EINVAL; |
223 | } | |
224 | if (sg_list[i].addr + ((u64) sg_list[i].length) < | |
225 | sg_list[i].addr) { | |
33718363 | 226 | PDBG("%s %d\n", __func__, __LINE__); |
b038ced7 SW |
227 | return -EINVAL; |
228 | } | |
229 | if (sg_list[i].addr + ((u64) sg_list[i].length) > | |
230 | mhp->attr.va_fbo + ((u64) mhp->attr.len)) { | |
33718363 | 231 | PDBG("%s %d\n", __func__, __LINE__); |
b038ced7 SW |
232 | return -EINVAL; |
233 | } | |
234 | offset = sg_list[i].addr - mhp->attr.va_fbo; | |
900f4c16 SW |
235 | offset += mhp->attr.va_fbo & |
236 | ((1UL << (12 + mhp->attr.page_size)) - 1); | |
b038ced7 SW |
237 | pbl_addr[i] = ((mhp->attr.pbl_addr - |
238 | rhp->rdev.rnic_info.pbl_base) >> 3) + | |
239 | (offset >> (12 + mhp->attr.page_size)); | |
240 | page_size[i] = mhp->attr.page_size; | |
241 | } | |
242 | return 0; | |
243 | } | |
244 | ||
4ab928f6 | 245 | static int build_rdma_recv(struct iwch_qp *qhp, union t3_wr *wqe, |
2b540355 | 246 | struct ib_recv_wr *wr) |
b038ced7 | 247 | { |
4ab928f6 SW |
248 | int i, err = 0; |
249 | u32 pbl_addr[T3_MAX_SGE]; | |
250 | u8 page_size[T3_MAX_SGE]; | |
251 | ||
252 | err = iwch_sgl2pbl_map(qhp->rhp, wr->sg_list, wr->num_sge, pbl_addr, | |
253 | page_size); | |
254 | if (err) | |
255 | return err; | |
256 | wqe->recv.pagesz[0] = page_size[0]; | |
257 | wqe->recv.pagesz[1] = page_size[1]; | |
258 | wqe->recv.pagesz[2] = page_size[2]; | |
259 | wqe->recv.pagesz[3] = page_size[3]; | |
b038ced7 SW |
260 | wqe->recv.num_sgle = cpu_to_be32(wr->num_sge); |
261 | for (i = 0; i < wr->num_sge; i++) { | |
262 | wqe->recv.sgl[i].stag = cpu_to_be32(wr->sg_list[i].lkey); | |
263 | wqe->recv.sgl[i].len = cpu_to_be32(wr->sg_list[i].length); | |
4ab928f6 SW |
264 | |
265 | /* to in the WQE == the offset into the page */ | |
42632896 SW |
266 | wqe->recv.sgl[i].to = cpu_to_be64(((u32)wr->sg_list[i].addr) & |
267 | ((1UL << (12 + page_size[i])) - 1)); | |
4ab928f6 SW |
268 | |
269 | /* pbl_addr is the adapters address in the PBL */ | |
270 | wqe->recv.pbl_addr[i] = cpu_to_be32(pbl_addr[i]); | |
271 | } | |
272 | for (; i < T3_MAX_SGE; i++) { | |
273 | wqe->recv.sgl[i].stag = 0; | |
274 | wqe->recv.sgl[i].len = 0; | |
275 | wqe->recv.sgl[i].to = 0; | |
276 | wqe->recv.pbl_addr[i] = 0; | |
277 | } | |
278 | qhp->wq.rq[Q_PTR2IDX(qhp->wq.rq_wptr, | |
279 | qhp->wq.rq_size_log2)].wr_id = wr->wr_id; | |
280 | qhp->wq.rq[Q_PTR2IDX(qhp->wq.rq_wptr, | |
281 | qhp->wq.rq_size_log2)].pbl_addr = 0; | |
282 | return 0; | |
283 | } | |
284 | ||
285 | static int build_zero_stag_recv(struct iwch_qp *qhp, union t3_wr *wqe, | |
286 | struct ib_recv_wr *wr) | |
287 | { | |
288 | int i; | |
289 | u32 pbl_addr; | |
290 | u32 pbl_offset; | |
291 | ||
292 | ||
293 | /* | |
294 | * The T3 HW requires the PBL in the HW recv descriptor to reference | |
295 | * a PBL entry. So we allocate the max needed PBL memory here and pass | |
296 | * it to the uP in the recv WR. The uP will build the PBL and setup | |
297 | * the HW recv descriptor. | |
298 | */ | |
299 | pbl_addr = cxio_hal_pblpool_alloc(&qhp->rhp->rdev, T3_STAG0_PBL_SIZE); | |
300 | if (!pbl_addr) | |
301 | return -ENOMEM; | |
302 | ||
303 | /* | |
304 | * Compute the 8B aligned offset. | |
305 | */ | |
306 | pbl_offset = (pbl_addr - qhp->rhp->rdev.rnic_info.pbl_base) >> 3; | |
307 | ||
308 | wqe->recv.num_sgle = cpu_to_be32(wr->num_sge); | |
309 | ||
310 | for (i = 0; i < wr->num_sge; i++) { | |
311 | ||
312 | /* | |
313 | * Use a 128MB page size. This and an imposed 128MB | |
314 | * sge length limit allows us to require only a 2-entry HW | |
315 | * PBL for each SGE. This restriction is acceptable since | |
316 | * since it is not possible to allocate 128MB of contiguous | |
317 | * DMA coherent memory! | |
318 | */ | |
319 | if (wr->sg_list[i].length > T3_STAG0_MAX_PBE_LEN) | |
320 | return -EINVAL; | |
321 | wqe->recv.pagesz[i] = T3_STAG0_PAGE_SHIFT; | |
322 | ||
323 | /* | |
324 | * T3 restricts a recv to all zero-stag or all non-zero-stag. | |
325 | */ | |
326 | if (wr->sg_list[i].lkey != 0) | |
327 | return -EINVAL; | |
328 | wqe->recv.sgl[i].stag = 0; | |
329 | wqe->recv.sgl[i].len = cpu_to_be32(wr->sg_list[i].length); | |
457fe7b8 | 330 | wqe->recv.sgl[i].to = cpu_to_be64(wr->sg_list[i].addr); |
4ab928f6 SW |
331 | wqe->recv.pbl_addr[i] = cpu_to_be32(pbl_offset); |
332 | pbl_offset += 2; | |
b038ced7 SW |
333 | } |
334 | for (; i < T3_MAX_SGE; i++) { | |
4ab928f6 | 335 | wqe->recv.pagesz[i] = 0; |
b038ced7 SW |
336 | wqe->recv.sgl[i].stag = 0; |
337 | wqe->recv.sgl[i].len = 0; | |
338 | wqe->recv.sgl[i].to = 0; | |
4ab928f6 | 339 | wqe->recv.pbl_addr[i] = 0; |
b038ced7 | 340 | } |
4ab928f6 SW |
341 | qhp->wq.rq[Q_PTR2IDX(qhp->wq.rq_wptr, |
342 | qhp->wq.rq_size_log2)].wr_id = wr->wr_id; | |
343 | qhp->wq.rq[Q_PTR2IDX(qhp->wq.rq_wptr, | |
344 | qhp->wq.rq_size_log2)].pbl_addr = pbl_addr; | |
b038ced7 SW |
345 | return 0; |
346 | } | |
347 | ||
348 | int iwch_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr, | |
349 | struct ib_send_wr **bad_wr) | |
350 | { | |
351 | int err = 0; | |
21609ae3 | 352 | u8 uninitialized_var(t3_wr_flit_cnt); |
b038ced7 SW |
353 | enum t3_wr_opcode t3_wr_opcode = 0; |
354 | enum t3_wr_flags t3_wr_flags; | |
355 | struct iwch_qp *qhp; | |
356 | u32 idx; | |
357 | union t3_wr *wqe; | |
358 | u32 num_wrs; | |
359 | unsigned long flag; | |
360 | struct t3_swsq *sqp; | |
e7e55829 | 361 | int wr_cnt = 1; |
b038ced7 SW |
362 | |
363 | qhp = to_iwch_qp(ibqp); | |
364 | spin_lock_irqsave(&qhp->lock, flag); | |
365 | if (qhp->attr.state > IWCH_QP_STATE_RTS) { | |
366 | spin_unlock_irqrestore(&qhp->lock, flag); | |
367 | return -EINVAL; | |
368 | } | |
369 | num_wrs = Q_FREECNT(qhp->wq.sq_rptr, qhp->wq.sq_wptr, | |
370 | qhp->wq.sq_size_log2); | |
371 | if (num_wrs <= 0) { | |
372 | spin_unlock_irqrestore(&qhp->lock, flag); | |
373 | return -ENOMEM; | |
374 | } | |
375 | while (wr) { | |
376 | if (num_wrs == 0) { | |
377 | err = -ENOMEM; | |
378 | *bad_wr = wr; | |
379 | break; | |
380 | } | |
381 | idx = Q_PTR2IDX(qhp->wq.wptr, qhp->wq.size_log2); | |
382 | wqe = (union t3_wr *) (qhp->wq.queue + idx); | |
383 | t3_wr_flags = 0; | |
384 | if (wr->send_flags & IB_SEND_SOLICITED) | |
385 | t3_wr_flags |= T3_SOLICITED_EVENT_FLAG; | |
b038ced7 SW |
386 | if (wr->send_flags & IB_SEND_SIGNALED) |
387 | t3_wr_flags |= T3_COMPLETION_FLAG; | |
388 | sqp = qhp->wq.sq + | |
389 | Q_PTR2IDX(qhp->wq.sq_wptr, qhp->wq.sq_size_log2); | |
390 | switch (wr->opcode) { | |
391 | case IB_WR_SEND: | |
e7e55829 SW |
392 | case IB_WR_SEND_WITH_INV: |
393 | if (wr->send_flags & IB_SEND_FENCE) | |
394 | t3_wr_flags |= T3_READ_FENCE_FLAG; | |
b038ced7 | 395 | t3_wr_opcode = T3_WR_SEND; |
4ab928f6 | 396 | err = build_rdma_send(wqe, wr, &t3_wr_flit_cnt); |
b038ced7 SW |
397 | break; |
398 | case IB_WR_RDMA_WRITE: | |
399 | case IB_WR_RDMA_WRITE_WITH_IMM: | |
400 | t3_wr_opcode = T3_WR_WRITE; | |
4ab928f6 | 401 | err = build_rdma_write(wqe, wr, &t3_wr_flit_cnt); |
b038ced7 SW |
402 | break; |
403 | case IB_WR_RDMA_READ: | |
e7e55829 | 404 | case IB_WR_RDMA_READ_WITH_INV: |
b038ced7 SW |
405 | t3_wr_opcode = T3_WR_READ; |
406 | t3_wr_flags = 0; /* T3 reads are always signaled */ | |
4ab928f6 | 407 | err = build_rdma_read(wqe, wr, &t3_wr_flit_cnt); |
b038ced7 SW |
408 | if (err) |
409 | break; | |
410 | sqp->read_len = wqe->read.local_len; | |
411 | if (!qhp->wq.oldest_read) | |
412 | qhp->wq.oldest_read = sqp; | |
413 | break; | |
e7e55829 SW |
414 | case IB_WR_FAST_REG_MR: |
415 | t3_wr_opcode = T3_WR_FASTREG; | |
4ab928f6 | 416 | err = build_fastreg(wqe, wr, &t3_wr_flit_cnt, |
e7e55829 SW |
417 | &wr_cnt, &qhp->wq); |
418 | break; | |
419 | case IB_WR_LOCAL_INV: | |
420 | if (wr->send_flags & IB_SEND_FENCE) | |
421 | t3_wr_flags |= T3_LOCAL_FENCE_FLAG; | |
422 | t3_wr_opcode = T3_WR_INV_STAG; | |
4ab928f6 | 423 | err = build_inv_stag(wqe, wr, &t3_wr_flit_cnt); |
e7e55829 | 424 | break; |
b038ced7 | 425 | default: |
33718363 | 426 | PDBG("%s post of type=%d TBD!\n", __func__, |
b038ced7 SW |
427 | wr->opcode); |
428 | err = -EINVAL; | |
429 | } | |
430 | if (err) { | |
431 | *bad_wr = wr; | |
432 | break; | |
433 | } | |
434 | wqe->send.wrid.id0.hi = qhp->wq.sq_wptr; | |
435 | sqp->wr_id = wr->wr_id; | |
436 | sqp->opcode = wr2opcode(t3_wr_opcode); | |
437 | sqp->sq_wptr = qhp->wq.sq_wptr; | |
438 | sqp->complete = 0; | |
439 | sqp->signaled = (wr->send_flags & IB_SEND_SIGNALED); | |
440 | ||
441 | build_fw_riwrh((void *) wqe, t3_wr_opcode, t3_wr_flags, | |
442 | Q_GENBIT(qhp->wq.wptr, qhp->wq.size_log2), | |
e7e55829 SW |
443 | 0, t3_wr_flit_cnt, |
444 | (wr_cnt == 1) ? T3_SOPEOP : T3_SOP); | |
b038ced7 | 445 | PDBG("%s cookie 0x%llx wq idx 0x%x swsq idx %ld opcode %d\n", |
33718363 | 446 | __func__, (unsigned long long) wr->wr_id, idx, |
b038ced7 SW |
447 | Q_PTR2IDX(qhp->wq.sq_wptr, qhp->wq.sq_size_log2), |
448 | sqp->opcode); | |
449 | wr = wr->next; | |
450 | num_wrs--; | |
e7e55829 | 451 | qhp->wq.wptr += wr_cnt; |
b038ced7 SW |
452 | ++(qhp->wq.sq_wptr); |
453 | } | |
454 | spin_unlock_irqrestore(&qhp->lock, flag); | |
455 | ring_doorbell(qhp->wq.doorbell, qhp->wq.qpid); | |
456 | return err; | |
457 | } | |
458 | ||
459 | int iwch_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr, | |
460 | struct ib_recv_wr **bad_wr) | |
461 | { | |
462 | int err = 0; | |
463 | struct iwch_qp *qhp; | |
464 | u32 idx; | |
465 | union t3_wr *wqe; | |
466 | u32 num_wrs; | |
467 | unsigned long flag; | |
468 | ||
469 | qhp = to_iwch_qp(ibqp); | |
470 | spin_lock_irqsave(&qhp->lock, flag); | |
471 | if (qhp->attr.state > IWCH_QP_STATE_RTS) { | |
472 | spin_unlock_irqrestore(&qhp->lock, flag); | |
473 | return -EINVAL; | |
474 | } | |
475 | num_wrs = Q_FREECNT(qhp->wq.rq_rptr, qhp->wq.rq_wptr, | |
476 | qhp->wq.rq_size_log2) - 1; | |
477 | if (!wr) { | |
478 | spin_unlock_irqrestore(&qhp->lock, flag); | |
479 | return -EINVAL; | |
480 | } | |
481 | while (wr) { | |
4ab928f6 SW |
482 | if (wr->num_sge > T3_MAX_SGE) { |
483 | err = -EINVAL; | |
484 | *bad_wr = wr; | |
485 | break; | |
486 | } | |
b038ced7 SW |
487 | idx = Q_PTR2IDX(qhp->wq.wptr, qhp->wq.size_log2); |
488 | wqe = (union t3_wr *) (qhp->wq.queue + idx); | |
489 | if (num_wrs) | |
4ab928f6 SW |
490 | if (wr->sg_list[0].lkey) |
491 | err = build_rdma_recv(qhp, wqe, wr); | |
492 | else | |
493 | err = build_zero_stag_recv(qhp, wqe, wr); | |
b038ced7 SW |
494 | else |
495 | err = -ENOMEM; | |
496 | if (err) { | |
497 | *bad_wr = wr; | |
498 | break; | |
499 | } | |
b038ced7 SW |
500 | build_fw_riwrh((void *) wqe, T3_WR_RCV, T3_COMPLETION_FLAG, |
501 | Q_GENBIT(qhp->wq.wptr, qhp->wq.size_log2), | |
e7e55829 | 502 | 0, sizeof(struct t3_receive_wr) >> 3, T3_SOPEOP); |
b038ced7 | 503 | PDBG("%s cookie 0x%llx idx 0x%x rq_wptr 0x%x rw_rptr 0x%x " |
33718363 | 504 | "wqe %p \n", __func__, (unsigned long long) wr->wr_id, |
b038ced7 SW |
505 | idx, qhp->wq.rq_wptr, qhp->wq.rq_rptr, wqe); |
506 | ++(qhp->wq.rq_wptr); | |
507 | ++(qhp->wq.wptr); | |
508 | wr = wr->next; | |
509 | num_wrs--; | |
510 | } | |
511 | spin_unlock_irqrestore(&qhp->lock, flag); | |
512 | ring_doorbell(qhp->wq.doorbell, qhp->wq.qpid); | |
513 | return err; | |
514 | } | |
515 | ||
516 | int iwch_bind_mw(struct ib_qp *qp, | |
517 | struct ib_mw *mw, | |
518 | struct ib_mw_bind *mw_bind) | |
519 | { | |
520 | struct iwch_dev *rhp; | |
521 | struct iwch_mw *mhp; | |
522 | struct iwch_qp *qhp; | |
523 | union t3_wr *wqe; | |
524 | u32 pbl_addr; | |
525 | u8 page_size; | |
526 | u32 num_wrs; | |
527 | unsigned long flag; | |
528 | struct ib_sge sgl; | |
529 | int err=0; | |
530 | enum t3_wr_flags t3_wr_flags; | |
531 | u32 idx; | |
532 | struct t3_swsq *sqp; | |
533 | ||
534 | qhp = to_iwch_qp(qp); | |
535 | mhp = to_iwch_mw(mw); | |
536 | rhp = qhp->rhp; | |
537 | ||
538 | spin_lock_irqsave(&qhp->lock, flag); | |
539 | if (qhp->attr.state > IWCH_QP_STATE_RTS) { | |
540 | spin_unlock_irqrestore(&qhp->lock, flag); | |
541 | return -EINVAL; | |
542 | } | |
543 | num_wrs = Q_FREECNT(qhp->wq.sq_rptr, qhp->wq.sq_wptr, | |
544 | qhp->wq.sq_size_log2); | |
545 | if ((num_wrs) <= 0) { | |
546 | spin_unlock_irqrestore(&qhp->lock, flag); | |
547 | return -ENOMEM; | |
548 | } | |
549 | idx = Q_PTR2IDX(qhp->wq.wptr, qhp->wq.size_log2); | |
33718363 | 550 | PDBG("%s: idx 0x%0x, mw 0x%p, mw_bind 0x%p\n", __func__, idx, |
b038ced7 SW |
551 | mw, mw_bind); |
552 | wqe = (union t3_wr *) (qhp->wq.queue + idx); | |
553 | ||
554 | t3_wr_flags = 0; | |
555 | if (mw_bind->send_flags & IB_SEND_SIGNALED) | |
556 | t3_wr_flags = T3_COMPLETION_FLAG; | |
557 | ||
558 | sgl.addr = mw_bind->addr; | |
559 | sgl.lkey = mw_bind->mr->lkey; | |
560 | sgl.length = mw_bind->length; | |
561 | wqe->bind.reserved = 0; | |
e7e55829 | 562 | wqe->bind.type = TPT_VATO; |
b038ced7 SW |
563 | |
564 | /* TBD: check perms */ | |
1c355a6e | 565 | wqe->bind.perms = iwch_ib_to_tpt_bind_access(mw_bind->mw_access_flags); |
b038ced7 SW |
566 | wqe->bind.mr_stag = cpu_to_be32(mw_bind->mr->lkey); |
567 | wqe->bind.mw_stag = cpu_to_be32(mw->rkey); | |
568 | wqe->bind.mw_len = cpu_to_be32(mw_bind->length); | |
569 | wqe->bind.mw_va = cpu_to_be64(mw_bind->addr); | |
570 | err = iwch_sgl2pbl_map(rhp, &sgl, 1, &pbl_addr, &page_size); | |
571 | if (err) { | |
572 | spin_unlock_irqrestore(&qhp->lock, flag); | |
e7e55829 | 573 | return err; |
b038ced7 SW |
574 | } |
575 | wqe->send.wrid.id0.hi = qhp->wq.sq_wptr; | |
576 | sqp = qhp->wq.sq + Q_PTR2IDX(qhp->wq.sq_wptr, qhp->wq.sq_size_log2); | |
577 | sqp->wr_id = mw_bind->wr_id; | |
578 | sqp->opcode = T3_BIND_MW; | |
579 | sqp->sq_wptr = qhp->wq.sq_wptr; | |
580 | sqp->complete = 0; | |
581 | sqp->signaled = (mw_bind->send_flags & IB_SEND_SIGNALED); | |
582 | wqe->bind.mr_pbl_addr = cpu_to_be32(pbl_addr); | |
583 | wqe->bind.mr_pagesz = page_size; | |
b038ced7 SW |
584 | build_fw_riwrh((void *)wqe, T3_WR_BIND, t3_wr_flags, |
585 | Q_GENBIT(qhp->wq.wptr, qhp->wq.size_log2), 0, | |
e7e55829 | 586 | sizeof(struct t3_bind_mw_wr) >> 3, T3_SOPEOP); |
b038ced7 SW |
587 | ++(qhp->wq.wptr); |
588 | ++(qhp->wq.sq_wptr); | |
589 | spin_unlock_irqrestore(&qhp->lock, flag); | |
590 | ||
591 | ring_doorbell(qhp->wq.doorbell, qhp->wq.qpid); | |
592 | ||
593 | return err; | |
594 | } | |
595 | ||
4a97d47e SW |
596 | static inline void build_term_codes(struct respQ_msg_t *rsp_msg, |
597 | u8 *layer_type, u8 *ecode) | |
b038ced7 | 598 | { |
4a97d47e SW |
599 | int status = TPT_ERR_INTERNAL_ERR; |
600 | int tagged = 0; | |
601 | int opcode = -1; | |
602 | int rqtype = 0; | |
603 | int send_inv = 0; | |
604 | ||
605 | if (rsp_msg) { | |
606 | status = CQE_STATUS(rsp_msg->cqe); | |
607 | opcode = CQE_OPCODE(rsp_msg->cqe); | |
608 | rqtype = RQ_TYPE(rsp_msg->cqe); | |
609 | send_inv = (opcode == T3_SEND_WITH_INV) || | |
610 | (opcode == T3_SEND_WITH_SE_INV); | |
611 | tagged = (opcode == T3_RDMA_WRITE) || | |
612 | (rqtype && (opcode == T3_READ_RESP)); | |
613 | } | |
614 | ||
615 | switch (status) { | |
b038ced7 | 616 | case TPT_ERR_STAG: |
4a97d47e SW |
617 | if (send_inv) { |
618 | *layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP; | |
619 | *ecode = RDMAP_CANT_INV_STAG; | |
620 | } else { | |
b038ced7 SW |
621 | *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; |
622 | *ecode = RDMAP_INV_STAG; | |
623 | } | |
624 | break; | |
625 | case TPT_ERR_PDID: | |
4a97d47e SW |
626 | *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; |
627 | if ((opcode == T3_SEND_WITH_INV) || | |
628 | (opcode == T3_SEND_WITH_SE_INV)) | |
629 | *ecode = RDMAP_CANT_INV_STAG; | |
630 | else | |
631 | *ecode = RDMAP_STAG_NOT_ASSOC; | |
632 | break; | |
b038ced7 | 633 | case TPT_ERR_QPID: |
4a97d47e SW |
634 | *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; |
635 | *ecode = RDMAP_STAG_NOT_ASSOC; | |
636 | break; | |
b038ced7 | 637 | case TPT_ERR_ACCESS: |
4a97d47e SW |
638 | *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; |
639 | *ecode = RDMAP_ACC_VIOL; | |
b038ced7 SW |
640 | break; |
641 | case TPT_ERR_WRAP: | |
642 | *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; | |
643 | *ecode = RDMAP_TO_WRAP; | |
644 | break; | |
645 | case TPT_ERR_BOUND: | |
4a97d47e | 646 | if (tagged) { |
b038ced7 SW |
647 | *layer_type = LAYER_DDP|DDP_TAGGED_ERR; |
648 | *ecode = DDPT_BASE_BOUNDS; | |
4a97d47e | 649 | } else { |
b038ced7 SW |
650 | *layer_type = LAYER_RDMAP|RDMAP_REMOTE_PROT; |
651 | *ecode = RDMAP_BASE_BOUNDS; | |
b038ced7 SW |
652 | } |
653 | break; | |
654 | case TPT_ERR_INVALIDATE_SHARED_MR: | |
655 | case TPT_ERR_INVALIDATE_MR_WITH_MW_BOUND: | |
656 | *layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP; | |
657 | *ecode = RDMAP_CANT_INV_STAG; | |
658 | break; | |
659 | case TPT_ERR_ECC: | |
660 | case TPT_ERR_ECC_PSTAG: | |
661 | case TPT_ERR_INTERNAL_ERR: | |
662 | *layer_type = LAYER_RDMAP|RDMAP_LOCAL_CATA; | |
663 | *ecode = 0; | |
664 | break; | |
665 | case TPT_ERR_OUT_OF_RQE: | |
666 | *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; | |
667 | *ecode = DDPU_INV_MSN_NOBUF; | |
668 | break; | |
669 | case TPT_ERR_PBL_ADDR_BOUND: | |
670 | *layer_type = LAYER_DDP|DDP_TAGGED_ERR; | |
671 | *ecode = DDPT_BASE_BOUNDS; | |
672 | break; | |
673 | case TPT_ERR_CRC: | |
674 | *layer_type = LAYER_MPA|DDP_LLP; | |
675 | *ecode = MPA_CRC_ERR; | |
676 | break; | |
677 | case TPT_ERR_MARKER: | |
678 | *layer_type = LAYER_MPA|DDP_LLP; | |
679 | *ecode = MPA_MARKER_ERR; | |
680 | break; | |
681 | case TPT_ERR_PDU_LEN_ERR: | |
682 | *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; | |
683 | *ecode = DDPU_MSG_TOOBIG; | |
684 | break; | |
685 | case TPT_ERR_DDP_VERSION: | |
686 | if (tagged) { | |
687 | *layer_type = LAYER_DDP|DDP_TAGGED_ERR; | |
688 | *ecode = DDPT_INV_VERS; | |
689 | } else { | |
690 | *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; | |
691 | *ecode = DDPU_INV_VERS; | |
692 | } | |
693 | break; | |
694 | case TPT_ERR_RDMA_VERSION: | |
695 | *layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP; | |
696 | *ecode = RDMAP_INV_VERS; | |
697 | break; | |
698 | case TPT_ERR_OPCODE: | |
699 | *layer_type = LAYER_RDMAP|RDMAP_REMOTE_OP; | |
700 | *ecode = RDMAP_INV_OPCODE; | |
701 | break; | |
702 | case TPT_ERR_DDP_QUEUE_NUM: | |
703 | *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; | |
704 | *ecode = DDPU_INV_QN; | |
705 | break; | |
706 | case TPT_ERR_MSN: | |
707 | case TPT_ERR_MSN_GAP: | |
708 | case TPT_ERR_MSN_RANGE: | |
709 | case TPT_ERR_IRD_OVERFLOW: | |
710 | *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; | |
711 | *ecode = DDPU_INV_MSN_RANGE; | |
712 | break; | |
713 | case TPT_ERR_TBIT: | |
714 | *layer_type = LAYER_DDP|DDP_LOCAL_CATA; | |
715 | *ecode = 0; | |
716 | break; | |
717 | case TPT_ERR_MO: | |
718 | *layer_type = LAYER_DDP|DDP_UNTAGGED_ERR; | |
719 | *ecode = DDPU_INV_MO; | |
720 | break; | |
721 | default: | |
722 | *layer_type = LAYER_RDMAP|DDP_LOCAL_CATA; | |
723 | *ecode = 0; | |
724 | break; | |
725 | } | |
726 | } | |
727 | ||
f8b0dfd1 SW |
728 | int iwch_post_zb_read(struct iwch_qp *qhp) |
729 | { | |
730 | union t3_wr *wqe; | |
731 | struct sk_buff *skb; | |
732 | u8 flit_cnt = sizeof(struct t3_rdma_read_wr) >> 3; | |
733 | ||
734 | PDBG("%s enter\n", __func__); | |
735 | skb = alloc_skb(40, GFP_KERNEL); | |
736 | if (!skb) { | |
737 | printk(KERN_ERR "%s cannot send zb_read!!\n", __func__); | |
738 | return -ENOMEM; | |
739 | } | |
740 | wqe = (union t3_wr *)skb_put(skb, sizeof(struct t3_rdma_read_wr)); | |
741 | memset(wqe, 0, sizeof(struct t3_rdma_read_wr)); | |
742 | wqe->read.rdmaop = T3_READ_REQ; | |
743 | wqe->read.reserved[0] = 0; | |
744 | wqe->read.reserved[1] = 0; | |
f8b0dfd1 SW |
745 | wqe->read.rem_stag = cpu_to_be32(1); |
746 | wqe->read.rem_to = cpu_to_be64(1); | |
747 | wqe->read.local_stag = cpu_to_be32(1); | |
748 | wqe->read.local_len = cpu_to_be32(0); | |
749 | wqe->read.local_to = cpu_to_be64(1); | |
750 | wqe->send.wrh.op_seop_flags = cpu_to_be32(V_FW_RIWR_OP(T3_WR_READ)); | |
751 | wqe->send.wrh.gen_tid_len = cpu_to_be32(V_FW_RIWR_TID(qhp->ep->hwtid)| | |
752 | V_FW_RIWR_LEN(flit_cnt)); | |
753 | skb->priority = CPL_PRIORITY_DATA; | |
04b5d028 | 754 | return iwch_cxgb3_ofld_send(qhp->rhp->rdev.t3cdev_p, skb); |
f8b0dfd1 SW |
755 | } |
756 | ||
b038ced7 SW |
757 | /* |
758 | * This posts a TERMINATE with layer=RDMA, type=catastrophic. | |
759 | */ | |
760 | int iwch_post_terminate(struct iwch_qp *qhp, struct respQ_msg_t *rsp_msg) | |
761 | { | |
762 | union t3_wr *wqe; | |
763 | struct terminate_message *term; | |
b038ced7 SW |
764 | struct sk_buff *skb; |
765 | ||
33718363 | 766 | PDBG("%s %d\n", __func__, __LINE__); |
b038ced7 SW |
767 | skb = alloc_skb(40, GFP_ATOMIC); |
768 | if (!skb) { | |
33718363 | 769 | printk(KERN_ERR "%s cannot send TERMINATE!\n", __func__); |
b038ced7 SW |
770 | return -ENOMEM; |
771 | } | |
772 | wqe = (union t3_wr *)skb_put(skb, 40); | |
773 | memset(wqe, 0, 40); | |
774 | wqe->send.rdmaop = T3_TERMINATE; | |
775 | ||
776 | /* immediate data length */ | |
777 | wqe->send.plen = htonl(4); | |
778 | ||
779 | /* immediate data starts here. */ | |
780 | term = (struct terminate_message *)wqe->send.sgl; | |
4a97d47e | 781 | build_term_codes(rsp_msg, &term->layer_etype, &term->ecode); |
fb497d72 SW |
782 | wqe->send.wrh.op_seop_flags = cpu_to_be32(V_FW_RIWR_OP(T3_WR_SEND) | |
783 | V_FW_RIWR_FLAGS(T3_COMPLETION_FLAG | T3_NOTIFY_FLAG)); | |
784 | wqe->send.wrh.gen_tid_len = cpu_to_be32(V_FW_RIWR_TID(qhp->ep->hwtid)); | |
b038ced7 | 785 | skb->priority = CPL_PRIORITY_DATA; |
04b5d028 | 786 | return iwch_cxgb3_ofld_send(qhp->rhp->rdev.t3cdev_p, skb); |
b038ced7 SW |
787 | } |
788 | ||
789 | /* | |
790 | * Assumes qhp lock is held. | |
791 | */ | |
792 | static void __flush_qp(struct iwch_qp *qhp, unsigned long *flag) | |
793 | { | |
794 | struct iwch_cq *rchp, *schp; | |
795 | int count; | |
c8286944 | 796 | int flushed; |
b038ced7 SW |
797 | |
798 | rchp = get_chp(qhp->rhp, qhp->attr.rcq); | |
799 | schp = get_chp(qhp->rhp, qhp->attr.scq); | |
800 | ||
33718363 | 801 | PDBG("%s qhp %p rchp %p schp %p\n", __func__, qhp, rchp, schp); |
b038ced7 SW |
802 | /* take a ref on the qhp since we must release the lock */ |
803 | atomic_inc(&qhp->refcnt); | |
804 | spin_unlock_irqrestore(&qhp->lock, *flag); | |
805 | ||
806 | /* locking heirarchy: cq lock first, then qp lock. */ | |
807 | spin_lock_irqsave(&rchp->lock, *flag); | |
808 | spin_lock(&qhp->lock); | |
809 | cxio_flush_hw_cq(&rchp->cq); | |
810 | cxio_count_rcqes(&rchp->cq, &qhp->wq, &count); | |
c8286944 | 811 | flushed = cxio_flush_rq(&qhp->wq, &rchp->cq, count); |
b038ced7 SW |
812 | spin_unlock(&qhp->lock); |
813 | spin_unlock_irqrestore(&rchp->lock, *flag); | |
c8286944 SW |
814 | if (flushed) |
815 | (*rchp->ibcq.comp_handler)(&rchp->ibcq, rchp->ibcq.cq_context); | |
b038ced7 SW |
816 | |
817 | /* locking heirarchy: cq lock first, then qp lock. */ | |
818 | spin_lock_irqsave(&schp->lock, *flag); | |
819 | spin_lock(&qhp->lock); | |
820 | cxio_flush_hw_cq(&schp->cq); | |
821 | cxio_count_scqes(&schp->cq, &qhp->wq, &count); | |
c8286944 | 822 | flushed = cxio_flush_sq(&qhp->wq, &schp->cq, count); |
b038ced7 SW |
823 | spin_unlock(&qhp->lock); |
824 | spin_unlock_irqrestore(&schp->lock, *flag); | |
c8286944 SW |
825 | if (flushed) |
826 | (*schp->ibcq.comp_handler)(&schp->ibcq, schp->ibcq.cq_context); | |
b038ced7 SW |
827 | |
828 | /* deref */ | |
829 | if (atomic_dec_and_test(&qhp->refcnt)) | |
830 | wake_up(&qhp->wait); | |
831 | ||
832 | spin_lock_irqsave(&qhp->lock, *flag); | |
833 | } | |
834 | ||
2b540355 | 835 | static void flush_qp(struct iwch_qp *qhp, unsigned long *flag) |
b038ced7 | 836 | { |
856b5925 | 837 | if (qhp->ibqp.uobject) |
b038ced7 SW |
838 | cxio_set_wq_in_error(&qhp->wq); |
839 | else | |
840 | __flush_qp(qhp, flag); | |
841 | } | |
842 | ||
843 | ||
844 | /* | |
f8b0dfd1 | 845 | * Return count of RECV WRs posted |
b038ced7 | 846 | */ |
f8b0dfd1 | 847 | u16 iwch_rqes_posted(struct iwch_qp *qhp) |
b038ced7 | 848 | { |
f8b0dfd1 SW |
849 | union t3_wr *wqe = qhp->wq.queue; |
850 | u16 count = 0; | |
851 | while ((count+1) != 0 && fw_riwrh_opcode((struct fw_riwrh *)wqe) == T3_WR_RCV) { | |
852 | count++; | |
853 | wqe++; | |
854 | } | |
855 | PDBG("%s qhp %p count %u\n", __func__, qhp, count); | |
856 | return count; | |
b038ced7 SW |
857 | } |
858 | ||
859 | static int rdma_init(struct iwch_dev *rhp, struct iwch_qp *qhp, | |
860 | enum iwch_qp_attr_mask mask, | |
861 | struct iwch_qp_attributes *attrs) | |
862 | { | |
863 | struct t3_rdma_init_attr init_attr; | |
864 | int ret; | |
865 | ||
866 | init_attr.tid = qhp->ep->hwtid; | |
867 | init_attr.qpid = qhp->wq.qpid; | |
868 | init_attr.pdid = qhp->attr.pd; | |
869 | init_attr.scqid = qhp->attr.scq; | |
870 | init_attr.rcqid = qhp->attr.rcq; | |
871 | init_attr.rq_addr = qhp->wq.rq_addr; | |
872 | init_attr.rq_size = 1 << qhp->wq.rq_size_log2; | |
873 | init_attr.mpaattrs = uP_RI_MPA_IETF_ENABLE | | |
874 | qhp->attr.mpa_attr.recv_marker_enabled | | |
875 | (qhp->attr.mpa_attr.xmit_marker_enabled << 1) | | |
876 | (qhp->attr.mpa_attr.crc_enabled << 2); | |
877 | ||
5f0f66b0 SW |
878 | init_attr.qpcaps = uP_RI_QP_RDMA_READ_ENABLE | |
879 | uP_RI_QP_RDMA_WRITE_ENABLE | | |
880 | uP_RI_QP_BIND_ENABLE; | |
881 | if (!qhp->ibqp.uobject) | |
882 | init_attr.qpcaps |= uP_RI_QP_STAG0_ENABLE | | |
883 | uP_RI_QP_FAST_REGISTER_ENABLE; | |
884 | ||
b038ced7 SW |
885 | init_attr.tcp_emss = qhp->ep->emss; |
886 | init_attr.ord = qhp->attr.max_ord; | |
887 | init_attr.ird = qhp->attr.max_ird; | |
888 | init_attr.qp_dma_addr = qhp->wq.dma_addr; | |
889 | init_attr.qp_dma_size = (1UL << qhp->wq.size_log2); | |
f8b0dfd1 SW |
890 | init_attr.rqe_count = iwch_rqes_posted(qhp); |
891 | init_attr.flags = qhp->attr.mpa_attr.initiator ? MPA_INITIATOR : 0; | |
b496fe82 | 892 | init_attr.chan = qhp->ep->l2t->smt_idx; |
f8b0dfd1 SW |
893 | if (peer2peer) { |
894 | init_attr.rtr_type = RTR_READ; | |
895 | if (init_attr.ord == 0 && qhp->attr.mpa_attr.initiator) | |
896 | init_attr.ord = 1; | |
897 | if (init_attr.ird == 0 && !qhp->attr.mpa_attr.initiator) | |
898 | init_attr.ird = 1; | |
899 | } else | |
900 | init_attr.rtr_type = 0; | |
de3d3530 | 901 | init_attr.irs = qhp->ep->rcv_seq; |
b038ced7 | 902 | PDBG("%s init_attr.rq_addr 0x%x init_attr.rq_size = %d " |
33718363 | 903 | "flags 0x%x qpcaps 0x%x\n", __func__, |
b038ced7 SW |
904 | init_attr.rq_addr, init_attr.rq_size, |
905 | init_attr.flags, init_attr.qpcaps); | |
906 | ret = cxio_rdma_init(&rhp->rdev, &init_attr); | |
33718363 | 907 | PDBG("%s ret %d\n", __func__, ret); |
b038ced7 SW |
908 | return ret; |
909 | } | |
910 | ||
911 | int iwch_modify_qp(struct iwch_dev *rhp, struct iwch_qp *qhp, | |
912 | enum iwch_qp_attr_mask mask, | |
913 | struct iwch_qp_attributes *attrs, | |
914 | int internal) | |
915 | { | |
916 | int ret = 0; | |
917 | struct iwch_qp_attributes newattr = qhp->attr; | |
918 | unsigned long flag; | |
919 | int disconnect = 0; | |
920 | int terminate = 0; | |
921 | int abort = 0; | |
922 | int free = 0; | |
923 | struct iwch_ep *ep = NULL; | |
924 | ||
33718363 | 925 | PDBG("%s qhp %p qpid 0x%x ep %p state %d -> %d\n", __func__, |
b038ced7 SW |
926 | qhp, qhp->wq.qpid, qhp->ep, qhp->attr.state, |
927 | (mask & IWCH_QP_ATTR_NEXT_STATE) ? attrs->next_state : -1); | |
928 | ||
929 | spin_lock_irqsave(&qhp->lock, flag); | |
930 | ||
931 | /* Process attr changes if in IDLE */ | |
932 | if (mask & IWCH_QP_ATTR_VALID_MODIFY) { | |
933 | if (qhp->attr.state != IWCH_QP_STATE_IDLE) { | |
934 | ret = -EIO; | |
935 | goto out; | |
936 | } | |
937 | if (mask & IWCH_QP_ATTR_ENABLE_RDMA_READ) | |
938 | newattr.enable_rdma_read = attrs->enable_rdma_read; | |
939 | if (mask & IWCH_QP_ATTR_ENABLE_RDMA_WRITE) | |
940 | newattr.enable_rdma_write = attrs->enable_rdma_write; | |
941 | if (mask & IWCH_QP_ATTR_ENABLE_RDMA_BIND) | |
942 | newattr.enable_bind = attrs->enable_bind; | |
943 | if (mask & IWCH_QP_ATTR_MAX_ORD) { | |
944 | if (attrs->max_ord > | |
945 | rhp->attr.max_rdma_read_qp_depth) { | |
946 | ret = -EINVAL; | |
947 | goto out; | |
948 | } | |
949 | newattr.max_ord = attrs->max_ord; | |
950 | } | |
951 | if (mask & IWCH_QP_ATTR_MAX_IRD) { | |
952 | if (attrs->max_ird > | |
953 | rhp->attr.max_rdma_reads_per_qp) { | |
954 | ret = -EINVAL; | |
955 | goto out; | |
956 | } | |
957 | newattr.max_ird = attrs->max_ird; | |
958 | } | |
959 | qhp->attr = newattr; | |
960 | } | |
961 | ||
962 | if (!(mask & IWCH_QP_ATTR_NEXT_STATE)) | |
963 | goto out; | |
964 | if (qhp->attr.state == attrs->next_state) | |
965 | goto out; | |
966 | ||
967 | switch (qhp->attr.state) { | |
968 | case IWCH_QP_STATE_IDLE: | |
969 | switch (attrs->next_state) { | |
970 | case IWCH_QP_STATE_RTS: | |
971 | if (!(mask & IWCH_QP_ATTR_LLP_STREAM_HANDLE)) { | |
972 | ret = -EINVAL; | |
973 | goto out; | |
974 | } | |
975 | if (!(mask & IWCH_QP_ATTR_MPA_ATTR)) { | |
976 | ret = -EINVAL; | |
977 | goto out; | |
978 | } | |
979 | qhp->attr.mpa_attr = attrs->mpa_attr; | |
980 | qhp->attr.llp_stream_handle = attrs->llp_stream_handle; | |
981 | qhp->ep = qhp->attr.llp_stream_handle; | |
982 | qhp->attr.state = IWCH_QP_STATE_RTS; | |
983 | ||
984 | /* | |
985 | * Ref the endpoint here and deref when we | |
986 | * disassociate the endpoint from the QP. This | |
987 | * happens in CLOSING->IDLE transition or *->ERROR | |
988 | * transition. | |
989 | */ | |
990 | get_ep(&qhp->ep->com); | |
991 | spin_unlock_irqrestore(&qhp->lock, flag); | |
992 | ret = rdma_init(rhp, qhp, mask, attrs); | |
993 | spin_lock_irqsave(&qhp->lock, flag); | |
994 | if (ret) | |
995 | goto err; | |
996 | break; | |
997 | case IWCH_QP_STATE_ERROR: | |
998 | qhp->attr.state = IWCH_QP_STATE_ERROR; | |
999 | flush_qp(qhp, &flag); | |
1000 | break; | |
1001 | default: | |
1002 | ret = -EINVAL; | |
1003 | goto out; | |
1004 | } | |
1005 | break; | |
1006 | case IWCH_QP_STATE_RTS: | |
1007 | switch (attrs->next_state) { | |
1008 | case IWCH_QP_STATE_CLOSING: | |
1009 | BUG_ON(atomic_read(&qhp->ep->com.kref.refcount) < 2); | |
1010 | qhp->attr.state = IWCH_QP_STATE_CLOSING; | |
1011 | if (!internal) { | |
1012 | abort=0; | |
1013 | disconnect = 1; | |
1014 | ep = qhp->ep; | |
989a1780 | 1015 | get_ep(&ep->com); |
b038ced7 SW |
1016 | } |
1017 | break; | |
1018 | case IWCH_QP_STATE_TERMINATE: | |
1019 | qhp->attr.state = IWCH_QP_STATE_TERMINATE; | |
856b5925 | 1020 | if (qhp->ibqp.uobject) |
a1a75052 | 1021 | cxio_set_wq_in_error(&qhp->wq); |
b038ced7 SW |
1022 | if (!internal) |
1023 | terminate = 1; | |
1024 | break; | |
1025 | case IWCH_QP_STATE_ERROR: | |
1026 | qhp->attr.state = IWCH_QP_STATE_ERROR; | |
1027 | if (!internal) { | |
1028 | abort=1; | |
1029 | disconnect = 1; | |
1030 | ep = qhp->ep; | |
989a1780 | 1031 | get_ep(&ep->com); |
b038ced7 SW |
1032 | } |
1033 | goto err; | |
1034 | break; | |
1035 | default: | |
1036 | ret = -EINVAL; | |
1037 | goto out; | |
1038 | } | |
1039 | break; | |
1040 | case IWCH_QP_STATE_CLOSING: | |
1041 | if (!internal) { | |
1042 | ret = -EINVAL; | |
1043 | goto out; | |
1044 | } | |
1045 | switch (attrs->next_state) { | |
1046 | case IWCH_QP_STATE_IDLE: | |
c8286944 | 1047 | flush_qp(qhp, &flag); |
b038ced7 SW |
1048 | qhp->attr.state = IWCH_QP_STATE_IDLE; |
1049 | qhp->attr.llp_stream_handle = NULL; | |
1050 | put_ep(&qhp->ep->com); | |
1051 | qhp->ep = NULL; | |
1052 | wake_up(&qhp->wait); | |
1053 | break; | |
1054 | case IWCH_QP_STATE_ERROR: | |
1055 | goto err; | |
1056 | default: | |
1057 | ret = -EINVAL; | |
1058 | goto err; | |
1059 | } | |
1060 | break; | |
1061 | case IWCH_QP_STATE_ERROR: | |
1062 | if (attrs->next_state != IWCH_QP_STATE_IDLE) { | |
1063 | ret = -EINVAL; | |
1064 | goto out; | |
1065 | } | |
1066 | ||
1067 | if (!Q_EMPTY(qhp->wq.sq_rptr, qhp->wq.sq_wptr) || | |
1068 | !Q_EMPTY(qhp->wq.rq_rptr, qhp->wq.rq_wptr)) { | |
1069 | ret = -EINVAL; | |
1070 | goto out; | |
1071 | } | |
1072 | qhp->attr.state = IWCH_QP_STATE_IDLE; | |
b038ced7 SW |
1073 | break; |
1074 | case IWCH_QP_STATE_TERMINATE: | |
1075 | if (!internal) { | |
1076 | ret = -EINVAL; | |
1077 | goto out; | |
1078 | } | |
1079 | goto err; | |
1080 | break; | |
1081 | default: | |
1082 | printk(KERN_ERR "%s in a bad state %d\n", | |
33718363 | 1083 | __func__, qhp->attr.state); |
b038ced7 SW |
1084 | ret = -EINVAL; |
1085 | goto err; | |
1086 | break; | |
1087 | } | |
1088 | goto out; | |
1089 | err: | |
33718363 | 1090 | PDBG("%s disassociating ep %p qpid 0x%x\n", __func__, qhp->ep, |
b038ced7 SW |
1091 | qhp->wq.qpid); |
1092 | ||
1093 | /* disassociate the LLP connection */ | |
1094 | qhp->attr.llp_stream_handle = NULL; | |
1095 | ep = qhp->ep; | |
1096 | qhp->ep = NULL; | |
1097 | qhp->attr.state = IWCH_QP_STATE_ERROR; | |
1098 | free=1; | |
1099 | wake_up(&qhp->wait); | |
1100 | BUG_ON(!ep); | |
1101 | flush_qp(qhp, &flag); | |
1102 | out: | |
1103 | spin_unlock_irqrestore(&qhp->lock, flag); | |
1104 | ||
1105 | if (terminate) | |
1106 | iwch_post_terminate(qhp, NULL); | |
1107 | ||
1108 | /* | |
1109 | * If disconnect is 1, then we need to initiate a disconnect | |
1110 | * on the EP. This can be a normal close (RTS->CLOSING) or | |
1111 | * an abnormal close (RTS/CLOSING->ERROR). | |
1112 | */ | |
989a1780 | 1113 | if (disconnect) { |
b038ced7 | 1114 | iwch_ep_disconnect(ep, abort, GFP_KERNEL); |
989a1780 SW |
1115 | put_ep(&ep->com); |
1116 | } | |
b038ced7 SW |
1117 | |
1118 | /* | |
1119 | * If free is 1, then we've disassociated the EP from the QP | |
1120 | * and we need to dereference the EP. | |
1121 | */ | |
1122 | if (free) | |
1123 | put_ep(&ep->com); | |
1124 | ||
33718363 | 1125 | PDBG("%s exit state %d\n", __func__, qhp->attr.state); |
b038ced7 SW |
1126 | return ret; |
1127 | } | |
1128 | ||
1129 | static int quiesce_qp(struct iwch_qp *qhp) | |
1130 | { | |
1131 | spin_lock_irq(&qhp->lock); | |
1132 | iwch_quiesce_tid(qhp->ep); | |
1133 | qhp->flags |= QP_QUIESCED; | |
1134 | spin_unlock_irq(&qhp->lock); | |
1135 | return 0; | |
1136 | } | |
1137 | ||
1138 | static int resume_qp(struct iwch_qp *qhp) | |
1139 | { | |
1140 | spin_lock_irq(&qhp->lock); | |
1141 | iwch_resume_tid(qhp->ep); | |
1142 | qhp->flags &= ~QP_QUIESCED; | |
1143 | spin_unlock_irq(&qhp->lock); | |
1144 | return 0; | |
1145 | } | |
1146 | ||
1147 | int iwch_quiesce_qps(struct iwch_cq *chp) | |
1148 | { | |
1149 | int i; | |
1150 | struct iwch_qp *qhp; | |
1151 | ||
1152 | for (i=0; i < T3_MAX_NUM_QP; i++) { | |
1153 | qhp = get_qhp(chp->rhp, i); | |
1154 | if (!qhp) | |
1155 | continue; | |
1156 | if ((qhp->attr.rcq == chp->cq.cqid) && !qp_quiesced(qhp)) { | |
1157 | quiesce_qp(qhp); | |
1158 | continue; | |
1159 | } | |
1160 | if ((qhp->attr.scq == chp->cq.cqid) && !qp_quiesced(qhp)) | |
1161 | quiesce_qp(qhp); | |
1162 | } | |
1163 | return 0; | |
1164 | } | |
1165 | ||
1166 | int iwch_resume_qps(struct iwch_cq *chp) | |
1167 | { | |
1168 | int i; | |
1169 | struct iwch_qp *qhp; | |
1170 | ||
1171 | for (i=0; i < T3_MAX_NUM_QP; i++) { | |
1172 | qhp = get_qhp(chp->rhp, i); | |
1173 | if (!qhp) | |
1174 | continue; | |
1175 | if ((qhp->attr.rcq == chp->cq.cqid) && qp_quiesced(qhp)) { | |
1176 | resume_qp(qhp); | |
1177 | continue; | |
1178 | } | |
1179 | if ((qhp->attr.scq == chp->cq.cqid) && qp_quiesced(qhp)) | |
1180 | resume_qp(qhp); | |
1181 | } | |
1182 | return 0; | |
1183 | } |