Commit | Line | Data |
---|---|---|
e85b24b5 OG |
1 | /* |
2 | * Copyright (c) 2004, 2005, 2006 Voltaire, Inc. All rights reserved. | |
28f292e8 | 3 | * Copyright (c) 2013 Mellanox Technologies. All rights reserved. |
e85b24b5 OG |
4 | * |
5 | * This software is available to you under a choice of one of two | |
6 | * licenses. You may choose to be licensed under the terms of the GNU | |
7 | * General Public License (GPL) Version 2, available from the file | |
8 | * COPYING in the main directory of this source tree, or the | |
9 | * OpenIB.org BSD license below: | |
10 | * | |
11 | * Redistribution and use in source and binary forms, with or | |
12 | * without modification, are permitted provided that the following | |
13 | * conditions are met: | |
14 | * | |
15 | * - Redistributions of source code must retain the above | |
16 | * copyright notice, this list of conditions and the following | |
17 | * disclaimer. | |
18 | * | |
19 | * - Redistributions in binary form must reproduce the above | |
20 | * copyright notice, this list of conditions and the following | |
21 | * disclaimer in the documentation and/or other materials | |
22 | * provided with the distribution. | |
23 | * | |
24 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
25 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
26 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
27 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
28 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
29 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
30 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
31 | * SOFTWARE. | |
e85b24b5 OG |
32 | */ |
33 | #include <linux/kernel.h> | |
34 | #include <linux/slab.h> | |
35 | #include <linux/mm.h> | |
e85b24b5 OG |
36 | #include <linux/scatterlist.h> |
37 | #include <linux/kfifo.h> | |
38 | #include <scsi/scsi_cmnd.h> | |
39 | #include <scsi/scsi_host.h> | |
40 | ||
41 | #include "iscsi_iser.h" | |
42 | ||
e85b24b5 OG |
43 | /* Register user buffer memory and initialize passive rdma |
44 | * dto descriptor. Total data size is stored in | |
2261ec3d | 45 | * iser_task->data[ISER_DIR_IN].data_len |
e85b24b5 | 46 | */ |
2261ec3d | 47 | static int iser_prepare_read_cmd(struct iscsi_task *task, |
e85b24b5 OG |
48 | unsigned int edtl) |
49 | ||
50 | { | |
2261ec3d | 51 | struct iscsi_iser_task *iser_task = task->dd_data; |
b4e155ff | 52 | struct iser_device *device = iser_task->iser_conn->ib_conn->device; |
e85b24b5 OG |
53 | struct iser_regd_buf *regd_buf; |
54 | int err; | |
2261ec3d MC |
55 | struct iser_hdr *hdr = &iser_task->desc.iser_header; |
56 | struct iser_data_buf *buf_in = &iser_task->data[ISER_DIR_IN]; | |
e85b24b5 | 57 | |
2261ec3d | 58 | err = iser_dma_map_task_data(iser_task, |
e85b24b5 OG |
59 | buf_in, |
60 | ISER_DIR_IN, | |
61 | DMA_FROM_DEVICE); | |
62 | if (err) | |
63 | return err; | |
64 | ||
2261ec3d | 65 | if (edtl > iser_task->data[ISER_DIR_IN].data_len) { |
e85b24b5 OG |
66 | iser_err("Total data length: %ld, less than EDTL: " |
67 | "%d, in READ cmd BHS itt: %d, conn: 0x%p\n", | |
2261ec3d MC |
68 | iser_task->data[ISER_DIR_IN].data_len, edtl, |
69 | task->itt, iser_task->iser_conn); | |
e85b24b5 OG |
70 | return -EINVAL; |
71 | } | |
72 | ||
b4e155ff | 73 | err = device->iser_reg_rdma_mem(iser_task, ISER_DIR_IN); |
e85b24b5 OG |
74 | if (err) { |
75 | iser_err("Failed to set up Data-IN RDMA\n"); | |
76 | return err; | |
77 | } | |
2261ec3d | 78 | regd_buf = &iser_task->rdma_regd[ISER_DIR_IN]; |
e85b24b5 OG |
79 | |
80 | hdr->flags |= ISER_RSV; | |
81 | hdr->read_stag = cpu_to_be32(regd_buf->reg.rkey); | |
82 | hdr->read_va = cpu_to_be64(regd_buf->reg.va); | |
83 | ||
84 | iser_dbg("Cmd itt:%d READ tags RKEY:%#.4X VA:%#llX\n", | |
2261ec3d | 85 | task->itt, regd_buf->reg.rkey, |
e85b24b5 OG |
86 | (unsigned long long)regd_buf->reg.va); |
87 | ||
88 | return 0; | |
89 | } | |
90 | ||
91 | /* Register user buffer memory and initialize passive rdma | |
92 | * dto descriptor. Total data size is stored in | |
2261ec3d | 93 | * task->data[ISER_DIR_OUT].data_len |
e85b24b5 OG |
94 | */ |
95 | static int | |
2261ec3d | 96 | iser_prepare_write_cmd(struct iscsi_task *task, |
e85b24b5 OG |
97 | unsigned int imm_sz, |
98 | unsigned int unsol_sz, | |
99 | unsigned int edtl) | |
100 | { | |
2261ec3d | 101 | struct iscsi_iser_task *iser_task = task->dd_data; |
b4e155ff | 102 | struct iser_device *device = iser_task->iser_conn->ib_conn->device; |
e85b24b5 OG |
103 | struct iser_regd_buf *regd_buf; |
104 | int err; | |
2261ec3d MC |
105 | struct iser_hdr *hdr = &iser_task->desc.iser_header; |
106 | struct iser_data_buf *buf_out = &iser_task->data[ISER_DIR_OUT]; | |
f19624aa | 107 | struct ib_sge *tx_dsg = &iser_task->desc.tx_sg[1]; |
e85b24b5 | 108 | |
2261ec3d | 109 | err = iser_dma_map_task_data(iser_task, |
e85b24b5 OG |
110 | buf_out, |
111 | ISER_DIR_OUT, | |
112 | DMA_TO_DEVICE); | |
113 | if (err) | |
114 | return err; | |
115 | ||
2261ec3d | 116 | if (edtl > iser_task->data[ISER_DIR_OUT].data_len) { |
e85b24b5 OG |
117 | iser_err("Total data length: %ld, less than EDTL: %d, " |
118 | "in WRITE cmd BHS itt: %d, conn: 0x%p\n", | |
2261ec3d MC |
119 | iser_task->data[ISER_DIR_OUT].data_len, |
120 | edtl, task->itt, task->conn); | |
e85b24b5 OG |
121 | return -EINVAL; |
122 | } | |
123 | ||
b4e155ff | 124 | err = device->iser_reg_rdma_mem(iser_task, ISER_DIR_OUT); |
e85b24b5 OG |
125 | if (err != 0) { |
126 | iser_err("Failed to register write cmd RDMA mem\n"); | |
127 | return err; | |
128 | } | |
129 | ||
2261ec3d | 130 | regd_buf = &iser_task->rdma_regd[ISER_DIR_OUT]; |
e85b24b5 OG |
131 | |
132 | if (unsol_sz < edtl) { | |
133 | hdr->flags |= ISER_WSV; | |
134 | hdr->write_stag = cpu_to_be32(regd_buf->reg.rkey); | |
135 | hdr->write_va = cpu_to_be64(regd_buf->reg.va + unsol_sz); | |
136 | ||
137 | iser_dbg("Cmd itt:%d, WRITE tags, RKEY:%#.4X " | |
138 | "VA:%#llX + unsol:%d\n", | |
2261ec3d | 139 | task->itt, regd_buf->reg.rkey, |
e85b24b5 OG |
140 | (unsigned long long)regd_buf->reg.va, unsol_sz); |
141 | } | |
142 | ||
143 | if (imm_sz > 0) { | |
144 | iser_dbg("Cmd itt:%d, WRITE, adding imm.data sz: %d\n", | |
2261ec3d | 145 | task->itt, imm_sz); |
f19624aa OG |
146 | tx_dsg->addr = regd_buf->reg.va; |
147 | tx_dsg->length = imm_sz; | |
148 | tx_dsg->lkey = regd_buf->reg.lkey; | |
149 | iser_task->desc.num_sge = 2; | |
e85b24b5 OG |
150 | } |
151 | ||
152 | return 0; | |
153 | } | |
154 | ||
e85b24b5 | 155 | /* creates a new tx descriptor and adds header regd buffer */ |
f19624aa OG |
156 | static void iser_create_send_desc(struct iser_conn *ib_conn, |
157 | struct iser_tx_desc *tx_desc) | |
e85b24b5 | 158 | { |
f19624aa | 159 | struct iser_device *device = ib_conn->device; |
e85b24b5 | 160 | |
f19624aa OG |
161 | ib_dma_sync_single_for_cpu(device->ib_device, |
162 | tx_desc->dma_addr, ISER_HEADERS_LEN, DMA_TO_DEVICE); | |
e85b24b5 OG |
163 | |
164 | memset(&tx_desc->iser_header, 0, sizeof(struct iser_hdr)); | |
165 | tx_desc->iser_header.flags = ISER_VER; | |
166 | ||
f19624aa OG |
167 | tx_desc->num_sge = 1; |
168 | ||
169 | if (tx_desc->tx_sg[0].lkey != device->mr->lkey) { | |
170 | tx_desc->tx_sg[0].lkey = device->mr->lkey; | |
171 | iser_dbg("sdesc %p lkey mismatch, fixing\n", tx_desc); | |
172 | } | |
e85b24b5 OG |
173 | } |
174 | ||
986db0d6 SP |
175 | static void iser_free_login_buf(struct iser_conn *ib_conn) |
176 | { | |
177 | if (!ib_conn->login_buf) | |
178 | return; | |
179 | ||
180 | if (ib_conn->login_req_dma) | |
181 | ib_dma_unmap_single(ib_conn->device->ib_device, | |
182 | ib_conn->login_req_dma, | |
183 | ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_TO_DEVICE); | |
184 | ||
185 | if (ib_conn->login_resp_dma) | |
186 | ib_dma_unmap_single(ib_conn->device->ib_device, | |
187 | ib_conn->login_resp_dma, | |
188 | ISER_RX_LOGIN_SIZE, DMA_FROM_DEVICE); | |
189 | ||
190 | kfree(ib_conn->login_buf); | |
191 | ||
192 | /* make sure we never redo any unmapping */ | |
193 | ib_conn->login_req_dma = 0; | |
194 | ib_conn->login_resp_dma = 0; | |
195 | ib_conn->login_buf = NULL; | |
196 | } | |
197 | ||
198 | static int iser_alloc_login_buf(struct iser_conn *ib_conn) | |
199 | { | |
200 | struct iser_device *device; | |
201 | int req_err, resp_err; | |
202 | ||
203 | BUG_ON(ib_conn->device == NULL); | |
204 | ||
205 | device = ib_conn->device; | |
206 | ||
207 | ib_conn->login_buf = kmalloc(ISCSI_DEF_MAX_RECV_SEG_LEN + | |
208 | ISER_RX_LOGIN_SIZE, GFP_KERNEL); | |
209 | if (!ib_conn->login_buf) | |
210 | goto out_err; | |
211 | ||
212 | ib_conn->login_req_buf = ib_conn->login_buf; | |
213 | ib_conn->login_resp_buf = ib_conn->login_buf + | |
214 | ISCSI_DEF_MAX_RECV_SEG_LEN; | |
215 | ||
216 | ib_conn->login_req_dma = ib_dma_map_single(ib_conn->device->ib_device, | |
217 | (void *)ib_conn->login_req_buf, | |
218 | ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_TO_DEVICE); | |
219 | ||
220 | ib_conn->login_resp_dma = ib_dma_map_single(ib_conn->device->ib_device, | |
221 | (void *)ib_conn->login_resp_buf, | |
222 | ISER_RX_LOGIN_SIZE, DMA_FROM_DEVICE); | |
223 | ||
224 | req_err = ib_dma_mapping_error(device->ib_device, | |
225 | ib_conn->login_req_dma); | |
226 | resp_err = ib_dma_mapping_error(device->ib_device, | |
227 | ib_conn->login_resp_dma); | |
f19624aa | 228 | |
986db0d6 SP |
229 | if (req_err || resp_err) { |
230 | if (req_err) | |
231 | ib_conn->login_req_dma = 0; | |
232 | if (resp_err) | |
233 | ib_conn->login_resp_dma = 0; | |
234 | goto free_login_buf; | |
235 | } | |
236 | return 0; | |
237 | ||
238 | free_login_buf: | |
239 | iser_free_login_buf(ib_conn); | |
240 | ||
241 | out_err: | |
242 | iser_err("unable to alloc or map login buf\n"); | |
243 | return -ENOMEM; | |
244 | } | |
f19624aa | 245 | |
b7f04513 | 246 | int iser_alloc_rx_descriptors(struct iser_conn *ib_conn, struct iscsi_session *session) |
bcc60c38 OG |
247 | { |
248 | int i, j; | |
249 | u64 dma_addr; | |
250 | struct iser_rx_desc *rx_desc; | |
251 | struct ib_sge *rx_sg; | |
252 | struct iser_device *device = ib_conn->device; | |
253 | ||
b7f04513 SP |
254 | ib_conn->qp_max_recv_dtos = session->cmds_max; |
255 | ib_conn->qp_max_recv_dtos_mask = session->cmds_max - 1; /* cmds_max is 2^N */ | |
256 | ib_conn->min_posted_rx = ib_conn->qp_max_recv_dtos >> 2; | |
257 | ||
b4e155ff SG |
258 | if (device->iser_alloc_rdma_reg_res(ib_conn, session->scsi_cmds_max)) |
259 | goto create_rdma_reg_res_failed; | |
986db0d6 SP |
260 | |
261 | if (iser_alloc_login_buf(ib_conn)) | |
262 | goto alloc_login_buf_fail; | |
263 | ||
b7f04513 | 264 | ib_conn->rx_descs = kmalloc(session->cmds_max * |
bcc60c38 OG |
265 | sizeof(struct iser_rx_desc), GFP_KERNEL); |
266 | if (!ib_conn->rx_descs) | |
267 | goto rx_desc_alloc_fail; | |
268 | ||
269 | rx_desc = ib_conn->rx_descs; | |
270 | ||
b7f04513 | 271 | for (i = 0; i < ib_conn->qp_max_recv_dtos; i++, rx_desc++) { |
bcc60c38 OG |
272 | dma_addr = ib_dma_map_single(device->ib_device, (void *)rx_desc, |
273 | ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); | |
274 | if (ib_dma_mapping_error(device->ib_device, dma_addr)) | |
275 | goto rx_desc_dma_map_failed; | |
276 | ||
277 | rx_desc->dma_addr = dma_addr; | |
278 | ||
279 | rx_sg = &rx_desc->rx_sg; | |
280 | rx_sg->addr = rx_desc->dma_addr; | |
281 | rx_sg->length = ISER_RX_PAYLOAD_SIZE; | |
282 | rx_sg->lkey = device->mr->lkey; | |
283 | } | |
284 | ||
285 | ib_conn->rx_desc_head = 0; | |
286 | return 0; | |
287 | ||
288 | rx_desc_dma_map_failed: | |
289 | rx_desc = ib_conn->rx_descs; | |
290 | for (j = 0; j < i; j++, rx_desc++) | |
291 | ib_dma_unmap_single(device->ib_device, rx_desc->dma_addr, | |
986db0d6 | 292 | ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); |
bcc60c38 OG |
293 | kfree(ib_conn->rx_descs); |
294 | ib_conn->rx_descs = NULL; | |
295 | rx_desc_alloc_fail: | |
986db0d6 SP |
296 | iser_free_login_buf(ib_conn); |
297 | alloc_login_buf_fail: | |
b4e155ff SG |
298 | device->iser_free_rdma_reg_res(ib_conn); |
299 | create_rdma_reg_res_failed: | |
bcc60c38 OG |
300 | iser_err("failed allocating rx descriptors / data buffers\n"); |
301 | return -ENOMEM; | |
302 | } | |
303 | ||
304 | void iser_free_rx_descriptors(struct iser_conn *ib_conn) | |
305 | { | |
306 | int i; | |
307 | struct iser_rx_desc *rx_desc; | |
308 | struct iser_device *device = ib_conn->device; | |
309 | ||
bcc60c38 | 310 | if (!ib_conn->rx_descs) |
986db0d6 | 311 | goto free_login_buf; |
bcc60c38 | 312 | |
2e02d653 | 313 | if (device->iser_free_rdma_reg_res) |
b4e155ff | 314 | device->iser_free_rdma_reg_res(ib_conn); |
bcc60c38 OG |
315 | |
316 | rx_desc = ib_conn->rx_descs; | |
b7f04513 | 317 | for (i = 0; i < ib_conn->qp_max_recv_dtos; i++, rx_desc++) |
bcc60c38 | 318 | ib_dma_unmap_single(device->ib_device, rx_desc->dma_addr, |
986db0d6 | 319 | ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); |
bcc60c38 | 320 | kfree(ib_conn->rx_descs); |
986db0d6 SP |
321 | /* make sure we never redo any unmapping */ |
322 | ib_conn->rx_descs = NULL; | |
323 | ||
324 | free_login_buf: | |
325 | iser_free_login_buf(ib_conn); | |
bcc60c38 OG |
326 | } |
327 | ||
89e984e2 | 328 | static int iser_post_rx_bufs(struct iscsi_conn *conn, struct iscsi_hdr *req) |
e85b24b5 OG |
329 | { |
330 | struct iscsi_iser_conn *iser_conn = conn->dd_data; | |
6a06a4b8 | 331 | struct iscsi_session *session = conn->session; |
e85b24b5 | 332 | |
89e984e2 OG |
333 | iser_dbg("req op %x flags %x\n", req->opcode, req->flags); |
334 | /* check if this is the last login - going to full feature phase */ | |
335 | if ((req->flags & ISCSI_FULL_FEATURE_PHASE) != ISCSI_FULL_FEATURE_PHASE) | |
336 | return 0; | |
e85b24b5 | 337 | |
89e984e2 OG |
338 | /* |
339 | * Check that there is one posted recv buffer (for the last login | |
340 | * response) and no posted send buffers left - they must have been | |
341 | * consumed during previous login phases. | |
342 | */ | |
343 | WARN_ON(iser_conn->ib_conn->post_recv_buf_count != 1); | |
344 | WARN_ON(atomic_read(&iser_conn->ib_conn->post_send_buf_count) != 0); | |
bcc60c38 | 345 | |
6a06a4b8 OG |
346 | if (session->discovery_sess) { |
347 | iser_info("Discovery session, re-using login RX buffer\n"); | |
348 | return 0; | |
349 | } else | |
350 | iser_info("Normal session, posting batch of RX %d buffers\n", | |
7c049d08 | 351 | iser_conn->ib_conn->min_posted_rx); |
6a06a4b8 | 352 | |
e85b24b5 | 353 | /* Initial post receive buffers */ |
b7f04513 SP |
354 | if (iser_post_recvm(iser_conn->ib_conn, |
355 | iser_conn->ib_conn->min_posted_rx)) | |
bcc60c38 OG |
356 | return -ENOMEM; |
357 | ||
e85b24b5 OG |
358 | return 0; |
359 | } | |
360 | ||
e85b24b5 OG |
361 | /** |
362 | * iser_send_command - send command PDU | |
363 | */ | |
2747fdb2 | 364 | int iser_send_command(struct iscsi_conn *conn, |
2261ec3d | 365 | struct iscsi_task *task) |
e85b24b5 OG |
366 | { |
367 | struct iscsi_iser_conn *iser_conn = conn->dd_data; | |
2261ec3d | 368 | struct iscsi_iser_task *iser_task = task->dd_data; |
e85b24b5 | 369 | unsigned long edtl; |
bcc60c38 | 370 | int err; |
e85b24b5 | 371 | struct iser_data_buf *data_buf; |
12352183 | 372 | struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)task->hdr; |
2261ec3d | 373 | struct scsi_cmnd *sc = task->sc; |
f19624aa | 374 | struct iser_tx_desc *tx_desc = &iser_task->desc; |
e85b24b5 | 375 | |
e85b24b5 OG |
376 | edtl = ntohl(hdr->data_length); |
377 | ||
378 | /* build the tx desc regd header and add it to the tx desc dto */ | |
f19624aa OG |
379 | tx_desc->type = ISCSI_TX_SCSI_COMMAND; |
380 | iser_create_send_desc(iser_conn->ib_conn, tx_desc); | |
e85b24b5 OG |
381 | |
382 | if (hdr->flags & ISCSI_FLAG_CMD_READ) | |
2261ec3d | 383 | data_buf = &iser_task->data[ISER_DIR_IN]; |
e85b24b5 | 384 | else |
2261ec3d | 385 | data_buf = &iser_task->data[ISER_DIR_OUT]; |
e85b24b5 | 386 | |
da9c0c77 FT |
387 | if (scsi_sg_count(sc)) { /* using a scatter list */ |
388 | data_buf->buf = scsi_sglist(sc); | |
389 | data_buf->size = scsi_sg_count(sc); | |
e85b24b5 OG |
390 | } |
391 | ||
da9c0c77 | 392 | data_buf->data_len = scsi_bufflen(sc); |
e85b24b5 OG |
393 | |
394 | if (hdr->flags & ISCSI_FLAG_CMD_READ) { | |
2261ec3d | 395 | err = iser_prepare_read_cmd(task, edtl); |
e85b24b5 OG |
396 | if (err) |
397 | goto send_command_error; | |
398 | } | |
399 | if (hdr->flags & ISCSI_FLAG_CMD_WRITE) { | |
2261ec3d MC |
400 | err = iser_prepare_write_cmd(task, |
401 | task->imm_count, | |
402 | task->imm_count + | |
0f9c7449 | 403 | task->unsol_r2t.data_length, |
e85b24b5 OG |
404 | edtl); |
405 | if (err) | |
406 | goto send_command_error; | |
407 | } | |
408 | ||
2261ec3d | 409 | iser_task->status = ISER_TASK_STATUS_STARTED; |
e85b24b5 | 410 | |
f19624aa | 411 | err = iser_post_send(iser_conn->ib_conn, tx_desc); |
e85b24b5 OG |
412 | if (!err) |
413 | return 0; | |
414 | ||
415 | send_command_error: | |
2261ec3d | 416 | iser_err("conn %p failed task->itt %d err %d\n",conn, task->itt, err); |
e85b24b5 OG |
417 | return err; |
418 | } | |
419 | ||
420 | /** | |
421 | * iser_send_data_out - send data out PDU | |
422 | */ | |
2747fdb2 | 423 | int iser_send_data_out(struct iscsi_conn *conn, |
2261ec3d | 424 | struct iscsi_task *task, |
e85b24b5 OG |
425 | struct iscsi_data *hdr) |
426 | { | |
427 | struct iscsi_iser_conn *iser_conn = conn->dd_data; | |
2261ec3d | 428 | struct iscsi_iser_task *iser_task = task->dd_data; |
f19624aa OG |
429 | struct iser_tx_desc *tx_desc = NULL; |
430 | struct iser_regd_buf *regd_buf; | |
e85b24b5 OG |
431 | unsigned long buf_offset; |
432 | unsigned long data_seg_len; | |
0a22ab92 | 433 | uint32_t itt; |
e85b24b5 | 434 | int err = 0; |
f19624aa OG |
435 | struct ib_sge *tx_dsg; |
436 | ||
0a22ab92 | 437 | itt = (__force uint32_t)hdr->itt; |
e85b24b5 OG |
438 | data_seg_len = ntoh24(hdr->dlength); |
439 | buf_offset = ntohl(hdr->offset); | |
440 | ||
441 | iser_dbg("%s itt %d dseg_len %d offset %d\n", | |
442 | __func__,(int)itt,(int)data_seg_len,(int)buf_offset); | |
443 | ||
528f4e8c | 444 | tx_desc = kmem_cache_zalloc(ig.desc_cache, GFP_ATOMIC); |
e85b24b5 OG |
445 | if (tx_desc == NULL) { |
446 | iser_err("Failed to alloc desc for post dataout\n"); | |
447 | return -ENOMEM; | |
448 | } | |
449 | ||
450 | tx_desc->type = ISCSI_TX_DATAOUT; | |
f19624aa | 451 | tx_desc->iser_header.flags = ISER_VER; |
e85b24b5 OG |
452 | memcpy(&tx_desc->iscsi_header, hdr, sizeof(struct iscsi_hdr)); |
453 | ||
f19624aa OG |
454 | /* build the tx desc */ |
455 | iser_initialize_task_headers(task, tx_desc); | |
e85b24b5 | 456 | |
f19624aa OG |
457 | regd_buf = &iser_task->rdma_regd[ISER_DIR_OUT]; |
458 | tx_dsg = &tx_desc->tx_sg[1]; | |
459 | tx_dsg->addr = regd_buf->reg.va + buf_offset; | |
460 | tx_dsg->length = data_seg_len; | |
461 | tx_dsg->lkey = regd_buf->reg.lkey; | |
462 | tx_desc->num_sge = 2; | |
e85b24b5 | 463 | |
2261ec3d | 464 | if (buf_offset + data_seg_len > iser_task->data[ISER_DIR_OUT].data_len) { |
e85b24b5 OG |
465 | iser_err("Offset:%ld & DSL:%ld in Data-Out " |
466 | "inconsistent with total len:%ld, itt:%d\n", | |
467 | buf_offset, data_seg_len, | |
2261ec3d | 468 | iser_task->data[ISER_DIR_OUT].data_len, itt); |
e85b24b5 OG |
469 | err = -EINVAL; |
470 | goto send_data_out_error; | |
471 | } | |
472 | iser_dbg("data-out itt: %d, offset: %ld, sz: %ld\n", | |
473 | itt, buf_offset, data_seg_len); | |
474 | ||
475 | ||
f19624aa | 476 | err = iser_post_send(iser_conn->ib_conn, tx_desc); |
e85b24b5 OG |
477 | if (!err) |
478 | return 0; | |
479 | ||
480 | send_data_out_error: | |
e85b24b5 OG |
481 | kmem_cache_free(ig.desc_cache, tx_desc); |
482 | iser_err("conn %p failed err %d\n",conn, err); | |
483 | return err; | |
484 | } | |
485 | ||
486 | int iser_send_control(struct iscsi_conn *conn, | |
2261ec3d | 487 | struct iscsi_task *task) |
e85b24b5 OG |
488 | { |
489 | struct iscsi_iser_conn *iser_conn = conn->dd_data; | |
2261ec3d | 490 | struct iscsi_iser_task *iser_task = task->dd_data; |
f19624aa | 491 | struct iser_tx_desc *mdesc = &iser_task->desc; |
e85b24b5 | 492 | unsigned long data_seg_len; |
f19624aa | 493 | int err = 0; |
e85b24b5 | 494 | struct iser_device *device; |
2c4ce609 | 495 | struct iser_conn *ib_conn = iser_conn->ib_conn; |
e85b24b5 | 496 | |
e85b24b5 OG |
497 | /* build the tx desc regd header and add it to the tx desc dto */ |
498 | mdesc->type = ISCSI_TX_CONTROL; | |
f19624aa | 499 | iser_create_send_desc(iser_conn->ib_conn, mdesc); |
e85b24b5 OG |
500 | |
501 | device = iser_conn->ib_conn->device; | |
502 | ||
2261ec3d | 503 | data_seg_len = ntoh24(task->hdr->dlength); |
e85b24b5 OG |
504 | |
505 | if (data_seg_len > 0) { | |
f19624aa OG |
506 | struct ib_sge *tx_dsg = &mdesc->tx_sg[1]; |
507 | if (task != conn->login_task) { | |
508 | iser_err("data present on non login task!!!\n"); | |
509 | goto send_control_error; | |
510 | } | |
2c4ce609 OG |
511 | |
512 | ib_dma_sync_single_for_cpu(device->ib_device, | |
513 | ib_conn->login_req_dma, task->data_count, | |
514 | DMA_TO_DEVICE); | |
515 | ||
516 | memcpy(iser_conn->ib_conn->login_req_buf, task->data, | |
f19624aa | 517 | task->data_count); |
2c4ce609 OG |
518 | |
519 | ib_dma_sync_single_for_device(device->ib_device, | |
520 | ib_conn->login_req_dma, task->data_count, | |
521 | DMA_TO_DEVICE); | |
522 | ||
523 | tx_dsg->addr = iser_conn->ib_conn->login_req_dma; | |
200ae1a0 | 524 | tx_dsg->length = task->data_count; |
f19624aa OG |
525 | tx_dsg->lkey = device->mr->lkey; |
526 | mdesc->num_sge = 2; | |
e85b24b5 OG |
527 | } |
528 | ||
bcc60c38 | 529 | if (task == conn->login_task) { |
6a06a4b8 OG |
530 | iser_dbg("op %x dsl %lx, posting login rx buffer\n", |
531 | task->hdr->opcode, data_seg_len); | |
bcc60c38 OG |
532 | err = iser_post_recvl(iser_conn->ib_conn); |
533 | if (err) | |
534 | goto send_control_error; | |
89e984e2 OG |
535 | err = iser_post_rx_bufs(conn, task->hdr); |
536 | if (err) | |
537 | goto send_control_error; | |
e85b24b5 OG |
538 | } |
539 | ||
f19624aa | 540 | err = iser_post_send(iser_conn->ib_conn, mdesc); |
e85b24b5 OG |
541 | if (!err) |
542 | return 0; | |
543 | ||
544 | send_control_error: | |
e85b24b5 OG |
545 | iser_err("conn %p failed err %d\n",conn, err); |
546 | return err; | |
547 | } | |
548 | ||
549 | /** | |
550 | * iser_rcv_dto_completion - recv DTO completion | |
551 | */ | |
bcc60c38 OG |
552 | void iser_rcv_completion(struct iser_rx_desc *rx_desc, |
553 | unsigned long rx_xfer_len, | |
554 | struct iser_conn *ib_conn) | |
e85b24b5 | 555 | { |
bcc60c38 | 556 | struct iscsi_iser_conn *conn = ib_conn->iser_conn; |
e85b24b5 | 557 | struct iscsi_hdr *hdr; |
bcc60c38 OG |
558 | u64 rx_dma; |
559 | int rx_buflen, outstanding, count, err; | |
560 | ||
561 | /* differentiate between login to all other PDUs */ | |
2c4ce609 OG |
562 | if ((char *)rx_desc == ib_conn->login_resp_buf) { |
563 | rx_dma = ib_conn->login_resp_dma; | |
bcc60c38 OG |
564 | rx_buflen = ISER_RX_LOGIN_SIZE; |
565 | } else { | |
566 | rx_dma = rx_desc->dma_addr; | |
567 | rx_buflen = ISER_RX_PAYLOAD_SIZE; | |
568 | } | |
e85b24b5 | 569 | |
bcc60c38 OG |
570 | ib_dma_sync_single_for_cpu(ib_conn->device->ib_device, rx_dma, |
571 | rx_buflen, DMA_FROM_DEVICE); | |
e85b24b5 | 572 | |
bcc60c38 | 573 | hdr = &rx_desc->iscsi_header; |
e85b24b5 | 574 | |
bcc60c38 OG |
575 | iser_dbg("op 0x%x itt 0x%x dlen %d\n", hdr->opcode, |
576 | hdr->itt, (int)(rx_xfer_len - ISER_HEADERS_LEN)); | |
e85b24b5 | 577 | |
bcc60c38 OG |
578 | iscsi_iser_recv(conn->iscsi_conn, hdr, |
579 | rx_desc->data, rx_xfer_len - ISER_HEADERS_LEN); | |
e85b24b5 | 580 | |
bcc60c38 OG |
581 | ib_dma_sync_single_for_device(ib_conn->device->ib_device, rx_dma, |
582 | rx_buflen, DMA_FROM_DEVICE); | |
e85b24b5 OG |
583 | |
584 | /* decrementing conn->post_recv_buf_count only --after-- freeing the * | |
585 | * task eliminates the need to worry on tasks which are completed in * | |
586 | * parallel to the execution of iser_conn_term. So the code that waits * | |
587 | * for the posted rx bufs refcount to become zero handles everything */ | |
704315f0 | 588 | conn->ib_conn->post_recv_buf_count--; |
bcc60c38 | 589 | |
2c4ce609 | 590 | if (rx_dma == ib_conn->login_resp_dma) |
bcc60c38 OG |
591 | return; |
592 | ||
704315f0 | 593 | outstanding = ib_conn->post_recv_buf_count; |
b7f04513 SP |
594 | if (outstanding + ib_conn->min_posted_rx <= ib_conn->qp_max_recv_dtos) { |
595 | count = min(ib_conn->qp_max_recv_dtos - outstanding, | |
596 | ib_conn->min_posted_rx); | |
bcc60c38 OG |
597 | err = iser_post_recvm(ib_conn, count); |
598 | if (err) | |
599 | iser_err("posting %d rx bufs err %d\n", count, err); | |
600 | } | |
e85b24b5 OG |
601 | } |
602 | ||
f19624aa OG |
603 | void iser_snd_completion(struct iser_tx_desc *tx_desc, |
604 | struct iser_conn *ib_conn) | |
e85b24b5 | 605 | { |
2261ec3d | 606 | struct iscsi_task *task; |
f19624aa | 607 | struct iser_device *device = ib_conn->device; |
e85b24b5 | 608 | |
f19624aa OG |
609 | if (tx_desc->type == ISCSI_TX_DATAOUT) { |
610 | ib_dma_unmap_single(device->ib_device, tx_desc->dma_addr, | |
611 | ISER_HEADERS_LEN, DMA_TO_DEVICE); | |
e85b24b5 | 612 | kmem_cache_free(ig.desc_cache, tx_desc); |
fd8b48b2 | 613 | tx_desc = NULL; |
f19624aa | 614 | } |
e85b24b5 | 615 | |
87e8df7a | 616 | atomic_dec(&ib_conn->post_send_buf_count); |
e85b24b5 | 617 | |
fd8b48b2 | 618 | if (tx_desc && tx_desc->type == ISCSI_TX_CONTROL) { |
e85b24b5 | 619 | /* this arithmetic is legal by libiscsi dd_data allocation */ |
2261ec3d MC |
620 | task = (void *) ((long)(void *)tx_desc - |
621 | sizeof(struct iscsi_task)); | |
622 | if (task->hdr->itt == RESERVED_ITT) | |
623 | iscsi_put_task(task); | |
e85b24b5 OG |
624 | } |
625 | } | |
626 | ||
2261ec3d | 627 | void iser_task_rdma_init(struct iscsi_iser_task *iser_task) |
e85b24b5 OG |
628 | |
629 | { | |
2261ec3d | 630 | iser_task->status = ISER_TASK_STATUS_INIT; |
e85b24b5 | 631 | |
2261ec3d MC |
632 | iser_task->dir[ISER_DIR_IN] = 0; |
633 | iser_task->dir[ISER_DIR_OUT] = 0; | |
e85b24b5 | 634 | |
2261ec3d MC |
635 | iser_task->data[ISER_DIR_IN].data_len = 0; |
636 | iser_task->data[ISER_DIR_OUT].data_len = 0; | |
e85b24b5 | 637 | |
2261ec3d | 638 | memset(&iser_task->rdma_regd[ISER_DIR_IN], 0, |
e85b24b5 | 639 | sizeof(struct iser_regd_buf)); |
2261ec3d | 640 | memset(&iser_task->rdma_regd[ISER_DIR_OUT], 0, |
e85b24b5 OG |
641 | sizeof(struct iser_regd_buf)); |
642 | } | |
643 | ||
2261ec3d | 644 | void iser_task_rdma_finalize(struct iscsi_iser_task *iser_task) |
e85b24b5 | 645 | { |
b4e155ff | 646 | struct iser_device *device = iser_task->iser_conn->ib_conn->device; |
74a20780 | 647 | int is_rdma_aligned = 1; |
e85b24b5 OG |
648 | |
649 | /* if we were reading, copy back to unaligned sglist, | |
650 | * anyway dma_unmap and free the copy | |
651 | */ | |
2261ec3d | 652 | if (iser_task->data_copy[ISER_DIR_IN].copy_buf != NULL) { |
74a20780 | 653 | is_rdma_aligned = 0; |
2261ec3d | 654 | iser_finalize_rdma_unaligned_sg(iser_task, ISER_DIR_IN); |
74a20780 | 655 | } |
2261ec3d | 656 | if (iser_task->data_copy[ISER_DIR_OUT].copy_buf != NULL) { |
74a20780 | 657 | is_rdma_aligned = 0; |
2261ec3d | 658 | iser_finalize_rdma_unaligned_sg(iser_task, ISER_DIR_OUT); |
74a20780 | 659 | } |
e85b24b5 | 660 | |
b4e155ff SG |
661 | if (iser_task->dir[ISER_DIR_IN]) |
662 | device->iser_unreg_rdma_mem(iser_task, ISER_DIR_IN); | |
e85b24b5 | 663 | |
b4e155ff SG |
664 | if (iser_task->dir[ISER_DIR_OUT]) |
665 | device->iser_unreg_rdma_mem(iser_task, ISER_DIR_OUT); | |
e85b24b5 | 666 | |
74a20780 EZ |
667 | /* if the data was unaligned, it was already unmapped and then copied */ |
668 | if (is_rdma_aligned) | |
2261ec3d | 669 | iser_dma_unmap_task_data(iser_task); |
e85b24b5 | 670 | } |