Commit | Line | Data |
---|---|---|
6461f64a OG |
1 | /* |
2 | * Copyright (c) 2004, 2005, 2006 Voltaire, Inc. All rights reserved. | |
3ee07d27 | 3 | * Copyright (c) 2013-2014 Mellanox Technologies. All rights reserved. |
6461f64a OG |
4 | * |
5 | * This software is available to you under a choice of one of two | |
6 | * licenses. You may choose to be licensed under the terms of the GNU | |
7 | * General Public License (GPL) Version 2, available from the file | |
8 | * COPYING in the main directory of this source tree, or the | |
9 | * OpenIB.org BSD license below: | |
10 | * | |
11 | * Redistribution and use in source and binary forms, with or | |
12 | * without modification, are permitted provided that the following | |
13 | * conditions are met: | |
14 | * | |
15 | * - Redistributions of source code must retain the above | |
16 | * copyright notice, this list of conditions and the following | |
17 | * disclaimer. | |
18 | * | |
19 | * - Redistributions in binary form must reproduce the above | |
20 | * copyright notice, this list of conditions and the following | |
21 | * disclaimer in the documentation and/or other materials | |
22 | * provided with the distribution. | |
23 | * | |
24 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
25 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
26 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
27 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
28 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
29 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
30 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
31 | * SOFTWARE. | |
6461f64a | 32 | */ |
6461f64a OG |
33 | #include <linux/kernel.h> |
34 | #include <linux/slab.h> | |
35 | #include <linux/mm.h> | |
a1f8e7f7 | 36 | #include <linux/highmem.h> |
6461f64a OG |
37 | #include <linux/scatterlist.h> |
38 | ||
39 | #include "iscsi_iser.h" | |
48afbff6 | 40 | |
cfeb91b3 CH |
41 | void iser_reg_comp(struct ib_cq *cq, struct ib_wc *wc) |
42 | { | |
43 | iser_err_comp(wc, "memreg"); | |
44 | } | |
45 | ||
ca2770c6 | 46 | static struct iser_fr_desc *iser_reg_desc_get_fr(struct ib_conn *ib_conn) |
bd8b944e | 47 | { |
385ad87d | 48 | struct iser_fr_pool *fr_pool = &ib_conn->fr_pool; |
5190cc26 | 49 | struct iser_fr_desc *desc; |
bd8b944e SG |
50 | unsigned long flags; |
51 | ||
385ad87d | 52 | spin_lock_irqsave(&fr_pool->lock, flags); |
2b3bf958 | 53 | desc = list_first_entry(&fr_pool->list, |
5190cc26 | 54 | struct iser_fr_desc, list); |
bd8b944e | 55 | list_del(&desc->list); |
385ad87d | 56 | spin_unlock_irqrestore(&fr_pool->lock, flags); |
bd8b944e SG |
57 | |
58 | return desc; | |
59 | } | |
60 | ||
ca2770c6 MG |
61 | static void iser_reg_desc_put_fr(struct ib_conn *ib_conn, |
62 | struct iser_fr_desc *desc) | |
bd8b944e | 63 | { |
385ad87d | 64 | struct iser_fr_pool *fr_pool = &ib_conn->fr_pool; |
bd8b944e SG |
65 | unsigned long flags; |
66 | ||
385ad87d | 67 | spin_lock_irqsave(&fr_pool->lock, flags); |
2b3bf958 | 68 | list_add(&desc->list, &fr_pool->list); |
385ad87d | 69 | spin_unlock_irqrestore(&fr_pool->lock, flags); |
bd8b944e SG |
70 | } |
71 | ||
2261ec3d | 72 | int iser_dma_map_task_data(struct iscsi_iser_task *iser_task, |
ca2770c6 MG |
73 | enum iser_data_dir iser_dir, |
74 | enum dma_data_direction dma_dir) | |
74a20780 | 75 | { |
80303ee2 | 76 | struct iser_data_buf *data = &iser_task->data[iser_dir]; |
5180311f | 77 | struct ib_device *dev; |
74a20780 | 78 | |
2261ec3d | 79 | iser_task->dir[iser_dir] = 1; |
a4ee3539 | 80 | dev = iser_task->iser_conn->ib_conn.device->ib_device; |
74a20780 | 81 | |
e3784bd1 | 82 | data->dma_nents = ib_dma_map_sg(dev, data->sg, data->size, dma_dir); |
70bcc63f | 83 | if (unlikely(data->dma_nents == 0)) { |
74a20780 EZ |
84 | iser_err("dma_map_sg failed!!!\n"); |
85 | return -EINVAL; | |
86 | } | |
80303ee2 MG |
87 | |
88 | if (scsi_prot_sg_count(iser_task->sc)) { | |
89 | struct iser_data_buf *pdata = &iser_task->prot[iser_dir]; | |
90 | ||
91 | pdata->dma_nents = ib_dma_map_sg(dev, pdata->sg, pdata->size, dma_dir); | |
92 | if (unlikely(pdata->dma_nents == 0)) { | |
93 | iser_err("protection dma_map_sg failed!!!\n"); | |
94 | goto out_unmap; | |
95 | } | |
96 | } | |
97 | ||
74a20780 | 98 | return 0; |
80303ee2 MG |
99 | |
100 | out_unmap: | |
101 | ib_dma_unmap_sg(dev, data->sg, data->size, dma_dir); | |
102 | return -EINVAL; | |
74a20780 EZ |
103 | } |
104 | ||
80303ee2 | 105 | |
9a8b08fa | 106 | void iser_dma_unmap_task_data(struct iscsi_iser_task *iser_task, |
80303ee2 MG |
107 | enum iser_data_dir iser_dir, |
108 | enum dma_data_direction dma_dir) | |
74a20780 | 109 | { |
80303ee2 | 110 | struct iser_data_buf *data = &iser_task->data[iser_dir]; |
5180311f | 111 | struct ib_device *dev; |
74a20780 | 112 | |
a4ee3539 | 113 | dev = iser_task->iser_conn->ib_conn.device->ib_device; |
80303ee2 MG |
114 | ib_dma_unmap_sg(dev, data->sg, data->size, dma_dir); |
115 | ||
116 | if (scsi_prot_sg_count(iser_task->sc)) { | |
117 | struct iser_data_buf *pdata = &iser_task->prot[iser_dir]; | |
118 | ||
119 | ib_dma_unmap_sg(dev, pdata->sg, pdata->size, dma_dir); | |
120 | } | |
74a20780 EZ |
121 | } |
122 | ||
ca2770c6 MG |
123 | static int iser_reg_dma(struct iser_device *device, struct iser_data_buf *mem, |
124 | struct iser_mem_reg *reg) | |
ad1e5672 SG |
125 | { |
126 | struct scatterlist *sg = mem->sg; | |
127 | ||
256b7ad2 | 128 | reg->sge.lkey = device->pd->local_dma_lkey; |
b5f04b00 JD |
129 | /* |
130 | * FIXME: rework the registration code path to differentiate | |
131 | * rkey/lkey use cases | |
132 | */ | |
8e61212d CH |
133 | |
134 | if (device->pd->flags & IB_PD_UNSAFE_GLOBAL_RKEY) | |
135 | reg->rkey = device->pd->unsafe_global_rkey; | |
136 | else | |
137 | reg->rkey = 0; | |
a163afc8 BVA |
138 | reg->sge.addr = sg_dma_address(&sg[0]); |
139 | reg->sge.length = sg_dma_len(&sg[0]); | |
ad1e5672 SG |
140 | |
141 | iser_dbg("Single DMA entry: lkey=0x%x, rkey=0x%x, addr=0x%llx," | |
142 | " length=0x%x\n", reg->sge.lkey, reg->rkey, | |
143 | reg->sge.addr, reg->sge.length); | |
144 | ||
145 | return 0; | |
146 | } | |
147 | ||
d03e61d0 SG |
148 | void iser_unreg_mem_fastreg(struct iscsi_iser_task *iser_task, |
149 | enum iser_data_dir cmd_dir) | |
150 | { | |
b130eded | 151 | struct iser_mem_reg *reg = &iser_task->rdma_reg[cmd_dir]; |
26e28deb SG |
152 | struct iser_fr_desc *desc; |
153 | struct ib_mr_status mr_status; | |
d03e61d0 | 154 | |
ee4efeae | 155 | desc = reg->desc; |
26e28deb | 156 | if (!desc) |
d03e61d0 SG |
157 | return; |
158 | ||
26e28deb SG |
159 | /* |
160 | * The signature MR cannot be invalidated and reused without checking. | |
161 | * libiscsi calls the check_protection transport handler only if | |
162 | * SCSI-Response is received. And the signature MR is not checked if | |
163 | * the task is completed for some other reason like a timeout or error | |
164 | * handling. That's why we must check the signature MR here before | |
165 | * putting it to the free pool. | |
166 | */ | |
167 | if (unlikely(desc->sig_protected)) { | |
168 | desc->sig_protected = false; | |
169 | ib_check_mr_status(desc->rsc.sig_mr, IB_MR_CHECK_SIG_STATUS, | |
170 | &mr_status); | |
171 | } | |
ee4efeae MG |
172 | iser_reg_desc_put_fr(&iser_task->iser_conn->ib_conn, reg->desc); |
173 | reg->desc = NULL; | |
d03e61d0 SG |
174 | } |
175 | ||
ca2770c6 MG |
176 | static void iser_set_dif_domain(struct scsi_cmnd *sc, |
177 | struct ib_sig_domain *domain) | |
92792c0a | 178 | { |
78eda2bb | 179 | domain->sig_type = IB_SIG_TYPE_T10_DIF; |
5bb6e543 | 180 | domain->sig.dif.pi_interval = scsi_prot_interval(sc); |
99247108 | 181 | domain->sig.dif.ref_tag = t10_pi_ref_tag(scsi_cmd_to_rq(sc)); |
78eda2bb SG |
182 | /* |
183 | * At the moment we hard code those, but in the future | |
184 | * we will take them from sc. | |
185 | */ | |
186 | domain->sig.dif.apptag_check_mask = 0xffff; | |
187 | domain->sig.dif.app_escape = true; | |
188 | domain->sig.dif.ref_escape = true; | |
5bb6e543 | 189 | if (sc->prot_flags & SCSI_PROT_REF_INCREMENT) |
78eda2bb | 190 | domain->sig.dif.ref_remap = true; |
5bf0e4b8 | 191 | } |
177e31bd | 192 | |
ca2770c6 MG |
193 | static int iser_set_sig_attrs(struct scsi_cmnd *sc, |
194 | struct ib_sig_attrs *sig_attrs) | |
177e31bd | 195 | { |
177e31bd SG |
196 | switch (scsi_get_prot_op(sc)) { |
197 | case SCSI_PROT_WRITE_INSERT: | |
198 | case SCSI_PROT_READ_STRIP: | |
78eda2bb | 199 | sig_attrs->mem.sig_type = IB_SIG_TYPE_NONE; |
0cc2896f | 200 | iser_set_dif_domain(sc, &sig_attrs->wire); |
177e31bd | 201 | sig_attrs->wire.sig.dif.bg_type = IB_T10DIF_CRC; |
177e31bd SG |
202 | break; |
203 | case SCSI_PROT_READ_INSERT: | |
204 | case SCSI_PROT_WRITE_STRIP: | |
78eda2bb | 205 | sig_attrs->wire.sig_type = IB_SIG_TYPE_NONE; |
0cc2896f | 206 | iser_set_dif_domain(sc, &sig_attrs->mem); |
5bb6e543 SG |
207 | sig_attrs->mem.sig.dif.bg_type = sc->prot_flags & SCSI_PROT_IP_CHECKSUM ? |
208 | IB_T10DIF_CSUM : IB_T10DIF_CRC; | |
177e31bd SG |
209 | break; |
210 | case SCSI_PROT_READ_PASS: | |
211 | case SCSI_PROT_WRITE_PASS: | |
0cc2896f | 212 | iser_set_dif_domain(sc, &sig_attrs->wire); |
177e31bd | 213 | sig_attrs->wire.sig.dif.bg_type = IB_T10DIF_CRC; |
0cc2896f | 214 | iser_set_dif_domain(sc, &sig_attrs->mem); |
5bb6e543 SG |
215 | sig_attrs->mem.sig.dif.bg_type = sc->prot_flags & SCSI_PROT_IP_CHECKSUM ? |
216 | IB_T10DIF_CSUM : IB_T10DIF_CRC; | |
177e31bd SG |
217 | break; |
218 | default: | |
219 | iser_err("Unsupported PI operation %d\n", | |
220 | scsi_get_prot_op(sc)); | |
221 | return -EINVAL; | |
222 | } | |
78eda2bb | 223 | |
177e31bd SG |
224 | return 0; |
225 | } | |
226 | ||
ca2770c6 | 227 | static inline void iser_set_prot_checks(struct scsi_cmnd *sc, u8 *mask) |
177e31bd | 228 | { |
5bb6e543 SG |
229 | *mask = 0; |
230 | if (sc->prot_flags & SCSI_PROT_REF_CHECK) | |
c6c2c03a | 231 | *mask |= IB_SIG_CHECK_REFTAG; |
5bb6e543 | 232 | if (sc->prot_flags & SCSI_PROT_GUARD_CHECK) |
c6c2c03a | 233 | *mask |= IB_SIG_CHECK_GUARD; |
177e31bd SG |
234 | } |
235 | ||
ca2770c6 MG |
236 | static inline void iser_inv_rkey(struct ib_send_wr *inv_wr, struct ib_mr *mr, |
237 | struct ib_cqe *cqe, struct ib_send_wr *next_wr) | |
a11b3e69 | 238 | { |
a11b3e69 | 239 | inv_wr->opcode = IB_WR_LOCAL_INV; |
cfeb91b3 | 240 | inv_wr->wr_cqe = cqe; |
a11b3e69 | 241 | inv_wr->ex.invalidate_rkey = mr->rkey; |
7332bed0 SG |
242 | inv_wr->send_flags = 0; |
243 | inv_wr->num_sge = 0; | |
b9294f8b | 244 | inv_wr->next = next_wr; |
a11b3e69 SG |
245 | } |
246 | ||
ca2770c6 MG |
247 | static int iser_reg_sig_mr(struct iscsi_iser_task *iser_task, |
248 | struct iser_data_buf *mem, | |
249 | struct iser_data_buf *sig_mem, | |
250 | struct iser_reg_resources *rsc, | |
251 | struct iser_mem_reg *sig_reg) | |
177e31bd | 252 | { |
7332bed0 | 253 | struct iser_tx_desc *tx_desc = &iser_task->desc; |
cfeb91b3 | 254 | struct ib_cqe *cqe = &iser_task->iser_conn->ib_conn.reg_cqe; |
b76a4399 IR |
255 | struct ib_mr *mr = rsc->sig_mr; |
256 | struct ib_sig_attrs *sig_attrs = mr->sig_attrs; | |
b9294f8b | 257 | struct ib_reg_wr *wr = &tx_desc->reg_wr; |
177e31bd | 258 | int ret; |
177e31bd | 259 | |
7332bed0 SG |
260 | memset(sig_attrs, 0, sizeof(*sig_attrs)); |
261 | ret = iser_set_sig_attrs(iser_task->sc, sig_attrs); | |
177e31bd SG |
262 | if (ret) |
263 | goto err; | |
264 | ||
7332bed0 | 265 | iser_set_prot_checks(iser_task->sc, &sig_attrs->check_mask); |
177e31bd | 266 | |
2f188828 | 267 | if (rsc->sig_mr->need_inval) |
b9294f8b | 268 | iser_inv_rkey(&tx_desc->inv_wr, mr, cqe, &wr->wr); |
e26d2d21 SG |
269 | |
270 | ib_update_fast_reg_key(mr, ib_inc_rkey(mr->rkey)); | |
e622f2f4 | 271 | |
b76a4399 IR |
272 | ret = ib_map_mr_sg_pi(mr, mem->sg, mem->dma_nents, NULL, |
273 | sig_mem->sg, sig_mem->dma_nents, NULL, SZ_4K); | |
274 | if (unlikely(ret)) { | |
275 | iser_err("failed to map PI sg (%d)\n", | |
276 | mem->dma_nents + sig_mem->dma_nents); | |
277 | goto err; | |
278 | } | |
279 | ||
b76a4399 | 280 | memset(wr, 0, sizeof(*wr)); |
b9294f8b | 281 | wr->wr.next = &tx_desc->send_wr; |
b76a4399 | 282 | wr->wr.opcode = IB_WR_REG_MR_INTEGRITY; |
cfeb91b3 | 283 | wr->wr.wr_cqe = cqe; |
b76a4399 | 284 | wr->wr.num_sge = 0; |
e622f2f4 | 285 | wr->wr.send_flags = 0; |
b76a4399 IR |
286 | wr->mr = mr; |
287 | wr->key = mr->rkey; | |
288 | wr->access = IB_ACCESS_LOCAL_WRITE | | |
289 | IB_ACCESS_REMOTE_READ | | |
290 | IB_ACCESS_REMOTE_WRITE; | |
2f188828 | 291 | rsc->sig_mr->need_inval = true; |
177e31bd | 292 | |
e26d2d21 SG |
293 | sig_reg->sge.lkey = mr->lkey; |
294 | sig_reg->rkey = mr->rkey; | |
b76a4399 IR |
295 | sig_reg->sge.addr = mr->iova; |
296 | sig_reg->sge.length = mr->length; | |
177e31bd | 297 | |
39405885 | 298 | iser_dbg("lkey=0x%x rkey=0x%x addr=0x%llx length=%u\n", |
6ef8bb83 SG |
299 | sig_reg->sge.lkey, sig_reg->rkey, sig_reg->sge.addr, |
300 | sig_reg->sge.length); | |
177e31bd SG |
301 | err: |
302 | return ret; | |
303 | } | |
304 | ||
d11ec4ec | 305 | static int iser_fast_reg_mr(struct iscsi_iser_task *iser_task, |
d11ec4ec | 306 | struct iser_data_buf *mem, |
d711d81d | 307 | struct iser_reg_resources *rsc, |
6ef8bb83 | 308 | struct iser_mem_reg *reg) |
5587856c | 309 | { |
7332bed0 | 310 | struct iser_tx_desc *tx_desc = &iser_task->desc; |
cfeb91b3 | 311 | struct ib_cqe *cqe = &iser_task->iser_conn->ib_conn.reg_cqe; |
39405885 | 312 | struct ib_mr *mr = rsc->mr; |
b9294f8b | 313 | struct ib_reg_wr *wr = &tx_desc->reg_wr; |
39405885 | 314 | int n; |
5587856c | 315 | |
2f188828 | 316 | if (rsc->mr->need_inval) |
b9294f8b | 317 | iser_inv_rkey(&tx_desc->inv_wr, mr, cqe, &wr->wr); |
e622f2f4 | 318 | |
e26d2d21 SG |
319 | ib_update_fast_reg_key(mr, ib_inc_rkey(mr->rkey)); |
320 | ||
6eeff06d | 321 | n = ib_map_mr_sg(mr, mem->sg, mem->dma_nents, NULL, SZ_4K); |
57b26497 | 322 | if (unlikely(n != mem->dma_nents)) { |
39405885 | 323 | iser_err("failed to map sg (%d/%d)\n", |
57b26497 | 324 | n, mem->dma_nents); |
39405885 SG |
325 | return n < 0 ? n : -EINVAL; |
326 | } | |
327 | ||
b9294f8b | 328 | wr->wr.next = &tx_desc->send_wr; |
39405885 | 329 | wr->wr.opcode = IB_WR_REG_MR; |
cfeb91b3 | 330 | wr->wr.wr_cqe = cqe; |
e622f2f4 | 331 | wr->wr.send_flags = 0; |
39405885 SG |
332 | wr->wr.num_sge = 0; |
333 | wr->mr = mr; | |
334 | wr->key = mr->rkey; | |
335 | wr->access = IB_ACCESS_LOCAL_WRITE | | |
336 | IB_ACCESS_REMOTE_WRITE | | |
337 | IB_ACCESS_REMOTE_READ; | |
338 | ||
2f188828 | 339 | rsc->mr->need_inval = true; |
5587856c | 340 | |
6ef8bb83 SG |
341 | reg->sge.lkey = mr->lkey; |
342 | reg->rkey = mr->rkey; | |
39405885 SG |
343 | reg->sge.addr = mr->iova; |
344 | reg->sge.length = mr->length; | |
5587856c | 345 | |
39405885 SG |
346 | iser_dbg("lkey=0x%x rkey=0x%x addr=0x%llx length=0x%x\n", |
347 | reg->sge.lkey, reg->rkey, reg->sge.addr, reg->sge.length); | |
1b16c989 | 348 | |
7332bed0 | 349 | return 0; |
5587856c SG |
350 | } |
351 | ||
1fc43132 IR |
352 | int iser_reg_mem_fastreg(struct iscsi_iser_task *task, |
353 | enum iser_data_dir dir, | |
354 | bool all_imm) | |
32467c42 SG |
355 | { |
356 | struct ib_conn *ib_conn = &task->iser_conn->ib_conn; | |
7f68d749 | 357 | struct iser_device *device = ib_conn->device; |
32467c42 SG |
358 | struct iser_data_buf *mem = &task->data[dir]; |
359 | struct iser_mem_reg *reg = &task->rdma_reg[dir]; | |
7f68d749 | 360 | struct iser_fr_desc *desc; |
3cffd930 | 361 | bool use_dma_key; |
32467c42 SG |
362 | int err; |
363 | ||
b5f04b00 JD |
364 | use_dma_key = mem->dma_nents == 1 && (all_imm || !iser_always_reg) && |
365 | scsi_get_prot_op(task->sc) == SCSI_PROT_NORMAL; | |
7f68d749 MG |
366 | if (use_dma_key) |
367 | return iser_reg_dma(device, mem, reg); | |
3cffd930 | 368 | |
7f68d749 | 369 | desc = iser_reg_desc_get_fr(ib_conn); |
b76a4399 | 370 | if (scsi_get_prot_op(task->sc) == SCSI_PROT_NORMAL) { |
7f68d749 | 371 | err = iser_fast_reg_mr(task, mem, &desc->rsc, reg); |
b76a4399 IR |
372 | if (unlikely(err)) |
373 | goto err_reg; | |
374 | } else { | |
375 | err = iser_reg_sig_mr(task, mem, &task->prot[dir], | |
376 | &desc->rsc, reg); | |
32467c42 SG |
377 | if (unlikely(err)) |
378 | goto err_reg; | |
379 | ||
c934833e | 380 | desc->sig_protected = true; |
177e31bd | 381 | } |
d11ec4ec | 382 | |
ee4efeae | 383 | reg->desc = desc; |
7f68d749 | 384 | |
5587856c | 385 | return 0; |
32467c42 | 386 | |
5587856c | 387 | err_reg: |
7f68d749 | 388 | iser_reg_desc_put_fr(ib_conn, desc); |
d11ec4ec | 389 | |
5587856c SG |
390 | return err; |
391 | } |