Commit | Line | Data |
---|---|---|
6461f64a OG |
1 | /* |
2 | * Copyright (c) 2004, 2005, 2006 Voltaire, Inc. All rights reserved. | |
3ee07d27 | 3 | * Copyright (c) 2013-2014 Mellanox Technologies. All rights reserved. |
6461f64a OG |
4 | * |
5 | * This software is available to you under a choice of one of two | |
6 | * licenses. You may choose to be licensed under the terms of the GNU | |
7 | * General Public License (GPL) Version 2, available from the file | |
8 | * COPYING in the main directory of this source tree, or the | |
9 | * OpenIB.org BSD license below: | |
10 | * | |
11 | * Redistribution and use in source and binary forms, with or | |
12 | * without modification, are permitted provided that the following | |
13 | * conditions are met: | |
14 | * | |
15 | * - Redistributions of source code must retain the above | |
16 | * copyright notice, this list of conditions and the following | |
17 | * disclaimer. | |
18 | * | |
19 | * - Redistributions in binary form must reproduce the above | |
20 | * copyright notice, this list of conditions and the following | |
21 | * disclaimer in the documentation and/or other materials | |
22 | * provided with the distribution. | |
23 | * | |
24 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
25 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
26 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
27 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
28 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
29 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
30 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
31 | * SOFTWARE. | |
6461f64a OG |
32 | */ |
33 | #include <linux/module.h> | |
34 | #include <linux/kernel.h> | |
35 | #include <linux/slab.h> | |
36 | #include <linux/mm.h> | |
a1f8e7f7 | 37 | #include <linux/highmem.h> |
6461f64a OG |
38 | #include <linux/scatterlist.h> |
39 | ||
40 | #include "iscsi_iser.h" | |
41 | ||
42 | #define ISER_KMALLOC_THRESHOLD 0x20000 /* 128K - kmalloc limit */ | |
8dfa0876 | 43 | |
6461f64a OG |
44 | /** |
45 | * iser_start_rdma_unaligned_sg | |
46 | */ | |
2261ec3d | 47 | static int iser_start_rdma_unaligned_sg(struct iscsi_iser_task *iser_task, |
5f588e3d SG |
48 | struct iser_data_buf *data, |
49 | struct iser_data_buf *data_copy, | |
41179e2d | 50 | enum iser_data_dir cmd_dir) |
6461f64a | 51 | { |
a4ee3539 | 52 | struct ib_device *dev = iser_task->iser_conn->ib_conn.device->ib_device; |
5f588e3d SG |
53 | struct scatterlist *sgl = (struct scatterlist *)data->buf; |
54 | struct scatterlist *sg; | |
6461f64a | 55 | char *mem = NULL; |
5f588e3d SG |
56 | unsigned long cmd_data_len = 0; |
57 | int dma_nents, i; | |
58 | ||
59 | for_each_sg(sgl, sg, data->size, i) | |
60 | cmd_data_len += ib_sg_dma_len(dev, sg); | |
6461f64a OG |
61 | |
62 | if (cmd_data_len > ISER_KMALLOC_THRESHOLD) | |
528f4e8c | 63 | mem = (void *)__get_free_pages(GFP_ATOMIC, |
f0d1b0b3 | 64 | ilog2(roundup_pow_of_two(cmd_data_len)) - PAGE_SHIFT); |
6461f64a | 65 | else |
528f4e8c | 66 | mem = kmalloc(cmd_data_len, GFP_ATOMIC); |
6461f64a OG |
67 | |
68 | if (mem == NULL) { | |
69 | iser_err("Failed to allocate mem size %d %d for copying sglist\n", | |
5f588e3d | 70 | data->size, (int)cmd_data_len); |
6461f64a OG |
71 | return -ENOMEM; |
72 | } | |
73 | ||
74 | if (cmd_dir == ISER_DIR_OUT) { | |
75 | /* copy the unaligned sg the buffer which is used for RDMA */ | |
6461f64a OG |
76 | char *p, *from; |
77 | ||
5f588e3d | 78 | sgl = (struct scatterlist *)data->buf; |
53d412fc JA |
79 | p = mem; |
80 | for_each_sg(sgl, sg, data->size, i) { | |
2a156d09 | 81 | from = kmap_atomic(sg_page(sg)); |
6461f64a | 82 | memcpy(p, |
53d412fc JA |
83 | from + sg->offset, |
84 | sg->length); | |
2a156d09 | 85 | kunmap_atomic(from); |
53d412fc | 86 | p += sg->length; |
6461f64a OG |
87 | } |
88 | } | |
89 | ||
5f588e3d SG |
90 | sg_init_one(&data_copy->sg_single, mem, cmd_data_len); |
91 | data_copy->buf = &data_copy->sg_single; | |
92 | data_copy->size = 1; | |
93 | data_copy->copy_buf = mem; | |
6461f64a | 94 | |
5f588e3d | 95 | dma_nents = ib_dma_map_sg(dev, &data_copy->sg_single, 1, |
5180311f RC |
96 | (cmd_dir == ISER_DIR_OUT) ? |
97 | DMA_TO_DEVICE : DMA_FROM_DEVICE); | |
6461f64a OG |
98 | BUG_ON(dma_nents == 0); |
99 | ||
5f588e3d SG |
100 | data_copy->dma_nents = dma_nents; |
101 | data_copy->data_len = cmd_data_len; | |
102 | ||
6461f64a OG |
103 | return 0; |
104 | } | |
105 | ||
106 | /** | |
107 | * iser_finalize_rdma_unaligned_sg | |
108 | */ | |
9a8b08fa | 109 | |
2261ec3d | 110 | void iser_finalize_rdma_unaligned_sg(struct iscsi_iser_task *iser_task, |
9a8b08fa SG |
111 | struct iser_data_buf *data, |
112 | struct iser_data_buf *data_copy, | |
113 | enum iser_data_dir cmd_dir) | |
6461f64a | 114 | { |
5180311f | 115 | struct ib_device *dev; |
6461f64a OG |
116 | unsigned long cmd_data_len; |
117 | ||
a4ee3539 | 118 | dev = iser_task->iser_conn->ib_conn.device->ib_device; |
6461f64a | 119 | |
9a8b08fa | 120 | ib_dma_unmap_sg(dev, &data_copy->sg_single, 1, |
5180311f RC |
121 | (cmd_dir == ISER_DIR_OUT) ? |
122 | DMA_TO_DEVICE : DMA_FROM_DEVICE); | |
6461f64a OG |
123 | |
124 | if (cmd_dir == ISER_DIR_IN) { | |
125 | char *mem; | |
53d412fc | 126 | struct scatterlist *sgl, *sg; |
6461f64a OG |
127 | unsigned char *p, *to; |
128 | unsigned int sg_size; | |
129 | int i; | |
130 | ||
131 | /* copy back read RDMA to unaligned sg */ | |
9a8b08fa | 132 | mem = data_copy->copy_buf; |
6461f64a | 133 | |
9a8b08fa SG |
134 | sgl = (struct scatterlist *)data->buf; |
135 | sg_size = data->size; | |
6461f64a | 136 | |
53d412fc JA |
137 | p = mem; |
138 | for_each_sg(sgl, sg, sg_size, i) { | |
2a156d09 | 139 | to = kmap_atomic(sg_page(sg)); |
53d412fc | 140 | memcpy(to + sg->offset, |
6461f64a | 141 | p, |
53d412fc | 142 | sg->length); |
2a156d09 | 143 | kunmap_atomic(to); |
53d412fc | 144 | p += sg->length; |
6461f64a OG |
145 | } |
146 | } | |
147 | ||
9a8b08fa | 148 | cmd_data_len = data->data_len; |
6461f64a OG |
149 | |
150 | if (cmd_data_len > ISER_KMALLOC_THRESHOLD) | |
9a8b08fa | 151 | free_pages((unsigned long)data_copy->copy_buf, |
f0d1b0b3 | 152 | ilog2(roundup_pow_of_two(cmd_data_len)) - PAGE_SHIFT); |
6461f64a | 153 | else |
9a8b08fa | 154 | kfree(data_copy->copy_buf); |
6461f64a | 155 | |
9a8b08fa | 156 | data_copy->copy_buf = NULL; |
6461f64a OG |
157 | } |
158 | ||
c1ccaf24 OG |
159 | #define IS_4K_ALIGNED(addr) ((((unsigned long)addr) & ~MASK_4K) == 0) |
160 | ||
6461f64a OG |
161 | /** |
162 | * iser_sg_to_page_vec - Translates scatterlist entries to physical addresses | |
163 | * and returns the length of resulting physical address array (may be less than | |
164 | * the original due to possible compaction). | |
165 | * | |
166 | * we build a "page vec" under the assumption that the SG meets the RDMA | |
167 | * alignment requirements. Other then the first and last SG elements, all | |
168 | * the "internal" elements can be compacted into a list whose elements are | |
169 | * dma addresses of physical pages. The code supports also the weird case | |
170 | * where --few fragments of the same page-- are present in the SG as | |
171 | * consecutive elements. Also, it handles one entry SG. | |
172 | */ | |
c1ccaf24 | 173 | |
6461f64a | 174 | static int iser_sg_to_page_vec(struct iser_data_buf *data, |
919fc274 SG |
175 | struct ib_device *ibdev, u64 *pages, |
176 | int *offset, int *data_size) | |
6461f64a | 177 | { |
c1ccaf24 OG |
178 | struct scatterlist *sg, *sgl = (struct scatterlist *)data->buf; |
179 | u64 start_addr, end_addr, page, chunk_start = 0; | |
6461f64a | 180 | unsigned long total_sz = 0; |
c1ccaf24 OG |
181 | unsigned int dma_len; |
182 | int i, new_chunk, cur_page, last_ent = data->dma_nents - 1; | |
6461f64a OG |
183 | |
184 | /* compute the offset of first element */ | |
919fc274 | 185 | *offset = (u64) sgl[0].offset & ~MASK_4K; |
6461f64a | 186 | |
c1ccaf24 OG |
187 | new_chunk = 1; |
188 | cur_page = 0; | |
53d412fc | 189 | for_each_sg(sgl, sg, data->dma_nents, i) { |
c1ccaf24 OG |
190 | start_addr = ib_sg_dma_address(ibdev, sg); |
191 | if (new_chunk) | |
192 | chunk_start = start_addr; | |
193 | dma_len = ib_sg_dma_len(ibdev, sg); | |
194 | end_addr = start_addr + dma_len; | |
5180311f | 195 | total_sz += dma_len; |
6461f64a | 196 | |
c1ccaf24 OG |
197 | /* collect page fragments until aligned or end of SG list */ |
198 | if (!IS_4K_ALIGNED(end_addr) && i < last_ent) { | |
199 | new_chunk = 0; | |
200 | continue; | |
6461f64a | 201 | } |
c1ccaf24 OG |
202 | new_chunk = 1; |
203 | ||
204 | /* address of the first page in the contiguous chunk; | |
205 | masking relevant for the very first SG entry, | |
206 | which might be unaligned */ | |
207 | page = chunk_start & MASK_4K; | |
208 | do { | |
919fc274 | 209 | pages[cur_page++] = page; |
8dfa0876 | 210 | page += SIZE_4K; |
c1ccaf24 | 211 | } while (page < end_addr); |
6461f64a | 212 | } |
c1ccaf24 | 213 | |
919fc274 SG |
214 | *data_size = total_sz; |
215 | iser_dbg("page_vec->data_size:%d cur_page %d\n", | |
216 | *data_size, cur_page); | |
6461f64a OG |
217 | return cur_page; |
218 | } | |
219 | ||
6461f64a OG |
220 | |
221 | /** | |
222 | * iser_data_buf_aligned_len - Tries to determine the maximal correctly aligned | |
223 | * for RDMA sub-list of a scatter-gather list of memory buffers, and returns | |
224 | * the number of entries which are aligned correctly. Supports the case where | |
225 | * consecutive SG elements are actually fragments of the same physcial page. | |
226 | */ | |
c1ccaf24 OG |
227 | static int iser_data_buf_aligned_len(struct iser_data_buf *data, |
228 | struct ib_device *ibdev) | |
6461f64a | 229 | { |
c1ccaf24 OG |
230 | struct scatterlist *sgl, *sg, *next_sg = NULL; |
231 | u64 start_addr, end_addr; | |
232 | int i, ret_len, start_check = 0; | |
233 | ||
234 | if (data->dma_nents == 1) | |
235 | return 1; | |
6461f64a | 236 | |
53d412fc | 237 | sgl = (struct scatterlist *)data->buf; |
c1ccaf24 | 238 | start_addr = ib_sg_dma_address(ibdev, sgl); |
6461f64a | 239 | |
53d412fc | 240 | for_each_sg(sgl, sg, data->dma_nents, i) { |
c1ccaf24 OG |
241 | if (start_check && !IS_4K_ALIGNED(start_addr)) |
242 | break; | |
243 | ||
244 | next_sg = sg_next(sg); | |
245 | if (!next_sg) | |
246 | break; | |
247 | ||
248 | end_addr = start_addr + ib_sg_dma_len(ibdev, sg); | |
249 | start_addr = ib_sg_dma_address(ibdev, next_sg); | |
250 | ||
251 | if (end_addr == start_addr) { | |
252 | start_check = 0; | |
253 | continue; | |
254 | } else | |
255 | start_check = 1; | |
256 | ||
257 | if (!IS_4K_ALIGNED(end_addr)) | |
258 | break; | |
6461f64a | 259 | } |
c1ccaf24 | 260 | ret_len = (next_sg) ? i : i+1; |
6461f64a OG |
261 | iser_dbg("Found %d aligned entries out of %d in sg:0x%p\n", |
262 | ret_len, data->dma_nents, data); | |
263 | return ret_len; | |
264 | } | |
265 | ||
5180311f RC |
266 | static void iser_data_buf_dump(struct iser_data_buf *data, |
267 | struct ib_device *ibdev) | |
6461f64a | 268 | { |
53d412fc JA |
269 | struct scatterlist *sgl = (struct scatterlist *)data->buf; |
270 | struct scatterlist *sg; | |
6461f64a OG |
271 | int i; |
272 | ||
53d412fc | 273 | for_each_sg(sgl, sg, data->dma_nents, i) |
f91424cf | 274 | iser_dbg("sg[%d] dma_addr:0x%lX page:0x%p " |
e981f1d4 | 275 | "off:0x%x sz:0x%x dma_len:0x%x\n", |
53d412fc | 276 | i, (unsigned long)ib_sg_dma_address(ibdev, sg), |
45711f1a | 277 | sg_page(sg), sg->offset, |
53d412fc | 278 | sg->length, ib_sg_dma_len(ibdev, sg)); |
6461f64a OG |
279 | } |
280 | ||
281 | static void iser_dump_page_vec(struct iser_page_vec *page_vec) | |
282 | { | |
283 | int i; | |
284 | ||
285 | iser_err("page vec length %d data size %d\n", | |
286 | page_vec->length, page_vec->data_size); | |
287 | for (i = 0; i < page_vec->length; i++) | |
288 | iser_err("%d %lx\n",i,(unsigned long)page_vec->pages[i]); | |
289 | } | |
290 | ||
291 | static void iser_page_vec_build(struct iser_data_buf *data, | |
5180311f RC |
292 | struct iser_page_vec *page_vec, |
293 | struct ib_device *ibdev) | |
6461f64a OG |
294 | { |
295 | int page_vec_len = 0; | |
296 | ||
297 | page_vec->length = 0; | |
298 | page_vec->offset = 0; | |
299 | ||
300 | iser_dbg("Translating sg sz: %d\n", data->dma_nents); | |
919fc274 SG |
301 | page_vec_len = iser_sg_to_page_vec(data, ibdev, page_vec->pages, |
302 | &page_vec->offset, | |
303 | &page_vec->data_size); | |
304 | iser_dbg("sg len %d page_vec_len %d\n", data->dma_nents, page_vec_len); | |
6461f64a OG |
305 | |
306 | page_vec->length = page_vec_len; | |
307 | ||
8dfa0876 | 308 | if (page_vec_len * SIZE_4K < page_vec->data_size) { |
6461f64a | 309 | iser_err("page_vec too short to hold this SG\n"); |
5180311f | 310 | iser_data_buf_dump(data, ibdev); |
6461f64a OG |
311 | iser_dump_page_vec(page_vec); |
312 | BUG(); | |
313 | } | |
314 | } | |
315 | ||
2261ec3d MC |
316 | int iser_dma_map_task_data(struct iscsi_iser_task *iser_task, |
317 | struct iser_data_buf *data, | |
318 | enum iser_data_dir iser_dir, | |
319 | enum dma_data_direction dma_dir) | |
74a20780 | 320 | { |
5180311f | 321 | struct ib_device *dev; |
74a20780 | 322 | |
2261ec3d | 323 | iser_task->dir[iser_dir] = 1; |
a4ee3539 | 324 | dev = iser_task->iser_conn->ib_conn.device->ib_device; |
74a20780 | 325 | |
5180311f | 326 | data->dma_nents = ib_dma_map_sg(dev, data->buf, data->size, dma_dir); |
74a20780 EZ |
327 | if (data->dma_nents == 0) { |
328 | iser_err("dma_map_sg failed!!!\n"); | |
329 | return -EINVAL; | |
330 | } | |
331 | return 0; | |
332 | } | |
333 | ||
9a8b08fa SG |
334 | void iser_dma_unmap_task_data(struct iscsi_iser_task *iser_task, |
335 | struct iser_data_buf *data) | |
74a20780 | 336 | { |
5180311f | 337 | struct ib_device *dev; |
74a20780 | 338 | |
a4ee3539 | 339 | dev = iser_task->iser_conn->ib_conn.device->ib_device; |
9a8b08fa | 340 | ib_dma_unmap_sg(dev, data->buf, data->size, DMA_FROM_DEVICE); |
74a20780 EZ |
341 | } |
342 | ||
919fc274 SG |
343 | static int fall_to_bounce_buf(struct iscsi_iser_task *iser_task, |
344 | struct ib_device *ibdev, | |
5f588e3d SG |
345 | struct iser_data_buf *mem, |
346 | struct iser_data_buf *mem_copy, | |
919fc274 SG |
347 | enum iser_data_dir cmd_dir, |
348 | int aligned_len) | |
349 | { | |
5716af6e | 350 | struct iscsi_conn *iscsi_conn = iser_task->iser_conn->iscsi_conn; |
919fc274 SG |
351 | |
352 | iscsi_conn->fmr_unalign_cnt++; | |
353 | iser_warn("rdma alignment violation (%d/%d aligned) or FMR not supported\n", | |
354 | aligned_len, mem->size); | |
355 | ||
356 | if (iser_debug_level > 0) | |
357 | iser_data_buf_dump(mem, ibdev); | |
358 | ||
359 | /* unmap the command data before accessing it */ | |
5f588e3d | 360 | iser_dma_unmap_task_data(iser_task, mem); |
919fc274 SG |
361 | |
362 | /* allocate copy buf, if we are writing, copy the */ | |
363 | /* unaligned scatterlist, dma map the copy */ | |
5f588e3d SG |
364 | if (iser_start_rdma_unaligned_sg(iser_task, mem, mem_copy, cmd_dir) != 0) |
365 | return -ENOMEM; | |
919fc274 SG |
366 | |
367 | return 0; | |
368 | } | |
369 | ||
6461f64a | 370 | /** |
e657571b SG |
371 | * iser_reg_rdma_mem_fmr - Registers memory intended for RDMA, |
372 | * using FMR (if possible) obtaining rkey and va | |
6461f64a OG |
373 | * |
374 | * returns 0 on success, errno code on failure | |
375 | */ | |
e657571b SG |
376 | int iser_reg_rdma_mem_fmr(struct iscsi_iser_task *iser_task, |
377 | enum iser_data_dir cmd_dir) | |
6461f64a | 378 | { |
a4ee3539 SG |
379 | struct ib_conn *ib_conn = &iser_task->iser_conn->ib_conn; |
380 | struct iser_device *device = ib_conn->device; | |
5180311f | 381 | struct ib_device *ibdev = device->ib_device; |
2261ec3d | 382 | struct iser_data_buf *mem = &iser_task->data[cmd_dir]; |
6461f64a OG |
383 | struct iser_regd_buf *regd_buf; |
384 | int aligned_len; | |
385 | int err; | |
e981f1d4 | 386 | int i; |
d8111028 | 387 | struct scatterlist *sg; |
6461f64a | 388 | |
2261ec3d | 389 | regd_buf = &iser_task->rdma_regd[cmd_dir]; |
6461f64a | 390 | |
5180311f | 391 | aligned_len = iser_data_buf_aligned_len(mem, ibdev); |
5587856c | 392 | if (aligned_len != mem->dma_nents) { |
5f588e3d SG |
393 | err = fall_to_bounce_buf(iser_task, ibdev, mem, |
394 | &iser_task->data_copy[cmd_dir], | |
919fc274 SG |
395 | cmd_dir, aligned_len); |
396 | if (err) { | |
397 | iser_err("failed to allocate bounce buffer\n"); | |
398 | return err; | |
399 | } | |
2261ec3d | 400 | mem = &iser_task->data_copy[cmd_dir]; |
6461f64a OG |
401 | } |
402 | ||
d8111028 EZ |
403 | /* if there a single dma entry, FMR is not needed */ |
404 | if (mem->dma_nents == 1) { | |
405 | sg = (struct scatterlist *)mem->buf; | |
406 | ||
407 | regd_buf->reg.lkey = device->mr->lkey; | |
408 | regd_buf->reg.rkey = device->mr->rkey; | |
5180311f RC |
409 | regd_buf->reg.len = ib_sg_dma_len(ibdev, &sg[0]); |
410 | regd_buf->reg.va = ib_sg_dma_address(ibdev, &sg[0]); | |
d8111028 EZ |
411 | |
412 | iser_dbg("PHYSICAL Mem.register: lkey: 0x%08X rkey: 0x%08X " | |
413 | "va: 0x%08lX sz: %ld]\n", | |
414 | (unsigned int)regd_buf->reg.lkey, | |
415 | (unsigned int)regd_buf->reg.rkey, | |
416 | (unsigned long)regd_buf->reg.va, | |
417 | (unsigned long)regd_buf->reg.len); | |
418 | } else { /* use FMR for multiple dma entries */ | |
a4ee3539 SG |
419 | iser_page_vec_build(mem, ib_conn->fmr.page_vec, ibdev); |
420 | err = iser_reg_page_vec(ib_conn, ib_conn->fmr.page_vec, | |
e657571b | 421 | ®d_buf->reg); |
819a0873 | 422 | if (err && err != -EAGAIN) { |
5180311f | 423 | iser_data_buf_dump(mem, ibdev); |
2261ec3d MC |
424 | iser_err("mem->dma_nents = %d (dlength = 0x%x)\n", |
425 | mem->dma_nents, | |
426 | ntoh24(iser_task->desc.iscsi_header.dlength)); | |
d8111028 | 427 | iser_err("page_vec: data_size = 0x%x, length = %d, offset = 0x%x\n", |
a4ee3539 SG |
428 | ib_conn->fmr.page_vec->data_size, |
429 | ib_conn->fmr.page_vec->length, | |
430 | ib_conn->fmr.page_vec->offset); | |
431 | for (i = 0; i < ib_conn->fmr.page_vec->length; i++) | |
d8111028 | 432 | iser_err("page_vec[%d] = 0x%llx\n", i, |
a4ee3539 | 433 | (unsigned long long)ib_conn->fmr.page_vec->pages[i]); |
e981f1d4 | 434 | } |
450d1e40 OG |
435 | if (err) |
436 | return err; | |
e981f1d4 | 437 | } |
6461f64a OG |
438 | return 0; |
439 | } | |
5587856c | 440 | |
5bb6e543 | 441 | static void |
92792c0a SG |
442 | iser_set_dif_domain(struct scsi_cmnd *sc, struct ib_sig_attrs *sig_attrs, |
443 | struct ib_sig_domain *domain) | |
444 | { | |
78eda2bb | 445 | domain->sig_type = IB_SIG_TYPE_T10_DIF; |
5bb6e543 SG |
446 | domain->sig.dif.pi_interval = scsi_prot_interval(sc); |
447 | domain->sig.dif.ref_tag = scsi_prot_ref_tag(sc); | |
78eda2bb SG |
448 | /* |
449 | * At the moment we hard code those, but in the future | |
450 | * we will take them from sc. | |
451 | */ | |
452 | domain->sig.dif.apptag_check_mask = 0xffff; | |
453 | domain->sig.dif.app_escape = true; | |
454 | domain->sig.dif.ref_escape = true; | |
5bb6e543 | 455 | if (sc->prot_flags & SCSI_PROT_REF_INCREMENT) |
78eda2bb | 456 | domain->sig.dif.ref_remap = true; |
92792c0a | 457 | }; |
177e31bd SG |
458 | |
459 | static int | |
460 | iser_set_sig_attrs(struct scsi_cmnd *sc, struct ib_sig_attrs *sig_attrs) | |
461 | { | |
177e31bd SG |
462 | switch (scsi_get_prot_op(sc)) { |
463 | case SCSI_PROT_WRITE_INSERT: | |
464 | case SCSI_PROT_READ_STRIP: | |
78eda2bb | 465 | sig_attrs->mem.sig_type = IB_SIG_TYPE_NONE; |
92792c0a | 466 | iser_set_dif_domain(sc, sig_attrs, &sig_attrs->wire); |
177e31bd | 467 | sig_attrs->wire.sig.dif.bg_type = IB_T10DIF_CRC; |
177e31bd SG |
468 | break; |
469 | case SCSI_PROT_READ_INSERT: | |
470 | case SCSI_PROT_WRITE_STRIP: | |
78eda2bb | 471 | sig_attrs->wire.sig_type = IB_SIG_TYPE_NONE; |
92792c0a | 472 | iser_set_dif_domain(sc, sig_attrs, &sig_attrs->mem); |
5bb6e543 SG |
473 | sig_attrs->mem.sig.dif.bg_type = sc->prot_flags & SCSI_PROT_IP_CHECKSUM ? |
474 | IB_T10DIF_CSUM : IB_T10DIF_CRC; | |
177e31bd SG |
475 | break; |
476 | case SCSI_PROT_READ_PASS: | |
477 | case SCSI_PROT_WRITE_PASS: | |
92792c0a | 478 | iser_set_dif_domain(sc, sig_attrs, &sig_attrs->wire); |
177e31bd | 479 | sig_attrs->wire.sig.dif.bg_type = IB_T10DIF_CRC; |
92792c0a | 480 | iser_set_dif_domain(sc, sig_attrs, &sig_attrs->mem); |
5bb6e543 SG |
481 | sig_attrs->mem.sig.dif.bg_type = sc->prot_flags & SCSI_PROT_IP_CHECKSUM ? |
482 | IB_T10DIF_CSUM : IB_T10DIF_CRC; | |
177e31bd SG |
483 | break; |
484 | default: | |
485 | iser_err("Unsupported PI operation %d\n", | |
486 | scsi_get_prot_op(sc)); | |
487 | return -EINVAL; | |
488 | } | |
78eda2bb | 489 | |
177e31bd SG |
490 | return 0; |
491 | } | |
492 | ||
5bb6e543 | 493 | static inline void |
177e31bd SG |
494 | iser_set_prot_checks(struct scsi_cmnd *sc, u8 *mask) |
495 | { | |
5bb6e543 SG |
496 | *mask = 0; |
497 | if (sc->prot_flags & SCSI_PROT_REF_CHECK) | |
498 | *mask |= ISER_CHECK_REFTAG; | |
499 | if (sc->prot_flags & SCSI_PROT_GUARD_CHECK) | |
500 | *mask |= ISER_CHECK_GUARD; | |
177e31bd SG |
501 | } |
502 | ||
a11b3e69 SG |
503 | static void |
504 | iser_inv_rkey(struct ib_send_wr *inv_wr, struct ib_mr *mr) | |
505 | { | |
506 | u32 rkey; | |
507 | ||
508 | memset(inv_wr, 0, sizeof(*inv_wr)); | |
509 | inv_wr->opcode = IB_WR_LOCAL_INV; | |
510 | inv_wr->wr_id = ISER_FASTREG_LI_WRID; | |
511 | inv_wr->ex.invalidate_rkey = mr->rkey; | |
512 | ||
513 | rkey = ib_inc_rkey(mr->rkey); | |
514 | ib_update_fast_reg_key(mr, rkey); | |
515 | } | |
516 | ||
177e31bd SG |
517 | static int |
518 | iser_reg_sig_mr(struct iscsi_iser_task *iser_task, | |
519 | struct fast_reg_descriptor *desc, struct ib_sge *data_sge, | |
520 | struct ib_sge *prot_sge, struct ib_sge *sig_sge) | |
521 | { | |
a4ee3539 | 522 | struct ib_conn *ib_conn = &iser_task->iser_conn->ib_conn; |
177e31bd SG |
523 | struct iser_pi_context *pi_ctx = desc->pi_ctx; |
524 | struct ib_send_wr sig_wr, inv_wr; | |
525 | struct ib_send_wr *bad_wr, *wr = NULL; | |
526 | struct ib_sig_attrs sig_attrs; | |
527 | int ret; | |
177e31bd SG |
528 | |
529 | memset(&sig_attrs, 0, sizeof(sig_attrs)); | |
530 | ret = iser_set_sig_attrs(iser_task->sc, &sig_attrs); | |
531 | if (ret) | |
532 | goto err; | |
533 | ||
5bb6e543 | 534 | iser_set_prot_checks(iser_task->sc, &sig_attrs.check_mask); |
177e31bd SG |
535 | |
536 | if (!(desc->reg_indicators & ISER_SIG_KEY_VALID)) { | |
a11b3e69 | 537 | iser_inv_rkey(&inv_wr, pi_ctx->sig_mr); |
177e31bd | 538 | wr = &inv_wr; |
177e31bd SG |
539 | } |
540 | ||
541 | memset(&sig_wr, 0, sizeof(sig_wr)); | |
542 | sig_wr.opcode = IB_WR_REG_SIG_MR; | |
543 | sig_wr.wr_id = ISER_FASTREG_LI_WRID; | |
544 | sig_wr.sg_list = data_sge; | |
545 | sig_wr.num_sge = 1; | |
546 | sig_wr.wr.sig_handover.sig_attrs = &sig_attrs; | |
547 | sig_wr.wr.sig_handover.sig_mr = pi_ctx->sig_mr; | |
548 | if (scsi_prot_sg_count(iser_task->sc)) | |
549 | sig_wr.wr.sig_handover.prot = prot_sge; | |
550 | sig_wr.wr.sig_handover.access_flags = IB_ACCESS_LOCAL_WRITE | | |
551 | IB_ACCESS_REMOTE_READ | | |
552 | IB_ACCESS_REMOTE_WRITE; | |
553 | ||
554 | if (!wr) | |
555 | wr = &sig_wr; | |
556 | else | |
557 | wr->next = &sig_wr; | |
558 | ||
a4ee3539 | 559 | ret = ib_post_send(ib_conn->qp, wr, &bad_wr); |
177e31bd SG |
560 | if (ret) { |
561 | iser_err("reg_sig_mr failed, ret:%d\n", ret); | |
562 | goto err; | |
563 | } | |
564 | desc->reg_indicators &= ~ISER_SIG_KEY_VALID; | |
565 | ||
566 | sig_sge->lkey = pi_ctx->sig_mr->lkey; | |
567 | sig_sge->addr = 0; | |
5bb6e543 | 568 | sig_sge->length = scsi_transfer_length(iser_task->sc); |
177e31bd SG |
569 | |
570 | iser_dbg("sig_sge: addr: 0x%llx length: %u lkey: 0x%x\n", | |
571 | sig_sge->addr, sig_sge->length, | |
572 | sig_sge->lkey); | |
573 | err: | |
574 | return ret; | |
575 | } | |
576 | ||
d11ec4ec | 577 | static int iser_fast_reg_mr(struct iscsi_iser_task *iser_task, |
5587856c | 578 | struct iser_regd_buf *regd_buf, |
d11ec4ec | 579 | struct iser_data_buf *mem, |
177e31bd | 580 | enum iser_reg_indicator ind, |
d11ec4ec | 581 | struct ib_sge *sge) |
5587856c | 582 | { |
d11ec4ec | 583 | struct fast_reg_descriptor *desc = regd_buf->reg.mem_h; |
a4ee3539 SG |
584 | struct ib_conn *ib_conn = &iser_task->iser_conn->ib_conn; |
585 | struct iser_device *device = ib_conn->device; | |
d11ec4ec | 586 | struct ib_device *ibdev = device->ib_device; |
177e31bd SG |
587 | struct ib_mr *mr; |
588 | struct ib_fast_reg_page_list *frpl; | |
5587856c SG |
589 | struct ib_send_wr fastreg_wr, inv_wr; |
590 | struct ib_send_wr *bad_wr, *wr = NULL; | |
d11ec4ec SG |
591 | int ret, offset, size, plen; |
592 | ||
593 | /* if there a single dma entry, dma mr suffices */ | |
594 | if (mem->dma_nents == 1) { | |
595 | struct scatterlist *sg = (struct scatterlist *)mem->buf; | |
596 | ||
597 | sge->lkey = device->mr->lkey; | |
598 | sge->addr = ib_sg_dma_address(ibdev, &sg[0]); | |
599 | sge->length = ib_sg_dma_len(ibdev, &sg[0]); | |
600 | ||
601 | iser_dbg("Single DMA entry: lkey=0x%x, addr=0x%llx, length=0x%x\n", | |
602 | sge->lkey, sge->addr, sge->length); | |
603 | return 0; | |
604 | } | |
605 | ||
177e31bd SG |
606 | if (ind == ISER_DATA_KEY_VALID) { |
607 | mr = desc->data_mr; | |
608 | frpl = desc->data_frpl; | |
609 | } else { | |
610 | mr = desc->pi_ctx->prot_mr; | |
611 | frpl = desc->pi_ctx->prot_frpl; | |
612 | } | |
613 | ||
614 | plen = iser_sg_to_page_vec(mem, device->ib_device, frpl->page_list, | |
d11ec4ec SG |
615 | &offset, &size); |
616 | if (plen * SIZE_4K < size) { | |
617 | iser_err("fast reg page_list too short to hold this SG\n"); | |
618 | return -EINVAL; | |
619 | } | |
5587856c | 620 | |
177e31bd | 621 | if (!(desc->reg_indicators & ind)) { |
a11b3e69 | 622 | iser_inv_rkey(&inv_wr, mr); |
5587856c | 623 | wr = &inv_wr; |
5587856c SG |
624 | } |
625 | ||
626 | /* Prepare FASTREG WR */ | |
627 | memset(&fastreg_wr, 0, sizeof(fastreg_wr)); | |
7306b8fa | 628 | fastreg_wr.wr_id = ISER_FASTREG_LI_WRID; |
5587856c | 629 | fastreg_wr.opcode = IB_WR_FAST_REG_MR; |
177e31bd SG |
630 | fastreg_wr.wr.fast_reg.iova_start = frpl->page_list[0] + offset; |
631 | fastreg_wr.wr.fast_reg.page_list = frpl; | |
d11ec4ec | 632 | fastreg_wr.wr.fast_reg.page_list_len = plen; |
5587856c | 633 | fastreg_wr.wr.fast_reg.page_shift = SHIFT_4K; |
d11ec4ec | 634 | fastreg_wr.wr.fast_reg.length = size; |
177e31bd | 635 | fastreg_wr.wr.fast_reg.rkey = mr->rkey; |
5587856c SG |
636 | fastreg_wr.wr.fast_reg.access_flags = (IB_ACCESS_LOCAL_WRITE | |
637 | IB_ACCESS_REMOTE_WRITE | | |
638 | IB_ACCESS_REMOTE_READ); | |
639 | ||
db523b8d | 640 | if (!wr) |
5587856c | 641 | wr = &fastreg_wr; |
db523b8d | 642 | else |
5587856c | 643 | wr->next = &fastreg_wr; |
5587856c | 644 | |
a4ee3539 | 645 | ret = ib_post_send(ib_conn->qp, wr, &bad_wr); |
5587856c | 646 | if (ret) { |
5587856c SG |
647 | iser_err("fast registration failed, ret:%d\n", ret); |
648 | return ret; | |
649 | } | |
177e31bd | 650 | desc->reg_indicators &= ~ind; |
5587856c | 651 | |
177e31bd SG |
652 | sge->lkey = mr->lkey; |
653 | sge->addr = frpl->page_list[0] + offset; | |
d11ec4ec | 654 | sge->length = size; |
5587856c SG |
655 | |
656 | return ret; | |
657 | } | |
658 | ||
659 | /** | |
7306b8fa | 660 | * iser_reg_rdma_mem_fastreg - Registers memory intended for RDMA, |
5587856c SG |
661 | * using Fast Registration WR (if possible) obtaining rkey and va |
662 | * | |
663 | * returns 0 on success, errno code on failure | |
664 | */ | |
7306b8fa SG |
665 | int iser_reg_rdma_mem_fastreg(struct iscsi_iser_task *iser_task, |
666 | enum iser_data_dir cmd_dir) | |
5587856c | 667 | { |
a4ee3539 SG |
668 | struct ib_conn *ib_conn = &iser_task->iser_conn->ib_conn; |
669 | struct iser_device *device = ib_conn->device; | |
5587856c SG |
670 | struct ib_device *ibdev = device->ib_device; |
671 | struct iser_data_buf *mem = &iser_task->data[cmd_dir]; | |
672 | struct iser_regd_buf *regd_buf = &iser_task->rdma_regd[cmd_dir]; | |
d11ec4ec SG |
673 | struct fast_reg_descriptor *desc = NULL; |
674 | struct ib_sge data_sge; | |
5587856c SG |
675 | int err, aligned_len; |
676 | unsigned long flags; | |
5587856c SG |
677 | |
678 | aligned_len = iser_data_buf_aligned_len(mem, ibdev); | |
679 | if (aligned_len != mem->dma_nents) { | |
5f588e3d SG |
680 | err = fall_to_bounce_buf(iser_task, ibdev, mem, |
681 | &iser_task->data_copy[cmd_dir], | |
5587856c SG |
682 | cmd_dir, aligned_len); |
683 | if (err) { | |
684 | iser_err("failed to allocate bounce buffer\n"); | |
685 | return err; | |
686 | } | |
687 | mem = &iser_task->data_copy[cmd_dir]; | |
688 | } | |
689 | ||
177e31bd SG |
690 | if (mem->dma_nents != 1 || |
691 | scsi_get_prot_op(iser_task->sc) != SCSI_PROT_NORMAL) { | |
a4ee3539 SG |
692 | spin_lock_irqsave(&ib_conn->lock, flags); |
693 | desc = list_first_entry(&ib_conn->fastreg.pool, | |
5587856c SG |
694 | struct fast_reg_descriptor, list); |
695 | list_del(&desc->list); | |
a4ee3539 | 696 | spin_unlock_irqrestore(&ib_conn->lock, flags); |
d11ec4ec SG |
697 | regd_buf->reg.mem_h = desc; |
698 | } | |
5587856c | 699 | |
177e31bd SG |
700 | err = iser_fast_reg_mr(iser_task, regd_buf, mem, |
701 | ISER_DATA_KEY_VALID, &data_sge); | |
d11ec4ec SG |
702 | if (err) |
703 | goto err_reg; | |
704 | ||
177e31bd SG |
705 | if (scsi_get_prot_op(iser_task->sc) != SCSI_PROT_NORMAL) { |
706 | struct ib_sge prot_sge, sig_sge; | |
707 | ||
708 | memset(&prot_sge, 0, sizeof(prot_sge)); | |
709 | if (scsi_prot_sg_count(iser_task->sc)) { | |
710 | mem = &iser_task->prot[cmd_dir]; | |
711 | aligned_len = iser_data_buf_aligned_len(mem, ibdev); | |
712 | if (aligned_len != mem->dma_nents) { | |
713 | err = fall_to_bounce_buf(iser_task, ibdev, mem, | |
714 | &iser_task->prot_copy[cmd_dir], | |
715 | cmd_dir, aligned_len); | |
716 | if (err) { | |
717 | iser_err("failed to allocate bounce buffer\n"); | |
718 | return err; | |
719 | } | |
720 | mem = &iser_task->prot_copy[cmd_dir]; | |
721 | } | |
722 | ||
723 | err = iser_fast_reg_mr(iser_task, regd_buf, mem, | |
724 | ISER_PROT_KEY_VALID, &prot_sge); | |
725 | if (err) | |
726 | goto err_reg; | |
727 | } | |
728 | ||
729 | err = iser_reg_sig_mr(iser_task, desc, &data_sge, | |
730 | &prot_sge, &sig_sge); | |
731 | if (err) { | |
732 | iser_err("Failed to register signature mr\n"); | |
733 | return err; | |
734 | } | |
735 | desc->reg_indicators |= ISER_FASTREG_PROTECTED; | |
736 | ||
737 | regd_buf->reg.lkey = sig_sge.lkey; | |
738 | regd_buf->reg.rkey = desc->pi_ctx->sig_mr->rkey; | |
739 | regd_buf->reg.va = sig_sge.addr; | |
740 | regd_buf->reg.len = sig_sge.length; | |
d11ec4ec | 741 | } else { |
7e1fd4d1 | 742 | if (desc) |
177e31bd | 743 | regd_buf->reg.rkey = desc->data_mr->rkey; |
7e1fd4d1 | 744 | else |
177e31bd | 745 | regd_buf->reg.rkey = device->mr->rkey; |
5587856c | 746 | |
177e31bd SG |
747 | regd_buf->reg.lkey = data_sge.lkey; |
748 | regd_buf->reg.va = data_sge.addr; | |
749 | regd_buf->reg.len = data_sge.length; | |
750 | } | |
d11ec4ec | 751 | |
5587856c SG |
752 | return 0; |
753 | err_reg: | |
d11ec4ec | 754 | if (desc) { |
a4ee3539 SG |
755 | spin_lock_irqsave(&ib_conn->lock, flags); |
756 | list_add_tail(&desc->list, &ib_conn->fastreg.pool); | |
757 | spin_unlock_irqrestore(&ib_conn->lock, flags); | |
d11ec4ec SG |
758 | } |
759 | ||
5587856c SG |
760 | return err; |
761 | } |