projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
treewide: Use array_size() in vzalloc()
[linux-block.git] / drivers / infiniband / hw / hns / hns_roce_mr.c
CommitLineData
9a443537 1/*
2 * Copyright (c) 2016 Hisilicon Limited.
3 * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
4 *
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
10 *
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
13 * conditions are met:
14 *
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
17 * disclaimer.
18 *
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
23 *
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31 * SOFTWARE.
32 */
33
34#include <linux/platform_device.h>
e89bf462 35#include <linux/vmalloc.h>
9a443537 36#include <rdma/ib_umem.h>
37#include "hns_roce_device.h"
38#include "hns_roce_cmd.h"
39#include "hns_roce_hem.h"
40
41static u32 hw_index_to_key(unsigned long ind)
42{
43 return (u32)(ind >> 24) | (ind << 8);
44}
45
bfcc681b 46unsigned long key_to_hw_index(u32 key)
9a443537 47{
48 return (key << 24) | (key >> 8);
49}
08805fdb 50EXPORT_SYMBOL_GPL(key_to_hw_index);
9a443537 51
52static int hns_roce_sw2hw_mpt(struct hns_roce_dev *hr_dev,
53 struct hns_roce_cmd_mailbox *mailbox,
54 unsigned long mpt_index)
55{
56 return hns_roce_cmd_mbox(hr_dev, mailbox->dma, 0, mpt_index, 0,
57 HNS_ROCE_CMD_SW2HW_MPT,
6b877c32 58 HNS_ROCE_CMD_TIMEOUT_MSECS);
9a443537 59}
60
bfcc681b 61int hns_roce_hw2sw_mpt(struct hns_roce_dev *hr_dev,
9a443537 62 struct hns_roce_cmd_mailbox *mailbox,
63 unsigned long mpt_index)
64{
65 return hns_roce_cmd_mbox(hr_dev, 0, mailbox ? mailbox->dma : 0,
66 mpt_index, !mailbox, HNS_ROCE_CMD_HW2SW_MPT,
6b877c32 67 HNS_ROCE_CMD_TIMEOUT_MSECS);
9a443537 68}
08805fdb 69EXPORT_SYMBOL_GPL(hns_roce_hw2sw_mpt);
9a443537 70
71static int hns_roce_buddy_alloc(struct hns_roce_buddy *buddy, int order,
72 unsigned long *seg)
73{
74 int o;
75 u32 m;
76
77 spin_lock(&buddy->lock);
78
79 for (o = order; o <= buddy->max_order; ++o) {
80 if (buddy->num_free[o]) {
81 m = 1 << (buddy->max_order - o);
82 *seg = find_first_bit(buddy->bits[o], m);
83 if (*seg < m)
84 goto found;
85 }
86 }
87 spin_unlock(&buddy->lock);
88 return -1;
89
90 found:
91 clear_bit(*seg, buddy->bits[o]);
92 --buddy->num_free[o];
93
94 while (o > order) {
95 --o;
96 *seg <<= 1;
97 set_bit(*seg ^ 1, buddy->bits[o]);
98 ++buddy->num_free[o];
99 }
100
101 spin_unlock(&buddy->lock);
102
103 *seg <<= order;
104 return 0;
105}
106
107static void hns_roce_buddy_free(struct hns_roce_buddy *buddy, unsigned long seg,
108 int order)
109{
110 seg >>= order;
111
112 spin_lock(&buddy->lock);
113
114 while (test_bit(seg ^ 1, buddy->bits[order])) {
115 clear_bit(seg ^ 1, buddy->bits[order]);
116 --buddy->num_free[order];
117 seg >>= 1;
118 ++order;
119 }
120
121 set_bit(seg, buddy->bits[order]);
122 ++buddy->num_free[order];
123
124 spin_unlock(&buddy->lock);
125}
126
127static int hns_roce_buddy_init(struct hns_roce_buddy *buddy, int max_order)
128{
129 int i, s;
130
131 buddy->max_order = max_order;
132 spin_lock_init(&buddy->lock);
4418b27b
ME		 133 buddy->bits = kcalloc(buddy->max_order + 1,
134 sizeof(*buddy->bits),
135 GFP_KERNEL);
136 buddy->num_free = kcalloc(buddy->max_order + 1,
137 sizeof(*buddy->num_free),
138 GFP_KERNEL);
9a443537 139 if (!buddy->bits || !buddy->num_free)
140 goto err_out;
141
142 for (i = 0; i <= buddy->max_order; ++i) {
143 s = BITS_TO_LONGS(1 << (buddy->max_order - i));
8d497eb0
WHX		 144 buddy->bits[i] = kcalloc(s, sizeof(long), GFP_KERNEL |
145 __GFP_NOWARN);
146 if (!buddy->bits[i]) {
fad953ce 147 buddy->bits[i] = vzalloc(array_size(s, sizeof(long)));
8d497eb0
WHX		 148 if (!buddy->bits[i])
149 goto err_out_free;
150 }
9a443537 151 }
152
153 set_bit(0, buddy->bits[buddy->max_order]);
154 buddy->num_free[buddy->max_order] = 1;
155
156 return 0;
157
158err_out_free:
159 for (i = 0; i <= buddy->max_order; ++i)
8d497eb0 160 kvfree(buddy->bits[i]);
9a443537 161
162err_out:
163 kfree(buddy->bits);
164 kfree(buddy->num_free);
165 return -ENOMEM;
166}
167
168static void hns_roce_buddy_cleanup(struct hns_roce_buddy *buddy)
169{
170 int i;
171
172 for (i = 0; i <= buddy->max_order; ++i)
8d497eb0 173 kvfree(buddy->bits[i]);
9a443537 174
175 kfree(buddy->bits);
176 kfree(buddy->num_free);
177}
178
179static int hns_roce_alloc_mtt_range(struct hns_roce_dev *hr_dev, int order,
9766edc3 180 unsigned long *seg, u32 mtt_type)
9a443537 181{
182 struct hns_roce_mr_table *mr_table = &hr_dev->mr_table;
9766edc3
SX		 183 struct hns_roce_hem_table *table;
184 struct hns_roce_buddy *buddy;
185 int ret;
186
187 if (mtt_type == MTT_TYPE_WQE) {
188 buddy = &mr_table->mtt_buddy;
189 table = &mr_table->mtt_table;
190 } else {
191 buddy = &mr_table->mtt_cqe_buddy;
192 table = &mr_table->mtt_cqe_table;
193 }
9a443537 194
9766edc3 195 ret = hns_roce_buddy_alloc(buddy, order, seg);
9a443537 196 if (ret == -1)
197 return -1;
198
9766edc3 199 if (hns_roce_table_get_range(hr_dev, table, *seg,
9a443537 200 *seg + (1 << order) - 1)) {
9766edc3 201 hns_roce_buddy_free(buddy, *seg, order);
9a443537 202 return -1;
203 }
204
205 return 0;
206}
207
208int hns_roce_mtt_init(struct hns_roce_dev *hr_dev, int npages, int page_shift,
209 struct hns_roce_mtt *mtt)
210{
9766edc3 211 int ret;
9a443537 212 int i;
213
214 /* Page num is zero, correspond to DMA memory register */
215 if (!npages) {
216 mtt->order = -1;
217 mtt->page_shift = HNS_ROCE_HEM_PAGE_SHIFT;
218 return 0;
219 }
220
ad61dd30 221 /* Note: if page_shift is zero, FAST memory register */
9a443537 222 mtt->page_shift = page_shift;
223
224 /* Compute MTT entry necessary */
225 for (mtt->order = 0, i = HNS_ROCE_MTT_ENTRY_PER_SEG; i < npages;
226 i <<= 1)
227 ++mtt->order;
228
229 /* Allocate MTT entry */
9766edc3
SX		 230 ret = hns_roce_alloc_mtt_range(hr_dev, mtt->order, &mtt->first_seg,
231 mtt->mtt_type);
9a443537 232 if (ret == -1)
233 return -ENOMEM;
234
235 return 0;
236}
237
238void hns_roce_mtt_cleanup(struct hns_roce_dev *hr_dev, struct hns_roce_mtt *mtt)
239{
240 struct hns_roce_mr_table *mr_table = &hr_dev->mr_table;
241
242 if (mtt->order < 0)
243 return;
244
9766edc3
SX		 245 if (mtt->mtt_type == MTT_TYPE_WQE) {
246 hns_roce_buddy_free(&mr_table->mtt_buddy, mtt->first_seg,
247 mtt->order);
248 hns_roce_table_put_range(hr_dev, &mr_table->mtt_table,
249 mtt->first_seg,
250 mtt->first_seg + (1 << mtt->order) - 1);
251 } else {
252 hns_roce_buddy_free(&mr_table->mtt_cqe_buddy, mtt->first_seg,
253 mtt->order);
254 hns_roce_table_put_range(hr_dev, &mr_table->mtt_cqe_table,
255 mtt->first_seg,
256 mtt->first_seg + (1 << mtt->order) - 1);
257 }
9a443537 258}
08805fdb 259EXPORT_SYMBOL_GPL(hns_roce_mtt_cleanup);
9a443537 260
ff795f71
WHX		 261static void hns_roce_loop_free(struct hns_roce_dev *hr_dev,
262 struct hns_roce_mr *mr, int err_loop_index,
263 int loop_i, int loop_j)
264{
265 struct device *dev = hr_dev->dev;
266 u32 mhop_num;
267 u32 pbl_bt_sz;
268 u64 bt_idx;
269 int i, j;
270
271 pbl_bt_sz = 1 << (hr_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT);
272 mhop_num = hr_dev->caps.pbl_hop_num;
273
274 i = loop_i;
ff795f71
WHX		 275 if (mhop_num == 3 && err_loop_index == 2) {
276 for (; i >= 0; i--) {
277 dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l1[i],
278 mr->pbl_l1_dma_addr[i]);
279
280 for (j = 0; j < pbl_bt_sz / 8; j++) {
281 if (i == loop_i && j >= loop_j)
282 break;
283
284 bt_idx = i * pbl_bt_sz / 8 + j;
285 dma_free_coherent(dev, pbl_bt_sz,
286 mr->pbl_bt_l2[bt_idx],
287 mr->pbl_l2_dma_addr[bt_idx]);
288 }
289 }
290 } else if (mhop_num == 3 && err_loop_index == 1) {
291 for (i -= 1; i >= 0; i--) {
292 dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l1[i],
293 mr->pbl_l1_dma_addr[i]);
294
295 for (j = 0; j < pbl_bt_sz / 8; j++) {
296 bt_idx = i * pbl_bt_sz / 8 + j;
297 dma_free_coherent(dev, pbl_bt_sz,
298 mr->pbl_bt_l2[bt_idx],
299 mr->pbl_l2_dma_addr[bt_idx]);
300 }
301 }
302 } else if (mhop_num == 2 && err_loop_index == 1) {
303 for (i -= 1; i >= 0; i--)
304 dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l1[i],
305 mr->pbl_l1_dma_addr[i]);
306 } else {
307 dev_warn(dev, "not support: mhop_num=%d, err_loop_index=%d.",
308 mhop_num, err_loop_index);
309 return;
310 }
311
312 dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l0, mr->pbl_l0_dma_addr);
313 mr->pbl_bt_l0 = NULL;
314 mr->pbl_l0_dma_addr = 0;
315}
316
317/* PBL multi hop addressing */
318static int hns_roce_mhop_alloc(struct hns_roce_dev *hr_dev, int npages,
319 struct hns_roce_mr *mr)
320{
321 struct device *dev = hr_dev->dev;
322 int mr_alloc_done = 0;
323 int npages_allocated;
324 int i = 0, j = 0;
325 u32 pbl_bt_sz;
326 u32 mhop_num;
327 u64 pbl_last_bt_num;
328 u64 pbl_bt_cnt = 0;
329 u64 bt_idx;
330 u64 size;
331
332 mhop_num = hr_dev->caps.pbl_hop_num;
333 pbl_bt_sz = 1 << (hr_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT);
334 pbl_last_bt_num = (npages + pbl_bt_sz / 8 - 1) / (pbl_bt_sz / 8);
335
336 if (mhop_num == HNS_ROCE_HOP_NUM_0)
337 return 0;
338
339 /* hop_num = 1 */
340 if (mhop_num == 1) {
341 if (npages > pbl_bt_sz / 8) {
342 dev_err(dev, "npages %d is larger than buf_pg_sz!",
343 npages);
344 return -EINVAL;
345 }
346 mr->pbl_buf = dma_alloc_coherent(dev, npages * 8,
347 &(mr->pbl_dma_addr),
348 GFP_KERNEL);
349 if (!mr->pbl_buf)
350 return -ENOMEM;
351
352 mr->pbl_size = npages;
353 mr->pbl_ba = mr->pbl_dma_addr;
354 mr->pbl_hop_num = hr_dev->caps.pbl_hop_num;
355 mr->pbl_ba_pg_sz = hr_dev->caps.pbl_ba_pg_sz;
356 mr->pbl_buf_pg_sz = hr_dev->caps.pbl_buf_pg_sz;
357 return 0;
358 }
359
360 mr->pbl_l1_dma_addr = kcalloc(pbl_bt_sz / 8,
361 sizeof(*mr->pbl_l1_dma_addr),
362 GFP_KERNEL);
363 if (!mr->pbl_l1_dma_addr)
364 return -ENOMEM;
365
366 mr->pbl_bt_l1 = kcalloc(pbl_bt_sz / 8, sizeof(*mr->pbl_bt_l1),
367 GFP_KERNEL);
368 if (!mr->pbl_bt_l1)
369 goto err_kcalloc_bt_l1;
370
371 if (mhop_num == 3) {
372 mr->pbl_l2_dma_addr = kcalloc(pbl_last_bt_num,
373 sizeof(*mr->pbl_l2_dma_addr),
374 GFP_KERNEL);
375 if (!mr->pbl_l2_dma_addr)
376 goto err_kcalloc_l2_dma;
377
378 mr->pbl_bt_l2 = kcalloc(pbl_last_bt_num,
379 sizeof(*mr->pbl_bt_l2),
380 GFP_KERNEL);
381 if (!mr->pbl_bt_l2)
382 goto err_kcalloc_bt_l2;
383 }
384
385 /* alloc L0 BT */
386 mr->pbl_bt_l0 = dma_alloc_coherent(dev, pbl_bt_sz,
387 &(mr->pbl_l0_dma_addr),
388 GFP_KERNEL);
389 if (!mr->pbl_bt_l0)
390 goto err_dma_alloc_l0;
391
392 if (mhop_num == 2) {
393 /* alloc L1 BT */
394 for (i = 0; i < pbl_bt_sz / 8; i++) {
395 if (pbl_bt_cnt + 1 < pbl_last_bt_num) {
396 size = pbl_bt_sz;
397 } else {
398 npages_allocated = i * (pbl_bt_sz / 8);
399 size = (npages - npages_allocated) * 8;
400 }
401 mr->pbl_bt_l1[i] = dma_alloc_coherent(dev, size,
402 &(mr->pbl_l1_dma_addr[i]),
403 GFP_KERNEL);
404 if (!mr->pbl_bt_l1[i]) {
405 hns_roce_loop_free(hr_dev, mr, 1, i, 0);
406 goto err_dma_alloc_l0;
407 }
408
409 *(mr->pbl_bt_l0 + i) = mr->pbl_l1_dma_addr[i];
410
411 pbl_bt_cnt++;
412 if (pbl_bt_cnt >= pbl_last_bt_num)
413 break;
414 }
415 } else if (mhop_num == 3) {
416 /* alloc L1, L2 BT */
417 for (i = 0; i < pbl_bt_sz / 8; i++) {
418 mr->pbl_bt_l1[i] = dma_alloc_coherent(dev, pbl_bt_sz,
419 &(mr->pbl_l1_dma_addr[i]),
420 GFP_KERNEL);
421 if (!mr->pbl_bt_l1[i]) {
422 hns_roce_loop_free(hr_dev, mr, 1, i, 0);
423 goto err_dma_alloc_l0;
424 }
425
426 *(mr->pbl_bt_l0 + i) = mr->pbl_l1_dma_addr[i];
427
428 for (j = 0; j < pbl_bt_sz / 8; j++) {
429 bt_idx = i * pbl_bt_sz / 8 + j;
430
431 if (pbl_bt_cnt + 1 < pbl_last_bt_num) {
432 size = pbl_bt_sz;
433 } else {
434 npages_allocated = bt_idx *
435 (pbl_bt_sz / 8);
436 size = (npages - npages_allocated) * 8;
437 }
438 mr->pbl_bt_l2[bt_idx] = dma_alloc_coherent(
439 dev, size,
440 &(mr->pbl_l2_dma_addr[bt_idx]),
441 GFP_KERNEL);
442 if (!mr->pbl_bt_l2[bt_idx]) {
443 hns_roce_loop_free(hr_dev, mr, 2, i, j);
444 goto err_dma_alloc_l0;
445 }
446
447 *(mr->pbl_bt_l1[i] + j) =
448 mr->pbl_l2_dma_addr[bt_idx];
449
450 pbl_bt_cnt++;
451 if (pbl_bt_cnt >= pbl_last_bt_num) {
452 mr_alloc_done = 1;
453 break;
454 }
455 }
456
457 if (mr_alloc_done)
458 break;
459 }
460 }
461
462 mr->l0_chunk_last_num = i + 1;
463 if (mhop_num == 3)
464 mr->l1_chunk_last_num = j + 1;
465
466 mr->pbl_size = npages;
467 mr->pbl_ba = mr->pbl_l0_dma_addr;
468 mr->pbl_hop_num = hr_dev->caps.pbl_hop_num;
469 mr->pbl_ba_pg_sz = hr_dev->caps.pbl_ba_pg_sz;
470 mr->pbl_buf_pg_sz = hr_dev->caps.pbl_buf_pg_sz;
471
472 return 0;
473
474err_dma_alloc_l0:
475 kfree(mr->pbl_bt_l2);
476 mr->pbl_bt_l2 = NULL;
477
478err_kcalloc_bt_l2:
479 kfree(mr->pbl_l2_dma_addr);
480 mr->pbl_l2_dma_addr = NULL;
481
482err_kcalloc_l2_dma:
483 kfree(mr->pbl_bt_l1);
484 mr->pbl_bt_l1 = NULL;
485
486err_kcalloc_bt_l1:
487 kfree(mr->pbl_l1_dma_addr);
488 mr->pbl_l1_dma_addr = NULL;
489
490 return -ENOMEM;
491}
492
9a443537 493static int hns_roce_mr_alloc(struct hns_roce_dev *hr_dev, u32 pd, u64 iova,
494 u64 size, u32 access, int npages,
495 struct hns_roce_mr *mr)
496{
13ca970e 497 struct device *dev = hr_dev->dev;
9a443537 498 unsigned long index = 0;
499 int ret = 0;
9a443537 500
501 /* Allocate a key for mr from mr_table */
502 ret = hns_roce_bitmap_alloc(&hr_dev->mr_table.mtpt_bitmap, &index);
503 if (ret == -1)
504 return -ENOMEM;
505
506 mr->iova = iova; /* MR va starting addr */
507 mr->size = size; /* MR addr range */
508 mr->pd = pd; /* MR num */
509 mr->access = access; /* MR access permit */
510 mr->enabled = 0; /* MR active status */
511 mr->key = hw_index_to_key(index); /* MR key */
512
513 if (size == ~0ull) {
514 mr->type = MR_TYPE_DMA;
515 mr->pbl_buf = NULL;
516 mr->pbl_dma_addr = 0;
ff795f71
WHX		 517 /* PBL multi-hop addressing parameters */
518 mr->pbl_bt_l2 = NULL;
519 mr->pbl_bt_l1 = NULL;
520 mr->pbl_bt_l0 = NULL;
521 mr->pbl_l2_dma_addr = NULL;
522 mr->pbl_l1_dma_addr = NULL;
523 mr->pbl_l0_dma_addr = 0;
9a443537 524 } else {
525 mr->type = MR_TYPE_MR;
ff795f71
WHX		 526 if (!hr_dev->caps.pbl_hop_num) {
527 mr->pbl_buf = dma_alloc_coherent(dev, npages * 8,
528 &(mr->pbl_dma_addr),
529 GFP_KERNEL);
530 if (!mr->pbl_buf)
531 return -ENOMEM;
532 } else {
533 ret = hns_roce_mhop_alloc(hr_dev, npages, mr);
534 }
9a443537 535 }
536
ff795f71
WHX		 537 return ret;
538}
539
540static void hns_roce_mhop_free(struct hns_roce_dev *hr_dev,
541 struct hns_roce_mr *mr)
542{
543 struct device *dev = hr_dev->dev;
544 int npages_allocated;
545 int npages;
546 int i, j;
547 u32 pbl_bt_sz;
548 u32 mhop_num;
549 u64 bt_idx;
550
551 npages = ib_umem_page_count(mr->umem);
552 pbl_bt_sz = 1 << (hr_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT);
553 mhop_num = hr_dev->caps.pbl_hop_num;
554
555 if (mhop_num == HNS_ROCE_HOP_NUM_0)
556 return;
557
558 /* hop_num = 1 */
559 if (mhop_num == 1) {
560 dma_free_coherent(dev, (unsigned int)(npages * 8),
561 mr->pbl_buf, mr->pbl_dma_addr);
562 return;
563 }
564
565 dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l0,
566 mr->pbl_l0_dma_addr);
567
568 if (mhop_num == 2) {
569 for (i = 0; i < mr->l0_chunk_last_num; i++) {
570 if (i == mr->l0_chunk_last_num - 1) {
571 npages_allocated = i * (pbl_bt_sz / 8);
572
573 dma_free_coherent(dev,
574 (npages - npages_allocated) * 8,
575 mr->pbl_bt_l1[i],
576 mr->pbl_l1_dma_addr[i]);
577
578 break;
579 }
580
581 dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l1[i],
582 mr->pbl_l1_dma_addr[i]);
583 }
584 } else if (mhop_num == 3) {
585 for (i = 0; i < mr->l0_chunk_last_num; i++) {
586 dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l1[i],
587 mr->pbl_l1_dma_addr[i]);
588
589 for (j = 0; j < pbl_bt_sz / 8; j++) {
590 bt_idx = i * (pbl_bt_sz / 8) + j;
591
592 if ((i == mr->l0_chunk_last_num - 1)
593 && j == mr->l1_chunk_last_num - 1) {
594 npages_allocated = bt_idx *
595 (pbl_bt_sz / 8);
596
597 dma_free_coherent(dev,
598 (npages - npages_allocated) * 8,
599 mr->pbl_bt_l2[bt_idx],
600 mr->pbl_l2_dma_addr[bt_idx]);
601
602 break;
603 }
604
605 dma_free_coherent(dev, pbl_bt_sz,
606 mr->pbl_bt_l2[bt_idx],
607 mr->pbl_l2_dma_addr[bt_idx]);
608 }
609 }
610 }
611
612 kfree(mr->pbl_bt_l1);
613 kfree(mr->pbl_l1_dma_addr);
614 mr->pbl_bt_l1 = NULL;
615 mr->pbl_l1_dma_addr = NULL;
616 if (mhop_num == 3) {
617 kfree(mr->pbl_bt_l2);
618 kfree(mr->pbl_l2_dma_addr);
619 mr->pbl_bt_l2 = NULL;
620 mr->pbl_l2_dma_addr = NULL;
621 }
9a443537 622}
623
624static void hns_roce_mr_free(struct hns_roce_dev *hr_dev,
625 struct hns_roce_mr *mr)
626{
13ca970e 627 struct device *dev = hr_dev->dev;
9a443537 628 int npages = 0;
629 int ret;
630
631 if (mr->enabled) {
632 ret = hns_roce_hw2sw_mpt(hr_dev, NULL, key_to_hw_index(mr->key)
633 & (hr_dev->caps.num_mtpts - 1));
634 if (ret)
635 dev_warn(dev, "HW2SW_MPT failed (%d)\n", ret);
636 }
637
638 if (mr->size != ~0ULL) {
639 npages = ib_umem_page_count(mr->umem);
ff795f71
WHX		 640
641 if (!hr_dev->caps.pbl_hop_num)
642 dma_free_coherent(dev, (unsigned int)(npages * 8),
643 mr->pbl_buf, mr->pbl_dma_addr);
644 else
645 hns_roce_mhop_free(hr_dev, mr);
9a443537 646 }
647
ff795f71
WHX		 648 if (mr->enabled)
649 hns_roce_table_put(hr_dev, &hr_dev->mr_table.mtpt_table,
650 key_to_hw_index(mr->key));
651
9a443537 652 hns_roce_bitmap_free(&hr_dev->mr_table.mtpt_bitmap,
5e6ff78a 653 key_to_hw_index(mr->key), BITMAP_NO_RR);
9a443537 654}
655
656static int hns_roce_mr_enable(struct hns_roce_dev *hr_dev,
657 struct hns_roce_mr *mr)
658{
659 int ret;
660 unsigned long mtpt_idx = key_to_hw_index(mr->key);
13ca970e 661 struct device *dev = hr_dev->dev;
9a443537 662 struct hns_roce_cmd_mailbox *mailbox;
663 struct hns_roce_mr_table *mr_table = &hr_dev->mr_table;
664
665 /* Prepare HEM entry memory */
666 ret = hns_roce_table_get(hr_dev, &mr_table->mtpt_table, mtpt_idx);
667 if (ret)
668 return ret;
669
670 /* Allocate mailbox memory */
671 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
672 if (IS_ERR(mailbox)) {
673 ret = PTR_ERR(mailbox);
674 goto err_table;
675 }
676
677 ret = hr_dev->hw->write_mtpt(mailbox->buf, mr, mtpt_idx);
678 if (ret) {
679 dev_err(dev, "Write mtpt fail!\n");
680 goto err_page;
681 }
682
683 ret = hns_roce_sw2hw_mpt(hr_dev, mailbox,
684 mtpt_idx & (hr_dev->caps.num_mtpts - 1));
685 if (ret) {
686 dev_err(dev, "SW2HW_MPT failed (%d)\n", ret);
687 goto err_page;
688 }
689
690 mr->enabled = 1;
691 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
692
693 return 0;
694
695err_page:
696 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
697
698err_table:
699 hns_roce_table_put(hr_dev, &mr_table->mtpt_table, mtpt_idx);
700 return ret;
701}
702
703static int hns_roce_write_mtt_chunk(struct hns_roce_dev *hr_dev,
704 struct hns_roce_mtt *mtt, u32 start_index,
705 u32 npages, u64 *page_list)
706{
6a93c77a 707 struct hns_roce_hem_table *table;
9a443537 708 dma_addr_t dma_handle;
6a93c77a 709 __le64 *mtts;
9a443537 710 u32 s = start_index * sizeof(u64);
9a8982dc 711 u32 bt_page_size;
6a93c77a 712 u32 i;
9a443537 713
9a8982dc
WHX		 714 if (mtt->mtt_type == MTT_TYPE_WQE)
715 bt_page_size = 1 << (hr_dev->caps.mtt_ba_pg_sz + PAGE_SHIFT);
716 else
717 bt_page_size = 1 << (hr_dev->caps.cqe_ba_pg_sz + PAGE_SHIFT);
718
9a443537 719 /* All MTTs must fit in the same page */
9a8982dc
WHX		 720 if (start_index / (bt_page_size / sizeof(u64)) !=
721 (start_index + npages - 1) / (bt_page_size / sizeof(u64)))
9a443537 722 return -EINVAL;
723
724 if (start_index & (HNS_ROCE_MTT_ENTRY_PER_SEG - 1))
725 return -EINVAL;
726
9766edc3
SX		 727 if (mtt->mtt_type == MTT_TYPE_WQE)
728 table = &hr_dev->mr_table.mtt_table;
729 else
730 table = &hr_dev->mr_table.mtt_cqe_table;
731
6a93c77a 732 mtts = hns_roce_table_find(hr_dev, table,
9a443537 733 mtt->first_seg + s / hr_dev->caps.mtt_entry_sz,
734 &dma_handle);
735 if (!mtts)
736 return -ENOMEM;
737
738 /* Save page addr, low 12 bits : 0 */
6a93c77a
SX		 739 for (i = 0; i < npages; ++i) {
740 if (!hr_dev->caps.mtt_hop_num)
741 mtts[i] = cpu_to_le64(page_list[i] >> PAGE_ADDR_SHIFT);
742 else
743 mtts[i] = cpu_to_le64(page_list[i]);
744 }
9a443537 745
746 return 0;
747}
748
749static int hns_roce_write_mtt(struct hns_roce_dev *hr_dev,
750 struct hns_roce_mtt *mtt, u32 start_index,
751 u32 npages, u64 *page_list)
752{
753 int chunk;
754 int ret;
9a8982dc 755 u32 bt_page_size;
9a443537 756
757 if (mtt->order < 0)
758 return -EINVAL;
759
9a8982dc
WHX		 760 if (mtt->mtt_type == MTT_TYPE_WQE)
761 bt_page_size = 1 << (hr_dev->caps.mtt_ba_pg_sz + PAGE_SHIFT);
762 else
763 bt_page_size = 1 << (hr_dev->caps.cqe_ba_pg_sz + PAGE_SHIFT);
764
9a443537 765 while (npages > 0) {
9a8982dc 766 chunk = min_t(int, bt_page_size / sizeof(u64), npages);
9a443537 767
768 ret = hns_roce_write_mtt_chunk(hr_dev, mtt, start_index, chunk,
769 page_list);
770 if (ret)
771 return ret;
772
773 npages -= chunk;
774 start_index += chunk;
775 page_list += chunk;
776 }
777
778 return 0;
779}
780
781int hns_roce_buf_write_mtt(struct hns_roce_dev *hr_dev,
782 struct hns_roce_mtt *mtt, struct hns_roce_buf *buf)
783{
9766edc3
SX		 784 u64 *page_list;
785 int ret;
786 u32 i;
9a443537 787
788 page_list = kmalloc_array(buf->npages, sizeof(*page_list), GFP_KERNEL);
789 if (!page_list)
790 return -ENOMEM;
791
792 for (i = 0; i < buf->npages; ++i) {
793 if (buf->nbufs == 1)
794 page_list[i] = buf->direct.map + (i << buf->page_shift);
795 else
796 page_list[i] = buf->page_list[i].map;
797
798 }
799 ret = hns_roce_write_mtt(hr_dev, mtt, 0, buf->npages, page_list);
800
801 kfree(page_list);
802
803 return ret;
804}
805
806int hns_roce_init_mr_table(struct hns_roce_dev *hr_dev)
807{
808 struct hns_roce_mr_table *mr_table = &hr_dev->mr_table;
9766edc3 809 int ret;
9a443537 810
811 ret = hns_roce_bitmap_init(&mr_table->mtpt_bitmap,
812 hr_dev->caps.num_mtpts,
813 hr_dev->caps.num_mtpts - 1,
814 hr_dev->caps.reserved_mrws, 0);
815 if (ret)
816 return ret;
817
818 ret = hns_roce_buddy_init(&mr_table->mtt_buddy,
819 ilog2(hr_dev->caps.num_mtt_segs));
820 if (ret)
821 goto err_buddy;
822
9766edc3
SX		 823 if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE)) {
824 ret = hns_roce_buddy_init(&mr_table->mtt_cqe_buddy,
825 ilog2(hr_dev->caps.num_cqe_segs));
826 if (ret)
827 goto err_buddy_cqe;
828 }
9a443537 829 return 0;
830
9766edc3
SX		 831err_buddy_cqe:
832 hns_roce_buddy_cleanup(&mr_table->mtt_buddy);
833
9a443537 834err_buddy:
835 hns_roce_bitmap_cleanup(&mr_table->mtpt_bitmap);
836 return ret;
837}
838
839void hns_roce_cleanup_mr_table(struct hns_roce_dev *hr_dev)
840{
841 struct hns_roce_mr_table *mr_table = &hr_dev->mr_table;
842
843 hns_roce_buddy_cleanup(&mr_table->mtt_buddy);
9766edc3
SX		 844 if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE))
845 hns_roce_buddy_cleanup(&mr_table->mtt_cqe_buddy);
9a443537 846 hns_roce_bitmap_cleanup(&mr_table->mtpt_bitmap);
847}
848
849struct ib_mr *hns_roce_get_dma_mr(struct ib_pd *pd, int acc)
850{
ff795f71
WHX		 851 struct hns_roce_mr *mr;
852 int ret;
9a443537 853
854 mr = kmalloc(sizeof(*mr), GFP_KERNEL);
855 if (mr == NULL)
856 return ERR_PTR(-ENOMEM);
857
858 /* Allocate memory region key */
859 ret = hns_roce_mr_alloc(to_hr_dev(pd->device), to_hr_pd(pd)->pdn, 0,
860 ~0ULL, acc, 0, mr);
861 if (ret)
862 goto err_free;
863
864 ret = hns_roce_mr_enable(to_hr_dev(pd->device), mr);
865 if (ret)
866 goto err_mr;
867
868 mr->ibmr.rkey = mr->ibmr.lkey = mr->key;
869 mr->umem = NULL;
870
871 return &mr->ibmr;
872
873err_mr:
874 hns_roce_mr_free(to_hr_dev(pd->device), mr);
875
876err_free:
877 kfree(mr);
878 return ERR_PTR(ret);
879}
880
881int hns_roce_ib_umem_write_mtt(struct hns_roce_dev *hr_dev,
882 struct hns_roce_mtt *mtt, struct ib_umem *umem)
883{
9a8982dc 884 struct device *dev = hr_dev->dev;
9a443537 885 struct scatterlist *sg;
9a8982dc 886 unsigned int order;
9a443537 887 int i, k, entry;
9a8982dc 888 int npage = 0;
9a443537 889 int ret = 0;
9a8982dc
WHX		 890 int len;
891 u64 page_addr;
9a443537 892 u64 *pages;
9a8982dc 893 u32 bt_page_size;
9a443537 894 u32 n;
9a443537 895
9a8982dc
WHX		 896 order = mtt->mtt_type == MTT_TYPE_WQE ? hr_dev->caps.mtt_ba_pg_sz :
897 hr_dev->caps.cqe_ba_pg_sz;
898 bt_page_size = 1 << (order + PAGE_SHIFT);
899
900 pages = (u64 *) __get_free_pages(GFP_KERNEL, order);
9a443537 901 if (!pages)
902 return -ENOMEM;
903
904 i = n = 0;
905
906 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
9a8982dc 907 len = sg_dma_len(sg) >> PAGE_SHIFT;
9a443537 908 for (k = 0; k < len; ++k) {
9a8982dc
WHX		 909 page_addr =
910 sg_dma_address(sg) + (k << umem->page_shift);
911 if (!(npage % (1 << (mtt->page_shift - PAGE_SHIFT)))) {
912 if (page_addr & ((1 << mtt->page_shift) - 1)) {
913 dev_err(dev, "page_addr 0x%llx is not page_shift %d alignment!\n",
914 page_addr, mtt->page_shift);
915 ret = -EINVAL;
916 goto out;
917 }
918 pages[i++] = page_addr;
919 }
920 npage++;
921 if (i == bt_page_size / sizeof(u64)) {
9a443537 922 ret = hns_roce_write_mtt(hr_dev, mtt, n, i,
923 pages);
924 if (ret)
925 goto out;
926 n += i;
927 i = 0;
928 }
929 }
930 }
931
932 if (i)
933 ret = hns_roce_write_mtt(hr_dev, mtt, n, i, pages);
934
935out:
d480bb50 936 free_pages((unsigned long) pages, order);
9a443537 937 return ret;
938}
939
ff795f71
WHX		 940static int hns_roce_ib_umem_write_mr(struct hns_roce_dev *hr_dev,
941 struct hns_roce_mr *mr,
9a443537 942 struct ib_umem *umem)
943{
9a443537 944 struct scatterlist *sg;
9a8982dc 945 int i = 0, j = 0, k;
ff795f71 946 int entry;
9a8982dc
WHX		 947 int len;
948 u64 page_addr;
949 u32 pbl_bt_sz;
ff795f71
WHX		 950
951 if (hr_dev->caps.pbl_hop_num == HNS_ROCE_HOP_NUM_0)
952 return 0;
9a443537 953
9a8982dc 954 pbl_bt_sz = 1 << (hr_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT);
9a443537 955 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
9a8982dc
WHX		 956 len = sg_dma_len(sg) >> PAGE_SHIFT;
957 for (k = 0; k < len; ++k) {
958 page_addr = sg_dma_address(sg) +
959 (k << umem->page_shift);
960
961 if (!hr_dev->caps.pbl_hop_num) {
962 mr->pbl_buf[i++] = page_addr >> 12;
963 } else if (hr_dev->caps.pbl_hop_num == 1) {
964 mr->pbl_buf[i++] = page_addr;
965 } else {
966 if (hr_dev->caps.pbl_hop_num == 2)
967 mr->pbl_bt_l1[i][j] = page_addr;
968 else if (hr_dev->caps.pbl_hop_num == 3)
969 mr->pbl_bt_l2[i][j] = page_addr;
970
971 j++;
972 if (j >= (pbl_bt_sz / 8)) {
973 i++;
974 j = 0;
975 }
ff795f71
WHX		 976 }
977 }
9a443537 978 }
979
980 /* Memory barrier */
981 mb();
982
983 return 0;
984}
985
986struct ib_mr *hns_roce_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
987 u64 virt_addr, int access_flags,
988 struct ib_udata *udata)
989{
990 struct hns_roce_dev *hr_dev = to_hr_dev(pd->device);
13ca970e 991 struct device *dev = hr_dev->dev;
ff795f71
WHX		 992 struct hns_roce_mr *mr;
993 int bt_size;
994 int ret;
995 int n;
996 int i;
9a443537 997
998 mr = kmalloc(sizeof(*mr), GFP_KERNEL);
999 if (!mr)
1000 return ERR_PTR(-ENOMEM);
1001
1002 mr->umem = ib_umem_get(pd->uobject->context, start, length,
1003 access_flags, 0);
1004 if (IS_ERR(mr->umem)) {
1005 ret = PTR_ERR(mr->umem);
1006 goto err_free;
1007 }
1008
1009 n = ib_umem_page_count(mr->umem);
9a443537 1010
ff795f71
WHX		 1011 if (!hr_dev->caps.pbl_hop_num) {
1012 if (n > HNS_ROCE_MAX_MTPT_PBL_NUM) {
1013 dev_err(dev,
1014 " MR len %lld err. MR is limited to 4G at most!\n",
1015 length);
1016 ret = -EINVAL;
1017 goto err_umem;
1018 }
1019 } else {
1020 int pbl_size = 1;
1021
9a8982dc 1022 bt_size = (1 << (hr_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT)) / 8;
ff795f71
WHX		 1023 for (i = 0; i < hr_dev->caps.pbl_hop_num; i++)
1024 pbl_size *= bt_size;
1025 if (n > pbl_size) {
1026 dev_err(dev,
1027 " MR len %lld err. MR page num is limited to %d!\n",
1028 length, pbl_size);
1029 ret = -EINVAL;
1030 goto err_umem;
1031 }
9a443537 1032 }
1033
1034 ret = hns_roce_mr_alloc(hr_dev, to_hr_pd(pd)->pdn, virt_addr, length,
1035 access_flags, n, mr);
1036 if (ret)
1037 goto err_umem;
1038
ff795f71 1039 ret = hns_roce_ib_umem_write_mr(hr_dev, mr, mr->umem);
9a443537 1040 if (ret)
1041 goto err_mr;
1042
1043 ret = hns_roce_mr_enable(hr_dev, mr);
1044 if (ret)
1045 goto err_mr;
1046
1047 mr->ibmr.rkey = mr->ibmr.lkey = mr->key;
1048
1049 return &mr->ibmr;
1050
1051err_mr:
1052 hns_roce_mr_free(hr_dev, mr);
1053
1054err_umem:
1055 ib_umem_release(mr->umem);
1056
1057err_free:
1058 kfree(mr);
1059 return ERR_PTR(ret);
1060}
1061
a2c80b7b
WHX		 1062int hns_roce_rereg_user_mr(struct ib_mr *ibmr, int flags, u64 start, u64 length,
1063 u64 virt_addr, int mr_access_flags, struct ib_pd *pd,
1064 struct ib_udata *udata)
1065{
1066 struct hns_roce_dev *hr_dev = to_hr_dev(ibmr->device);
1067 struct hns_roce_mr *mr = to_hr_mr(ibmr);
1068 struct hns_roce_cmd_mailbox *mailbox;
1069 struct device *dev = hr_dev->dev;
1070 unsigned long mtpt_idx;
1071 u32 pdn = 0;
1072 int npages;
1073 int ret;
1074
1075 if (!mr->enabled)
1076 return -EINVAL;
1077
1078 mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
1079 if (IS_ERR(mailbox))
1080 return PTR_ERR(mailbox);
1081
1082 mtpt_idx = key_to_hw_index(mr->key) & (hr_dev->caps.num_mtpts - 1);
1083 ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma, mtpt_idx, 0,
1084 HNS_ROCE_CMD_QUERY_MPT,
1085 HNS_ROCE_CMD_TIMEOUT_MSECS);
1086 if (ret)
1087 goto free_cmd_mbox;
1088
1089 ret = hns_roce_hw2sw_mpt(hr_dev, NULL, mtpt_idx);
1090 if (ret)
1091 dev_warn(dev, "HW2SW_MPT failed (%d)\n", ret);
1092
1093 mr->enabled = 0;
1094
1095 if (flags & IB_MR_REREG_PD)
1096 pdn = to_hr_pd(pd)->pdn;
1097
1098 if (flags & IB_MR_REREG_TRANS) {
1099 if (mr->size != ~0ULL) {
1100 npages = ib_umem_page_count(mr->umem);
1101
1102 if (hr_dev->caps.pbl_hop_num)
1103 hns_roce_mhop_free(hr_dev, mr);
1104 else
1105 dma_free_coherent(dev, npages * 8, mr->pbl_buf,
1106 mr->pbl_dma_addr);
1107 }
1108 ib_umem_release(mr->umem);
1109
1110 mr->umem = ib_umem_get(ibmr->uobject->context, start, length,
1111 mr_access_flags, 0);
1112 if (IS_ERR(mr->umem)) {
1113 ret = PTR_ERR(mr->umem);
1114 mr->umem = NULL;
1115 goto free_cmd_mbox;
1116 }
1117 npages = ib_umem_page_count(mr->umem);
1118
1119 if (hr_dev->caps.pbl_hop_num) {
1120 ret = hns_roce_mhop_alloc(hr_dev, npages, mr);
1121 if (ret)
1122 goto release_umem;
1123 } else {
1124 mr->pbl_buf = dma_alloc_coherent(dev, npages * 8,
1125 &(mr->pbl_dma_addr),
1126 GFP_KERNEL);
1127 if (!mr->pbl_buf) {
1128 ret = -ENOMEM;
1129 goto release_umem;
1130 }
1131 }
1132 }
1133
1134 ret = hr_dev->hw->rereg_write_mtpt(hr_dev, mr, flags, pdn,
1135 mr_access_flags, virt_addr,
1136 length, mailbox->buf);
1137 if (ret) {
1138 if (flags & IB_MR_REREG_TRANS)
1139 goto release_umem;
1140 else
1141 goto free_cmd_mbox;
1142 }
1143
1144 if (flags & IB_MR_REREG_TRANS) {
1145 ret = hns_roce_ib_umem_write_mr(hr_dev, mr, mr->umem);
1146 if (ret) {
1147 if (mr->size != ~0ULL) {
1148 npages = ib_umem_page_count(mr->umem);
1149
1150 if (hr_dev->caps.pbl_hop_num)
1151 hns_roce_mhop_free(hr_dev, mr);
1152 else
1153 dma_free_coherent(dev, npages * 8,
1154 mr->pbl_buf,
1155 mr->pbl_dma_addr);
1156 }
1157
1158 goto release_umem;
1159 }
1160 }
1161
1162 ret = hns_roce_sw2hw_mpt(hr_dev, mailbox, mtpt_idx);
1163 if (ret) {
1164 dev_err(dev, "SW2HW_MPT failed (%d)\n", ret);
1165 goto release_umem;
1166 }
1167
1168 mr->enabled = 1;
1169 if (flags & IB_MR_REREG_ACCESS)
1170 mr->access = mr_access_flags;
1171
1172 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
1173
1174 return 0;
1175
1176release_umem:
1177 ib_umem_release(mr->umem);
1178
1179free_cmd_mbox:
1180 hns_roce_free_cmd_mailbox(hr_dev, mailbox);
1181
1182 return ret;
1183}
1184
9a443537 1185int hns_roce_dereg_mr(struct ib_mr *ibmr)
1186{
bfcc681b 1187 struct hns_roce_dev *hr_dev = to_hr_dev(ibmr->device);
9a443537 1188 struct hns_roce_mr *mr = to_hr_mr(ibmr);
bfcc681b 1189 int ret = 0;
9a443537 1190
bfcc681b
SX		 1191 if (hr_dev->hw->dereg_mr) {
1192 ret = hr_dev->hw->dereg_mr(hr_dev, mr);
1193 } else {
1194 hns_roce_mr_free(hr_dev, mr);
9a443537 1195
bfcc681b
SX		 1196 if (mr->umem)
1197 ib_umem_release(mr->umem);
9a443537 1198
bfcc681b
SX		 1199 kfree(mr);
1200 }
1201
1202 return ret;
9a443537 1203}
Linux 6.x block layer and io_uring tree(s)