drivers/infiniband/hw/hns/hns_roce_hem.c

   1 /*
   2  * Copyright (c) 2016 Hisilicon Limited.
   3  * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
   4  *
   5  * This software is available to you under a choice of one of two
   6  * licenses.  You may choose to be licensed under the terms of the GNU
   7  * General Public License (GPL) Version 2, available from the file
   8  * COPYING in the main directory of this source tree, or the
   9  * OpenIB.org BSD license below:
  10  *
  11  *     Redistribution and use in source and binary forms, with or
  12  *     without modification, are permitted provided that the following
  13  *     conditions are met:
  14  *
  15  *      - Redistributions of source code must retain the above
  16  *        copyright notice, this list of conditions and the following
  17  *        disclaimer.
  18  *
  19  *      - Redistributions in binary form must reproduce the above
  20  *        copyright notice, this list of conditions and the following
  21  *        disclaimer in the documentation and/or other materials
  22  *        provided with the distribution.
  23  *
  24  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  25  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  26  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  27  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  28  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  29  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  30  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  31  * SOFTWARE.
  32  */
  33
  34 #include <linux/platform_device.h>
  35 #include "hns_roce_device.h"
  36 #include "hns_roce_hem.h"
  37 #include "hns_roce_common.h"
  38
  39 #define DMA_ADDR_T_SHIFT                12
  40 #define BT_BA_SHIFT                     32
  41
  42 bool hns_roce_check_whether_mhop(struct hns_roce_dev *hr_dev, u32 type)
  43 {
  44         if ((hr_dev->caps.qpc_hop_num && type == HEM_TYPE_QPC) ||
  45             (hr_dev->caps.mpt_hop_num && type == HEM_TYPE_MTPT) ||
  46             (hr_dev->caps.cqc_hop_num && type == HEM_TYPE_CQC) ||
  47             (hr_dev->caps.srqc_hop_num && type == HEM_TYPE_SRQC) ||
  48             (hr_dev->caps.cqe_hop_num && type == HEM_TYPE_CQE) ||
  49             (hr_dev->caps.mtt_hop_num && type == HEM_TYPE_MTT))
  50                 return true;
  51
  52         return false;
  53 }
  54 EXPORT_SYMBOL_GPL(hns_roce_check_whether_mhop);
  55
  56 static bool hns_roce_check_hem_null(struct hns_roce_hem **hem, u64 start_idx,
  57                             u32 bt_chunk_num)
  58 {
  59         int i;
  60
  61         for (i = 0; i < bt_chunk_num; i++)
  62                 if (hem[start_idx + i])
  63                         return false;
  64
  65         return true;
  66 }
  67
  68 static bool hns_roce_check_bt_null(u64 **bt, u64 start_idx, u32 bt_chunk_num)
  69 {
  70         int i;
  71
  72         for (i = 0; i < bt_chunk_num; i++)
  73                 if (bt[start_idx + i])
  74                         return false;
  75
  76         return true;
  77 }
  78
  79 static int hns_roce_get_bt_num(u32 table_type, u32 hop_num)
  80 {
  81         if (check_whether_bt_num_3(table_type, hop_num))
  82                 return 3;
  83         else if (check_whether_bt_num_2(table_type, hop_num))
  84                 return 2;
  85         else if (check_whether_bt_num_1(table_type, hop_num))
  86                 return 1;
  87         else
  88                 return 0;
  89 }
  90
  91 int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev,
  92                            struct hns_roce_hem_table *table, unsigned long *obj,
  93                            struct hns_roce_hem_mhop *mhop)
  94 {
  95         struct device *dev = hr_dev->dev;
  96         u32 chunk_ba_num;
  97         u32 table_idx;
  98         u32 bt_num;
  99         u32 chunk_size;
 100
 101         switch (table->type) {
 102         case HEM_TYPE_QPC:
 103                 mhop->buf_chunk_size = 1 << (hr_dev->caps.qpc_buf_pg_sz
 104                                              + PAGE_SHIFT);
 105                 mhop->bt_chunk_size = 1 << (hr_dev->caps.qpc_ba_pg_sz
 106                                              + PAGE_SHIFT);
 107                 mhop->ba_l0_num = hr_dev->caps.qpc_bt_num;
 108                 mhop->hop_num = hr_dev->caps.qpc_hop_num;
 109                 break;
 110         case HEM_TYPE_MTPT:
 111                 mhop->buf_chunk_size = 1 << (hr_dev->caps.mpt_buf_pg_sz
 112                                              + PAGE_SHIFT);
 113                 mhop->bt_chunk_size = 1 << (hr_dev->caps.mpt_ba_pg_sz
 114                                              + PAGE_SHIFT);
 115                 mhop->ba_l0_num = hr_dev->caps.mpt_bt_num;
 116                 mhop->hop_num = hr_dev->caps.mpt_hop_num;
 117                 break;
 118         case HEM_TYPE_CQC:
 119                 mhop->buf_chunk_size = 1 << (hr_dev->caps.cqc_buf_pg_sz
 120                                              + PAGE_SHIFT);
 121                 mhop->bt_chunk_size = 1 << (hr_dev->caps.cqc_ba_pg_sz
 122                                             + PAGE_SHIFT);
 123                 mhop->ba_l0_num = hr_dev->caps.cqc_bt_num;
 124                 mhop->hop_num = hr_dev->caps.cqc_hop_num;
 125                 break;
 126         case HEM_TYPE_SRQC:
 127                 mhop->buf_chunk_size = 1 << (hr_dev->caps.srqc_buf_pg_sz
 128                                              + PAGE_SHIFT);
 129                 mhop->bt_chunk_size = 1 << (hr_dev->caps.srqc_ba_pg_sz
 130                                              + PAGE_SHIFT);
 131                 mhop->ba_l0_num = hr_dev->caps.srqc_bt_num;
 132                 mhop->hop_num = hr_dev->caps.srqc_hop_num;
 133                 break;
 134         case HEM_TYPE_MTT:
 135                 mhop->buf_chunk_size = 1 << (hr_dev->caps.mtt_buf_pg_sz
 136                                              + PAGE_SHIFT);
 137                 mhop->bt_chunk_size = 1 << (hr_dev->caps.mtt_ba_pg_sz
 138                                              + PAGE_SHIFT);
 139                 mhop->ba_l0_num = mhop->bt_chunk_size / 8;
 140                 mhop->hop_num = hr_dev->caps.mtt_hop_num;
 141                 break;
 142         case HEM_TYPE_CQE:
 143                 mhop->buf_chunk_size = 1 << (hr_dev->caps.cqe_buf_pg_sz
 144                                              + PAGE_SHIFT);
 145                 mhop->bt_chunk_size = 1 << (hr_dev->caps.cqe_ba_pg_sz
 146                                              + PAGE_SHIFT);
 147                 mhop->ba_l0_num = mhop->bt_chunk_size / 8;
 148                 mhop->hop_num = hr_dev->caps.cqe_hop_num;
 149                 break;
 150         default:
 151                 dev_err(dev, "Table %d not support multi-hop addressing!\n",
 152                          table->type);
 153                 return -EINVAL;
 154         }
 155
 156         if (!obj)
 157                 return 0;
 158
 159         /*
 160          * QPC/MTPT/CQC/SRQC alloc hem for buffer pages.
 161          * MTT/CQE alloc hem for bt pages.
 162          */
 163         bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num);
 164         chunk_ba_num = mhop->bt_chunk_size / 8;
 165         chunk_size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size :
 166                               mhop->bt_chunk_size;
 167         table_idx = (*obj & (table->num_obj - 1)) /
 168                      (chunk_size / table->obj_size);
 169         switch (bt_num) {
 170         case 3:
 171                 mhop->l2_idx = table_idx & (chunk_ba_num - 1);
 172                 mhop->l1_idx = table_idx / chunk_ba_num & (chunk_ba_num - 1);
 173                 mhop->l0_idx = table_idx / chunk_ba_num / chunk_ba_num;
 174                 break;
 175         case 2:
 176                 mhop->l1_idx = table_idx & (chunk_ba_num - 1);
 177                 mhop->l0_idx = table_idx / chunk_ba_num;
 178                 break;
 179         case 1:
 180                 mhop->l0_idx = table_idx;
 181                 break;
 182         default:
 183                 dev_err(dev, "Table %d not support hop_num = %d!\n",
 184                              table->type, mhop->hop_num);
 185                 return -EINVAL;
 186         }
 187         if (mhop->l0_idx >= mhop->ba_l0_num)
 188                 mhop->l0_idx %= mhop->ba_l0_num;
 189
 190         return 0;
 191 }
 192 EXPORT_SYMBOL_GPL(hns_roce_calc_hem_mhop);
 193
 194 static struct hns_roce_hem *hns_roce_alloc_hem(struct hns_roce_dev *hr_dev,
 195                                                int npages,
 196                                                unsigned long hem_alloc_size,
 197                                                gfp_t gfp_mask)
 198 {
 199         struct hns_roce_hem_chunk *chunk = NULL;
 200         struct hns_roce_hem *hem;
 201         struct scatterlist *mem;
 202         int order;
 203         void *buf;
 204
 205         WARN_ON(gfp_mask & __GFP_HIGHMEM);
 206
 207         hem = kmalloc(sizeof(*hem),
 208                       gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN));
 209         if (!hem)
 210                 return NULL;
 211
 212         hem->refcount = 0;
 213         INIT_LIST_HEAD(&hem->chunk_list);
 214
 215         order = get_order(hem_alloc_size);
 216
 217         while (npages > 0) {
 218                 if (!chunk) {
 219                         chunk = kmalloc(sizeof(*chunk),
 220                                 gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN));
 221                         if (!chunk)
 222                                 goto fail;
 223
 224                         sg_init_table(chunk->mem, HNS_ROCE_HEM_CHUNK_LEN);
 225                         chunk->npages = 0;
 226                         chunk->nsg = 0;
 227                         list_add_tail(&chunk->list, &hem->chunk_list);
 228                 }
 229
 230                 while (1 << order > npages)
 231                         --order;
 232
 233                 /*
 234                  * Alloc memory one time. If failed, don't alloc small block
 235                  * memory, directly return fail.
 236                  */
 237                 mem = &chunk->mem[chunk->npages];
 238                 buf = dma_alloc_coherent(hr_dev->dev, PAGE_SIZE << order,
 239                                 &sg_dma_address(mem), gfp_mask);
 240                 if (!buf)
 241                         goto fail;
 242
 243                 sg_set_buf(mem, buf, PAGE_SIZE << order);
 244                 WARN_ON(mem->offset);
 245                 sg_dma_len(mem) = PAGE_SIZE << order;
 246
 247                 ++chunk->npages;
 248                 ++chunk->nsg;
 249                 npages -= 1 << order;
 250         }
 251
 252         return hem;
 253
 254 fail:
 255         hns_roce_free_hem(hr_dev, hem);
 256         return NULL;
 257 }
 258
 259 void hns_roce_free_hem(struct hns_roce_dev *hr_dev, struct hns_roce_hem *hem)
 260 {
 261         struct hns_roce_hem_chunk *chunk, *tmp;
 262         int i;
 263
 264         if (!hem)
 265                 return;
 266
 267         list_for_each_entry_safe(chunk, tmp, &hem->chunk_list, list) {
 268                 for (i = 0; i < chunk->npages; ++i)
 269                         dma_free_coherent(hr_dev->dev,
 270                                    chunk->mem[i].length,
 271                                    lowmem_page_address(sg_page(&chunk->mem[i])),
 272                                    sg_dma_address(&chunk->mem[i]));
 273                 kfree(chunk);
 274         }
 275
 276         kfree(hem);
 277 }
 278
 279 static int hns_roce_set_hem(struct hns_roce_dev *hr_dev,
 280                             struct hns_roce_hem_table *table, unsigned long obj)
 281 {
 282         spinlock_t *lock = &hr_dev->bt_cmd_lock;
 283         struct device *dev = hr_dev->dev;
 284         unsigned long end = 0;
 285         unsigned long flags;
 286         struct hns_roce_hem_iter iter;
 287         void __iomem *bt_cmd;
 288         u32 bt_cmd_h_val = 0;
 289         u32 bt_cmd_val[2];
 290         u32 bt_cmd_l = 0;
 291         u64 bt_ba = 0;
 292         int ret = 0;
 293
 294         /* Find the HEM(Hardware Entry Memory) entry */
 295         unsigned long i = (obj & (table->num_obj - 1)) /
 296                           (table->table_chunk_size / table->obj_size);
 297
 298         switch (table->type) {
 299         case HEM_TYPE_QPC:
 300                 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
 301                                ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, HEM_TYPE_QPC);
 302                 break;
 303         case HEM_TYPE_MTPT:
 304                 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
 305                                ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S,
 306                                HEM_TYPE_MTPT);
 307                 break;
 308         case HEM_TYPE_CQC:
 309                 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
 310                                ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, HEM_TYPE_CQC);
 311                 break;
 312         case HEM_TYPE_SRQC:
 313                 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
 314                                ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S,
 315                                HEM_TYPE_SRQC);
 316                 break;
 317         default:
 318                 return ret;
 319         }
 320         roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_M,
 321                        ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_S, obj);
 322         roce_set_bit(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_S, 0);
 323         roce_set_bit(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_HW_SYNS_S, 1);
 324
 325         /* Currently iter only a chunk */
 326         for (hns_roce_hem_first(table->hem[i], &iter);
 327              !hns_roce_hem_last(&iter); hns_roce_hem_next(&iter)) {
 328                 bt_ba = hns_roce_hem_addr(&iter) >> DMA_ADDR_T_SHIFT;
 329
 330                 spin_lock_irqsave(lock, flags);
 331
 332                 bt_cmd = hr_dev->reg_base + ROCEE_BT_CMD_H_REG;
 333
 334                 end = msecs_to_jiffies(HW_SYNC_TIMEOUT_MSECS) + jiffies;
 335                 while (1) {
 336                         if (readl(bt_cmd) >> BT_CMD_SYNC_SHIFT) {
 337                                 if (!(time_before(jiffies, end))) {
 338                                         dev_err(dev, "Write bt_cmd err,hw_sync is not zero.\n");
 339                                         spin_unlock_irqrestore(lock, flags);
 340                                         return -EBUSY;
 341                                 }
 342                         } else {
 343                                 break;
 344                         }
 345                         msleep(HW_SYNC_SLEEP_TIME_INTERVAL);
 346                 }
 347
 348                 bt_cmd_l = (u32)bt_ba;
 349                 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_M,
 350                                ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_S,
 351                                bt_ba >> BT_BA_SHIFT);
 352
 353                 bt_cmd_val[0] = bt_cmd_l;
 354                 bt_cmd_val[1] = bt_cmd_h_val;
 355                 hns_roce_write64_k(bt_cmd_val,
 356                                    hr_dev->reg_base + ROCEE_BT_CMD_L_REG);
 357                 spin_unlock_irqrestore(lock, flags);
 358         }
 359
 360         return ret;
 361 }
 362
 363 static int hns_roce_table_mhop_get(struct hns_roce_dev *hr_dev,
 364                                    struct hns_roce_hem_table *table,
 365                                    unsigned long obj)
 366 {
 367         struct device *dev = hr_dev->dev;
 368         struct hns_roce_hem_mhop mhop;
 369         struct hns_roce_hem_iter iter;
 370         u32 buf_chunk_size;
 371         u32 bt_chunk_size;
 372         u32 chunk_ba_num;
 373         u32 hop_num;
 374         u32 size;
 375         u32 bt_num;
 376         u64 hem_idx;
 377         u64 bt_l1_idx = 0;
 378         u64 bt_l0_idx = 0;
 379         u64 bt_ba;
 380         unsigned long mhop_obj = obj;
 381         int bt_l1_allocated = 0;
 382         int bt_l0_allocated = 0;
 383         int step_idx;
 384         int ret;
 385
 386         ret = hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop);
 387         if (ret)
 388                 return ret;
 389
 390         buf_chunk_size = mhop.buf_chunk_size;
 391         bt_chunk_size = mhop.bt_chunk_size;
 392         hop_num = mhop.hop_num;
 393         chunk_ba_num = bt_chunk_size / 8;
 394
 395         bt_num = hns_roce_get_bt_num(table->type, hop_num);
 396         switch (bt_num) {
 397         case 3:
 398                 hem_idx = mhop.l0_idx * chunk_ba_num * chunk_ba_num +
 399                           mhop.l1_idx * chunk_ba_num + mhop.l2_idx;
 400                 bt_l1_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx;
 401                 bt_l0_idx = mhop.l0_idx;
 402                 break;
 403         case 2:
 404                 hem_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx;
 405                 bt_l0_idx = mhop.l0_idx;
 406                 break;
 407         case 1:
 408                 hem_idx = mhop.l0_idx;
 409                 break;
 410         default:
 411                 dev_err(dev, "Table %d not support hop_num = %d!\n",
 412                              table->type, hop_num);
 413                 return -EINVAL;
 414         }
 415
 416         mutex_lock(&table->mutex);
 417
 418         if (table->hem[hem_idx]) {
 419                 ++table->hem[hem_idx]->refcount;
 420                 goto out;
 421         }
 422
 423         /* alloc L1 BA's chunk */
 424         if ((check_whether_bt_num_3(table->type, hop_num) ||
 425                 check_whether_bt_num_2(table->type, hop_num)) &&
 426                 !table->bt_l0[bt_l0_idx]) {
 427                 table->bt_l0[bt_l0_idx] = dma_alloc_coherent(dev, bt_chunk_size,
 428                                             &(table->bt_l0_dma_addr[bt_l0_idx]),
 429                                             GFP_KERNEL);
 430                 if (!table->bt_l0[bt_l0_idx]) {
 431                         ret = -ENOMEM;
 432                         goto out;
 433                 }
 434                 bt_l0_allocated = 1;
 435
 436                 /* set base address to hardware */
 437                 if (table->type < HEM_TYPE_MTT) {
 438                         step_idx = 0;
 439                         if (hr_dev->hw->set_hem(hr_dev, table, obj, step_idx)) {
 440                                 ret = -ENODEV;
 441                                 dev_err(dev, "set HEM base address to HW failed!\n");
 442                                 goto err_dma_alloc_l1;
 443                         }
 444                 }
 445         }
 446
 447         /* alloc L2 BA's chunk */
 448         if (check_whether_bt_num_3(table->type, hop_num) &&
 449             !table->bt_l1[bt_l1_idx])  {
 450                 table->bt_l1[bt_l1_idx] = dma_alloc_coherent(dev, bt_chunk_size,
 451                                             &(table->bt_l1_dma_addr[bt_l1_idx]),
 452                                             GFP_KERNEL);
 453                 if (!table->bt_l1[bt_l1_idx]) {
 454                         ret = -ENOMEM;
 455                         goto err_dma_alloc_l1;
 456                 }
 457                 bt_l1_allocated = 1;
 458                 *(table->bt_l0[bt_l0_idx] + mhop.l1_idx) =
 459                                                table->bt_l1_dma_addr[bt_l1_idx];
 460
 461                 /* set base address to hardware */
 462                 step_idx = 1;
 463                 if (hr_dev->hw->set_hem(hr_dev, table, obj, step_idx)) {
 464                         ret = -ENODEV;
 465                         dev_err(dev, "set HEM base address to HW failed!\n");
 466                         goto err_alloc_hem_buf;
 467                 }
 468         }
 469
 470         /*
 471          * alloc buffer space chunk for QPC/MTPT/CQC/SRQC.
 472          * alloc bt space chunk for MTT/CQE.
 473          */
 474         size = table->type < HEM_TYPE_MTT ? buf_chunk_size : bt_chunk_size;
 475         table->hem[hem_idx] = hns_roce_alloc_hem(hr_dev,
 476                                                 size >> PAGE_SHIFT,
 477                                                 size,
 478                                                 (table->lowmem ? GFP_KERNEL :
 479                                                 GFP_HIGHUSER) | __GFP_NOWARN);
 480         if (!table->hem[hem_idx]) {
 481                 ret = -ENOMEM;
 482                 goto err_alloc_hem_buf;
 483         }
 484
 485         hns_roce_hem_first(table->hem[hem_idx], &iter);
 486         bt_ba = hns_roce_hem_addr(&iter);
 487
 488         if (table->type < HEM_TYPE_MTT) {
 489                 if (hop_num == 2) {
 490                         *(table->bt_l1[bt_l1_idx] + mhop.l2_idx) = bt_ba;
 491                         step_idx = 2;
 492                 } else if (hop_num == 1) {
 493                         *(table->bt_l0[bt_l0_idx] + mhop.l1_idx) = bt_ba;
 494                         step_idx = 1;
 495                 } else if (hop_num == HNS_ROCE_HOP_NUM_0) {
 496                         step_idx = 0;
 497                 }
 498
 499                 /* set HEM base address to hardware */
 500                 if (hr_dev->hw->set_hem(hr_dev, table, obj, step_idx)) {
 501                         ret = -ENODEV;
 502                         dev_err(dev, "set HEM base address to HW failed!\n");
 503                         goto err_alloc_hem_buf;
 504                 }
 505         } else if (hop_num == 2) {
 506                 *(table->bt_l0[bt_l0_idx] + mhop.l1_idx) = bt_ba;
 507         }
 508
 509         ++table->hem[hem_idx]->refcount;
 510         goto out;
 511
 512 err_alloc_hem_buf:
 513         if (bt_l1_allocated) {
 514                 dma_free_coherent(dev, bt_chunk_size, table->bt_l1[bt_l1_idx],
 515                                   table->bt_l1_dma_addr[bt_l1_idx]);
 516                 table->bt_l1[bt_l1_idx] = NULL;
 517         }
 518
 519 err_dma_alloc_l1:
 520         if (bt_l0_allocated) {
 521                 dma_free_coherent(dev, bt_chunk_size, table->bt_l0[bt_l0_idx],
 522                                   table->bt_l0_dma_addr[bt_l0_idx]);
 523                 table->bt_l0[bt_l0_idx] = NULL;
 524         }
 525
 526 out:
 527         mutex_unlock(&table->mutex);
 528         return ret;
 529 }
 530
 531 int hns_roce_table_get(struct hns_roce_dev *hr_dev,
 532                        struct hns_roce_hem_table *table, unsigned long obj)
 533 {
 534         struct device *dev = hr_dev->dev;
 535         int ret = 0;
 536         unsigned long i;
 537
 538         if (hns_roce_check_whether_mhop(hr_dev, table->type))
 539                 return hns_roce_table_mhop_get(hr_dev, table, obj);
 540
 541         i = (obj & (table->num_obj - 1)) / (table->table_chunk_size /
 542              table->obj_size);
 543
 544         mutex_lock(&table->mutex);
 545
 546         if (table->hem[i]) {
 547                 ++table->hem[i]->refcount;
 548                 goto out;
 549         }
 550
 551         table->hem[i] = hns_roce_alloc_hem(hr_dev,
 552                                        table->table_chunk_size >> PAGE_SHIFT,
 553                                        table->table_chunk_size,
 554                                        (table->lowmem ? GFP_KERNEL :
 555                                         GFP_HIGHUSER) | __GFP_NOWARN);
 556         if (!table->hem[i]) {
 557                 ret = -ENOMEM;
 558                 goto out;
 559         }
 560
 561         /* Set HEM base address(128K/page, pa) to Hardware */
 562         if (hns_roce_set_hem(hr_dev, table, obj)) {
 563                 hns_roce_free_hem(hr_dev, table->hem[i]);
 564                 table->hem[i] = NULL;
 565                 ret = -ENODEV;
 566                 dev_err(dev, "set HEM base address to HW failed.\n");
 567                 goto out;
 568         }
 569
 570         ++table->hem[i]->refcount;
 571 out:
 572         mutex_unlock(&table->mutex);
 573         return ret;
 574 }
 575
 576 static void hns_roce_table_mhop_put(struct hns_roce_dev *hr_dev,
 577                                     struct hns_roce_hem_table *table,
 578                                     unsigned long obj,
 579                                     int check_refcount)
 580 {
 581         struct device *dev = hr_dev->dev;
 582         struct hns_roce_hem_mhop mhop;
 583         unsigned long mhop_obj = obj;
 584         u32 bt_chunk_size;
 585         u32 chunk_ba_num;
 586         u32 hop_num;
 587         u32 start_idx;
 588         u32 bt_num;
 589         u64 hem_idx;
 590         u64 bt_l1_idx = 0;
 591         int ret;
 592
 593         ret = hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop);
 594         if (ret)
 595                 return;
 596
 597         bt_chunk_size = mhop.bt_chunk_size;
 598         hop_num = mhop.hop_num;
 599         chunk_ba_num = bt_chunk_size / 8;
 600
 601         bt_num = hns_roce_get_bt_num(table->type, hop_num);
 602         switch (bt_num) {
 603         case 3:
 604                 hem_idx = mhop.l0_idx * chunk_ba_num * chunk_ba_num +
 605                           mhop.l1_idx * chunk_ba_num + mhop.l2_idx;
 606                 bt_l1_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx;
 607                 break;
 608         case 2:
 609                 hem_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx;
 610                 break;
 611         case 1:
 612                 hem_idx = mhop.l0_idx;
 613                 break;
 614         default:
 615                 dev_err(dev, "Table %d not support hop_num = %d!\n",
 616                              table->type, hop_num);
 617                 return;
 618         }
 619
 620         mutex_lock(&table->mutex);
 621
 622         if (check_refcount && (--table->hem[hem_idx]->refcount > 0)) {
 623                 mutex_unlock(&table->mutex);
 624                 return;
 625         }
 626
 627         if (table->type < HEM_TYPE_MTT && hop_num == 1) {
 628                 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 1))
 629                         dev_warn(dev, "Clear HEM base address failed.\n");
 630         } else if (table->type < HEM_TYPE_MTT && hop_num == 2) {
 631                 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 2))
 632                         dev_warn(dev, "Clear HEM base address failed.\n");
 633         } else if (table->type < HEM_TYPE_MTT &&
 634                    hop_num == HNS_ROCE_HOP_NUM_0) {
 635                 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 0))
 636                         dev_warn(dev, "Clear HEM base address failed.\n");
 637         }
 638
 639         /*
 640          * free buffer space chunk for QPC/MTPT/CQC/SRQC.
 641          * free bt space chunk for MTT/CQE.
 642          */
 643         hns_roce_free_hem(hr_dev, table->hem[hem_idx]);
 644         table->hem[hem_idx] = NULL;
 645
 646         if (check_whether_bt_num_2(table->type, hop_num)) {
 647                 start_idx = mhop.l0_idx * chunk_ba_num;
 648                 if (hns_roce_check_hem_null(table->hem, start_idx,
 649                                             chunk_ba_num)) {
 650                         if (table->type < HEM_TYPE_MTT &&
 651                             hr_dev->hw->clear_hem(hr_dev, table, obj, 0))
 652                                 dev_warn(dev, "Clear HEM base address failed.\n");
 653
 654                         dma_free_coherent(dev, bt_chunk_size,
 655                                           table->bt_l0[mhop.l0_idx],
 656                                           table->bt_l0_dma_addr[mhop.l0_idx]);
 657                         table->bt_l0[mhop.l0_idx] = NULL;
 658                 }
 659         } else if (check_whether_bt_num_3(table->type, hop_num)) {
 660                 start_idx = mhop.l0_idx * chunk_ba_num * chunk_ba_num +
 661                             mhop.l1_idx * chunk_ba_num;
 662                 if (hns_roce_check_hem_null(table->hem, start_idx,
 663                                             chunk_ba_num)) {
 664                         if (hr_dev->hw->clear_hem(hr_dev, table, obj, 1))
 665                                 dev_warn(dev, "Clear HEM base address failed.\n");
 666
 667                         dma_free_coherent(dev, bt_chunk_size,
 668                                           table->bt_l1[bt_l1_idx],
 669                                           table->bt_l1_dma_addr[bt_l1_idx]);
 670                         table->bt_l1[bt_l1_idx] = NULL;
 671
 672                         start_idx = mhop.l0_idx * chunk_ba_num;
 673                         if (hns_roce_check_bt_null(table->bt_l1, start_idx,
 674                                                    chunk_ba_num)) {
 675                                 if (hr_dev->hw->clear_hem(hr_dev, table, obj,
 676                                                           0))
 677                                         dev_warn(dev, "Clear HEM base address failed.\n");
 678
 679                                 dma_free_coherent(dev, bt_chunk_size,
 680                                             table->bt_l0[mhop.l0_idx],
 681                                             table->bt_l0_dma_addr[mhop.l0_idx]);
 682                                 table->bt_l0[mhop.l0_idx] = NULL;
 683                         }
 684                 }
 685         }
 686
 687         mutex_unlock(&table->mutex);
 688 }
 689
 690 void hns_roce_table_put(struct hns_roce_dev *hr_dev,
 691                         struct hns_roce_hem_table *table, unsigned long obj)
 692 {
 693         struct device *dev = hr_dev->dev;
 694         unsigned long i;
 695
 696         if (hns_roce_check_whether_mhop(hr_dev, table->type)) {
 697                 hns_roce_table_mhop_put(hr_dev, table, obj, 1);
 698                 return;
 699         }
 700
 701         i = (obj & (table->num_obj - 1)) /
 702             (table->table_chunk_size / table->obj_size);
 703
 704         mutex_lock(&table->mutex);
 705
 706         if (--table->hem[i]->refcount == 0) {
 707                 /* Clear HEM base address */
 708                 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 0))
 709                         dev_warn(dev, "Clear HEM base address failed.\n");
 710
 711                 hns_roce_free_hem(hr_dev, table->hem[i]);
 712                 table->hem[i] = NULL;
 713         }
 714
 715         mutex_unlock(&table->mutex);
 716 }
 717
 718 void *hns_roce_table_find(struct hns_roce_dev *hr_dev,
 719                           struct hns_roce_hem_table *table,
 720                           unsigned long obj, dma_addr_t *dma_handle)
 721 {
 722         struct hns_roce_hem_chunk *chunk;
 723         struct hns_roce_hem_mhop mhop;
 724         struct hns_roce_hem *hem;
 725         struct page *page = NULL;
 726         unsigned long mhop_obj = obj;
 727         unsigned long obj_per_chunk;
 728         unsigned long idx_offset;
 729         int offset, dma_offset;
 730         int i, j;
 731         u32 hem_idx = 0;
 732
 733         if (!table->lowmem)
 734                 return NULL;
 735
 736         mutex_lock(&table->mutex);
 737
 738         if (!hns_roce_check_whether_mhop(hr_dev, table->type)) {
 739                 obj_per_chunk = table->table_chunk_size / table->obj_size;
 740                 hem = table->hem[(obj & (table->num_obj - 1)) / obj_per_chunk];
 741                 idx_offset = (obj & (table->num_obj - 1)) % obj_per_chunk;
 742                 dma_offset = offset = idx_offset * table->obj_size;
 743         } else {
 744                 hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop);
 745                 /* mtt mhop */
 746                 i = mhop.l0_idx;
 747                 j = mhop.l1_idx;
 748                 if (mhop.hop_num == 2)
 749                         hem_idx = i * (mhop.bt_chunk_size / 8) + j;
 750                 else if (mhop.hop_num == 1 ||
 751                          mhop.hop_num == HNS_ROCE_HOP_NUM_0)
 752                         hem_idx = i;
 753
 754                 hem = table->hem[hem_idx];
 755                 dma_offset = offset = (obj & (table->num_obj - 1)) *
 756                                        table->obj_size % mhop.bt_chunk_size;
 757                 if (mhop.hop_num == 2)
 758                         dma_offset = offset = 0;
 759         }
 760
 761         if (!hem)
 762                 goto out;
 763
 764         list_for_each_entry(chunk, &hem->chunk_list, list) {
 765                 for (i = 0; i < chunk->npages; ++i) {
 766                         if (dma_handle && dma_offset >= 0) {
 767                                 if (sg_dma_len(&chunk->mem[i]) >
 768                                     (u32)dma_offset)
 769                                         *dma_handle = sg_dma_address(
 770                                                 &chunk->mem[i]) + dma_offset;
 771                                 dma_offset -= sg_dma_len(&chunk->mem[i]);
 772                         }
 773
 774                         if (chunk->mem[i].length > (u32)offset) {
 775                                 page = sg_page(&chunk->mem[i]);
 776                                 goto out;
 777                         }
 778                         offset -= chunk->mem[i].length;
 779                 }
 780         }
 781
 782 out:
 783         mutex_unlock(&table->mutex);
 784         return page ? lowmem_page_address(page) + offset : NULL;
 785 }
 786 EXPORT_SYMBOL_GPL(hns_roce_table_find);
 787
 788 int hns_roce_table_get_range(struct hns_roce_dev *hr_dev,
 789                              struct hns_roce_hem_table *table,
 790                              unsigned long start, unsigned long end)
 791 {
 792         struct hns_roce_hem_mhop mhop;
 793         unsigned long inc = table->table_chunk_size / table->obj_size;
 794         unsigned long i;
 795         int ret;
 796
 797         if (hns_roce_check_whether_mhop(hr_dev, table->type)) {
 798                 hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop);
 799                 inc = mhop.bt_chunk_size / table->obj_size;
 800         }
 801
 802         /* Allocate MTT entry memory according to chunk(128K) */
 803         for (i = start; i <= end; i += inc) {
 804                 ret = hns_roce_table_get(hr_dev, table, i);
 805                 if (ret)
 806                         goto fail;
 807         }
 808
 809         return 0;
 810
 811 fail:
 812         while (i > start) {
 813                 i -= inc;
 814                 hns_roce_table_put(hr_dev, table, i);
 815         }
 816         return ret;
 817 }
 818
 819 void hns_roce_table_put_range(struct hns_roce_dev *hr_dev,
 820                               struct hns_roce_hem_table *table,
 821                               unsigned long start, unsigned long end)
 822 {
 823         struct hns_roce_hem_mhop mhop;
 824         unsigned long inc = table->table_chunk_size / table->obj_size;
 825         unsigned long i;
 826
 827         if (hns_roce_check_whether_mhop(hr_dev, table->type)) {
 828                 hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop);
 829                 inc = mhop.bt_chunk_size / table->obj_size;
 830         }
 831
 832         for (i = start; i <= end; i += inc)
 833                 hns_roce_table_put(hr_dev, table, i);
 834 }
 835
 836 int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev,
 837                             struct hns_roce_hem_table *table, u32 type,
 838                             unsigned long obj_size, unsigned long nobj,
 839                             int use_lowmem)
 840 {
 841         struct device *dev = hr_dev->dev;
 842         unsigned long obj_per_chunk;
 843         unsigned long num_hem;
 844
 845         if (!hns_roce_check_whether_mhop(hr_dev, type)) {
 846                 table->table_chunk_size = hr_dev->caps.chunk_sz;
 847                 obj_per_chunk = table->table_chunk_size / obj_size;
 848                 num_hem = (nobj + obj_per_chunk - 1) / obj_per_chunk;
 849
 850                 table->hem = kcalloc(num_hem, sizeof(*table->hem), GFP_KERNEL);
 851                 if (!table->hem)
 852                         return -ENOMEM;
 853         } else {
 854                 unsigned long buf_chunk_size;
 855                 unsigned long bt_chunk_size;
 856                 unsigned long bt_chunk_num;
 857                 unsigned long num_bt_l0 = 0;
 858                 u32 hop_num;
 859
 860                 switch (type) {
 861                 case HEM_TYPE_QPC:
 862                         buf_chunk_size = 1 << (hr_dev->caps.qpc_buf_pg_sz
 863                                         + PAGE_SHIFT);
 864                         bt_chunk_size = 1 << (hr_dev->caps.qpc_ba_pg_sz
 865                                         + PAGE_SHIFT);
 866                         num_bt_l0 = hr_dev->caps.qpc_bt_num;
 867                         hop_num = hr_dev->caps.qpc_hop_num;
 868                         break;
 869                 case HEM_TYPE_MTPT:
 870                         buf_chunk_size = 1 << (hr_dev->caps.mpt_buf_pg_sz
 871                                         + PAGE_SHIFT);
 872                         bt_chunk_size = 1 << (hr_dev->caps.mpt_ba_pg_sz
 873                                         + PAGE_SHIFT);
 874                         num_bt_l0 = hr_dev->caps.mpt_bt_num;
 875                         hop_num = hr_dev->caps.mpt_hop_num;
 876                         break;
 877                 case HEM_TYPE_CQC:
 878                         buf_chunk_size = 1 << (hr_dev->caps.cqc_buf_pg_sz
 879                                         + PAGE_SHIFT);
 880                         bt_chunk_size = 1 << (hr_dev->caps.cqc_ba_pg_sz
 881                                         + PAGE_SHIFT);
 882                         num_bt_l0 = hr_dev->caps.cqc_bt_num;
 883                         hop_num = hr_dev->caps.cqc_hop_num;
 884                         break;
 885                 case HEM_TYPE_SRQC:
 886                         buf_chunk_size = 1 << (hr_dev->caps.srqc_buf_pg_sz
 887                                         + PAGE_SHIFT);
 888                         bt_chunk_size = 1 << (hr_dev->caps.srqc_ba_pg_sz
 889                                         + PAGE_SHIFT);
 890                         num_bt_l0 = hr_dev->caps.srqc_bt_num;
 891                         hop_num = hr_dev->caps.srqc_hop_num;
 892                         break;
 893                 case HEM_TYPE_MTT:
 894                         buf_chunk_size = 1 << (hr_dev->caps.mtt_ba_pg_sz
 895                                         + PAGE_SHIFT);
 896                         bt_chunk_size = buf_chunk_size;
 897                         hop_num = hr_dev->caps.mtt_hop_num;
 898                         break;
 899                 case HEM_TYPE_CQE:
 900                         buf_chunk_size = 1 << (hr_dev->caps.cqe_ba_pg_sz
 901                                         + PAGE_SHIFT);
 902                         bt_chunk_size = buf_chunk_size;
 903                         hop_num = hr_dev->caps.cqe_hop_num;
 904                         break;
 905                 default:
 906                         dev_err(dev,
 907                           "Table %d not support to init hem table here!\n",
 908                           type);
 909                         return -EINVAL;
 910                 }
 911                 obj_per_chunk = buf_chunk_size / obj_size;
 912                 num_hem = (nobj + obj_per_chunk - 1) / obj_per_chunk;
 913                 bt_chunk_num = bt_chunk_size / 8;
 914                 if (table->type >= HEM_TYPE_MTT)
 915                         num_bt_l0 = bt_chunk_num;
 916
 917                 table->hem = kcalloc(num_hem, sizeof(*table->hem),
 918                                          GFP_KERNEL);
 919                 if (!table->hem)
 920                         goto err_kcalloc_hem_buf;
 921
 922                 if (check_whether_bt_num_3(table->type, hop_num)) {
 923                         unsigned long num_bt_l1;
 924
 925                         num_bt_l1 = (num_hem + bt_chunk_num - 1) /
 926                                              bt_chunk_num;
 927                         table->bt_l1 = kcalloc(num_bt_l1,
 928                                                sizeof(*table->bt_l1),
 929                                                GFP_KERNEL);
 930                         if (!table->bt_l1)
 931                                 goto err_kcalloc_bt_l1;
 932
 933                         table->bt_l1_dma_addr = kcalloc(num_bt_l1,
 934                                                  sizeof(*table->bt_l1_dma_addr),
 935                                                  GFP_KERNEL);
 936
 937                         if (!table->bt_l1_dma_addr)
 938                                 goto err_kcalloc_l1_dma;
 939                 }
 940
 941                 if (check_whether_bt_num_2(table->type, hop_num) ||
 942                         check_whether_bt_num_3(table->type, hop_num)) {
 943                         table->bt_l0 = kcalloc(num_bt_l0, sizeof(*table->bt_l0),
 944                                                GFP_KERNEL);
 945                         if (!table->bt_l0)
 946                                 goto err_kcalloc_bt_l0;
 947
 948                         table->bt_l0_dma_addr = kcalloc(num_bt_l0,
 949                                                  sizeof(*table->bt_l0_dma_addr),
 950                                                  GFP_KERNEL);
 951                         if (!table->bt_l0_dma_addr)
 952                                 goto err_kcalloc_l0_dma;
 953                 }
 954         }
 955
 956         table->type = type;
 957         table->num_hem = num_hem;
 958         table->num_obj = nobj;
 959         table->obj_size = obj_size;
 960         table->lowmem = use_lowmem;
 961         mutex_init(&table->mutex);
 962
 963         return 0;
 964
 965 err_kcalloc_l0_dma:
 966         kfree(table->bt_l0);
 967         table->bt_l0 = NULL;
 968
 969 err_kcalloc_bt_l0:
 970         kfree(table->bt_l1_dma_addr);
 971         table->bt_l1_dma_addr = NULL;
 972
 973 err_kcalloc_l1_dma:
 974         kfree(table->bt_l1);
 975         table->bt_l1 = NULL;
 976
 977 err_kcalloc_bt_l1:
 978         kfree(table->hem);
 979         table->hem = NULL;
 980
 981 err_kcalloc_hem_buf:
 982         return -ENOMEM;
 983 }
 984
 985 static void hns_roce_cleanup_mhop_hem_table(struct hns_roce_dev *hr_dev,
 986                                             struct hns_roce_hem_table *table)
 987 {
 988         struct hns_roce_hem_mhop mhop;
 989         u32 buf_chunk_size;
 990         int i;
 991         u64 obj;
 992
 993         hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop);
 994         buf_chunk_size = table->type < HEM_TYPE_MTT ? mhop.buf_chunk_size :
 995                                         mhop.bt_chunk_size;
 996
 997         for (i = 0; i < table->num_hem; ++i) {
 998                 obj = i * buf_chunk_size / table->obj_size;
 999                 if (table->hem[i])
1000                         hns_roce_table_mhop_put(hr_dev, table, obj, 0);
1001         }
1002
1003         kfree(table->hem);
1004         table->hem = NULL;
1005         kfree(table->bt_l1);
1006         table->bt_l1 = NULL;
1007         kfree(table->bt_l1_dma_addr);
1008         table->bt_l1_dma_addr = NULL;
1009         kfree(table->bt_l0);
1010         table->bt_l0 = NULL;
1011         kfree(table->bt_l0_dma_addr);
1012         table->bt_l0_dma_addr = NULL;
1013 }
1014
1015 void hns_roce_cleanup_hem_table(struct hns_roce_dev *hr_dev,
1016                                 struct hns_roce_hem_table *table)
1017 {
1018         struct device *dev = hr_dev->dev;
1019         unsigned long i;
1020
1021         if (hns_roce_check_whether_mhop(hr_dev, table->type)) {
1022                 hns_roce_cleanup_mhop_hem_table(hr_dev, table);
1023                 return;
1024         }
1025
1026         for (i = 0; i < table->num_hem; ++i)
1027                 if (table->hem[i]) {
1028                         if (hr_dev->hw->clear_hem(hr_dev, table,
1029                             i * table->table_chunk_size / table->obj_size, 0))
1030                                 dev_err(dev, "Clear HEM base address failed.\n");
1031
1032                         hns_roce_free_hem(hr_dev, table->hem[i]);
1033                 }
1034
1035         kfree(table->hem);
1036 }
1037
1038 void hns_roce_cleanup_hem(struct hns_roce_dev *hr_dev)
1039 {
1040         hns_roce_cleanup_hem_table(hr_dev, &hr_dev->cq_table.table);
1041         hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.irrl_table);
1042         if (hr_dev->caps.trrl_entry_sz)
1043                 hns_roce_cleanup_hem_table(hr_dev,
1044                                            &hr_dev->qp_table.trrl_table);
1045         hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.qp_table);
1046         hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtpt_table);
1047         hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtt_table);
1048         if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE))
1049                 hns_roce_cleanup_hem_table(hr_dev,
1050                                            &hr_dev->mr_table.mtt_cqe_table);
1051 }