drivers/infiniband/hw/ehca/ehca_mrmw.c

   1 /*
   2  *  IBM eServer eHCA Infiniband device driver for Linux on POWER
   3  *
   4  *  MR/MW functions
   5  *
   6  *  Authors: Dietmar Decker <ddecker@de.ibm.com>
   7  *           Christoph Raisch <raisch@de.ibm.com>
   8  *
   9  *  Copyright (c) 2005 IBM Corporation
  10  *
  11  *  All rights reserved.
  12  *
  13  *  This source code is distributed under a dual license of GPL v2.0 and OpenIB
  14  *  BSD.
  15  *
  16  * OpenIB BSD License
  17  *
  18  * Redistribution and use in source and binary forms, with or without
  19  * modification, are permitted provided that the following conditions are met:
  20  *
  21  * Redistributions of source code must retain the above copyright notice, this
  22  * list of conditions and the following disclaimer.
  23  *
  24  * Redistributions in binary form must reproduce the above copyright notice,
  25  * this list of conditions and the following disclaimer in the documentation
  26  * and/or other materials
  27  * provided with the distribution.
  28  *
  29  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  30  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  31  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  32  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  33  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  34  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  35  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
  36  * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
  37  * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  38  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
  39  * POSSIBILITY OF SUCH DAMAGE.
  40  */
  41
  42 #include <rdma/ib_umem.h>
  43
  44 #include <asm/current.h>
  45
  46 #include "ehca_iverbs.h"
  47 #include "ehca_mrmw.h"
  48 #include "hcp_if.h"
  49 #include "hipz_hw.h"
  50
  51 #define NUM_CHUNKS(length, chunk_size) \
  52         (((length) + (chunk_size - 1)) / (chunk_size))
  53 /* max number of rpages (per hcall register_rpages) */
  54 #define MAX_RPAGES 512
  55
  56 static struct kmem_cache *mr_cache;
  57 static struct kmem_cache *mw_cache;
  58
  59 static struct ehca_mr *ehca_mr_new(void)
  60 {
  61         struct ehca_mr *me;
  62
  63         me = kmem_cache_zalloc(mr_cache, GFP_KERNEL);
  64         if (me)
  65                 spin_lock_init(&me->mrlock);
  66         else
  67                 ehca_gen_err("alloc failed");
  68
  69         return me;
  70 }
  71
  72 static void ehca_mr_delete(struct ehca_mr *me)
  73 {
  74         kmem_cache_free(mr_cache, me);
  75 }
  76
  77 static struct ehca_mw *ehca_mw_new(void)
  78 {
  79         struct ehca_mw *me;
  80
  81         me = kmem_cache_zalloc(mw_cache, GFP_KERNEL);
  82         if (me)
  83                 spin_lock_init(&me->mwlock);
  84         else
  85                 ehca_gen_err("alloc failed");
  86
  87         return me;
  88 }
  89
  90 static void ehca_mw_delete(struct ehca_mw *me)
  91 {
  92         kmem_cache_free(mw_cache, me);
  93 }
  94
  95 /*----------------------------------------------------------------------*/
  96
  97 struct ib_mr *ehca_get_dma_mr(struct ib_pd *pd, int mr_access_flags)
  98 {
  99         struct ib_mr *ib_mr;
 100         int ret;
 101         struct ehca_mr *e_maxmr;
 102         struct ehca_pd *e_pd = container_of(pd, struct ehca_pd, ib_pd);
 103         struct ehca_shca *shca =
 104                 container_of(pd->device, struct ehca_shca, ib_device);
 105
 106         if (shca->maxmr) {
 107                 e_maxmr = ehca_mr_new();
 108                 if (!e_maxmr) {
 109                         ehca_err(&shca->ib_device, "out of memory");
 110                         ib_mr = ERR_PTR(-ENOMEM);
 111                         goto get_dma_mr_exit0;
 112                 }
 113
 114                 ret = ehca_reg_maxmr(shca, e_maxmr, (u64 *)KERNELBASE,
 115                                      mr_access_flags, e_pd,
 116                                      &e_maxmr->ib.ib_mr.lkey,
 117                                      &e_maxmr->ib.ib_mr.rkey);
 118                 if (ret) {
 119                         ehca_mr_delete(e_maxmr);
 120                         ib_mr = ERR_PTR(ret);
 121                         goto get_dma_mr_exit0;
 122                 }
 123                 ib_mr = &e_maxmr->ib.ib_mr;
 124         } else {
 125                 ehca_err(&shca->ib_device, "no internal max-MR exist!");
 126                 ib_mr = ERR_PTR(-EINVAL);
 127                 goto get_dma_mr_exit0;
 128         }
 129
 130 get_dma_mr_exit0:
 131         if (IS_ERR(ib_mr))
 132                 ehca_err(&shca->ib_device, "rc=%lx pd=%p mr_access_flags=%x ",
 133                          PTR_ERR(ib_mr), pd, mr_access_flags);
 134         return ib_mr;
 135 } /* end ehca_get_dma_mr() */
 136
 137 /*----------------------------------------------------------------------*/
 138
 139 struct ib_mr *ehca_reg_phys_mr(struct ib_pd *pd,
 140                                struct ib_phys_buf *phys_buf_array,
 141                                int num_phys_buf,
 142                                int mr_access_flags,
 143                                u64 *iova_start)
 144 {
 145         struct ib_mr *ib_mr;
 146         int ret;
 147         struct ehca_mr *e_mr;
 148         struct ehca_shca *shca =
 149                 container_of(pd->device, struct ehca_shca, ib_device);
 150         struct ehca_pd *e_pd = container_of(pd, struct ehca_pd, ib_pd);
 151
 152         u64 size;
 153
 154         if ((num_phys_buf <= 0) || !phys_buf_array) {
 155                 ehca_err(pd->device, "bad input values: num_phys_buf=%x "
 156                          "phys_buf_array=%p", num_phys_buf, phys_buf_array);
 157                 ib_mr = ERR_PTR(-EINVAL);
 158                 goto reg_phys_mr_exit0;
 159         }
 160         if (((mr_access_flags & IB_ACCESS_REMOTE_WRITE) &&
 161              !(mr_access_flags & IB_ACCESS_LOCAL_WRITE)) ||
 162             ((mr_access_flags & IB_ACCESS_REMOTE_ATOMIC) &&
 163              !(mr_access_flags & IB_ACCESS_LOCAL_WRITE))) {
 164                 /*
 165                  * Remote Write Access requires Local Write Access
 166                  * Remote Atomic Access requires Local Write Access
 167                  */
 168                 ehca_err(pd->device, "bad input values: mr_access_flags=%x",
 169                          mr_access_flags);
 170                 ib_mr = ERR_PTR(-EINVAL);
 171                 goto reg_phys_mr_exit0;
 172         }
 173
 174         /* check physical buffer list and calculate size */
 175         ret = ehca_mr_chk_buf_and_calc_size(phys_buf_array, num_phys_buf,
 176                                             iova_start, &size);
 177         if (ret) {
 178                 ib_mr = ERR_PTR(ret);
 179                 goto reg_phys_mr_exit0;
 180         }
 181         if ((size == 0) ||
 182             (((u64)iova_start + size) < (u64)iova_start)) {
 183                 ehca_err(pd->device, "bad input values: size=%lx iova_start=%p",
 184                          size, iova_start);
 185                 ib_mr = ERR_PTR(-EINVAL);
 186                 goto reg_phys_mr_exit0;
 187         }
 188
 189         e_mr = ehca_mr_new();
 190         if (!e_mr) {
 191                 ehca_err(pd->device, "out of memory");
 192                 ib_mr = ERR_PTR(-ENOMEM);
 193                 goto reg_phys_mr_exit0;
 194         }
 195
 196         /* register MR on HCA */
 197         if (ehca_mr_is_maxmr(size, iova_start)) {
 198                 e_mr->flags |= EHCA_MR_FLAG_MAXMR;
 199                 ret = ehca_reg_maxmr(shca, e_mr, iova_start, mr_access_flags,
 200                                      e_pd, &e_mr->ib.ib_mr.lkey,
 201                                      &e_mr->ib.ib_mr.rkey);
 202                 if (ret) {
 203                         ib_mr = ERR_PTR(ret);
 204                         goto reg_phys_mr_exit1;
 205                 }
 206         } else {
 207                 struct ehca_mr_pginfo pginfo;
 208                 u32 num_kpages;
 209                 u32 num_hwpages;
 210
 211                 num_kpages = NUM_CHUNKS(((u64)iova_start % PAGE_SIZE) + size,
 212                                         PAGE_SIZE);
 213                 num_hwpages = NUM_CHUNKS(((u64)iova_start % EHCA_PAGESIZE) +
 214                                          size, EHCA_PAGESIZE);
 215                 memset(&pginfo, 0, sizeof(pginfo));
 216                 pginfo.type = EHCA_MR_PGI_PHYS;
 217                 pginfo.num_kpages = num_kpages;
 218                 pginfo.num_hwpages = num_hwpages;
 219                 pginfo.u.phy.num_phys_buf = num_phys_buf;
 220                 pginfo.u.phy.phys_buf_array = phys_buf_array;
 221                 pginfo.next_hwpage = (((u64)iova_start & ~PAGE_MASK) /
 222                                       EHCA_PAGESIZE);
 223
 224                 ret = ehca_reg_mr(shca, e_mr, iova_start, size, mr_access_flags,
 225                                   e_pd, &pginfo, &e_mr->ib.ib_mr.lkey,
 226                                   &e_mr->ib.ib_mr.rkey);
 227                 if (ret) {
 228                         ib_mr = ERR_PTR(ret);
 229                         goto reg_phys_mr_exit1;
 230                 }
 231         }
 232
 233         /* successful registration of all pages */
 234         return &e_mr->ib.ib_mr;
 235
 236 reg_phys_mr_exit1:
 237         ehca_mr_delete(e_mr);
 238 reg_phys_mr_exit0:
 239         if (IS_ERR(ib_mr))
 240                 ehca_err(pd->device, "rc=%lx pd=%p phys_buf_array=%p "
 241                          "num_phys_buf=%x mr_access_flags=%x iova_start=%p",
 242                          PTR_ERR(ib_mr), pd, phys_buf_array,
 243                          num_phys_buf, mr_access_flags, iova_start);
 244         return ib_mr;
 245 } /* end ehca_reg_phys_mr() */
 246
 247 /*----------------------------------------------------------------------*/
 248
 249 struct ib_mr *ehca_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
 250                                u64 virt, int mr_access_flags,
 251                                struct ib_udata *udata)
 252 {
 253         struct ib_mr *ib_mr;
 254         struct ehca_mr *e_mr;
 255         struct ehca_shca *shca =
 256                 container_of(pd->device, struct ehca_shca, ib_device);
 257         struct ehca_pd *e_pd = container_of(pd, struct ehca_pd, ib_pd);
 258         struct ehca_mr_pginfo pginfo;
 259         int ret;
 260         u32 num_kpages;
 261         u32 num_hwpages;
 262
 263         if (!pd) {
 264                 ehca_gen_err("bad pd=%p", pd);
 265                 return ERR_PTR(-EFAULT);
 266         }
 267
 268         if (((mr_access_flags & IB_ACCESS_REMOTE_WRITE) &&
 269              !(mr_access_flags & IB_ACCESS_LOCAL_WRITE)) ||
 270             ((mr_access_flags & IB_ACCESS_REMOTE_ATOMIC) &&
 271              !(mr_access_flags & IB_ACCESS_LOCAL_WRITE))) {
 272                 /*
 273                  * Remote Write Access requires Local Write Access
 274                  * Remote Atomic Access requires Local Write Access
 275                  */
 276                 ehca_err(pd->device, "bad input values: mr_access_flags=%x",
 277                          mr_access_flags);
 278                 ib_mr = ERR_PTR(-EINVAL);
 279                 goto reg_user_mr_exit0;
 280         }
 281
 282         if (length == 0 || virt + length < virt) {
 283                 ehca_err(pd->device, "bad input values: length=%lx "
 284                          "virt_base=%lx", length, virt);
 285                 ib_mr = ERR_PTR(-EINVAL);
 286                 goto reg_user_mr_exit0;
 287         }
 288
 289         e_mr = ehca_mr_new();
 290         if (!e_mr) {
 291                 ehca_err(pd->device, "out of memory");
 292                 ib_mr = ERR_PTR(-ENOMEM);
 293                 goto reg_user_mr_exit0;
 294         }
 295
 296         e_mr->umem = ib_umem_get(pd->uobject->context, start, length,
 297                                  mr_access_flags);
 298         if (IS_ERR(e_mr->umem)) {
 299                 ib_mr = (void *)e_mr->umem;
 300                 goto reg_user_mr_exit1;
 301         }
 302
 303         if (e_mr->umem->page_size != PAGE_SIZE) {
 304                 ehca_err(pd->device, "page size not supported, "
 305                          "e_mr->umem->page_size=%x", e_mr->umem->page_size);
 306                 ib_mr = ERR_PTR(-EINVAL);
 307                 goto reg_user_mr_exit2;
 308         }
 309
 310         /* determine number of MR pages */
 311         num_kpages = NUM_CHUNKS((virt % PAGE_SIZE) + length, PAGE_SIZE);
 312         num_hwpages = NUM_CHUNKS((virt % EHCA_PAGESIZE) + length,
 313                                  EHCA_PAGESIZE);
 314
 315         /* register MR on HCA */
 316         memset(&pginfo, 0, sizeof(pginfo));
 317         pginfo.type = EHCA_MR_PGI_USER;
 318         pginfo.num_kpages = num_kpages;
 319         pginfo.num_hwpages = num_hwpages;
 320         pginfo.u.usr.region = e_mr->umem;
 321         pginfo.next_hwpage = e_mr->umem->offset / EHCA_PAGESIZE;
 322         pginfo.u.usr.next_chunk = list_prepare_entry(pginfo.u.usr.next_chunk,
 323                                                      (&e_mr->umem->chunk_list),
 324                                                      list);
 325
 326         ret = ehca_reg_mr(shca, e_mr, (u64 *)virt, length, mr_access_flags,
 327                           e_pd, &pginfo, &e_mr->ib.ib_mr.lkey,
 328                           &e_mr->ib.ib_mr.rkey);
 329         if (ret) {
 330                 ib_mr = ERR_PTR(ret);
 331                 goto reg_user_mr_exit2;
 332         }
 333
 334         /* successful registration of all pages */
 335         return &e_mr->ib.ib_mr;
 336
 337 reg_user_mr_exit2:
 338         ib_umem_release(e_mr->umem);
 339 reg_user_mr_exit1:
 340         ehca_mr_delete(e_mr);
 341 reg_user_mr_exit0:
 342         if (IS_ERR(ib_mr))
 343                 ehca_err(pd->device, "rc=%lx pd=%p mr_access_flags=%x"
 344                          " udata=%p",
 345                          PTR_ERR(ib_mr), pd, mr_access_flags, udata);
 346         return ib_mr;
 347 } /* end ehca_reg_user_mr() */
 348
 349 /*----------------------------------------------------------------------*/
 350
 351 int ehca_rereg_phys_mr(struct ib_mr *mr,
 352                        int mr_rereg_mask,
 353                        struct ib_pd *pd,
 354                        struct ib_phys_buf *phys_buf_array,
 355                        int num_phys_buf,
 356                        int mr_access_flags,
 357                        u64 *iova_start)
 358 {
 359         int ret;
 360
 361         struct ehca_shca *shca =
 362                 container_of(mr->device, struct ehca_shca, ib_device);
 363         struct ehca_mr *e_mr = container_of(mr, struct ehca_mr, ib.ib_mr);
 364         struct ehca_pd *my_pd = container_of(mr->pd, struct ehca_pd, ib_pd);
 365         u64 new_size;
 366         u64 *new_start;
 367         u32 new_acl;
 368         struct ehca_pd *new_pd;
 369         u32 tmp_lkey, tmp_rkey;
 370         unsigned long sl_flags;
 371         u32 num_kpages = 0;
 372         u32 num_hwpages = 0;
 373         struct ehca_mr_pginfo pginfo;
 374         u32 cur_pid = current->tgid;
 375
 376         if (my_pd->ib_pd.uobject && my_pd->ib_pd.uobject->context &&
 377             (my_pd->ownpid != cur_pid)) {
 378                 ehca_err(mr->device, "Invalid caller pid=%x ownpid=%x",
 379                          cur_pid, my_pd->ownpid);
 380                 ret = -EINVAL;
 381                 goto rereg_phys_mr_exit0;
 382         }
 383
 384         if (!(mr_rereg_mask & IB_MR_REREG_TRANS)) {
 385                 /* TODO not supported, because PHYP rereg hCall needs pages */
 386                 ehca_err(mr->device, "rereg without IB_MR_REREG_TRANS not "
 387                          "supported yet, mr_rereg_mask=%x", mr_rereg_mask);
 388                 ret = -EINVAL;
 389                 goto rereg_phys_mr_exit0;
 390         }
 391
 392         if (mr_rereg_mask & IB_MR_REREG_PD) {
 393                 if (!pd) {
 394                         ehca_err(mr->device, "rereg with bad pd, pd=%p "
 395                                  "mr_rereg_mask=%x", pd, mr_rereg_mask);
 396                         ret = -EINVAL;
 397                         goto rereg_phys_mr_exit0;
 398                 }
 399         }
 400
 401         if ((mr_rereg_mask &
 402              ~(IB_MR_REREG_TRANS | IB_MR_REREG_PD | IB_MR_REREG_ACCESS)) ||
 403             (mr_rereg_mask == 0)) {
 404                 ret = -EINVAL;
 405                 goto rereg_phys_mr_exit0;
 406         }
 407
 408         /* check other parameters */
 409         if (e_mr == shca->maxmr) {
 410                 /* should be impossible, however reject to be sure */
 411                 ehca_err(mr->device, "rereg internal max-MR impossible, mr=%p "
 412                          "shca->maxmr=%p mr->lkey=%x",
 413                          mr, shca->maxmr, mr->lkey);
 414                 ret = -EINVAL;
 415                 goto rereg_phys_mr_exit0;
 416         }
 417         if (mr_rereg_mask & IB_MR_REREG_TRANS) { /* transl., i.e. addr/size */
 418                 if (e_mr->flags & EHCA_MR_FLAG_FMR) {
 419                         ehca_err(mr->device, "not supported for FMR, mr=%p "
 420                                  "flags=%x", mr, e_mr->flags);
 421                         ret = -EINVAL;
 422                         goto rereg_phys_mr_exit0;
 423                 }
 424                 if (!phys_buf_array || num_phys_buf <= 0) {
 425                         ehca_err(mr->device, "bad input values mr_rereg_mask=%x"
 426                                  " phys_buf_array=%p num_phys_buf=%x",
 427                                  mr_rereg_mask, phys_buf_array, num_phys_buf);
 428                         ret = -EINVAL;
 429                         goto rereg_phys_mr_exit0;
 430                 }
 431         }
 432         if ((mr_rereg_mask & IB_MR_REREG_ACCESS) &&     /* change ACL */
 433             (((mr_access_flags & IB_ACCESS_REMOTE_WRITE) &&
 434               !(mr_access_flags & IB_ACCESS_LOCAL_WRITE)) ||
 435              ((mr_access_flags & IB_ACCESS_REMOTE_ATOMIC) &&
 436               !(mr_access_flags & IB_ACCESS_LOCAL_WRITE)))) {
 437                 /*
 438                  * Remote Write Access requires Local Write Access
 439                  * Remote Atomic Access requires Local Write Access
 440                  */
 441                 ehca_err(mr->device, "bad input values: mr_rereg_mask=%x "
 442                          "mr_access_flags=%x", mr_rereg_mask, mr_access_flags);
 443                 ret = -EINVAL;
 444                 goto rereg_phys_mr_exit0;
 445         }
 446
 447         /* set requested values dependent on rereg request */
 448         spin_lock_irqsave(&e_mr->mrlock, sl_flags);
 449         new_start = e_mr->start;
 450         new_size = e_mr->size;
 451         new_acl = e_mr->acl;
 452         new_pd = container_of(mr->pd, struct ehca_pd, ib_pd);
 453
 454         if (mr_rereg_mask & IB_MR_REREG_TRANS) {
 455                 new_start = iova_start; /* change address */
 456                 /* check physical buffer list and calculate size */
 457                 ret = ehca_mr_chk_buf_and_calc_size(phys_buf_array,
 458                                                     num_phys_buf, iova_start,
 459                                                     &new_size);
 460                 if (ret)
 461                         goto rereg_phys_mr_exit1;
 462                 if ((new_size == 0) ||
 463                     (((u64)iova_start + new_size) < (u64)iova_start)) {
 464                         ehca_err(mr->device, "bad input values: new_size=%lx "
 465                                  "iova_start=%p", new_size, iova_start);
 466                         ret = -EINVAL;
 467                         goto rereg_phys_mr_exit1;
 468                 }
 469                 num_kpages = NUM_CHUNKS(((u64)new_start % PAGE_SIZE) +
 470                                         new_size, PAGE_SIZE);
 471                 num_hwpages = NUM_CHUNKS(((u64)new_start % EHCA_PAGESIZE) +
 472                                          new_size, EHCA_PAGESIZE);
 473                 memset(&pginfo, 0, sizeof(pginfo));
 474                 pginfo.type = EHCA_MR_PGI_PHYS;
 475                 pginfo.num_kpages = num_kpages;
 476                 pginfo.num_hwpages = num_hwpages;
 477                 pginfo.u.phy.num_phys_buf = num_phys_buf;
 478                 pginfo.u.phy.phys_buf_array = phys_buf_array;
 479                 pginfo.next_hwpage = (((u64)iova_start & ~PAGE_MASK) /
 480                                       EHCA_PAGESIZE);
 481         }
 482         if (mr_rereg_mask & IB_MR_REREG_ACCESS)
 483                 new_acl = mr_access_flags;
 484         if (mr_rereg_mask & IB_MR_REREG_PD)
 485                 new_pd = container_of(pd, struct ehca_pd, ib_pd);
 486
 487         ret = ehca_rereg_mr(shca, e_mr, new_start, new_size, new_acl,
 488                             new_pd, &pginfo, &tmp_lkey, &tmp_rkey);
 489         if (ret)
 490                 goto rereg_phys_mr_exit1;
 491
 492         /* successful reregistration */
 493         if (mr_rereg_mask & IB_MR_REREG_PD)
 494                 mr->pd = pd;
 495         mr->lkey = tmp_lkey;
 496         mr->rkey = tmp_rkey;
 497
 498 rereg_phys_mr_exit1:
 499         spin_unlock_irqrestore(&e_mr->mrlock, sl_flags);
 500 rereg_phys_mr_exit0:
 501         if (ret)
 502                 ehca_err(mr->device, "ret=%x mr=%p mr_rereg_mask=%x pd=%p "
 503                          "phys_buf_array=%p num_phys_buf=%x mr_access_flags=%x "
 504                          "iova_start=%p",
 505                          ret, mr, mr_rereg_mask, pd, phys_buf_array,
 506                          num_phys_buf, mr_access_flags, iova_start);
 507         return ret;
 508 } /* end ehca_rereg_phys_mr() */
 509
 510 /*----------------------------------------------------------------------*/
 511
 512 int ehca_query_mr(struct ib_mr *mr, struct ib_mr_attr *mr_attr)
 513 {
 514         int ret = 0;
 515         u64 h_ret;
 516         struct ehca_shca *shca =
 517                 container_of(mr->device, struct ehca_shca, ib_device);
 518         struct ehca_mr *e_mr = container_of(mr, struct ehca_mr, ib.ib_mr);
 519         struct ehca_pd *my_pd = container_of(mr->pd, struct ehca_pd, ib_pd);
 520         u32 cur_pid = current->tgid;
 521         unsigned long sl_flags;
 522         struct ehca_mr_hipzout_parms hipzout;
 523
 524         if (my_pd->ib_pd.uobject && my_pd->ib_pd.uobject->context &&
 525             (my_pd->ownpid != cur_pid)) {
 526                 ehca_err(mr->device, "Invalid caller pid=%x ownpid=%x",
 527                          cur_pid, my_pd->ownpid);
 528                 ret = -EINVAL;
 529                 goto query_mr_exit0;
 530         }
 531
 532         if ((e_mr->flags & EHCA_MR_FLAG_FMR)) {
 533                 ehca_err(mr->device, "not supported for FMR, mr=%p e_mr=%p "
 534                          "e_mr->flags=%x", mr, e_mr, e_mr->flags);
 535                 ret = -EINVAL;
 536                 goto query_mr_exit0;
 537         }
 538
 539         memset(mr_attr, 0, sizeof(struct ib_mr_attr));
 540         spin_lock_irqsave(&e_mr->mrlock, sl_flags);
 541
 542         h_ret = hipz_h_query_mr(shca->ipz_hca_handle, e_mr, &hipzout);
 543         if (h_ret != H_SUCCESS) {
 544                 ehca_err(mr->device, "hipz_mr_query failed, h_ret=%lx mr=%p "
 545                          "hca_hndl=%lx mr_hndl=%lx lkey=%x",
 546                          h_ret, mr, shca->ipz_hca_handle.handle,
 547                          e_mr->ipz_mr_handle.handle, mr->lkey);
 548                 ret = ehca2ib_return_code(h_ret);
 549                 goto query_mr_exit1;
 550         }
 551         mr_attr->pd = mr->pd;
 552         mr_attr->device_virt_addr = hipzout.vaddr;
 553         mr_attr->size = hipzout.len;
 554         mr_attr->lkey = hipzout.lkey;
 555         mr_attr->rkey = hipzout.rkey;
 556         ehca_mrmw_reverse_map_acl(&hipzout.acl, &mr_attr->mr_access_flags);
 557
 558 query_mr_exit1:
 559         spin_unlock_irqrestore(&e_mr->mrlock, sl_flags);
 560 query_mr_exit0:
 561         if (ret)
 562                 ehca_err(mr->device, "ret=%x mr=%p mr_attr=%p",
 563                          ret, mr, mr_attr);
 564         return ret;
 565 } /* end ehca_query_mr() */
 566
 567 /*----------------------------------------------------------------------*/
 568
 569 int ehca_dereg_mr(struct ib_mr *mr)
 570 {
 571         int ret = 0;
 572         u64 h_ret;
 573         struct ehca_shca *shca =
 574                 container_of(mr->device, struct ehca_shca, ib_device);
 575         struct ehca_mr *e_mr = container_of(mr, struct ehca_mr, ib.ib_mr);
 576         struct ehca_pd *my_pd = container_of(mr->pd, struct ehca_pd, ib_pd);
 577         u32 cur_pid = current->tgid;
 578
 579         if (my_pd->ib_pd.uobject && my_pd->ib_pd.uobject->context &&
 580             (my_pd->ownpid != cur_pid)) {
 581                 ehca_err(mr->device, "Invalid caller pid=%x ownpid=%x",
 582                          cur_pid, my_pd->ownpid);
 583                 ret = -EINVAL;
 584                 goto dereg_mr_exit0;
 585         }
 586
 587         if ((e_mr->flags & EHCA_MR_FLAG_FMR)) {
 588                 ehca_err(mr->device, "not supported for FMR, mr=%p e_mr=%p "
 589                          "e_mr->flags=%x", mr, e_mr, e_mr->flags);
 590                 ret = -EINVAL;
 591                 goto dereg_mr_exit0;
 592         } else if (e_mr == shca->maxmr) {
 593                 /* should be impossible, however reject to be sure */
 594                 ehca_err(mr->device, "dereg internal max-MR impossible, mr=%p "
 595                          "shca->maxmr=%p mr->lkey=%x",
 596                          mr, shca->maxmr, mr->lkey);
 597                 ret = -EINVAL;
 598                 goto dereg_mr_exit0;
 599         }
 600
 601         /* TODO: BUSY: MR still has bound window(s) */
 602         h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_mr);
 603         if (h_ret != H_SUCCESS) {
 604                 ehca_err(mr->device, "hipz_free_mr failed, h_ret=%lx shca=%p "
 605                          "e_mr=%p hca_hndl=%lx mr_hndl=%lx mr->lkey=%x",
 606                          h_ret, shca, e_mr, shca->ipz_hca_handle.handle,
 607                          e_mr->ipz_mr_handle.handle, mr->lkey);
 608                 ret = ehca2ib_return_code(h_ret);
 609                 goto dereg_mr_exit0;
 610         }
 611
 612         if (e_mr->umem)
 613                 ib_umem_release(e_mr->umem);
 614
 615         /* successful deregistration */
 616         ehca_mr_delete(e_mr);
 617
 618 dereg_mr_exit0:
 619         if (ret)
 620                 ehca_err(mr->device, "ret=%x mr=%p", ret, mr);
 621         return ret;
 622 } /* end ehca_dereg_mr() */
 623
 624 /*----------------------------------------------------------------------*/
 625
 626 struct ib_mw *ehca_alloc_mw(struct ib_pd *pd)
 627 {
 628         struct ib_mw *ib_mw;
 629         u64 h_ret;
 630         struct ehca_mw *e_mw;
 631         struct ehca_pd *e_pd = container_of(pd, struct ehca_pd, ib_pd);
 632         struct ehca_shca *shca =
 633                 container_of(pd->device, struct ehca_shca, ib_device);
 634         struct ehca_mw_hipzout_parms hipzout;
 635
 636         e_mw = ehca_mw_new();
 637         if (!e_mw) {
 638                 ib_mw = ERR_PTR(-ENOMEM);
 639                 goto alloc_mw_exit0;
 640         }
 641
 642         h_ret = hipz_h_alloc_resource_mw(shca->ipz_hca_handle, e_mw,
 643                                          e_pd->fw_pd, &hipzout);
 644         if (h_ret != H_SUCCESS) {
 645                 ehca_err(pd->device, "hipz_mw_allocate failed, h_ret=%lx "
 646                          "shca=%p hca_hndl=%lx mw=%p",
 647                          h_ret, shca, shca->ipz_hca_handle.handle, e_mw);
 648                 ib_mw = ERR_PTR(ehca2ib_return_code(h_ret));
 649                 goto alloc_mw_exit1;
 650         }
 651         /* successful MW allocation */
 652         e_mw->ipz_mw_handle = hipzout.handle;
 653         e_mw->ib_mw.rkey    = hipzout.rkey;
 654         return &e_mw->ib_mw;
 655
 656 alloc_mw_exit1:
 657         ehca_mw_delete(e_mw);
 658 alloc_mw_exit0:
 659         if (IS_ERR(ib_mw))
 660                 ehca_err(pd->device, "rc=%lx pd=%p", PTR_ERR(ib_mw), pd);
 661         return ib_mw;
 662 } /* end ehca_alloc_mw() */
 663
 664 /*----------------------------------------------------------------------*/
 665
 666 int ehca_bind_mw(struct ib_qp *qp,
 667                  struct ib_mw *mw,
 668                  struct ib_mw_bind *mw_bind)
 669 {
 670         /* TODO: not supported up to now */
 671         ehca_gen_err("bind MW currently not supported by HCAD");
 672
 673         return -EPERM;
 674 } /* end ehca_bind_mw() */
 675
 676 /*----------------------------------------------------------------------*/
 677
 678 int ehca_dealloc_mw(struct ib_mw *mw)
 679 {
 680         u64 h_ret;
 681         struct ehca_shca *shca =
 682                 container_of(mw->device, struct ehca_shca, ib_device);
 683         struct ehca_mw *e_mw = container_of(mw, struct ehca_mw, ib_mw);
 684
 685         h_ret = hipz_h_free_resource_mw(shca->ipz_hca_handle, e_mw);
 686         if (h_ret != H_SUCCESS) {
 687                 ehca_err(mw->device, "hipz_free_mw failed, h_ret=%lx shca=%p "
 688                          "mw=%p rkey=%x hca_hndl=%lx mw_hndl=%lx",
 689                          h_ret, shca, mw, mw->rkey, shca->ipz_hca_handle.handle,
 690                          e_mw->ipz_mw_handle.handle);
 691                 return ehca2ib_return_code(h_ret);
 692         }
 693         /* successful deallocation */
 694         ehca_mw_delete(e_mw);
 695         return 0;
 696 } /* end ehca_dealloc_mw() */
 697
 698 /*----------------------------------------------------------------------*/
 699
 700 struct ib_fmr *ehca_alloc_fmr(struct ib_pd *pd,
 701                               int mr_access_flags,
 702                               struct ib_fmr_attr *fmr_attr)
 703 {
 704         struct ib_fmr *ib_fmr;
 705         struct ehca_shca *shca =
 706                 container_of(pd->device, struct ehca_shca, ib_device);
 707         struct ehca_pd *e_pd = container_of(pd, struct ehca_pd, ib_pd);
 708         struct ehca_mr *e_fmr;
 709         int ret;
 710         u32 tmp_lkey, tmp_rkey;
 711         struct ehca_mr_pginfo pginfo;
 712
 713         /* check other parameters */
 714         if (((mr_access_flags & IB_ACCESS_REMOTE_WRITE) &&
 715              !(mr_access_flags & IB_ACCESS_LOCAL_WRITE)) ||
 716             ((mr_access_flags & IB_ACCESS_REMOTE_ATOMIC) &&
 717              !(mr_access_flags & IB_ACCESS_LOCAL_WRITE))) {
 718                 /*
 719                  * Remote Write Access requires Local Write Access
 720                  * Remote Atomic Access requires Local Write Access
 721                  */
 722                 ehca_err(pd->device, "bad input values: mr_access_flags=%x",
 723                          mr_access_flags);
 724                 ib_fmr = ERR_PTR(-EINVAL);
 725                 goto alloc_fmr_exit0;
 726         }
 727         if (mr_access_flags & IB_ACCESS_MW_BIND) {
 728                 ehca_err(pd->device, "bad input values: mr_access_flags=%x",
 729                          mr_access_flags);
 730                 ib_fmr = ERR_PTR(-EINVAL);
 731                 goto alloc_fmr_exit0;
 732         }
 733         if ((fmr_attr->max_pages == 0) || (fmr_attr->max_maps == 0)) {
 734                 ehca_err(pd->device, "bad input values: fmr_attr->max_pages=%x "
 735                          "fmr_attr->max_maps=%x fmr_attr->page_shift=%x",
 736                          fmr_attr->max_pages, fmr_attr->max_maps,
 737                          fmr_attr->page_shift);
 738                 ib_fmr = ERR_PTR(-EINVAL);
 739                 goto alloc_fmr_exit0;
 740         }
 741         if (((1 << fmr_attr->page_shift) != EHCA_PAGESIZE) &&
 742             ((1 << fmr_attr->page_shift) != PAGE_SIZE)) {
 743                 ehca_err(pd->device, "unsupported fmr_attr->page_shift=%x",
 744                          fmr_attr->page_shift);
 745                 ib_fmr = ERR_PTR(-EINVAL);
 746                 goto alloc_fmr_exit0;
 747         }
 748
 749         e_fmr = ehca_mr_new();
 750         if (!e_fmr) {
 751                 ib_fmr = ERR_PTR(-ENOMEM);
 752                 goto alloc_fmr_exit0;
 753         }
 754         e_fmr->flags |= EHCA_MR_FLAG_FMR;
 755
 756         /* register MR on HCA */
 757         memset(&pginfo, 0, sizeof(pginfo));
 758         ret = ehca_reg_mr(shca, e_fmr, NULL,
 759                           fmr_attr->max_pages * (1 << fmr_attr->page_shift),
 760                           mr_access_flags, e_pd, &pginfo,
 761                           &tmp_lkey, &tmp_rkey);
 762         if (ret) {
 763                 ib_fmr = ERR_PTR(ret);
 764                 goto alloc_fmr_exit1;
 765         }
 766
 767         /* successful */
 768         e_fmr->fmr_page_size = 1 << fmr_attr->page_shift;
 769         e_fmr->fmr_max_pages = fmr_attr->max_pages;
 770         e_fmr->fmr_max_maps = fmr_attr->max_maps;
 771         e_fmr->fmr_map_cnt = 0;
 772         return &e_fmr->ib.ib_fmr;
 773
 774 alloc_fmr_exit1:
 775         ehca_mr_delete(e_fmr);
 776 alloc_fmr_exit0:
 777         if (IS_ERR(ib_fmr))
 778                 ehca_err(pd->device, "rc=%lx pd=%p mr_access_flags=%x "
 779                          "fmr_attr=%p", PTR_ERR(ib_fmr), pd,
 780                          mr_access_flags, fmr_attr);
 781         return ib_fmr;
 782 } /* end ehca_alloc_fmr() */
 783
 784 /*----------------------------------------------------------------------*/
 785
 786 int ehca_map_phys_fmr(struct ib_fmr *fmr,
 787                       u64 *page_list,
 788                       int list_len,
 789                       u64 iova)
 790 {
 791         int ret;
 792         struct ehca_shca *shca =
 793                 container_of(fmr->device, struct ehca_shca, ib_device);
 794         struct ehca_mr *e_fmr = container_of(fmr, struct ehca_mr, ib.ib_fmr);
 795         struct ehca_pd *e_pd = container_of(fmr->pd, struct ehca_pd, ib_pd);
 796         struct ehca_mr_pginfo pginfo;
 797         u32 tmp_lkey, tmp_rkey;
 798
 799         if (!(e_fmr->flags & EHCA_MR_FLAG_FMR)) {
 800                 ehca_err(fmr->device, "not a FMR, e_fmr=%p e_fmr->flags=%x",
 801                          e_fmr, e_fmr->flags);
 802                 ret = -EINVAL;
 803                 goto map_phys_fmr_exit0;
 804         }
 805         ret = ehca_fmr_check_page_list(e_fmr, page_list, list_len);
 806         if (ret)
 807                 goto map_phys_fmr_exit0;
 808         if (iova % e_fmr->fmr_page_size) {
 809                 /* only whole-numbered pages */
 810                 ehca_err(fmr->device, "bad iova, iova=%lx fmr_page_size=%x",
 811                          iova, e_fmr->fmr_page_size);
 812                 ret = -EINVAL;
 813                 goto map_phys_fmr_exit0;
 814         }
 815         if (e_fmr->fmr_map_cnt >= e_fmr->fmr_max_maps) {
 816                 /* HCAD does not limit the maps, however trace this anyway */
 817                 ehca_info(fmr->device, "map limit exceeded, fmr=%p "
 818                           "e_fmr->fmr_map_cnt=%x e_fmr->fmr_max_maps=%x",
 819                           fmr, e_fmr->fmr_map_cnt, e_fmr->fmr_max_maps);
 820         }
 821
 822         memset(&pginfo, 0, sizeof(pginfo));
 823         pginfo.type = EHCA_MR_PGI_FMR;
 824         pginfo.num_kpages = list_len;
 825         pginfo.num_hwpages = list_len * (e_fmr->fmr_page_size / EHCA_PAGESIZE);
 826         pginfo.u.fmr.page_list = page_list;
 827         pginfo.next_hwpage = ((iova & (e_fmr->fmr_page_size-1)) /
 828                               EHCA_PAGESIZE);
 829         pginfo.u.fmr.fmr_pgsize = e_fmr->fmr_page_size;
 830
 831         ret = ehca_rereg_mr(shca, e_fmr, (u64 *)iova,
 832                             list_len * e_fmr->fmr_page_size,
 833                             e_fmr->acl, e_pd, &pginfo, &tmp_lkey, &tmp_rkey);
 834         if (ret)
 835                 goto map_phys_fmr_exit0;
 836
 837         /* successful reregistration */
 838         e_fmr->fmr_map_cnt++;
 839         e_fmr->ib.ib_fmr.lkey = tmp_lkey;
 840         e_fmr->ib.ib_fmr.rkey = tmp_rkey;
 841         return 0;
 842
 843 map_phys_fmr_exit0:
 844         if (ret)
 845                 ehca_err(fmr->device, "ret=%x fmr=%p page_list=%p list_len=%x "
 846                          "iova=%lx", ret, fmr, page_list, list_len, iova);
 847         return ret;
 848 } /* end ehca_map_phys_fmr() */
 849
 850 /*----------------------------------------------------------------------*/
 851
 852 int ehca_unmap_fmr(struct list_head *fmr_list)
 853 {
 854         int ret = 0;
 855         struct ib_fmr *ib_fmr;
 856         struct ehca_shca *shca = NULL;
 857         struct ehca_shca *prev_shca;
 858         struct ehca_mr *e_fmr;
 859         u32 num_fmr = 0;
 860         u32 unmap_fmr_cnt = 0;
 861
 862         /* check all FMR belong to same SHCA, and check internal flag */
 863         list_for_each_entry(ib_fmr, fmr_list, list) {
 864                 prev_shca = shca;
 865                 if (!ib_fmr) {
 866                         ehca_gen_err("bad fmr=%p in list", ib_fmr);
 867                         ret = -EINVAL;
 868                         goto unmap_fmr_exit0;
 869                 }
 870                 shca = container_of(ib_fmr->device, struct ehca_shca,
 871                                     ib_device);
 872                 e_fmr = container_of(ib_fmr, struct ehca_mr, ib.ib_fmr);
 873                 if ((shca != prev_shca) && prev_shca) {
 874                         ehca_err(&shca->ib_device, "SHCA mismatch, shca=%p "
 875                                  "prev_shca=%p e_fmr=%p",
 876                                  shca, prev_shca, e_fmr);
 877                         ret = -EINVAL;
 878                         goto unmap_fmr_exit0;
 879                 }
 880                 if (!(e_fmr->flags & EHCA_MR_FLAG_FMR)) {
 881                         ehca_err(&shca->ib_device, "not a FMR, e_fmr=%p "
 882                                  "e_fmr->flags=%x", e_fmr, e_fmr->flags);
 883                         ret = -EINVAL;
 884                         goto unmap_fmr_exit0;
 885                 }
 886                 num_fmr++;
 887         }
 888
 889         /* loop over all FMRs to unmap */
 890         list_for_each_entry(ib_fmr, fmr_list, list) {
 891                 unmap_fmr_cnt++;
 892                 e_fmr = container_of(ib_fmr, struct ehca_mr, ib.ib_fmr);
 893                 shca = container_of(ib_fmr->device, struct ehca_shca,
 894                                     ib_device);
 895                 ret = ehca_unmap_one_fmr(shca, e_fmr);
 896                 if (ret) {
 897                         /* unmap failed, stop unmapping of rest of FMRs */
 898                         ehca_err(&shca->ib_device, "unmap of one FMR failed, "
 899                                  "stop rest, e_fmr=%p num_fmr=%x "
 900                                  "unmap_fmr_cnt=%x lkey=%x", e_fmr, num_fmr,
 901                                  unmap_fmr_cnt, e_fmr->ib.ib_fmr.lkey);
 902                         goto unmap_fmr_exit0;
 903                 }
 904         }
 905
 906 unmap_fmr_exit0:
 907         if (ret)
 908                 ehca_gen_err("ret=%x fmr_list=%p num_fmr=%x unmap_fmr_cnt=%x",
 909                              ret, fmr_list, num_fmr, unmap_fmr_cnt);
 910         return ret;
 911 } /* end ehca_unmap_fmr() */
 912
 913 /*----------------------------------------------------------------------*/
 914
 915 int ehca_dealloc_fmr(struct ib_fmr *fmr)
 916 {
 917         int ret;
 918         u64 h_ret;
 919         struct ehca_shca *shca =
 920                 container_of(fmr->device, struct ehca_shca, ib_device);
 921         struct ehca_mr *e_fmr = container_of(fmr, struct ehca_mr, ib.ib_fmr);
 922
 923         if (!(e_fmr->flags & EHCA_MR_FLAG_FMR)) {
 924                 ehca_err(fmr->device, "not a FMR, e_fmr=%p e_fmr->flags=%x",
 925                          e_fmr, e_fmr->flags);
 926                 ret = -EINVAL;
 927                 goto free_fmr_exit0;
 928         }
 929
 930         h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_fmr);
 931         if (h_ret != H_SUCCESS) {
 932                 ehca_err(fmr->device, "hipz_free_mr failed, h_ret=%lx e_fmr=%p "
 933                          "hca_hndl=%lx fmr_hndl=%lx fmr->lkey=%x",
 934                          h_ret, e_fmr, shca->ipz_hca_handle.handle,
 935                          e_fmr->ipz_mr_handle.handle, fmr->lkey);
 936                 ret = ehca2ib_return_code(h_ret);
 937                 goto free_fmr_exit0;
 938         }
 939         /* successful deregistration */
 940         ehca_mr_delete(e_fmr);
 941         return 0;
 942
 943 free_fmr_exit0:
 944         if (ret)
 945                 ehca_err(&shca->ib_device, "ret=%x fmr=%p", ret, fmr);
 946         return ret;
 947 } /* end ehca_dealloc_fmr() */
 948
 949 /*----------------------------------------------------------------------*/
 950
 951 int ehca_reg_mr(struct ehca_shca *shca,
 952                 struct ehca_mr *e_mr,
 953                 u64 *iova_start,
 954                 u64 size,
 955                 int acl,
 956                 struct ehca_pd *e_pd,
 957                 struct ehca_mr_pginfo *pginfo,
 958                 u32 *lkey, /*OUT*/
 959                 u32 *rkey) /*OUT*/
 960 {
 961         int ret;
 962         u64 h_ret;
 963         u32 hipz_acl;
 964         struct ehca_mr_hipzout_parms hipzout;
 965
 966         ehca_mrmw_map_acl(acl, &hipz_acl);
 967         ehca_mrmw_set_pgsize_hipz_acl(&hipz_acl);
 968         if (ehca_use_hp_mr == 1)
 969                 hipz_acl |= 0x00000001;
 970
 971         h_ret = hipz_h_alloc_resource_mr(shca->ipz_hca_handle, e_mr,
 972                                          (u64)iova_start, size, hipz_acl,
 973                                          e_pd->fw_pd, &hipzout);
 974         if (h_ret != H_SUCCESS) {
 975                 ehca_err(&shca->ib_device, "hipz_alloc_mr failed, h_ret=%lx "
 976                          "hca_hndl=%lx", h_ret, shca->ipz_hca_handle.handle);
 977                 ret = ehca2ib_return_code(h_ret);
 978                 goto ehca_reg_mr_exit0;
 979         }
 980
 981         e_mr->ipz_mr_handle = hipzout.handle;
 982
 983         ret = ehca_reg_mr_rpages(shca, e_mr, pginfo);
 984         if (ret)
 985                 goto ehca_reg_mr_exit1;
 986
 987         /* successful registration */
 988         e_mr->num_kpages = pginfo->num_kpages;
 989         e_mr->num_hwpages = pginfo->num_hwpages;
 990         e_mr->start = iova_start;
 991         e_mr->size = size;
 992         e_mr->acl = acl;
 993         *lkey = hipzout.lkey;
 994         *rkey = hipzout.rkey;
 995         return 0;
 996
 997 ehca_reg_mr_exit1:
 998         h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_mr);
 999         if (h_ret != H_SUCCESS) {
1000                 ehca_err(&shca->ib_device, "h_ret=%lx shca=%p e_mr=%p "
1001                          "iova_start=%p size=%lx acl=%x e_pd=%p lkey=%x "
1002                          "pginfo=%p num_kpages=%lx num_hwpages=%lx ret=%x",
1003                          h_ret, shca, e_mr, iova_start, size, acl, e_pd,
1004                          hipzout.lkey, pginfo, pginfo->num_kpages,
1005                          pginfo->num_hwpages, ret);
1006                 ehca_err(&shca->ib_device, "internal error in ehca_reg_mr, "
1007                          "not recoverable");
1008         }
1009 ehca_reg_mr_exit0:
1010         if (ret)
1011                 ehca_err(&shca->ib_device, "ret=%x shca=%p e_mr=%p "
1012                          "iova_start=%p size=%lx acl=%x e_pd=%p pginfo=%p "
1013                          "num_kpages=%lx num_hwpages=%lx",
1014                          ret, shca, e_mr, iova_start, size, acl, e_pd, pginfo,
1015                          pginfo->num_kpages, pginfo->num_hwpages);
1016         return ret;
1017 } /* end ehca_reg_mr() */
1018
1019 /*----------------------------------------------------------------------*/
1020
1021 int ehca_reg_mr_rpages(struct ehca_shca *shca,
1022                        struct ehca_mr *e_mr,
1023                        struct ehca_mr_pginfo *pginfo)
1024 {
1025         int ret = 0;
1026         u64 h_ret;
1027         u32 rnum;
1028         u64 rpage;
1029         u32 i;
1030         u64 *kpage;
1031
1032         kpage = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
1033         if (!kpage) {
1034                 ehca_err(&shca->ib_device, "kpage alloc failed");
1035                 ret = -ENOMEM;
1036                 goto ehca_reg_mr_rpages_exit0;
1037         }
1038
1039         /* max 512 pages per shot */
1040         for (i = 0; i < NUM_CHUNKS(pginfo->num_hwpages, MAX_RPAGES); i++) {
1041
1042                 if (i == NUM_CHUNKS(pginfo->num_hwpages, MAX_RPAGES) - 1) {
1043                         rnum = pginfo->num_hwpages % MAX_RPAGES; /* last shot */
1044                         if (rnum == 0)
1045                                 rnum = MAX_RPAGES;      /* last shot is full */
1046                 } else
1047                         rnum = MAX_RPAGES;
1048
1049                 ret = ehca_set_pagebuf(pginfo, rnum, kpage);
1050                 if (ret) {
1051                         ehca_err(&shca->ib_device, "ehca_set_pagebuf "
1052                                          "bad rc, ret=%x rnum=%x kpage=%p",
1053                                          ret, rnum, kpage);
1054                         goto ehca_reg_mr_rpages_exit1;
1055                 }
1056
1057                 if (rnum > 1) {
1058                         rpage = virt_to_abs(kpage);
1059                         if (!rpage) {
1060                                 ehca_err(&shca->ib_device, "kpage=%p i=%x",
1061                                          kpage, i);
1062                                 ret = -EFAULT;
1063                                 goto ehca_reg_mr_rpages_exit1;
1064                         }
1065                 } else
1066                         rpage = *kpage;
1067
1068                 h_ret = hipz_h_register_rpage_mr(shca->ipz_hca_handle, e_mr,
1069                                                  0, /* pagesize 4k */
1070                                                  0, rpage, rnum);
1071
1072                 if (i == NUM_CHUNKS(pginfo->num_hwpages, MAX_RPAGES) - 1) {
1073                         /*
1074                          * check for 'registration complete'==H_SUCCESS
1075                          * and for 'page registered'==H_PAGE_REGISTERED
1076                          */
1077                         if (h_ret != H_SUCCESS) {
1078                                 ehca_err(&shca->ib_device, "last "
1079                                          "hipz_reg_rpage_mr failed, h_ret=%lx "
1080                                          "e_mr=%p i=%x hca_hndl=%lx mr_hndl=%lx"
1081                                          " lkey=%x", h_ret, e_mr, i,
1082                                          shca->ipz_hca_handle.handle,
1083                                          e_mr->ipz_mr_handle.handle,
1084                                          e_mr->ib.ib_mr.lkey);
1085                                 ret = ehca2ib_return_code(h_ret);
1086                                 break;
1087                         } else
1088                                 ret = 0;
1089                 } else if (h_ret != H_PAGE_REGISTERED) {
1090                         ehca_err(&shca->ib_device, "hipz_reg_rpage_mr failed, "
1091                                  "h_ret=%lx e_mr=%p i=%x lkey=%x hca_hndl=%lx "
1092                                  "mr_hndl=%lx", h_ret, e_mr, i,
1093                                  e_mr->ib.ib_mr.lkey,
1094                                  shca->ipz_hca_handle.handle,
1095                                  e_mr->ipz_mr_handle.handle);
1096                         ret = ehca2ib_return_code(h_ret);
1097                         break;
1098                 } else
1099                         ret = 0;
1100         } /* end for(i) */
1101
1102
1103 ehca_reg_mr_rpages_exit1:
1104         ehca_free_fw_ctrlblock(kpage);
1105 ehca_reg_mr_rpages_exit0:
1106         if (ret)
1107                 ehca_err(&shca->ib_device, "ret=%x shca=%p e_mr=%p pginfo=%p "
1108                          "num_kpages=%lx num_hwpages=%lx", ret, shca, e_mr,
1109                          pginfo, pginfo->num_kpages, pginfo->num_hwpages);
1110         return ret;
1111 } /* end ehca_reg_mr_rpages() */
1112
1113 /*----------------------------------------------------------------------*/
1114
1115 inline int ehca_rereg_mr_rereg1(struct ehca_shca *shca,
1116                                 struct ehca_mr *e_mr,
1117                                 u64 *iova_start,
1118                                 u64 size,
1119                                 u32 acl,
1120                                 struct ehca_pd *e_pd,
1121                                 struct ehca_mr_pginfo *pginfo,
1122                                 u32 *lkey, /*OUT*/
1123                                 u32 *rkey) /*OUT*/
1124 {
1125         int ret;
1126         u64 h_ret;
1127         u32 hipz_acl;
1128         u64 *kpage;
1129         u64 rpage;
1130         struct ehca_mr_pginfo pginfo_save;
1131         struct ehca_mr_hipzout_parms hipzout;
1132
1133         ehca_mrmw_map_acl(acl, &hipz_acl);
1134         ehca_mrmw_set_pgsize_hipz_acl(&hipz_acl);
1135
1136         kpage = ehca_alloc_fw_ctrlblock(GFP_KERNEL);
1137         if (!kpage) {
1138                 ehca_err(&shca->ib_device, "kpage alloc failed");
1139                 ret = -ENOMEM;
1140                 goto ehca_rereg_mr_rereg1_exit0;
1141         }
1142
1143         pginfo_save = *pginfo;
1144         ret = ehca_set_pagebuf(pginfo, pginfo->num_hwpages, kpage);
1145         if (ret) {
1146                 ehca_err(&shca->ib_device, "set pagebuf failed, e_mr=%p "
1147                          "pginfo=%p type=%x num_kpages=%lx num_hwpages=%lx "
1148                          "kpage=%p", e_mr, pginfo, pginfo->type,
1149                          pginfo->num_kpages, pginfo->num_hwpages, kpage);
1150                 goto ehca_rereg_mr_rereg1_exit1;
1151         }
1152         rpage = virt_to_abs(kpage);
1153         if (!rpage) {
1154                 ehca_err(&shca->ib_device, "kpage=%p", kpage);
1155                 ret = -EFAULT;
1156                 goto ehca_rereg_mr_rereg1_exit1;
1157         }
1158         h_ret = hipz_h_reregister_pmr(shca->ipz_hca_handle, e_mr,
1159                                       (u64)iova_start, size, hipz_acl,
1160                                       e_pd->fw_pd, rpage, &hipzout);
1161         if (h_ret != H_SUCCESS) {
1162                 /*
1163                  * reregistration unsuccessful, try it again with the 3 hCalls,
1164                  * e.g. this is required in case H_MR_CONDITION
1165                  * (MW bound or MR is shared)
1166                  */
1167                 ehca_warn(&shca->ib_device, "hipz_h_reregister_pmr failed "
1168                           "(Rereg1), h_ret=%lx e_mr=%p", h_ret, e_mr);
1169                 *pginfo = pginfo_save;
1170                 ret = -EAGAIN;
1171         } else if ((u64 *)hipzout.vaddr != iova_start) {
1172                 ehca_err(&shca->ib_device, "PHYP changed iova_start in "
1173                          "rereg_pmr, iova_start=%p iova_start_out=%lx e_mr=%p "
1174                          "mr_handle=%lx lkey=%x lkey_out=%x", iova_start,
1175                          hipzout.vaddr, e_mr, e_mr->ipz_mr_handle.handle,
1176                          e_mr->ib.ib_mr.lkey, hipzout.lkey);
1177                 ret = -EFAULT;
1178         } else {
1179                 /*
1180                  * successful reregistration
1181                  * note: start and start_out are identical for eServer HCAs
1182                  */
1183                 e_mr->num_kpages = pginfo->num_kpages;
1184                 e_mr->num_hwpages = pginfo->num_hwpages;
1185                 e_mr->start = iova_start;
1186                 e_mr->size = size;
1187                 e_mr->acl = acl;
1188                 *lkey = hipzout.lkey;
1189                 *rkey = hipzout.rkey;
1190         }
1191
1192 ehca_rereg_mr_rereg1_exit1:
1193         ehca_free_fw_ctrlblock(kpage);
1194 ehca_rereg_mr_rereg1_exit0:
1195         if ( ret && (ret != -EAGAIN) )
1196                 ehca_err(&shca->ib_device, "ret=%x lkey=%x rkey=%x "
1197                          "pginfo=%p num_kpages=%lx num_hwpages=%lx",
1198                          ret, *lkey, *rkey, pginfo, pginfo->num_kpages,
1199                          pginfo->num_hwpages);
1200         return ret;
1201 } /* end ehca_rereg_mr_rereg1() */
1202
1203 /*----------------------------------------------------------------------*/
1204
1205 int ehca_rereg_mr(struct ehca_shca *shca,
1206                   struct ehca_mr *e_mr,
1207                   u64 *iova_start,
1208                   u64 size,
1209                   int acl,
1210                   struct ehca_pd *e_pd,
1211                   struct ehca_mr_pginfo *pginfo,
1212                   u32 *lkey,
1213                   u32 *rkey)
1214 {
1215         int ret = 0;
1216         u64 h_ret;
1217         int rereg_1_hcall = 1; /* 1: use hipz_h_reregister_pmr directly */
1218         int rereg_3_hcall = 0; /* 1: use 3 hipz calls for reregistration */
1219
1220         /* first determine reregistration hCall(s) */
1221         if ((pginfo->num_hwpages > MAX_RPAGES) ||
1222             (e_mr->num_hwpages > MAX_RPAGES) ||
1223             (pginfo->num_hwpages > e_mr->num_hwpages)) {
1224                 ehca_dbg(&shca->ib_device, "Rereg3 case, "
1225                          "pginfo->num_hwpages=%lx e_mr->num_hwpages=%x",
1226                          pginfo->num_hwpages, e_mr->num_hwpages);
1227                 rereg_1_hcall = 0;
1228                 rereg_3_hcall = 1;
1229         }
1230
1231         if (e_mr->flags & EHCA_MR_FLAG_MAXMR) { /* check for max-MR */
1232                 rereg_1_hcall = 0;
1233                 rereg_3_hcall = 1;
1234                 e_mr->flags &= ~EHCA_MR_FLAG_MAXMR;
1235                 ehca_err(&shca->ib_device, "Rereg MR for max-MR! e_mr=%p",
1236                          e_mr);
1237         }
1238
1239         if (rereg_1_hcall) {
1240                 ret = ehca_rereg_mr_rereg1(shca, e_mr, iova_start, size,
1241                                            acl, e_pd, pginfo, lkey, rkey);
1242                 if (ret) {
1243                         if (ret == -EAGAIN)
1244                                 rereg_3_hcall = 1;
1245                         else
1246                                 goto ehca_rereg_mr_exit0;
1247                 }
1248         }
1249
1250         if (rereg_3_hcall) {
1251                 struct ehca_mr save_mr;
1252
1253                 /* first deregister old MR */
1254                 h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_mr);
1255                 if (h_ret != H_SUCCESS) {
1256                         ehca_err(&shca->ib_device, "hipz_free_mr failed, "
1257                                  "h_ret=%lx e_mr=%p hca_hndl=%lx mr_hndl=%lx "
1258                                  "mr->lkey=%x",
1259                                  h_ret, e_mr, shca->ipz_hca_handle.handle,
1260                                  e_mr->ipz_mr_handle.handle,
1261                                  e_mr->ib.ib_mr.lkey);
1262                         ret = ehca2ib_return_code(h_ret);
1263                         goto ehca_rereg_mr_exit0;
1264                 }
1265                 /* clean ehca_mr_t, without changing struct ib_mr and lock */
1266                 save_mr = *e_mr;
1267                 ehca_mr_deletenew(e_mr);
1268
1269                 /* set some MR values */
1270                 e_mr->flags = save_mr.flags;
1271                 e_mr->fmr_page_size = save_mr.fmr_page_size;
1272                 e_mr->fmr_max_pages = save_mr.fmr_max_pages;
1273                 e_mr->fmr_max_maps = save_mr.fmr_max_maps;
1274                 e_mr->fmr_map_cnt = save_mr.fmr_map_cnt;
1275
1276                 ret = ehca_reg_mr(shca, e_mr, iova_start, size, acl,
1277                                       e_pd, pginfo, lkey, rkey);
1278                 if (ret) {
1279                         u32 offset = (u64)(&e_mr->flags) - (u64)e_mr;
1280                         memcpy(&e_mr->flags, &(save_mr.flags),
1281                                sizeof(struct ehca_mr) - offset);
1282                         goto ehca_rereg_mr_exit0;
1283                 }
1284         }
1285
1286 ehca_rereg_mr_exit0:
1287         if (ret)
1288                 ehca_err(&shca->ib_device, "ret=%x shca=%p e_mr=%p "
1289                          "iova_start=%p size=%lx acl=%x e_pd=%p pginfo=%p "
1290                          "num_kpages=%lx lkey=%x rkey=%x rereg_1_hcall=%x "
1291                          "rereg_3_hcall=%x", ret, shca, e_mr, iova_start, size,
1292                          acl, e_pd, pginfo, pginfo->num_kpages, *lkey, *rkey,
1293                          rereg_1_hcall, rereg_3_hcall);
1294         return ret;
1295 } /* end ehca_rereg_mr() */
1296
1297 /*----------------------------------------------------------------------*/
1298
1299 int ehca_unmap_one_fmr(struct ehca_shca *shca,
1300                        struct ehca_mr *e_fmr)
1301 {
1302         int ret = 0;
1303         u64 h_ret;
1304         struct ehca_pd *e_pd =
1305                 container_of(e_fmr->ib.ib_fmr.pd, struct ehca_pd, ib_pd);
1306         struct ehca_mr save_fmr;
1307         u32 tmp_lkey, tmp_rkey;
1308         struct ehca_mr_pginfo pginfo;
1309         struct ehca_mr_hipzout_parms hipzout;
1310         struct ehca_mr save_mr;
1311
1312         if (e_fmr->fmr_max_pages <= MAX_RPAGES) {
1313                 /*
1314                  * note: after using rereg hcall with len=0,
1315                  * rereg hcall must be used again for registering pages
1316                  */
1317                 h_ret = hipz_h_reregister_pmr(shca->ipz_hca_handle, e_fmr, 0,
1318                                               0, 0, e_pd->fw_pd, 0, &hipzout);
1319                 if (h_ret == H_SUCCESS) {
1320                         /* successful reregistration */
1321                         e_fmr->start = NULL;
1322                         e_fmr->size = 0;
1323                         tmp_lkey = hipzout.lkey;
1324                         tmp_rkey = hipzout.rkey;
1325                         return 0;
1326                 }
1327                 /*
1328                  * should not happen, because length checked above,
1329                  * FMRs are not shared and no MW bound to FMRs
1330                  */
1331                 ehca_err(&shca->ib_device, "hipz_reregister_pmr failed "
1332                          "(Rereg1), h_ret=%lx e_fmr=%p hca_hndl=%lx "
1333                          "mr_hndl=%lx lkey=%x lkey_out=%x",
1334                          h_ret, e_fmr, shca->ipz_hca_handle.handle,
1335                          e_fmr->ipz_mr_handle.handle,
1336                          e_fmr->ib.ib_fmr.lkey, hipzout.lkey);
1337                 /* try free and rereg */
1338         }
1339
1340         /* first free old FMR */
1341         h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_fmr);
1342         if (h_ret != H_SUCCESS) {
1343                 ehca_err(&shca->ib_device, "hipz_free_mr failed, "
1344                          "h_ret=%lx e_fmr=%p hca_hndl=%lx mr_hndl=%lx "
1345                          "lkey=%x",
1346                          h_ret, e_fmr, shca->ipz_hca_handle.handle,
1347                          e_fmr->ipz_mr_handle.handle,
1348                          e_fmr->ib.ib_fmr.lkey);
1349                 ret = ehca2ib_return_code(h_ret);
1350                 goto ehca_unmap_one_fmr_exit0;
1351         }
1352         /* clean ehca_mr_t, without changing lock */
1353         save_fmr = *e_fmr;
1354         ehca_mr_deletenew(e_fmr);
1355
1356         /* set some MR values */
1357         e_fmr->flags = save_fmr.flags;
1358         e_fmr->fmr_page_size = save_fmr.fmr_page_size;
1359         e_fmr->fmr_max_pages = save_fmr.fmr_max_pages;
1360         e_fmr->fmr_max_maps = save_fmr.fmr_max_maps;
1361         e_fmr->fmr_map_cnt = save_fmr.fmr_map_cnt;
1362         e_fmr->acl = save_fmr.acl;
1363
1364         memset(&pginfo, 0, sizeof(pginfo));
1365         pginfo.type = EHCA_MR_PGI_FMR;
1366         pginfo.num_kpages = 0;
1367         pginfo.num_hwpages = 0;
1368         ret = ehca_reg_mr(shca, e_fmr, NULL,
1369                           (e_fmr->fmr_max_pages * e_fmr->fmr_page_size),
1370                           e_fmr->acl, e_pd, &pginfo, &tmp_lkey,
1371                           &tmp_rkey);
1372         if (ret) {
1373                 u32 offset = (u64)(&e_fmr->flags) - (u64)e_fmr;
1374                 memcpy(&e_fmr->flags, &(save_mr.flags),
1375                        sizeof(struct ehca_mr) - offset);
1376                 goto ehca_unmap_one_fmr_exit0;
1377         }
1378
1379 ehca_unmap_one_fmr_exit0:
1380         if (ret)
1381                 ehca_err(&shca->ib_device, "ret=%x tmp_lkey=%x tmp_rkey=%x "
1382                          "fmr_max_pages=%x",
1383                          ret, tmp_lkey, tmp_rkey, e_fmr->fmr_max_pages);
1384         return ret;
1385 } /* end ehca_unmap_one_fmr() */
1386
1387 /*----------------------------------------------------------------------*/
1388
1389 int ehca_reg_smr(struct ehca_shca *shca,
1390                  struct ehca_mr *e_origmr,
1391                  struct ehca_mr *e_newmr,
1392                  u64 *iova_start,
1393                  int acl,
1394                  struct ehca_pd *e_pd,
1395                  u32 *lkey, /*OUT*/
1396                  u32 *rkey) /*OUT*/
1397 {
1398         int ret = 0;
1399         u64 h_ret;
1400         u32 hipz_acl;
1401         struct ehca_mr_hipzout_parms hipzout;
1402
1403         ehca_mrmw_map_acl(acl, &hipz_acl);
1404         ehca_mrmw_set_pgsize_hipz_acl(&hipz_acl);
1405
1406         h_ret = hipz_h_register_smr(shca->ipz_hca_handle, e_newmr, e_origmr,
1407                                     (u64)iova_start, hipz_acl, e_pd->fw_pd,
1408                                     &hipzout);
1409         if (h_ret != H_SUCCESS) {
1410                 ehca_err(&shca->ib_device, "hipz_reg_smr failed, h_ret=%lx "
1411                          "shca=%p e_origmr=%p e_newmr=%p iova_start=%p acl=%x "
1412                          "e_pd=%p hca_hndl=%lx mr_hndl=%lx lkey=%x",
1413                          h_ret, shca, e_origmr, e_newmr, iova_start, acl, e_pd,
1414                          shca->ipz_hca_handle.handle,
1415                          e_origmr->ipz_mr_handle.handle,
1416                          e_origmr->ib.ib_mr.lkey);
1417                 ret = ehca2ib_return_code(h_ret);
1418                 goto ehca_reg_smr_exit0;
1419         }
1420         /* successful registration */
1421         e_newmr->num_kpages = e_origmr->num_kpages;
1422         e_newmr->num_hwpages = e_origmr->num_hwpages;
1423         e_newmr->start = iova_start;
1424         e_newmr->size = e_origmr->size;
1425         e_newmr->acl = acl;
1426         e_newmr->ipz_mr_handle = hipzout.handle;
1427         *lkey = hipzout.lkey;
1428         *rkey = hipzout.rkey;
1429         return 0;
1430
1431 ehca_reg_smr_exit0:
1432         if (ret)
1433                 ehca_err(&shca->ib_device, "ret=%x shca=%p e_origmr=%p "
1434                          "e_newmr=%p iova_start=%p acl=%x e_pd=%p",
1435                          ret, shca, e_origmr, e_newmr, iova_start, acl, e_pd);
1436         return ret;
1437 } /* end ehca_reg_smr() */
1438
1439 /*----------------------------------------------------------------------*/
1440
1441 /* register internal max-MR to internal SHCA */
1442 int ehca_reg_internal_maxmr(
1443         struct ehca_shca *shca,
1444         struct ehca_pd *e_pd,
1445         struct ehca_mr **e_maxmr)  /*OUT*/
1446 {
1447         int ret;
1448         struct ehca_mr *e_mr;
1449         u64 *iova_start;
1450         u64 size_maxmr;
1451         struct ehca_mr_pginfo pginfo;
1452         struct ib_phys_buf ib_pbuf;
1453         u32 num_kpages;
1454         u32 num_hwpages;
1455
1456         e_mr = ehca_mr_new();
1457         if (!e_mr) {
1458                 ehca_err(&shca->ib_device, "out of memory");
1459                 ret = -ENOMEM;
1460                 goto ehca_reg_internal_maxmr_exit0;
1461         }
1462         e_mr->flags |= EHCA_MR_FLAG_MAXMR;
1463
1464         /* register internal max-MR on HCA */
1465         size_maxmr = (u64)high_memory - PAGE_OFFSET;
1466         iova_start = (u64 *)KERNELBASE;
1467         ib_pbuf.addr = 0;
1468         ib_pbuf.size = size_maxmr;
1469         num_kpages = NUM_CHUNKS(((u64)iova_start % PAGE_SIZE) + size_maxmr,
1470                                 PAGE_SIZE);
1471         num_hwpages = NUM_CHUNKS(((u64)iova_start % EHCA_PAGESIZE) + size_maxmr,
1472                                  EHCA_PAGESIZE);
1473
1474         memset(&pginfo, 0, sizeof(pginfo));
1475         pginfo.type = EHCA_MR_PGI_PHYS;
1476         pginfo.num_kpages = num_kpages;
1477         pginfo.num_hwpages = num_hwpages;
1478         pginfo.u.phy.num_phys_buf = 1;
1479         pginfo.u.phy.phys_buf_array = &ib_pbuf;
1480
1481         ret = ehca_reg_mr(shca, e_mr, iova_start, size_maxmr, 0, e_pd,
1482                           &pginfo, &e_mr->ib.ib_mr.lkey,
1483                           &e_mr->ib.ib_mr.rkey);
1484         if (ret) {
1485                 ehca_err(&shca->ib_device, "reg of internal max MR failed, "
1486                          "e_mr=%p iova_start=%p size_maxmr=%lx num_kpages=%x "
1487                          "num_hwpages=%x", e_mr, iova_start, size_maxmr,
1488                          num_kpages, num_hwpages);
1489                 goto ehca_reg_internal_maxmr_exit1;
1490         }
1491
1492         /* successful registration of all pages */
1493         e_mr->ib.ib_mr.device = e_pd->ib_pd.device;
1494         e_mr->ib.ib_mr.pd = &e_pd->ib_pd;
1495         e_mr->ib.ib_mr.uobject = NULL;
1496         atomic_inc(&(e_pd->ib_pd.usecnt));
1497         atomic_set(&(e_mr->ib.ib_mr.usecnt), 0);
1498         *e_maxmr = e_mr;
1499         return 0;
1500
1501 ehca_reg_internal_maxmr_exit1:
1502         ehca_mr_delete(e_mr);
1503 ehca_reg_internal_maxmr_exit0:
1504         if (ret)
1505                 ehca_err(&shca->ib_device, "ret=%x shca=%p e_pd=%p e_maxmr=%p",
1506                          ret, shca, e_pd, e_maxmr);
1507         return ret;
1508 } /* end ehca_reg_internal_maxmr() */
1509
1510 /*----------------------------------------------------------------------*/
1511
1512 int ehca_reg_maxmr(struct ehca_shca *shca,
1513                    struct ehca_mr *e_newmr,
1514                    u64 *iova_start,
1515                    int acl,
1516                    struct ehca_pd *e_pd,
1517                    u32 *lkey,
1518                    u32 *rkey)
1519 {
1520         u64 h_ret;
1521         struct ehca_mr *e_origmr = shca->maxmr;
1522         u32 hipz_acl;
1523         struct ehca_mr_hipzout_parms hipzout;
1524
1525         ehca_mrmw_map_acl(acl, &hipz_acl);
1526         ehca_mrmw_set_pgsize_hipz_acl(&hipz_acl);
1527
1528         h_ret = hipz_h_register_smr(shca->ipz_hca_handle, e_newmr, e_origmr,
1529                                     (u64)iova_start, hipz_acl, e_pd->fw_pd,
1530                                     &hipzout);
1531         if (h_ret != H_SUCCESS) {
1532                 ehca_err(&shca->ib_device, "hipz_reg_smr failed, h_ret=%lx "
1533                          "e_origmr=%p hca_hndl=%lx mr_hndl=%lx lkey=%x",
1534                          h_ret, e_origmr, shca->ipz_hca_handle.handle,
1535                          e_origmr->ipz_mr_handle.handle,
1536                          e_origmr->ib.ib_mr.lkey);
1537                 return ehca2ib_return_code(h_ret);
1538         }
1539         /* successful registration */
1540         e_newmr->num_kpages = e_origmr->num_kpages;
1541         e_newmr->num_hwpages = e_origmr->num_hwpages;
1542         e_newmr->start = iova_start;
1543         e_newmr->size = e_origmr->size;
1544         e_newmr->acl = acl;
1545         e_newmr->ipz_mr_handle = hipzout.handle;
1546         *lkey = hipzout.lkey;
1547         *rkey = hipzout.rkey;
1548         return 0;
1549 } /* end ehca_reg_maxmr() */
1550
1551 /*----------------------------------------------------------------------*/
1552
1553 int ehca_dereg_internal_maxmr(struct ehca_shca *shca)
1554 {
1555         int ret;
1556         struct ehca_mr *e_maxmr;
1557         struct ib_pd *ib_pd;
1558
1559         if (!shca->maxmr) {
1560                 ehca_err(&shca->ib_device, "bad call, shca=%p", shca);
1561                 ret = -EINVAL;
1562                 goto ehca_dereg_internal_maxmr_exit0;
1563         }
1564
1565         e_maxmr = shca->maxmr;
1566         ib_pd = e_maxmr->ib.ib_mr.pd;
1567         shca->maxmr = NULL; /* remove internal max-MR indication from SHCA */
1568
1569         ret = ehca_dereg_mr(&e_maxmr->ib.ib_mr);
1570         if (ret) {
1571                 ehca_err(&shca->ib_device, "dereg internal max-MR failed, "
1572                          "ret=%x e_maxmr=%p shca=%p lkey=%x",
1573                          ret, e_maxmr, shca, e_maxmr->ib.ib_mr.lkey);
1574                 shca->maxmr = e_maxmr;
1575                 goto ehca_dereg_internal_maxmr_exit0;
1576         }
1577
1578         atomic_dec(&ib_pd->usecnt);
1579
1580 ehca_dereg_internal_maxmr_exit0:
1581         if (ret)
1582                 ehca_err(&shca->ib_device, "ret=%x shca=%p shca->maxmr=%p",
1583                          ret, shca, shca->maxmr);
1584         return ret;
1585 } /* end ehca_dereg_internal_maxmr() */
1586
1587 /*----------------------------------------------------------------------*/
1588
1589 /*
1590  * check physical buffer array of MR verbs for validness and
1591  * calculates MR size
1592  */
1593 int ehca_mr_chk_buf_and_calc_size(struct ib_phys_buf *phys_buf_array,
1594                                   int num_phys_buf,
1595                                   u64 *iova_start,
1596                                   u64 *size)
1597 {
1598         struct ib_phys_buf *pbuf = phys_buf_array;
1599         u64 size_count = 0;
1600         u32 i;
1601
1602         if (num_phys_buf == 0) {
1603                 ehca_gen_err("bad phys buf array len, num_phys_buf=0");
1604                 return -EINVAL;
1605         }
1606         /* check first buffer */
1607         if (((u64)iova_start & ~PAGE_MASK) != (pbuf->addr & ~PAGE_MASK)) {
1608                 ehca_gen_err("iova_start/addr mismatch, iova_start=%p "
1609                              "pbuf->addr=%lx pbuf->size=%lx",
1610                              iova_start, pbuf->addr, pbuf->size);
1611                 return -EINVAL;
1612         }
1613         if (((pbuf->addr + pbuf->size) % PAGE_SIZE) &&
1614             (num_phys_buf > 1)) {
1615                 ehca_gen_err("addr/size mismatch in 1st buf, pbuf->addr=%lx "
1616                              "pbuf->size=%lx", pbuf->addr, pbuf->size);
1617                 return -EINVAL;
1618         }
1619
1620         for (i = 0; i < num_phys_buf; i++) {
1621                 if ((i > 0) && (pbuf->addr % PAGE_SIZE)) {
1622                         ehca_gen_err("bad address, i=%x pbuf->addr=%lx "
1623                                      "pbuf->size=%lx",
1624                                      i, pbuf->addr, pbuf->size);
1625                         return -EINVAL;
1626                 }
1627                 if (((i > 0) && /* not 1st */
1628                      (i < (num_phys_buf - 1)) &&        /* not last */
1629                      (pbuf->size % PAGE_SIZE)) || (pbuf->size == 0)) {
1630                         ehca_gen_err("bad size, i=%x pbuf->size=%lx",
1631                                      i, pbuf->size);
1632                         return -EINVAL;
1633                 }
1634                 size_count += pbuf->size;
1635                 pbuf++;
1636         }
1637
1638         *size = size_count;
1639         return 0;
1640 } /* end ehca_mr_chk_buf_and_calc_size() */
1641
1642 /*----------------------------------------------------------------------*/
1643
1644 /* check page list of map FMR verb for validness */
1645 int ehca_fmr_check_page_list(struct ehca_mr *e_fmr,
1646                              u64 *page_list,
1647                              int list_len)
1648 {
1649         u32 i;
1650         u64 *page;
1651
1652         if ((list_len == 0) || (list_len > e_fmr->fmr_max_pages)) {
1653                 ehca_gen_err("bad list_len, list_len=%x "
1654                              "e_fmr->fmr_max_pages=%x fmr=%p",
1655                              list_len, e_fmr->fmr_max_pages, e_fmr);
1656                 return -EINVAL;
1657         }
1658
1659         /* each page must be aligned */
1660         page = page_list;
1661         for (i = 0; i < list_len; i++) {
1662                 if (*page % e_fmr->fmr_page_size) {
1663                         ehca_gen_err("bad page, i=%x *page=%lx page=%p fmr=%p "
1664                                      "fmr_page_size=%x", i, *page, page, e_fmr,
1665                                      e_fmr->fmr_page_size);
1666                         return -EINVAL;
1667                 }
1668                 page++;
1669         }
1670
1671         return 0;
1672 } /* end ehca_fmr_check_page_list() */
1673
1674 /*----------------------------------------------------------------------*/
1675
1676 /* PAGE_SIZE >= pginfo->hwpage_size */
1677 static int ehca_set_pagebuf_user1(struct ehca_mr_pginfo *pginfo,
1678                                   u32 number,
1679                                   u64 *kpage)
1680 {
1681         int ret = 0;
1682         struct ib_umem_chunk *prev_chunk;
1683         struct ib_umem_chunk *chunk;
1684         u64 pgaddr;
1685         u32 i = 0;
1686         u32 j = 0;
1687
1688         /* loop over desired chunk entries */
1689         chunk      = pginfo->u.usr.next_chunk;
1690         prev_chunk = pginfo->u.usr.next_chunk;
1691         list_for_each_entry_continue(
1692                 chunk, (&(pginfo->u.usr.region->chunk_list)), list) {
1693                 for (i = pginfo->u.usr.next_nmap; i < chunk->nmap; ) {
1694                         pgaddr = page_to_pfn(chunk->page_list[i].page)
1695                                 << PAGE_SHIFT ;
1696                         *kpage = phys_to_abs(pgaddr +
1697                                              (pginfo->next_hwpage *
1698                                               EHCA_PAGESIZE));
1699                         if ( !(*kpage) ) {
1700                                 ehca_gen_err("pgaddr=%lx "
1701                                              "chunk->page_list[i]=%lx "
1702                                              "i=%x next_hwpage=%lx",
1703                                              pgaddr, (u64)sg_dma_address(
1704                                                      &chunk->page_list[i]),
1705                                              i, pginfo->next_hwpage);
1706                                 return -EFAULT;
1707                         }
1708                         (pginfo->hwpage_cnt)++;
1709                         (pginfo->next_hwpage)++;
1710                         kpage++;
1711                         if (pginfo->next_hwpage %
1712                             (PAGE_SIZE / EHCA_PAGESIZE) == 0) {
1713                                 (pginfo->kpage_cnt)++;
1714                                 (pginfo->u.usr.next_nmap)++;
1715                                 pginfo->next_hwpage = 0;
1716                                 i++;
1717                         }
1718                         j++;
1719                         if (j >= number) break;
1720                 }
1721                 if ((pginfo->u.usr.next_nmap >= chunk->nmap) &&
1722                     (j >= number)) {
1723                         pginfo->u.usr.next_nmap = 0;
1724                         prev_chunk = chunk;
1725                         break;
1726                 } else if (pginfo->u.usr.next_nmap >= chunk->nmap) {
1727                         pginfo->u.usr.next_nmap = 0;
1728                         prev_chunk = chunk;
1729                 } else if (j >= number)
1730                         break;
1731                 else
1732                         prev_chunk = chunk;
1733         }
1734         pginfo->u.usr.next_chunk =
1735                 list_prepare_entry(prev_chunk,
1736                                    (&(pginfo->u.usr.region->chunk_list)),
1737                                    list);
1738         return ret;
1739 }
1740
1741 int ehca_set_pagebuf_phys(struct ehca_mr_pginfo *pginfo,
1742                           u32 number,
1743                           u64 *kpage)
1744 {
1745         int ret = 0;
1746         struct ib_phys_buf *pbuf;
1747         u64 num_hw, offs_hw;
1748         u32 i = 0;
1749
1750         /* loop over desired phys_buf_array entries */
1751         while (i < number) {
1752                 pbuf   = pginfo->u.phy.phys_buf_array + pginfo->u.phy.next_buf;
1753                 num_hw  = NUM_CHUNKS((pbuf->addr % EHCA_PAGESIZE) +
1754                                      pbuf->size, EHCA_PAGESIZE);
1755                 offs_hw = (pbuf->addr & ~PAGE_MASK) / EHCA_PAGESIZE;
1756                 while (pginfo->next_hwpage < offs_hw + num_hw) {
1757                         /* sanity check */
1758                         if ((pginfo->kpage_cnt >= pginfo->num_kpages) ||
1759                             (pginfo->hwpage_cnt >= pginfo->num_hwpages)) {
1760                                 ehca_gen_err("kpage_cnt >= num_kpages, "
1761                                              "kpage_cnt=%lx num_kpages=%lx "
1762                                              "hwpage_cnt=%lx "
1763                                              "num_hwpages=%lx i=%x",
1764                                              pginfo->kpage_cnt,
1765                                              pginfo->num_kpages,
1766                                              pginfo->hwpage_cnt,
1767                                              pginfo->num_hwpages, i);
1768                                 return -EFAULT;
1769                         }
1770                         *kpage = phys_to_abs(
1771                                 (pbuf->addr & EHCA_PAGEMASK)
1772                                 + (pginfo->next_hwpage * EHCA_PAGESIZE));
1773                         if ( !(*kpage) && pbuf->addr ) {
1774                                 ehca_gen_err("pbuf->addr=%lx "
1775                                              "pbuf->size=%lx "
1776                                              "next_hwpage=%lx", pbuf->addr,
1777                                              pbuf->size,
1778                                              pginfo->next_hwpage);
1779                                 return -EFAULT;
1780                         }
1781                         (pginfo->hwpage_cnt)++;
1782                         (pginfo->next_hwpage)++;
1783                         if (pginfo->next_hwpage %
1784                             (PAGE_SIZE / EHCA_PAGESIZE) == 0)
1785                                 (pginfo->kpage_cnt)++;
1786                         kpage++;
1787                         i++;
1788                         if (i >= number) break;
1789                 }
1790                 if (pginfo->next_hwpage >= offs_hw + num_hw) {
1791                         (pginfo->u.phy.next_buf)++;
1792                         pginfo->next_hwpage = 0;
1793                 }
1794         }
1795         return ret;
1796 }
1797
1798 int ehca_set_pagebuf_fmr(struct ehca_mr_pginfo *pginfo,
1799                          u32 number,
1800                          u64 *kpage)
1801 {
1802         int ret = 0;
1803         u64 *fmrlist;
1804         u32 i;
1805
1806         /* loop over desired page_list entries */
1807         fmrlist = pginfo->u.fmr.page_list + pginfo->u.fmr.next_listelem;
1808         for (i = 0; i < number; i++) {
1809                 *kpage = phys_to_abs((*fmrlist & EHCA_PAGEMASK) +
1810                                      pginfo->next_hwpage * EHCA_PAGESIZE);
1811                 if ( !(*kpage) ) {
1812                         ehca_gen_err("*fmrlist=%lx fmrlist=%p "
1813                                      "next_listelem=%lx next_hwpage=%lx",
1814                                      *fmrlist, fmrlist,
1815                                      pginfo->u.fmr.next_listelem,
1816                                      pginfo->next_hwpage);
1817                         return -EFAULT;
1818                 }
1819                 (pginfo->hwpage_cnt)++;
1820                 (pginfo->next_hwpage)++;
1821                 kpage++;
1822                 if (pginfo->next_hwpage %
1823                     (pginfo->u.fmr.fmr_pgsize / EHCA_PAGESIZE) == 0) {
1824                         (pginfo->kpage_cnt)++;
1825                         (pginfo->u.fmr.next_listelem)++;
1826                         fmrlist++;
1827                         pginfo->next_hwpage = 0;
1828                 }
1829         }
1830         return ret;
1831 }
1832
1833 /* setup page buffer from page info */
1834 int ehca_set_pagebuf(struct ehca_mr_pginfo *pginfo,
1835                      u32 number,
1836                      u64 *kpage)
1837 {
1838         int ret;
1839
1840         switch (pginfo->type) {
1841         case EHCA_MR_PGI_PHYS:
1842                 ret = ehca_set_pagebuf_phys(pginfo, number, kpage);
1843                 break;
1844         case EHCA_MR_PGI_USER:
1845                 ret = ehca_set_pagebuf_user1(pginfo, number, kpage);
1846                 break;
1847         case EHCA_MR_PGI_FMR:
1848                 ret = ehca_set_pagebuf_fmr(pginfo, number, kpage);
1849                 break;
1850         default:
1851                 ehca_gen_err("bad pginfo->type=%x", pginfo->type);
1852                 ret = -EFAULT;
1853                 break;
1854         }
1855         return ret;
1856 } /* end ehca_set_pagebuf() */
1857
1858 /*----------------------------------------------------------------------*/
1859
1860 /*
1861  * check MR if it is a max-MR, i.e. uses whole memory
1862  * in case it's a max-MR 1 is returned, else 0
1863  */
1864 int ehca_mr_is_maxmr(u64 size,
1865                      u64 *iova_start)
1866 {
1867         /* a MR is treated as max-MR only if it fits following: */
1868         if ((size == ((u64)high_memory - PAGE_OFFSET)) &&
1869             (iova_start == (void *)KERNELBASE)) {
1870                 ehca_gen_dbg("this is a max-MR");
1871                 return 1;
1872         } else
1873                 return 0;
1874 } /* end ehca_mr_is_maxmr() */
1875
1876 /*----------------------------------------------------------------------*/
1877
1878 /* map access control for MR/MW. This routine is used for MR and MW. */
1879 void ehca_mrmw_map_acl(int ib_acl,
1880                        u32 *hipz_acl)
1881 {
1882         *hipz_acl = 0;
1883         if (ib_acl & IB_ACCESS_REMOTE_READ)
1884                 *hipz_acl |= HIPZ_ACCESSCTRL_R_READ;
1885         if (ib_acl & IB_ACCESS_REMOTE_WRITE)
1886                 *hipz_acl |= HIPZ_ACCESSCTRL_R_WRITE;
1887         if (ib_acl & IB_ACCESS_REMOTE_ATOMIC)
1888                 *hipz_acl |= HIPZ_ACCESSCTRL_R_ATOMIC;
1889         if (ib_acl & IB_ACCESS_LOCAL_WRITE)
1890                 *hipz_acl |= HIPZ_ACCESSCTRL_L_WRITE;
1891         if (ib_acl & IB_ACCESS_MW_BIND)
1892                 *hipz_acl |= HIPZ_ACCESSCTRL_MW_BIND;
1893 } /* end ehca_mrmw_map_acl() */
1894
1895 /*----------------------------------------------------------------------*/
1896
1897 /* sets page size in hipz access control for MR/MW. */
1898 void ehca_mrmw_set_pgsize_hipz_acl(u32 *hipz_acl) /*INOUT*/
1899 {
1900         return; /* HCA supports only 4k */
1901 } /* end ehca_mrmw_set_pgsize_hipz_acl() */
1902
1903 /*----------------------------------------------------------------------*/
1904
1905 /*
1906  * reverse map access control for MR/MW.
1907  * This routine is used for MR and MW.
1908  */
1909 void ehca_mrmw_reverse_map_acl(const u32 *hipz_acl,
1910                                int *ib_acl) /*OUT*/
1911 {
1912         *ib_acl = 0;
1913         if (*hipz_acl & HIPZ_ACCESSCTRL_R_READ)
1914                 *ib_acl |= IB_ACCESS_REMOTE_READ;
1915         if (*hipz_acl & HIPZ_ACCESSCTRL_R_WRITE)
1916                 *ib_acl |= IB_ACCESS_REMOTE_WRITE;
1917         if (*hipz_acl & HIPZ_ACCESSCTRL_R_ATOMIC)
1918                 *ib_acl |= IB_ACCESS_REMOTE_ATOMIC;
1919         if (*hipz_acl & HIPZ_ACCESSCTRL_L_WRITE)
1920                 *ib_acl |= IB_ACCESS_LOCAL_WRITE;
1921         if (*hipz_acl & HIPZ_ACCESSCTRL_MW_BIND)
1922                 *ib_acl |= IB_ACCESS_MW_BIND;
1923 } /* end ehca_mrmw_reverse_map_acl() */
1924
1925
1926 /*----------------------------------------------------------------------*/
1927
1928 /*
1929  * MR destructor and constructor
1930  * used in Reregister MR verb, sets all fields in ehca_mr_t to 0,
1931  * except struct ib_mr and spinlock
1932  */
1933 void ehca_mr_deletenew(struct ehca_mr *mr)
1934 {
1935         mr->flags = 0;
1936         mr->num_kpages = 0;
1937         mr->num_hwpages = 0;
1938         mr->acl = 0;
1939         mr->start = NULL;
1940         mr->fmr_page_size = 0;
1941         mr->fmr_max_pages = 0;
1942         mr->fmr_max_maps = 0;
1943         mr->fmr_map_cnt = 0;
1944         memset(&mr->ipz_mr_handle, 0, sizeof(mr->ipz_mr_handle));
1945         memset(&mr->galpas, 0, sizeof(mr->galpas));
1946 } /* end ehca_mr_deletenew() */
1947
1948 int ehca_init_mrmw_cache(void)
1949 {
1950         mr_cache = kmem_cache_create("ehca_cache_mr",
1951                                      sizeof(struct ehca_mr), 0,
1952                                      SLAB_HWCACHE_ALIGN,
1953                                      NULL);
1954         if (!mr_cache)
1955                 return -ENOMEM;
1956         mw_cache = kmem_cache_create("ehca_cache_mw",
1957                                      sizeof(struct ehca_mw), 0,
1958                                      SLAB_HWCACHE_ALIGN,
1959                                      NULL);
1960         if (!mw_cache) {
1961                 kmem_cache_destroy(mr_cache);
1962                 mr_cache = NULL;
1963                 return -ENOMEM;
1964         }
1965         return 0;
1966 }
1967
1968 void ehca_cleanup_mrmw_cache(void)
1969 {
1970         if (mr_cache)
1971                 kmem_cache_destroy(mr_cache);
1972         if (mw_cache)
1973                 kmem_cache_destroy(mw_cache);
1974 }