net/smc/smc_core.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  *  Shared Memory Communications over RDMA (SMC-R) and RoCE
   4  *
   5  *  Basic Transport Functions exploiting Infiniband API
   6  *
   7  *  Copyright IBM Corp. 2016
   8  *
   9  *  Author(s):  Ursula Braun <ubraun@linux.vnet.ibm.com>
  10  */
  11
  12 #include <linux/socket.h>
  13 #include <linux/if_vlan.h>
  14 #include <linux/random.h>
  15 #include <linux/workqueue.h>
  16 #include <net/tcp.h>
  17 #include <net/sock.h>
  18 #include <rdma/ib_verbs.h>
  19
  20 #include "smc.h"
  21 #include "smc_clc.h"
  22 #include "smc_core.h"
  23 #include "smc_ib.h"
  24 #include "smc_wr.h"
  25 #include "smc_llc.h"
  26 #include "smc_cdc.h"
  27 #include "smc_close.h"
  28
  29 #define SMC_LGR_NUM_INCR                256
  30 #define SMC_LGR_FREE_DELAY_SERV         (600 * HZ)
  31 #define SMC_LGR_FREE_DELAY_CLNT         (SMC_LGR_FREE_DELAY_SERV + 10)
  32
  33 static u32 smc_lgr_num;                 /* unique link group number */
  34
  35 static void smc_lgr_schedule_free_work(struct smc_link_group *lgr)
  36 {
  37         /* client link group creation always follows the server link group
  38          * creation. For client use a somewhat higher removal delay time,
  39          * otherwise there is a risk of out-of-sync link groups.
  40          */
  41         mod_delayed_work(system_wq, &lgr->free_work,
  42                          lgr->role == SMC_CLNT ? SMC_LGR_FREE_DELAY_CLNT :
  43                                                  SMC_LGR_FREE_DELAY_SERV);
  44 }
  45
  46 /* Register connection's alert token in our lookup structure.
  47  * To use rbtrees we have to implement our own insert core.
  48  * Requires @conns_lock
  49  * @smc         connection to register
  50  * Returns 0 on success, != otherwise.
  51  */
  52 static void smc_lgr_add_alert_token(struct smc_connection *conn)
  53 {
  54         struct rb_node **link, *parent = NULL;
  55         u32 token = conn->alert_token_local;
  56
  57         link = &conn->lgr->conns_all.rb_node;
  58         while (*link) {
  59                 struct smc_connection *cur = rb_entry(*link,
  60                                         struct smc_connection, alert_node);
  61
  62                 parent = *link;
  63                 if (cur->alert_token_local > token)
  64                         link = &parent->rb_left;
  65                 else
  66                         link = &parent->rb_right;
  67         }
  68         /* Put the new node there */
  69         rb_link_node(&conn->alert_node, parent, link);
  70         rb_insert_color(&conn->alert_node, &conn->lgr->conns_all);
  71 }
  72
  73 /* Register connection in link group by assigning an alert token
  74  * registered in a search tree.
  75  * Requires @conns_lock
  76  * Note that '0' is a reserved value and not assigned.
  77  */
  78 static void smc_lgr_register_conn(struct smc_connection *conn)
  79 {
  80         struct smc_sock *smc = container_of(conn, struct smc_sock, conn);
  81         static atomic_t nexttoken = ATOMIC_INIT(0);
  82
  83         /* find a new alert_token_local value not yet used by some connection
  84          * in this link group
  85          */
  86         sock_hold(&smc->sk); /* sock_put in smc_lgr_unregister_conn() */
  87         while (!conn->alert_token_local) {
  88                 conn->alert_token_local = atomic_inc_return(&nexttoken);
  89                 if (smc_lgr_find_conn(conn->alert_token_local, conn->lgr))
  90                         conn->alert_token_local = 0;
  91         }
  92         smc_lgr_add_alert_token(conn);
  93         conn->lgr->conns_num++;
  94 }
  95
  96 /* Unregister connection and reset the alert token of the given connection<
  97  */
  98 static void __smc_lgr_unregister_conn(struct smc_connection *conn)
  99 {
 100         struct smc_sock *smc = container_of(conn, struct smc_sock, conn);
 101         struct smc_link_group *lgr = conn->lgr;
 102
 103         rb_erase(&conn->alert_node, &lgr->conns_all);
 104         lgr->conns_num--;
 105         conn->alert_token_local = 0;
 106         conn->lgr = NULL;
 107         sock_put(&smc->sk); /* sock_hold in smc_lgr_register_conn() */
 108 }
 109
 110 /* Unregister connection and trigger lgr freeing if applicable
 111  */
 112 static void smc_lgr_unregister_conn(struct smc_connection *conn)
 113 {
 114         struct smc_link_group *lgr = conn->lgr;
 115         int reduced = 0;
 116
 117         write_lock_bh(&lgr->conns_lock);
 118         if (conn->alert_token_local) {
 119                 reduced = 1;
 120                 __smc_lgr_unregister_conn(conn);
 121         }
 122         write_unlock_bh(&lgr->conns_lock);
 123         if (!reduced || lgr->conns_num)
 124                 return;
 125         smc_lgr_schedule_free_work(lgr);
 126 }
 127
 128 static void smc_lgr_free_work(struct work_struct *work)
 129 {
 130         struct smc_link_group *lgr = container_of(to_delayed_work(work),
 131                                                   struct smc_link_group,
 132                                                   free_work);
 133         bool conns;
 134
 135         spin_lock_bh(&smc_lgr_list.lock);
 136         if (list_empty(&lgr->list))
 137                 goto free;
 138         read_lock_bh(&lgr->conns_lock);
 139         conns = RB_EMPTY_ROOT(&lgr->conns_all);
 140         read_unlock_bh(&lgr->conns_lock);
 141         if (!conns) { /* number of lgr connections is no longer zero */
 142                 spin_unlock_bh(&smc_lgr_list.lock);
 143                 return;
 144         }
 145         list_del_init(&lgr->list); /* remove from smc_lgr_list */
 146 free:
 147         spin_unlock_bh(&smc_lgr_list.lock);
 148         if (!delayed_work_pending(&lgr->free_work))
 149                 smc_lgr_free(lgr);
 150 }
 151
 152 /* create a new SMC link group */
 153 static int smc_lgr_create(struct smc_sock *smc,
 154                           struct smc_ib_device *smcibdev, u8 ibport,
 155                           char *peer_systemid, unsigned short vlan_id)
 156 {
 157         struct smc_link_group *lgr;
 158         struct smc_link *lnk;
 159         u8 rndvec[3];
 160         int rc = 0;
 161         int i;
 162
 163         lgr = kzalloc(sizeof(*lgr), GFP_KERNEL);
 164         if (!lgr) {
 165                 rc = -ENOMEM;
 166                 goto out;
 167         }
 168         lgr->role = smc->listen_smc ? SMC_SERV : SMC_CLNT;
 169         lgr->sync_err = false;
 170         memcpy(lgr->peer_systemid, peer_systemid, SMC_SYSTEMID_LEN);
 171         lgr->vlan_id = vlan_id;
 172         rwlock_init(&lgr->sndbufs_lock);
 173         rwlock_init(&lgr->rmbs_lock);
 174         for (i = 0; i < SMC_RMBE_SIZES; i++) {
 175                 INIT_LIST_HEAD(&lgr->sndbufs[i]);
 176                 INIT_LIST_HEAD(&lgr->rmbs[i]);
 177         }
 178         smc_lgr_num += SMC_LGR_NUM_INCR;
 179         memcpy(&lgr->id, (u8 *)&smc_lgr_num, SMC_LGR_ID_SIZE);
 180         INIT_DELAYED_WORK(&lgr->free_work, smc_lgr_free_work);
 181         lgr->conns_all = RB_ROOT;
 182
 183         lnk = &lgr->lnk[SMC_SINGLE_LINK];
 184         /* initialize link */
 185         lnk->state = SMC_LNK_ACTIVATING;
 186         lnk->link_id = SMC_SINGLE_LINK;
 187         lnk->smcibdev = smcibdev;
 188         lnk->ibport = ibport;
 189         lnk->path_mtu = smcibdev->pattr[ibport - 1].active_mtu;
 190         if (!smcibdev->initialized)
 191                 smc_ib_setup_per_ibdev(smcibdev);
 192         get_random_bytes(rndvec, sizeof(rndvec));
 193         lnk->psn_initial = rndvec[0] + (rndvec[1] << 8) + (rndvec[2] << 16);
 194         rc = smc_wr_alloc_link_mem(lnk);
 195         if (rc)
 196                 goto free_lgr;
 197         rc = smc_ib_create_protection_domain(lnk);
 198         if (rc)
 199                 goto free_link_mem;
 200         rc = smc_ib_create_queue_pair(lnk);
 201         if (rc)
 202                 goto dealloc_pd;
 203         rc = smc_wr_create_link(lnk);
 204         if (rc)
 205                 goto destroy_qp;
 206         init_completion(&lnk->llc_confirm);
 207         init_completion(&lnk->llc_confirm_resp);
 208         init_completion(&lnk->llc_add);
 209         init_completion(&lnk->llc_add_resp);
 210
 211         smc->conn.lgr = lgr;
 212         rwlock_init(&lgr->conns_lock);
 213         spin_lock_bh(&smc_lgr_list.lock);
 214         list_add(&lgr->list, &smc_lgr_list.list);
 215         spin_unlock_bh(&smc_lgr_list.lock);
 216         return 0;
 217
 218 destroy_qp:
 219         smc_ib_destroy_queue_pair(lnk);
 220 dealloc_pd:
 221         smc_ib_dealloc_protection_domain(lnk);
 222 free_link_mem:
 223         smc_wr_free_link_mem(lnk);
 224 free_lgr:
 225         kfree(lgr);
 226 out:
 227         return rc;
 228 }
 229
 230 static void smc_buf_unuse(struct smc_connection *conn)
 231 {
 232         if (conn->sndbuf_desc) {
 233                 conn->sndbuf_desc->used = 0;
 234                 conn->sndbuf_size = 0;
 235         }
 236         if (conn->rmb_desc) {
 237                 conn->rmb_desc->reused = true;
 238                 conn->rmb_desc->used = 0;
 239                 conn->rmbe_size = 0;
 240         }
 241 }
 242
 243 /* remove a finished connection from its link group */
 244 void smc_conn_free(struct smc_connection *conn)
 245 {
 246         if (!conn->lgr)
 247                 return;
 248         smc_cdc_tx_dismiss_slots(conn);
 249         smc_lgr_unregister_conn(conn);
 250         smc_buf_unuse(conn);
 251 }
 252
 253 static void smc_link_clear(struct smc_link *lnk)
 254 {
 255         lnk->peer_qpn = 0;
 256         smc_ib_modify_qp_reset(lnk);
 257         smc_wr_free_link(lnk);
 258         smc_ib_destroy_queue_pair(lnk);
 259         smc_ib_dealloc_protection_domain(lnk);
 260         smc_wr_free_link_mem(lnk);
 261 }
 262
 263 static void smc_buf_free(struct smc_buf_desc *buf_desc, struct smc_link *lnk,
 264                          bool is_rmb)
 265 {
 266         if (is_rmb) {
 267                 if (buf_desc->mr_rx[SMC_SINGLE_LINK])
 268                         smc_ib_put_memory_region(
 269                                         buf_desc->mr_rx[SMC_SINGLE_LINK]);
 270                 smc_ib_buf_unmap_sg(lnk->smcibdev, buf_desc,
 271                                     DMA_FROM_DEVICE);
 272         } else {
 273                 smc_ib_buf_unmap_sg(lnk->smcibdev, buf_desc,
 274                                     DMA_TO_DEVICE);
 275         }
 276         sg_free_table(&buf_desc->sgt[SMC_SINGLE_LINK]);
 277         if (buf_desc->cpu_addr)
 278                 free_pages((unsigned long)buf_desc->cpu_addr, buf_desc->order);
 279         kfree(buf_desc);
 280 }
 281
 282 static void __smc_lgr_free_bufs(struct smc_link_group *lgr, bool is_rmb)
 283 {
 284         struct smc_link *lnk = &lgr->lnk[SMC_SINGLE_LINK];
 285         struct smc_buf_desc *buf_desc, *bf_desc;
 286         struct list_head *buf_list;
 287         int i;
 288
 289         for (i = 0; i < SMC_RMBE_SIZES; i++) {
 290                 if (is_rmb)
 291                         buf_list = &lgr->rmbs[i];
 292                 else
 293                         buf_list = &lgr->sndbufs[i];
 294                 list_for_each_entry_safe(buf_desc, bf_desc, buf_list,
 295                                          list) {
 296                         list_del(&buf_desc->list);
 297                         smc_buf_free(buf_desc, lnk, is_rmb);
 298                 }
 299         }
 300 }
 301
 302 static void smc_lgr_free_bufs(struct smc_link_group *lgr)
 303 {
 304         /* free send buffers */
 305         __smc_lgr_free_bufs(lgr, false);
 306         /* free rmbs */
 307         __smc_lgr_free_bufs(lgr, true);
 308 }
 309
 310 /* remove a link group */
 311 void smc_lgr_free(struct smc_link_group *lgr)
 312 {
 313         smc_lgr_free_bufs(lgr);
 314         smc_link_clear(&lgr->lnk[SMC_SINGLE_LINK]);
 315         kfree(lgr);
 316 }
 317
 318 void smc_lgr_forget(struct smc_link_group *lgr)
 319 {
 320         spin_lock_bh(&smc_lgr_list.lock);
 321         /* do not use this link group for new connections */
 322         if (!list_empty(&lgr->list))
 323                 list_del_init(&lgr->list);
 324         spin_unlock_bh(&smc_lgr_list.lock);
 325 }
 326
 327 /* terminate linkgroup abnormally */
 328 void smc_lgr_terminate(struct smc_link_group *lgr)
 329 {
 330         struct smc_connection *conn;
 331         struct smc_sock *smc;
 332         struct rb_node *node;
 333
 334         smc_lgr_forget(lgr);
 335
 336         write_lock_bh(&lgr->conns_lock);
 337         node = rb_first(&lgr->conns_all);
 338         while (node) {
 339                 conn = rb_entry(node, struct smc_connection, alert_node);
 340                 smc = container_of(conn, struct smc_sock, conn);
 341                 sock_hold(&smc->sk); /* sock_put in close work */
 342                 conn->local_tx_ctrl.conn_state_flags.peer_conn_abort = 1;
 343                 __smc_lgr_unregister_conn(conn);
 344                 write_unlock_bh(&lgr->conns_lock);
 345                 if (!schedule_work(&conn->close_work))
 346                         sock_put(&smc->sk);
 347                 write_lock_bh(&lgr->conns_lock);
 348                 node = rb_first(&lgr->conns_all);
 349         }
 350         write_unlock_bh(&lgr->conns_lock);
 351         wake_up(&lgr->lnk[SMC_SINGLE_LINK].wr_reg_wait);
 352         smc_lgr_schedule_free_work(lgr);
 353 }
 354
 355 /* Determine vlan of internal TCP socket.
 356  * @vlan_id: address to store the determined vlan id into
 357  */
 358 static int smc_vlan_by_tcpsk(struct socket *clcsock, unsigned short *vlan_id)
 359 {
 360         struct dst_entry *dst = sk_dst_get(clcsock->sk);
 361         int rc = 0;
 362
 363         *vlan_id = 0;
 364         if (!dst) {
 365                 rc = -ENOTCONN;
 366                 goto out;
 367         }
 368         if (!dst->dev) {
 369                 rc = -ENODEV;
 370                 goto out_rel;
 371         }
 372
 373         if (is_vlan_dev(dst->dev))
 374                 *vlan_id = vlan_dev_vlan_id(dst->dev);
 375
 376 out_rel:
 377         dst_release(dst);
 378 out:
 379         return rc;
 380 }
 381
 382 /* determine the link gid matching the vlan id of the link group */
 383 static int smc_link_determine_gid(struct smc_link_group *lgr)
 384 {
 385         struct smc_link *lnk = &lgr->lnk[SMC_SINGLE_LINK];
 386         struct ib_gid_attr gattr;
 387         union ib_gid gid;
 388         int i;
 389
 390         if (!lgr->vlan_id) {
 391                 lnk->gid = lnk->smcibdev->gid[lnk->ibport - 1];
 392                 return 0;
 393         }
 394
 395         for (i = 0; i < lnk->smcibdev->pattr[lnk->ibport - 1].gid_tbl_len;
 396              i++) {
 397                 if (ib_query_gid(lnk->smcibdev->ibdev, lnk->ibport, i, &gid,
 398                                  &gattr))
 399                         continue;
 400                 if (gattr.ndev) {
 401                         if (is_vlan_dev(gattr.ndev) &&
 402                             vlan_dev_vlan_id(gattr.ndev) == lgr->vlan_id) {
 403                                 lnk->gid = gid;
 404                                 dev_put(gattr.ndev);
 405                                 return 0;
 406                         }
 407                         dev_put(gattr.ndev);
 408                 }
 409         }
 410         return -ENODEV;
 411 }
 412
 413 /* create a new SMC connection (and a new link group if necessary) */
 414 int smc_conn_create(struct smc_sock *smc,
 415                     struct smc_ib_device *smcibdev, u8 ibport,
 416                     struct smc_clc_msg_local *lcl, int srv_first_contact)
 417 {
 418         struct smc_connection *conn = &smc->conn;
 419         struct smc_link_group *lgr;
 420         unsigned short vlan_id;
 421         enum smc_lgr_role role;
 422         int local_contact = SMC_FIRST_CONTACT;
 423         int rc = 0;
 424
 425         role = smc->listen_smc ? SMC_SERV : SMC_CLNT;
 426         rc = smc_vlan_by_tcpsk(smc->clcsock, &vlan_id);
 427         if (rc)
 428                 return rc;
 429
 430         if ((role == SMC_CLNT) && srv_first_contact)
 431                 /* create new link group as well */
 432                 goto create;
 433
 434         /* determine if an existing link group can be reused */
 435         spin_lock_bh(&smc_lgr_list.lock);
 436         list_for_each_entry(lgr, &smc_lgr_list.list, list) {
 437                 write_lock_bh(&lgr->conns_lock);
 438                 if (!memcmp(lgr->peer_systemid, lcl->id_for_peer,
 439                             SMC_SYSTEMID_LEN) &&
 440                     !memcmp(lgr->lnk[SMC_SINGLE_LINK].peer_gid, &lcl->gid,
 441                             SMC_GID_SIZE) &&
 442                     !memcmp(lgr->lnk[SMC_SINGLE_LINK].peer_mac, lcl->mac,
 443                             sizeof(lcl->mac)) &&
 444                     !lgr->sync_err &&
 445                     (lgr->role == role) &&
 446                     (lgr->vlan_id == vlan_id) &&
 447                     ((role == SMC_CLNT) ||
 448                      (lgr->conns_num < SMC_RMBS_PER_LGR_MAX))) {
 449                         /* link group found */
 450                         local_contact = SMC_REUSE_CONTACT;
 451                         conn->lgr = lgr;
 452                         smc_lgr_register_conn(conn); /* add smc conn to lgr */
 453                         write_unlock_bh(&lgr->conns_lock);
 454                         break;
 455                 }
 456                 write_unlock_bh(&lgr->conns_lock);
 457         }
 458         spin_unlock_bh(&smc_lgr_list.lock);
 459
 460         if (role == SMC_CLNT && !srv_first_contact &&
 461             (local_contact == SMC_FIRST_CONTACT)) {
 462                 /* Server reuses a link group, but Client wants to start
 463                  * a new one
 464                  * send out_of_sync decline, reason synchr. error
 465                  */
 466                 return -ENOLINK;
 467         }
 468
 469 create:
 470         if (local_contact == SMC_FIRST_CONTACT) {
 471                 rc = smc_lgr_create(smc, smcibdev, ibport,
 472                                     lcl->id_for_peer, vlan_id);
 473                 if (rc)
 474                         goto out;
 475                 smc_lgr_register_conn(conn); /* add smc conn to lgr */
 476                 rc = smc_link_determine_gid(conn->lgr);
 477         }
 478         conn->local_tx_ctrl.common.type = SMC_CDC_MSG_TYPE;
 479         conn->local_tx_ctrl.len = SMC_WR_TX_SIZE;
 480 #ifndef KERNEL_HAS_ATOMIC64
 481         spin_lock_init(&conn->acurs_lock);
 482 #endif
 483
 484 out:
 485         return rc ? rc : local_contact;
 486 }
 487
 488 /* try to reuse a sndbuf or rmb description slot for a certain
 489  * buffer size; if not available, return NULL
 490  */
 491 static inline
 492 struct smc_buf_desc *smc_buf_get_slot(struct smc_link_group *lgr,
 493                                       int compressed_bufsize,
 494                                       rwlock_t *lock,
 495                                       struct list_head *buf_list)
 496 {
 497         struct smc_buf_desc *buf_slot;
 498
 499         read_lock_bh(lock);
 500         list_for_each_entry(buf_slot, buf_list, list) {
 501                 if (cmpxchg(&buf_slot->used, 0, 1) == 0) {
 502                         read_unlock_bh(lock);
 503                         return buf_slot;
 504                 }
 505         }
 506         read_unlock_bh(lock);
 507         return NULL;
 508 }
 509
 510 /* one of the conditions for announcing a receiver's current window size is
 511  * that it "results in a minimum increase in the window size of 10% of the
 512  * receive buffer space" [RFC7609]
 513  */
 514 static inline int smc_rmb_wnd_update_limit(int rmbe_size)
 515 {
 516         return min_t(int, rmbe_size / 10, SOCK_MIN_SNDBUF / 2);
 517 }
 518
 519 static struct smc_buf_desc *smc_new_buf_create(struct smc_link_group *lgr,
 520                                                bool is_rmb, int bufsize)
 521 {
 522         struct smc_buf_desc *buf_desc;
 523         struct smc_link *lnk;
 524         int rc;
 525
 526         /* try to alloc a new buffer */
 527         buf_desc = kzalloc(sizeof(*buf_desc), GFP_KERNEL);
 528         if (!buf_desc)
 529                 return ERR_PTR(-ENOMEM);
 530
 531         buf_desc->cpu_addr =
 532                 (void *)__get_free_pages(GFP_KERNEL | __GFP_NOWARN |
 533                                          __GFP_NOMEMALLOC |
 534                                          __GFP_NORETRY | __GFP_ZERO,
 535                                          get_order(bufsize));
 536         if (!buf_desc->cpu_addr) {
 537                 kfree(buf_desc);
 538                 return ERR_PTR(-EAGAIN);
 539         }
 540         buf_desc->order = get_order(bufsize);
 541
 542         /* build the sg table from the pages */
 543         lnk = &lgr->lnk[SMC_SINGLE_LINK];
 544         rc = sg_alloc_table(&buf_desc->sgt[SMC_SINGLE_LINK], 1,
 545                             GFP_KERNEL);
 546         if (rc) {
 547                 smc_buf_free(buf_desc, lnk, is_rmb);
 548                 return ERR_PTR(rc);
 549         }
 550         sg_set_buf(buf_desc->sgt[SMC_SINGLE_LINK].sgl,
 551                    buf_desc->cpu_addr, bufsize);
 552
 553         /* map sg table to DMA address */
 554         rc = smc_ib_buf_map_sg(lnk->smcibdev, buf_desc,
 555                                is_rmb ? DMA_FROM_DEVICE : DMA_TO_DEVICE);
 556         /* SMC protocol depends on mapping to one DMA address only */
 557         if (rc != 1)  {
 558                 smc_buf_free(buf_desc, lnk, is_rmb);
 559                 return ERR_PTR(-EAGAIN);
 560         }
 561
 562         /* create a new memory region for the RMB */
 563         if (is_rmb) {
 564                 rc = smc_ib_get_memory_region(lnk->roce_pd,
 565                                               IB_ACCESS_REMOTE_WRITE |
 566                                               IB_ACCESS_LOCAL_WRITE,
 567                                               buf_desc);
 568                 if (rc) {
 569                         smc_buf_free(buf_desc, lnk, is_rmb);
 570                         return ERR_PTR(rc);
 571                 }
 572         }
 573
 574         return buf_desc;
 575 }
 576
 577 static int __smc_buf_create(struct smc_sock *smc, bool is_rmb)
 578 {
 579         struct smc_connection *conn = &smc->conn;
 580         struct smc_link_group *lgr = conn->lgr;
 581         struct smc_buf_desc *buf_desc = ERR_PTR(-ENOMEM);
 582         struct list_head *buf_list;
 583         int bufsize, bufsize_short;
 584         int sk_buf_size;
 585         rwlock_t *lock;
 586
 587         if (is_rmb)
 588                 /* use socket recv buffer size (w/o overhead) as start value */
 589                 sk_buf_size = smc->sk.sk_rcvbuf / 2;
 590         else
 591                 /* use socket send buffer size (w/o overhead) as start value */
 592                 sk_buf_size = smc->sk.sk_sndbuf / 2;
 593
 594         for (bufsize_short = smc_compress_bufsize(sk_buf_size);
 595              bufsize_short >= 0; bufsize_short--) {
 596
 597                 if (is_rmb) {
 598                         lock = &lgr->rmbs_lock;
 599                         buf_list = &lgr->rmbs[bufsize_short];
 600                 } else {
 601                         lock = &lgr->sndbufs_lock;
 602                         buf_list = &lgr->sndbufs[bufsize_short];
 603                 }
 604                 bufsize = smc_uncompress_bufsize(bufsize_short);
 605                 if ((1 << get_order(bufsize)) > SG_MAX_SINGLE_ALLOC)
 606                         continue;
 607
 608                 /* check for reusable slot in the link group */
 609                 buf_desc = smc_buf_get_slot(lgr, bufsize_short, lock, buf_list);
 610                 if (buf_desc) {
 611                         memset(buf_desc->cpu_addr, 0, bufsize);
 612                         break; /* found reusable slot */
 613                 }
 614
 615                 buf_desc = smc_new_buf_create(lgr, is_rmb, bufsize);
 616                 if (PTR_ERR(buf_desc) == -ENOMEM)
 617                         break;
 618                 if (IS_ERR(buf_desc))
 619                         continue;
 620
 621                 buf_desc->used = 1;
 622                 write_lock_bh(lock);
 623                 list_add(&buf_desc->list, buf_list);
 624                 write_unlock_bh(lock);
 625                 break; /* found */
 626         }
 627
 628         if (IS_ERR(buf_desc))
 629                 return -ENOMEM;
 630
 631         if (is_rmb) {
 632                 conn->rmb_desc = buf_desc;
 633                 conn->rmbe_size = bufsize;
 634                 conn->rmbe_size_short = bufsize_short;
 635                 smc->sk.sk_rcvbuf = bufsize * 2;
 636                 atomic_set(&conn->bytes_to_rcv, 0);
 637                 conn->rmbe_update_limit = smc_rmb_wnd_update_limit(bufsize);
 638         } else {
 639                 conn->sndbuf_desc = buf_desc;
 640                 conn->sndbuf_size = bufsize;
 641                 smc->sk.sk_sndbuf = bufsize * 2;
 642                 atomic_set(&conn->sndbuf_space, bufsize);
 643         }
 644         return 0;
 645 }
 646
 647 void smc_sndbuf_sync_sg_for_cpu(struct smc_connection *conn)
 648 {
 649         struct smc_link_group *lgr = conn->lgr;
 650
 651         smc_ib_sync_sg_for_cpu(lgr->lnk[SMC_SINGLE_LINK].smcibdev,
 652                                conn->sndbuf_desc, DMA_TO_DEVICE);
 653 }
 654
 655 void smc_sndbuf_sync_sg_for_device(struct smc_connection *conn)
 656 {
 657         struct smc_link_group *lgr = conn->lgr;
 658
 659         smc_ib_sync_sg_for_device(lgr->lnk[SMC_SINGLE_LINK].smcibdev,
 660                                   conn->sndbuf_desc, DMA_TO_DEVICE);
 661 }
 662
 663 void smc_rmb_sync_sg_for_cpu(struct smc_connection *conn)
 664 {
 665         struct smc_link_group *lgr = conn->lgr;
 666
 667         smc_ib_sync_sg_for_cpu(lgr->lnk[SMC_SINGLE_LINK].smcibdev,
 668                                conn->rmb_desc, DMA_FROM_DEVICE);
 669 }
 670
 671 void smc_rmb_sync_sg_for_device(struct smc_connection *conn)
 672 {
 673         struct smc_link_group *lgr = conn->lgr;
 674
 675         smc_ib_sync_sg_for_device(lgr->lnk[SMC_SINGLE_LINK].smcibdev,
 676                                   conn->rmb_desc, DMA_FROM_DEVICE);
 677 }
 678
 679 /* create the send and receive buffer for an SMC socket;
 680  * receive buffers are called RMBs;
 681  * (even though the SMC protocol allows more than one RMB-element per RMB,
 682  * the Linux implementation uses just one RMB-element per RMB, i.e. uses an
 683  * extra RMB for every connection in a link group
 684  */
 685 int smc_buf_create(struct smc_sock *smc)
 686 {
 687         int rc;
 688
 689         /* create send buffer */
 690         rc = __smc_buf_create(smc, false);
 691         if (rc)
 692                 return rc;
 693         /* create rmb */
 694         rc = __smc_buf_create(smc, true);
 695         if (rc)
 696                 smc_buf_free(smc->conn.sndbuf_desc,
 697                              &smc->conn.lgr->lnk[SMC_SINGLE_LINK], false);
 698         return rc;
 699 }
 700
 701 static inline int smc_rmb_reserve_rtoken_idx(struct smc_link_group *lgr)
 702 {
 703         int i;
 704
 705         for_each_clear_bit(i, lgr->rtokens_used_mask, SMC_RMBS_PER_LGR_MAX) {
 706                 if (!test_and_set_bit(i, lgr->rtokens_used_mask))
 707                         return i;
 708         }
 709         return -ENOSPC;
 710 }
 711
 712 /* add a new rtoken from peer */
 713 int smc_rtoken_add(struct smc_link_group *lgr, __be64 nw_vaddr, __be32 nw_rkey)
 714 {
 715         u64 dma_addr = be64_to_cpu(nw_vaddr);
 716         u32 rkey = ntohl(nw_rkey);
 717         int i;
 718
 719         for (i = 0; i < SMC_RMBS_PER_LGR_MAX; i++) {
 720                 if ((lgr->rtokens[i][SMC_SINGLE_LINK].rkey == rkey) &&
 721                     (lgr->rtokens[i][SMC_SINGLE_LINK].dma_addr == dma_addr) &&
 722                     test_bit(i, lgr->rtokens_used_mask)) {
 723                         /* already in list */
 724                         return i;
 725                 }
 726         }
 727         i = smc_rmb_reserve_rtoken_idx(lgr);
 728         if (i < 0)
 729                 return i;
 730         lgr->rtokens[i][SMC_SINGLE_LINK].rkey = rkey;
 731         lgr->rtokens[i][SMC_SINGLE_LINK].dma_addr = dma_addr;
 732         return i;
 733 }
 734
 735 /* delete an rtoken */
 736 int smc_rtoken_delete(struct smc_link_group *lgr, __be32 nw_rkey)
 737 {
 738         u32 rkey = ntohl(nw_rkey);
 739         int i;
 740
 741         for (i = 0; i < SMC_RMBS_PER_LGR_MAX; i++) {
 742                 if (lgr->rtokens[i][SMC_SINGLE_LINK].rkey == rkey &&
 743                     test_bit(i, lgr->rtokens_used_mask)) {
 744                         lgr->rtokens[i][SMC_SINGLE_LINK].rkey = 0;
 745                         lgr->rtokens[i][SMC_SINGLE_LINK].dma_addr = 0;
 746
 747                         clear_bit(i, lgr->rtokens_used_mask);
 748                         return 0;
 749                 }
 750         }
 751         return -ENOENT;
 752 }
 753
 754 /* save rkey and dma_addr received from peer during clc handshake */
 755 int smc_rmb_rtoken_handling(struct smc_connection *conn,
 756                             struct smc_clc_msg_accept_confirm *clc)
 757 {
 758         conn->rtoken_idx = smc_rtoken_add(conn->lgr, clc->rmb_dma_addr,
 759                                           clc->rmb_rkey);
 760         if (conn->rtoken_idx < 0)
 761                 return conn->rtoken_idx;
 762         return 0;
 763 }