drivers/infiniband/core/cm.c

   1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
   2 /*
   3  * Copyright (c) 2004-2007 Intel Corporation.  All rights reserved.
   4  * Copyright (c) 2004 Topspin Corporation.  All rights reserved.
   5  * Copyright (c) 2004, 2005 Voltaire Corporation.  All rights reserved.
   6  * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
   7  * Copyright (c) 2019, Mellanox Technologies inc.  All rights reserved.
   8  */
   9
  10 #include <linux/completion.h>
  11 #include <linux/dma-mapping.h>
  12 #include <linux/device.h>
  13 #include <linux/module.h>
  14 #include <linux/err.h>
  15 #include <linux/idr.h>
  16 #include <linux/interrupt.h>
  17 #include <linux/random.h>
  18 #include <linux/rbtree.h>
  19 #include <linux/spinlock.h>
  20 #include <linux/slab.h>
  21 #include <linux/sysfs.h>
  22 #include <linux/workqueue.h>
  23 #include <linux/kdev_t.h>
  24 #include <linux/etherdevice.h>
  25
  26 #include <rdma/ib_cache.h>
  27 #include <rdma/ib_cm.h>
  28 #include <rdma/ib_sysfs.h>
  29 #include "cm_msgs.h"
  30 #include "core_priv.h"
  31 #include "cm_trace.h"
  32
  33 MODULE_AUTHOR("Sean Hefty");
  34 MODULE_DESCRIPTION("InfiniBand CM");
  35 MODULE_LICENSE("Dual BSD/GPL");
  36
  37 #define CM_DESTROY_ID_WAIT_TIMEOUT 10000 /* msecs */
  38 #define CM_DIRECT_RETRY_CTX ((void *) 1UL)
  39 #define CM_MRA_SETTING 24 /* 4.096us * 2^24 = ~68.7 seconds */
  40
  41 static const char * const ibcm_rej_reason_strs[] = {
  42         [IB_CM_REJ_NO_QP]                       = "no QP",
  43         [IB_CM_REJ_NO_EEC]                      = "no EEC",
  44         [IB_CM_REJ_NO_RESOURCES]                = "no resources",
  45         [IB_CM_REJ_TIMEOUT]                     = "timeout",
  46         [IB_CM_REJ_UNSUPPORTED]                 = "unsupported",
  47         [IB_CM_REJ_INVALID_COMM_ID]             = "invalid comm ID",
  48         [IB_CM_REJ_INVALID_COMM_INSTANCE]       = "invalid comm instance",
  49         [IB_CM_REJ_INVALID_SERVICE_ID]          = "invalid service ID",
  50         [IB_CM_REJ_INVALID_TRANSPORT_TYPE]      = "invalid transport type",
  51         [IB_CM_REJ_STALE_CONN]                  = "stale conn",
  52         [IB_CM_REJ_RDC_NOT_EXIST]               = "RDC not exist",
  53         [IB_CM_REJ_INVALID_GID]                 = "invalid GID",
  54         [IB_CM_REJ_INVALID_LID]                 = "invalid LID",
  55         [IB_CM_REJ_INVALID_SL]                  = "invalid SL",
  56         [IB_CM_REJ_INVALID_TRAFFIC_CLASS]       = "invalid traffic class",
  57         [IB_CM_REJ_INVALID_HOP_LIMIT]           = "invalid hop limit",
  58         [IB_CM_REJ_INVALID_PACKET_RATE]         = "invalid packet rate",
  59         [IB_CM_REJ_INVALID_ALT_GID]             = "invalid alt GID",
  60         [IB_CM_REJ_INVALID_ALT_LID]             = "invalid alt LID",
  61         [IB_CM_REJ_INVALID_ALT_SL]              = "invalid alt SL",
  62         [IB_CM_REJ_INVALID_ALT_TRAFFIC_CLASS]   = "invalid alt traffic class",
  63         [IB_CM_REJ_INVALID_ALT_HOP_LIMIT]       = "invalid alt hop limit",
  64         [IB_CM_REJ_INVALID_ALT_PACKET_RATE]     = "invalid alt packet rate",
  65         [IB_CM_REJ_PORT_CM_REDIRECT]            = "port CM redirect",
  66         [IB_CM_REJ_PORT_REDIRECT]               = "port redirect",
  67         [IB_CM_REJ_INVALID_MTU]                 = "invalid MTU",
  68         [IB_CM_REJ_INSUFFICIENT_RESP_RESOURCES] = "insufficient resp resources",
  69         [IB_CM_REJ_CONSUMER_DEFINED]            = "consumer defined",
  70         [IB_CM_REJ_INVALID_RNR_RETRY]           = "invalid RNR retry",
  71         [IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID]     = "duplicate local comm ID",
  72         [IB_CM_REJ_INVALID_CLASS_VERSION]       = "invalid class version",
  73         [IB_CM_REJ_INVALID_FLOW_LABEL]          = "invalid flow label",
  74         [IB_CM_REJ_INVALID_ALT_FLOW_LABEL]      = "invalid alt flow label",
  75         [IB_CM_REJ_VENDOR_OPTION_NOT_SUPPORTED] =
  76                 "vendor option is not supported",
  77 };
  78
  79 const char *__attribute_const__ ibcm_reject_msg(int reason)
  80 {
  81         size_t index = reason;
  82
  83         if (index < ARRAY_SIZE(ibcm_rej_reason_strs) &&
  84             ibcm_rej_reason_strs[index])
  85                 return ibcm_rej_reason_strs[index];
  86         else
  87                 return "unrecognized reason";
  88 }
  89 EXPORT_SYMBOL(ibcm_reject_msg);
  90
  91 struct cm_id_private;
  92 struct cm_work;
  93 static int cm_add_one(struct ib_device *device);
  94 static void cm_remove_one(struct ib_device *device, void *client_data);
  95 static void cm_process_work(struct cm_id_private *cm_id_priv,
  96                             struct cm_work *work);
  97 static int cm_send_sidr_rep_locked(struct cm_id_private *cm_id_priv,
  98                                    struct ib_cm_sidr_rep_param *param);
  99 static void cm_issue_dreq(struct cm_id_private *cm_id_priv);
 100 static int cm_send_drep_locked(struct cm_id_private *cm_id_priv,
 101                                void *private_data, u8 private_data_len);
 102 static int cm_send_rej_locked(struct cm_id_private *cm_id_priv,
 103                               enum ib_cm_rej_reason reason, void *ari,
 104                               u8 ari_length, const void *private_data,
 105                               u8 private_data_len);
 106
 107 static struct ib_client cm_client = {
 108         .name   = "cm",
 109         .add    = cm_add_one,
 110         .remove = cm_remove_one
 111 };
 112
 113 static struct ib_cm {
 114         spinlock_t lock;
 115         struct list_head device_list;
 116         rwlock_t device_lock;
 117         struct rb_root listen_service_table;
 118         u64 listen_service_id;
 119         /* struct rb_root peer_service_table; todo: fix peer to peer */
 120         struct rb_root remote_qp_table;
 121         struct rb_root remote_id_table;
 122         struct rb_root remote_sidr_table;
 123         struct xarray local_id_table;
 124         u32 local_id_next;
 125         __be32 random_id_operand;
 126         struct list_head timewait_list;
 127         struct workqueue_struct *wq;
 128 } cm;
 129
 130 /* Counter indexes ordered by attribute ID */
 131 enum {
 132         CM_REQ_COUNTER,
 133         CM_MRA_COUNTER,
 134         CM_REJ_COUNTER,
 135         CM_REP_COUNTER,
 136         CM_RTU_COUNTER,
 137         CM_DREQ_COUNTER,
 138         CM_DREP_COUNTER,
 139         CM_SIDR_REQ_COUNTER,
 140         CM_SIDR_REP_COUNTER,
 141         CM_LAP_COUNTER,
 142         CM_APR_COUNTER,
 143         CM_ATTR_COUNT,
 144         CM_ATTR_ID_OFFSET = 0x0010,
 145 };
 146
 147 enum {
 148         CM_XMIT,
 149         CM_XMIT_RETRIES,
 150         CM_RECV,
 151         CM_RECV_DUPLICATES,
 152         CM_COUNTER_GROUPS
 153 };
 154
 155 struct cm_counter_attribute {
 156         struct ib_port_attribute attr;
 157         unsigned short group;
 158         unsigned short index;
 159 };
 160
 161 struct cm_port {
 162         struct cm_device *cm_dev;
 163         struct ib_mad_agent *mad_agent;
 164         u32 port_num;
 165         atomic_long_t counters[CM_COUNTER_GROUPS][CM_ATTR_COUNT];
 166 };
 167
 168 struct cm_device {
 169         struct kref kref;
 170         struct list_head list;
 171         rwlock_t mad_agent_lock;
 172         struct ib_device *ib_device;
 173         u8 ack_delay;
 174         int going_down;
 175         struct cm_port *port[];
 176 };
 177
 178 struct cm_av {
 179         struct cm_port *port;
 180         struct rdma_ah_attr ah_attr;
 181         u16 dlid_datapath;
 182         u16 pkey_index;
 183         u8 timeout;
 184 };
 185
 186 struct cm_work {
 187         struct delayed_work work;
 188         struct list_head list;
 189         struct cm_port *port;
 190         struct ib_mad_recv_wc *mad_recv_wc;     /* Received MADs */
 191         __be32 local_id;                        /* Established / timewait */
 192         __be32 remote_id;
 193         struct ib_cm_event cm_event;
 194         struct sa_path_rec path[];
 195 };
 196
 197 struct cm_timewait_info {
 198         struct cm_work work;
 199         struct list_head list;
 200         struct rb_node remote_qp_node;
 201         struct rb_node remote_id_node;
 202         __be64 remote_ca_guid;
 203         __be32 remote_qpn;
 204         u8 inserted_remote_qp;
 205         u8 inserted_remote_id;
 206 };
 207
 208 struct cm_id_private {
 209         struct ib_cm_id id;
 210
 211         struct rb_node service_node;
 212         struct rb_node sidr_id_node;
 213         u32 sidr_slid;
 214         spinlock_t lock;        /* Do not acquire inside cm.lock */
 215         struct completion comp;
 216         refcount_t refcount;
 217         /* Number of clients sharing this ib_cm_id. Only valid for listeners.
 218          * Protected by the cm.lock spinlock.
 219          */
 220         int listen_sharecount;
 221         struct rcu_head rcu;
 222
 223         struct ib_mad_send_buf *msg;
 224         struct cm_timewait_info *timewait_info;
 225         /* todo: use alternate port on send failure */
 226         struct cm_av av;
 227         struct cm_av alt_av;
 228
 229         void *private_data;
 230         __be64 tid;
 231         __be32 local_qpn;
 232         __be32 remote_qpn;
 233         enum ib_qp_type qp_type;
 234         __be32 sq_psn;
 235         __be32 rq_psn;
 236         int timeout_ms;
 237         enum ib_mtu path_mtu;
 238         __be16 pkey;
 239         u8 private_data_len;
 240         u8 max_cm_retries;
 241         u8 responder_resources;
 242         u8 initiator_depth;
 243         u8 retry_count;
 244         u8 rnr_retry_count;
 245         u8 target_ack_delay;
 246
 247         struct list_head work_list;
 248         atomic_t work_count;
 249
 250         struct rdma_ucm_ece ece;
 251 };
 252
 253 static void cm_dev_release(struct kref *kref)
 254 {
 255         struct cm_device *cm_dev = container_of(kref, struct cm_device, kref);
 256         u32 i;
 257
 258         rdma_for_each_port(cm_dev->ib_device, i)
 259                 kfree(cm_dev->port[i - 1]);
 260
 261         kfree(cm_dev);
 262 }
 263
 264 static void cm_device_put(struct cm_device *cm_dev)
 265 {
 266         kref_put(&cm_dev->kref, cm_dev_release);
 267 }
 268
 269 static void cm_work_handler(struct work_struct *work);
 270
 271 static inline void cm_deref_id(struct cm_id_private *cm_id_priv)
 272 {
 273         if (refcount_dec_and_test(&cm_id_priv->refcount))
 274                 complete(&cm_id_priv->comp);
 275 }
 276
 277 static struct ib_mad_send_buf *cm_alloc_msg(struct cm_id_private *cm_id_priv)
 278 {
 279         struct ib_mad_agent *mad_agent;
 280         struct ib_mad_send_buf *m;
 281         struct ib_ah *ah;
 282
 283         lockdep_assert_held(&cm_id_priv->lock);
 284
 285         if (!cm_id_priv->av.port)
 286                 return ERR_PTR(-EINVAL);
 287
 288         read_lock(&cm_id_priv->av.port->cm_dev->mad_agent_lock);
 289         mad_agent = cm_id_priv->av.port->mad_agent;
 290         if (!mad_agent) {
 291                 m = ERR_PTR(-EINVAL);
 292                 goto out;
 293         }
 294
 295         ah = rdma_create_ah(mad_agent->qp->pd, &cm_id_priv->av.ah_attr, 0);
 296         if (IS_ERR(ah)) {
 297                 m = ERR_CAST(ah);
 298                 goto out;
 299         }
 300
 301         m = ib_create_send_mad(mad_agent, cm_id_priv->id.remote_cm_qpn,
 302                                cm_id_priv->av.pkey_index,
 303                                0, IB_MGMT_MAD_HDR, IB_MGMT_MAD_DATA,
 304                                GFP_ATOMIC,
 305                                IB_MGMT_BASE_VERSION);
 306         if (IS_ERR(m)) {
 307                 rdma_destroy_ah(ah, 0);
 308                 goto out;
 309         }
 310
 311         m->ah = ah;
 312
 313 out:
 314         read_unlock(&cm_id_priv->av.port->cm_dev->mad_agent_lock);
 315         return m;
 316 }
 317
 318 static void cm_free_msg(struct ib_mad_send_buf *msg)
 319 {
 320         if (msg->ah)
 321                 rdma_destroy_ah(msg->ah, 0);
 322         ib_free_send_mad(msg);
 323 }
 324
 325 static struct ib_mad_send_buf *
 326 cm_alloc_priv_msg(struct cm_id_private *cm_id_priv, enum ib_cm_state state)
 327 {
 328         struct ib_mad_send_buf *msg;
 329
 330         lockdep_assert_held(&cm_id_priv->lock);
 331
 332         msg = cm_alloc_msg(cm_id_priv);
 333         if (IS_ERR(msg))
 334                 return msg;
 335
 336         cm_id_priv->msg = msg;
 337         refcount_inc(&cm_id_priv->refcount);
 338         msg->context[0] = cm_id_priv;
 339         msg->context[1] = (void *) (unsigned long) state;
 340
 341         msg->retries = cm_id_priv->max_cm_retries;
 342         msg->timeout_ms = cm_id_priv->timeout_ms;
 343
 344         return msg;
 345 }
 346
 347 static void cm_free_priv_msg(struct ib_mad_send_buf *msg)
 348 {
 349         struct cm_id_private *cm_id_priv = msg->context[0];
 350
 351         lockdep_assert_held(&cm_id_priv->lock);
 352
 353         if (!WARN_ON(cm_id_priv->msg != msg))
 354                 cm_id_priv->msg = NULL;
 355
 356         if (msg->ah)
 357                 rdma_destroy_ah(msg->ah, 0);
 358         cm_deref_id(cm_id_priv);
 359         ib_free_send_mad(msg);
 360 }
 361
 362 static struct ib_mad_send_buf *
 363 cm_alloc_response_msg_no_ah(struct cm_port *port,
 364                             struct ib_mad_recv_wc *mad_recv_wc,
 365                             bool direct_retry)
 366 {
 367         struct ib_mad_send_buf *m;
 368
 369         m = ib_create_send_mad(port->mad_agent, 1, mad_recv_wc->wc->pkey_index,
 370                                0, IB_MGMT_MAD_HDR, IB_MGMT_MAD_DATA,
 371                                GFP_ATOMIC, IB_MGMT_BASE_VERSION);
 372         if (!IS_ERR(m))
 373                 m->context[0] = direct_retry ? CM_DIRECT_RETRY_CTX : NULL;
 374
 375         return m;
 376 }
 377
 378 static int cm_create_response_msg_ah(struct cm_port *port,
 379                                      struct ib_mad_recv_wc *mad_recv_wc,
 380                                      struct ib_mad_send_buf *msg)
 381 {
 382         struct ib_ah *ah;
 383
 384         ah = ib_create_ah_from_wc(port->mad_agent->qp->pd, mad_recv_wc->wc,
 385                                   mad_recv_wc->recv_buf.grh, port->port_num);
 386         if (IS_ERR(ah))
 387                 return PTR_ERR(ah);
 388
 389         msg->ah = ah;
 390         return 0;
 391 }
 392
 393 static int cm_alloc_response_msg(struct cm_port *port,
 394                                  struct ib_mad_recv_wc *mad_recv_wc,
 395                                  bool direct_retry,
 396                                  struct ib_mad_send_buf **msg)
 397 {
 398         struct ib_mad_send_buf *m;
 399         int ret;
 400
 401         m = cm_alloc_response_msg_no_ah(port, mad_recv_wc, direct_retry);
 402         if (IS_ERR(m))
 403                 return PTR_ERR(m);
 404
 405         ret = cm_create_response_msg_ah(port, mad_recv_wc, m);
 406         if (ret) {
 407                 ib_free_send_mad(m);
 408                 return ret;
 409         }
 410
 411         *msg = m;
 412         return 0;
 413 }
 414
 415 static void *cm_copy_private_data(const void *private_data, u8 private_data_len)
 416 {
 417         void *data;
 418
 419         if (!private_data || !private_data_len)
 420                 return NULL;
 421
 422         data = kmemdup(private_data, private_data_len, GFP_KERNEL);
 423         if (!data)
 424                 return ERR_PTR(-ENOMEM);
 425
 426         return data;
 427 }
 428
 429 static void cm_set_private_data(struct cm_id_private *cm_id_priv,
 430                                  void *private_data, u8 private_data_len)
 431 {
 432         if (cm_id_priv->private_data && cm_id_priv->private_data_len)
 433                 kfree(cm_id_priv->private_data);
 434
 435         cm_id_priv->private_data = private_data;
 436         cm_id_priv->private_data_len = private_data_len;
 437 }
 438
 439 static void cm_set_av_port(struct cm_av *av, struct cm_port *port)
 440 {
 441         struct cm_port *old_port = av->port;
 442
 443         if (old_port == port)
 444                 return;
 445
 446         av->port = port;
 447         if (old_port)
 448                 cm_device_put(old_port->cm_dev);
 449         if (port)
 450                 kref_get(&port->cm_dev->kref);
 451 }
 452
 453 static void cm_init_av_for_lap(struct cm_port *port, struct ib_wc *wc,
 454                                struct rdma_ah_attr *ah_attr, struct cm_av *av)
 455 {
 456         cm_set_av_port(av, port);
 457         av->pkey_index = wc->pkey_index;
 458         rdma_move_ah_attr(&av->ah_attr, ah_attr);
 459 }
 460
 461 static int cm_init_av_for_response(struct cm_port *port, struct ib_wc *wc,
 462                                    struct ib_grh *grh, struct cm_av *av)
 463 {
 464         cm_set_av_port(av, port);
 465         av->pkey_index = wc->pkey_index;
 466         return ib_init_ah_attr_from_wc(port->cm_dev->ib_device,
 467                                        port->port_num, wc,
 468                                        grh, &av->ah_attr);
 469 }
 470
 471 static struct cm_port *
 472 get_cm_port_from_path(struct sa_path_rec *path, const struct ib_gid_attr *attr)
 473 {
 474         struct cm_device *cm_dev;
 475         struct cm_port *port = NULL;
 476         unsigned long flags;
 477
 478         if (attr) {
 479                 read_lock_irqsave(&cm.device_lock, flags);
 480                 list_for_each_entry(cm_dev, &cm.device_list, list) {
 481                         if (cm_dev->ib_device == attr->device) {
 482                                 port = cm_dev->port[attr->port_num - 1];
 483                                 break;
 484                         }
 485                 }
 486                 read_unlock_irqrestore(&cm.device_lock, flags);
 487         } else {
 488                 /* SGID attribute can be NULL in following
 489                  * conditions.
 490                  * (a) Alternative path
 491                  * (b) IB link layer without GRH
 492                  * (c) LAP send messages
 493                  */
 494                 read_lock_irqsave(&cm.device_lock, flags);
 495                 list_for_each_entry(cm_dev, &cm.device_list, list) {
 496                         attr = rdma_find_gid(cm_dev->ib_device,
 497                                              &path->sgid,
 498                                              sa_conv_pathrec_to_gid_type(path),
 499                                              NULL);
 500                         if (!IS_ERR(attr)) {
 501                                 port = cm_dev->port[attr->port_num - 1];
 502                                 break;
 503                         }
 504                 }
 505                 read_unlock_irqrestore(&cm.device_lock, flags);
 506                 if (port)
 507                         rdma_put_gid_attr(attr);
 508         }
 509         return port;
 510 }
 511
 512 static int cm_init_av_by_path(struct sa_path_rec *path,
 513                               const struct ib_gid_attr *sgid_attr,
 514                               struct cm_av *av)
 515 {
 516         struct rdma_ah_attr new_ah_attr;
 517         struct cm_device *cm_dev;
 518         struct cm_port *port;
 519         int ret;
 520
 521         port = get_cm_port_from_path(path, sgid_attr);
 522         if (!port)
 523                 return -EINVAL;
 524         cm_dev = port->cm_dev;
 525
 526         ret = ib_find_cached_pkey(cm_dev->ib_device, port->port_num,
 527                                   be16_to_cpu(path->pkey), &av->pkey_index);
 528         if (ret)
 529                 return ret;
 530
 531         cm_set_av_port(av, port);
 532
 533         /*
 534          * av->ah_attr might be initialized based on wc or during
 535          * request processing time which might have reference to sgid_attr.
 536          * So initialize a new ah_attr on stack.
 537          * If initialization fails, old ah_attr is used for sending any
 538          * responses. If initialization is successful, than new ah_attr
 539          * is used by overwriting the old one. So that right ah_attr
 540          * can be used to return an error response.
 541          */
 542         ret = ib_init_ah_attr_from_path(cm_dev->ib_device, port->port_num, path,
 543                                         &new_ah_attr, sgid_attr);
 544         if (ret)
 545                 return ret;
 546
 547         av->timeout = path->packet_life_time + 1;
 548         rdma_move_ah_attr(&av->ah_attr, &new_ah_attr);
 549         return 0;
 550 }
 551
 552 /* Move av created by cm_init_av_by_path(), so av.dgid is not moved */
 553 static void cm_move_av_from_path(struct cm_av *dest, struct cm_av *src)
 554 {
 555         cm_set_av_port(dest, src->port);
 556         cm_set_av_port(src, NULL);
 557         dest->pkey_index = src->pkey_index;
 558         rdma_move_ah_attr(&dest->ah_attr, &src->ah_attr);
 559         dest->timeout = src->timeout;
 560 }
 561
 562 static void cm_destroy_av(struct cm_av *av)
 563 {
 564         rdma_destroy_ah_attr(&av->ah_attr);
 565         cm_set_av_port(av, NULL);
 566 }
 567
 568 static u32 cm_local_id(__be32 local_id)
 569 {
 570         return (__force u32) (local_id ^ cm.random_id_operand);
 571 }
 572
 573 static struct cm_id_private *cm_acquire_id(__be32 local_id, __be32 remote_id)
 574 {
 575         struct cm_id_private *cm_id_priv;
 576
 577         rcu_read_lock();
 578         cm_id_priv = xa_load(&cm.local_id_table, cm_local_id(local_id));
 579         if (!cm_id_priv || cm_id_priv->id.remote_id != remote_id ||
 580             !refcount_inc_not_zero(&cm_id_priv->refcount))
 581                 cm_id_priv = NULL;
 582         rcu_read_unlock();
 583
 584         return cm_id_priv;
 585 }
 586
 587 /*
 588  * Trivial helpers to strip endian annotation and compare; the
 589  * endianness doesn't actually matter since we just need a stable
 590  * order for the RB tree.
 591  */
 592 static int be32_lt(__be32 a, __be32 b)
 593 {
 594         return (__force u32) a < (__force u32) b;
 595 }
 596
 597 static int be32_gt(__be32 a, __be32 b)
 598 {
 599         return (__force u32) a > (__force u32) b;
 600 }
 601
 602 static int be64_lt(__be64 a, __be64 b)
 603 {
 604         return (__force u64) a < (__force u64) b;
 605 }
 606
 607 static int be64_gt(__be64 a, __be64 b)
 608 {
 609         return (__force u64) a > (__force u64) b;
 610 }
 611
 612 /*
 613  * Inserts a new cm_id_priv into the listen_service_table. Returns cm_id_priv
 614  * if the new ID was inserted, NULL if it could not be inserted due to a
 615  * collision, or the existing cm_id_priv ready for shared usage.
 616  */
 617 static struct cm_id_private *cm_insert_listen(struct cm_id_private *cm_id_priv,
 618                                               ib_cm_handler shared_handler)
 619 {
 620         struct rb_node **link = &cm.listen_service_table.rb_node;
 621         struct rb_node *parent = NULL;
 622         struct cm_id_private *cur_cm_id_priv;
 623         __be64 service_id = cm_id_priv->id.service_id;
 624         unsigned long flags;
 625
 626         spin_lock_irqsave(&cm.lock, flags);
 627         while (*link) {
 628                 parent = *link;
 629                 cur_cm_id_priv = rb_entry(parent, struct cm_id_private,
 630                                           service_node);
 631
 632                 if (cm_id_priv->id.device < cur_cm_id_priv->id.device)
 633                         link = &(*link)->rb_left;
 634                 else if (cm_id_priv->id.device > cur_cm_id_priv->id.device)
 635                         link = &(*link)->rb_right;
 636                 else if (be64_lt(service_id, cur_cm_id_priv->id.service_id))
 637                         link = &(*link)->rb_left;
 638                 else if (be64_gt(service_id, cur_cm_id_priv->id.service_id))
 639                         link = &(*link)->rb_right;
 640                 else {
 641                         /*
 642                          * Sharing an ib_cm_id with different handlers is not
 643                          * supported
 644                          */
 645                         if (cur_cm_id_priv->id.cm_handler != shared_handler ||
 646                             cur_cm_id_priv->id.context ||
 647                             WARN_ON(!cur_cm_id_priv->id.cm_handler)) {
 648                                 spin_unlock_irqrestore(&cm.lock, flags);
 649                                 return NULL;
 650                         }
 651                         refcount_inc(&cur_cm_id_priv->refcount);
 652                         cur_cm_id_priv->listen_sharecount++;
 653                         spin_unlock_irqrestore(&cm.lock, flags);
 654                         return cur_cm_id_priv;
 655                 }
 656         }
 657         cm_id_priv->listen_sharecount++;
 658         rb_link_node(&cm_id_priv->service_node, parent, link);
 659         rb_insert_color(&cm_id_priv->service_node, &cm.listen_service_table);
 660         spin_unlock_irqrestore(&cm.lock, flags);
 661         return cm_id_priv;
 662 }
 663
 664 static struct cm_id_private *cm_find_listen(struct ib_device *device,
 665                                             __be64 service_id)
 666 {
 667         struct rb_node *node = cm.listen_service_table.rb_node;
 668         struct cm_id_private *cm_id_priv;
 669
 670         while (node) {
 671                 cm_id_priv = rb_entry(node, struct cm_id_private, service_node);
 672
 673                 if (device < cm_id_priv->id.device)
 674                         node = node->rb_left;
 675                 else if (device > cm_id_priv->id.device)
 676                         node = node->rb_right;
 677                 else if (be64_lt(service_id, cm_id_priv->id.service_id))
 678                         node = node->rb_left;
 679                 else if (be64_gt(service_id, cm_id_priv->id.service_id))
 680                         node = node->rb_right;
 681                 else {
 682                         refcount_inc(&cm_id_priv->refcount);
 683                         return cm_id_priv;
 684                 }
 685         }
 686         return NULL;
 687 }
 688
 689 static struct cm_timewait_info *
 690 cm_insert_remote_id(struct cm_timewait_info *timewait_info)
 691 {
 692         struct rb_node **link = &cm.remote_id_table.rb_node;
 693         struct rb_node *parent = NULL;
 694         struct cm_timewait_info *cur_timewait_info;
 695         __be64 remote_ca_guid = timewait_info->remote_ca_guid;
 696         __be32 remote_id = timewait_info->work.remote_id;
 697
 698         while (*link) {
 699                 parent = *link;
 700                 cur_timewait_info = rb_entry(parent, struct cm_timewait_info,
 701                                              remote_id_node);
 702                 if (be32_lt(remote_id, cur_timewait_info->work.remote_id))
 703                         link = &(*link)->rb_left;
 704                 else if (be32_gt(remote_id, cur_timewait_info->work.remote_id))
 705                         link = &(*link)->rb_right;
 706                 else if (be64_lt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
 707                         link = &(*link)->rb_left;
 708                 else if (be64_gt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
 709                         link = &(*link)->rb_right;
 710                 else
 711                         return cur_timewait_info;
 712         }
 713         timewait_info->inserted_remote_id = 1;
 714         rb_link_node(&timewait_info->remote_id_node, parent, link);
 715         rb_insert_color(&timewait_info->remote_id_node, &cm.remote_id_table);
 716         return NULL;
 717 }
 718
 719 static struct cm_id_private *cm_find_remote_id(__be64 remote_ca_guid,
 720                                                __be32 remote_id)
 721 {
 722         struct rb_node *node = cm.remote_id_table.rb_node;
 723         struct cm_timewait_info *timewait_info;
 724         struct cm_id_private *res = NULL;
 725
 726         spin_lock_irq(&cm.lock);
 727         while (node) {
 728                 timewait_info = rb_entry(node, struct cm_timewait_info,
 729                                          remote_id_node);
 730                 if (be32_lt(remote_id, timewait_info->work.remote_id))
 731                         node = node->rb_left;
 732                 else if (be32_gt(remote_id, timewait_info->work.remote_id))
 733                         node = node->rb_right;
 734                 else if (be64_lt(remote_ca_guid, timewait_info->remote_ca_guid))
 735                         node = node->rb_left;
 736                 else if (be64_gt(remote_ca_guid, timewait_info->remote_ca_guid))
 737                         node = node->rb_right;
 738                 else {
 739                         res = cm_acquire_id(timewait_info->work.local_id,
 740                                              timewait_info->work.remote_id);
 741                         break;
 742                 }
 743         }
 744         spin_unlock_irq(&cm.lock);
 745         return res;
 746 }
 747
 748 static struct cm_timewait_info *
 749 cm_insert_remote_qpn(struct cm_timewait_info *timewait_info)
 750 {
 751         struct rb_node **link = &cm.remote_qp_table.rb_node;
 752         struct rb_node *parent = NULL;
 753         struct cm_timewait_info *cur_timewait_info;
 754         __be64 remote_ca_guid = timewait_info->remote_ca_guid;
 755         __be32 remote_qpn = timewait_info->remote_qpn;
 756
 757         while (*link) {
 758                 parent = *link;
 759                 cur_timewait_info = rb_entry(parent, struct cm_timewait_info,
 760                                              remote_qp_node);
 761                 if (be32_lt(remote_qpn, cur_timewait_info->remote_qpn))
 762                         link = &(*link)->rb_left;
 763                 else if (be32_gt(remote_qpn, cur_timewait_info->remote_qpn))
 764                         link = &(*link)->rb_right;
 765                 else if (be64_lt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
 766                         link = &(*link)->rb_left;
 767                 else if (be64_gt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
 768                         link = &(*link)->rb_right;
 769                 else
 770                         return cur_timewait_info;
 771         }
 772         timewait_info->inserted_remote_qp = 1;
 773         rb_link_node(&timewait_info->remote_qp_node, parent, link);
 774         rb_insert_color(&timewait_info->remote_qp_node, &cm.remote_qp_table);
 775         return NULL;
 776 }
 777
 778 static struct cm_id_private *
 779 cm_insert_remote_sidr(struct cm_id_private *cm_id_priv)
 780 {
 781         struct rb_node **link = &cm.remote_sidr_table.rb_node;
 782         struct rb_node *parent = NULL;
 783         struct cm_id_private *cur_cm_id_priv;
 784         __be32 remote_id = cm_id_priv->id.remote_id;
 785
 786         while (*link) {
 787                 parent = *link;
 788                 cur_cm_id_priv = rb_entry(parent, struct cm_id_private,
 789                                           sidr_id_node);
 790                 if (be32_lt(remote_id, cur_cm_id_priv->id.remote_id))
 791                         link = &(*link)->rb_left;
 792                 else if (be32_gt(remote_id, cur_cm_id_priv->id.remote_id))
 793                         link = &(*link)->rb_right;
 794                 else {
 795                         if (cur_cm_id_priv->sidr_slid < cm_id_priv->sidr_slid)
 796                                 link = &(*link)->rb_left;
 797                         else if (cur_cm_id_priv->sidr_slid > cm_id_priv->sidr_slid)
 798                                 link = &(*link)->rb_right;
 799                         else
 800                                 return cur_cm_id_priv;
 801                 }
 802         }
 803         rb_link_node(&cm_id_priv->sidr_id_node, parent, link);
 804         rb_insert_color(&cm_id_priv->sidr_id_node, &cm.remote_sidr_table);
 805         return NULL;
 806 }
 807
 808 static struct cm_id_private *cm_alloc_id_priv(struct ib_device *device,
 809                                               ib_cm_handler cm_handler,
 810                                               void *context)
 811 {
 812         struct cm_id_private *cm_id_priv;
 813         u32 id;
 814         int ret;
 815
 816         cm_id_priv = kzalloc(sizeof *cm_id_priv, GFP_KERNEL);
 817         if (!cm_id_priv)
 818                 return ERR_PTR(-ENOMEM);
 819
 820         cm_id_priv->id.state = IB_CM_IDLE;
 821         cm_id_priv->id.device = device;
 822         cm_id_priv->id.cm_handler = cm_handler;
 823         cm_id_priv->id.context = context;
 824         cm_id_priv->id.remote_cm_qpn = 1;
 825
 826         RB_CLEAR_NODE(&cm_id_priv->service_node);
 827         RB_CLEAR_NODE(&cm_id_priv->sidr_id_node);
 828         spin_lock_init(&cm_id_priv->lock);
 829         init_completion(&cm_id_priv->comp);
 830         INIT_LIST_HEAD(&cm_id_priv->work_list);
 831         atomic_set(&cm_id_priv->work_count, -1);
 832         refcount_set(&cm_id_priv->refcount, 1);
 833
 834         ret = xa_alloc_cyclic(&cm.local_id_table, &id, NULL, xa_limit_32b,
 835                               &cm.local_id_next, GFP_KERNEL);
 836         if (ret < 0)
 837                 goto error;
 838         cm_id_priv->id.local_id = (__force __be32)id ^ cm.random_id_operand;
 839
 840         return cm_id_priv;
 841
 842 error:
 843         kfree(cm_id_priv);
 844         return ERR_PTR(ret);
 845 }
 846
 847 /*
 848  * Make the ID visible to the MAD handlers and other threads that use the
 849  * xarray.
 850  */
 851 static void cm_finalize_id(struct cm_id_private *cm_id_priv)
 852 {
 853         xa_store(&cm.local_id_table, cm_local_id(cm_id_priv->id.local_id),
 854                  cm_id_priv, GFP_ATOMIC);
 855 }
 856
 857 struct ib_cm_id *ib_create_cm_id(struct ib_device *device,
 858                                  ib_cm_handler cm_handler,
 859                                  void *context)
 860 {
 861         struct cm_id_private *cm_id_priv;
 862
 863         cm_id_priv = cm_alloc_id_priv(device, cm_handler, context);
 864         if (IS_ERR(cm_id_priv))
 865                 return ERR_CAST(cm_id_priv);
 866
 867         cm_finalize_id(cm_id_priv);
 868         return &cm_id_priv->id;
 869 }
 870 EXPORT_SYMBOL(ib_create_cm_id);
 871
 872 static struct cm_work *cm_dequeue_work(struct cm_id_private *cm_id_priv)
 873 {
 874         struct cm_work *work;
 875
 876         if (list_empty(&cm_id_priv->work_list))
 877                 return NULL;
 878
 879         work = list_entry(cm_id_priv->work_list.next, struct cm_work, list);
 880         list_del(&work->list);
 881         return work;
 882 }
 883
 884 static void cm_free_work(struct cm_work *work)
 885 {
 886         if (work->mad_recv_wc)
 887                 ib_free_recv_mad(work->mad_recv_wc);
 888         kfree(work);
 889 }
 890
 891 static void cm_queue_work_unlock(struct cm_id_private *cm_id_priv,
 892                                  struct cm_work *work)
 893         __releases(&cm_id_priv->lock)
 894 {
 895         bool immediate;
 896
 897         /*
 898          * To deliver the event to the user callback we have the drop the
 899          * spinlock, however, we need to ensure that the user callback is single
 900          * threaded and receives events in the temporal order. If there are
 901          * already events being processed then thread new events onto a list,
 902          * the thread currently processing will pick them up.
 903          */
 904         immediate = atomic_inc_and_test(&cm_id_priv->work_count);
 905         if (!immediate) {
 906                 list_add_tail(&work->list, &cm_id_priv->work_list);
 907                 /*
 908                  * This routine always consumes incoming reference. Once queued
 909                  * to the work_list then a reference is held by the thread
 910                  * currently running cm_process_work() and this reference is not
 911                  * needed.
 912                  */
 913                 cm_deref_id(cm_id_priv);
 914         }
 915         spin_unlock_irq(&cm_id_priv->lock);
 916
 917         if (immediate)
 918                 cm_process_work(cm_id_priv, work);
 919 }
 920
 921 static inline int cm_convert_to_ms(int iba_time)
 922 {
 923         /* approximate conversion to ms from 4.096us x 2^iba_time */
 924         return 1 << max(iba_time - 8, 0);
 925 }
 926
 927 /*
 928  * calculate: 4.096x2^ack_timeout = 4.096x2^ack_delay + 2x4.096x2^life_time
 929  * Because of how ack_timeout is stored, adding one doubles the timeout.
 930  * To avoid large timeouts, select the max(ack_delay, life_time + 1), and
 931  * increment it (round up) only if the other is within 50%.
 932  */
 933 static u8 cm_ack_timeout(u8 ca_ack_delay, u8 packet_life_time)
 934 {
 935         int ack_timeout = packet_life_time + 1;
 936
 937         if (ack_timeout >= ca_ack_delay)
 938                 ack_timeout += (ca_ack_delay >= (ack_timeout - 1));
 939         else
 940                 ack_timeout = ca_ack_delay +
 941                               (ack_timeout >= (ca_ack_delay - 1));
 942
 943         return min(31, ack_timeout);
 944 }
 945
 946 static void cm_remove_remote(struct cm_id_private *cm_id_priv)
 947 {
 948         struct cm_timewait_info *timewait_info = cm_id_priv->timewait_info;
 949
 950         if (timewait_info->inserted_remote_id) {
 951                 rb_erase(&timewait_info->remote_id_node, &cm.remote_id_table);
 952                 timewait_info->inserted_remote_id = 0;
 953         }
 954
 955         if (timewait_info->inserted_remote_qp) {
 956                 rb_erase(&timewait_info->remote_qp_node, &cm.remote_qp_table);
 957                 timewait_info->inserted_remote_qp = 0;
 958         }
 959 }
 960
 961 static struct cm_timewait_info *cm_create_timewait_info(__be32 local_id)
 962 {
 963         struct cm_timewait_info *timewait_info;
 964
 965         timewait_info = kzalloc(sizeof *timewait_info, GFP_KERNEL);
 966         if (!timewait_info)
 967                 return ERR_PTR(-ENOMEM);
 968
 969         timewait_info->work.local_id = local_id;
 970         INIT_DELAYED_WORK(&timewait_info->work.work, cm_work_handler);
 971         timewait_info->work.cm_event.event = IB_CM_TIMEWAIT_EXIT;
 972         return timewait_info;
 973 }
 974
 975 static void cm_enter_timewait(struct cm_id_private *cm_id_priv)
 976 {
 977         int wait_time;
 978         unsigned long flags;
 979         struct cm_device *cm_dev;
 980
 981         lockdep_assert_held(&cm_id_priv->lock);
 982
 983         cm_dev = ib_get_client_data(cm_id_priv->id.device, &cm_client);
 984         if (!cm_dev)
 985                 return;
 986
 987         spin_lock_irqsave(&cm.lock, flags);
 988         cm_remove_remote(cm_id_priv);
 989         list_add_tail(&cm_id_priv->timewait_info->list, &cm.timewait_list);
 990         spin_unlock_irqrestore(&cm.lock, flags);
 991
 992         /*
 993          * The cm_id could be destroyed by the user before we exit timewait.
 994          * To protect against this, we search for the cm_id after exiting
 995          * timewait before notifying the user that we've exited timewait.
 996          */
 997         cm_id_priv->id.state = IB_CM_TIMEWAIT;
 998         wait_time = cm_convert_to_ms(cm_id_priv->av.timeout);
 999
1000         /* Check if the device started its remove_one */
1001         spin_lock_irqsave(&cm.lock, flags);
1002         if (!cm_dev->going_down)
1003                 queue_delayed_work(cm.wq, &cm_id_priv->timewait_info->work.work,
1004                                    msecs_to_jiffies(wait_time));
1005         spin_unlock_irqrestore(&cm.lock, flags);
1006
1007         /*
1008          * The timewait_info is converted into a work and gets freed during
1009          * cm_free_work() in cm_timewait_handler().
1010          */
1011         BUILD_BUG_ON(offsetof(struct cm_timewait_info, work) != 0);
1012         cm_id_priv->timewait_info = NULL;
1013 }
1014
1015 static void cm_reset_to_idle(struct cm_id_private *cm_id_priv)
1016 {
1017         unsigned long flags;
1018
1019         lockdep_assert_held(&cm_id_priv->lock);
1020
1021         cm_id_priv->id.state = IB_CM_IDLE;
1022         if (cm_id_priv->timewait_info) {
1023                 spin_lock_irqsave(&cm.lock, flags);
1024                 cm_remove_remote(cm_id_priv);
1025                 spin_unlock_irqrestore(&cm.lock, flags);
1026                 kfree(cm_id_priv->timewait_info);
1027                 cm_id_priv->timewait_info = NULL;
1028         }
1029 }
1030
1031 static noinline void cm_destroy_id_wait_timeout(struct ib_cm_id *cm_id,
1032                                                 enum ib_cm_state old_state)
1033 {
1034         struct cm_id_private *cm_id_priv;
1035
1036         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
1037         pr_err("%s: cm_id=%p timed out. state %d -> %d, refcnt=%d\n", __func__,
1038                cm_id, old_state, cm_id->state, refcount_read(&cm_id_priv->refcount));
1039 }
1040
1041 static void cm_destroy_id(struct ib_cm_id *cm_id, int err)
1042 {
1043         struct cm_id_private *cm_id_priv;
1044         enum ib_cm_state old_state;
1045         struct cm_work *work;
1046         int ret;
1047
1048         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
1049         spin_lock_irq(&cm_id_priv->lock);
1050         old_state = cm_id->state;
1051 retest:
1052         switch (cm_id->state) {
1053         case IB_CM_LISTEN:
1054                 spin_lock(&cm.lock);
1055                 if (--cm_id_priv->listen_sharecount > 0) {
1056                         /* The id is still shared. */
1057                         WARN_ON(refcount_read(&cm_id_priv->refcount) == 1);
1058                         spin_unlock(&cm.lock);
1059                         spin_unlock_irq(&cm_id_priv->lock);
1060                         cm_deref_id(cm_id_priv);
1061                         return;
1062                 }
1063                 cm_id->state = IB_CM_IDLE;
1064                 rb_erase(&cm_id_priv->service_node, &cm.listen_service_table);
1065                 RB_CLEAR_NODE(&cm_id_priv->service_node);
1066                 spin_unlock(&cm.lock);
1067                 break;
1068         case IB_CM_SIDR_REQ_SENT:
1069                 cm_id->state = IB_CM_IDLE;
1070                 ib_cancel_mad(cm_id_priv->msg);
1071                 break;
1072         case IB_CM_SIDR_REQ_RCVD:
1073                 cm_send_sidr_rep_locked(cm_id_priv,
1074                                         &(struct ib_cm_sidr_rep_param){
1075                                                 .status = IB_SIDR_REJECT });
1076                 /* cm_send_sidr_rep_locked will not move to IDLE if it fails */
1077                 cm_id->state = IB_CM_IDLE;
1078                 break;
1079         case IB_CM_REQ_SENT:
1080         case IB_CM_MRA_REQ_RCVD:
1081                 ib_cancel_mad(cm_id_priv->msg);
1082                 cm_send_rej_locked(cm_id_priv, IB_CM_REJ_TIMEOUT,
1083                                    &cm_id_priv->id.device->node_guid,
1084                                    sizeof(cm_id_priv->id.device->node_guid),
1085                                    NULL, 0);
1086                 break;
1087         case IB_CM_REQ_RCVD:
1088                 if (err == -ENOMEM) {
1089                         /* Do not reject to allow future retries. */
1090                         cm_reset_to_idle(cm_id_priv);
1091                 } else {
1092                         cm_send_rej_locked(cm_id_priv,
1093                                            IB_CM_REJ_CONSUMER_DEFINED, NULL, 0,
1094                                            NULL, 0);
1095                 }
1096                 break;
1097         case IB_CM_REP_SENT:
1098         case IB_CM_MRA_REP_RCVD:
1099                 ib_cancel_mad(cm_id_priv->msg);
1100                 cm_send_rej_locked(cm_id_priv, IB_CM_REJ_CONSUMER_DEFINED, NULL,
1101                                    0, NULL, 0);
1102                 goto retest;
1103         case IB_CM_MRA_REQ_SENT:
1104         case IB_CM_REP_RCVD:
1105         case IB_CM_MRA_REP_SENT:
1106                 cm_send_rej_locked(cm_id_priv, IB_CM_REJ_CONSUMER_DEFINED, NULL,
1107                                    0, NULL, 0);
1108                 break;
1109         case IB_CM_ESTABLISHED:
1110                 if (cm_id_priv->qp_type == IB_QPT_XRC_TGT) {
1111                         cm_id->state = IB_CM_IDLE;
1112                         break;
1113                 }
1114                 cm_issue_dreq(cm_id_priv);
1115                 cm_enter_timewait(cm_id_priv);
1116                 goto retest;
1117         case IB_CM_DREQ_SENT:
1118                 ib_cancel_mad(cm_id_priv->msg);
1119                 cm_enter_timewait(cm_id_priv);
1120                 goto retest;
1121         case IB_CM_DREQ_RCVD:
1122                 cm_send_drep_locked(cm_id_priv, NULL, 0);
1123                 WARN_ON(cm_id->state != IB_CM_TIMEWAIT);
1124                 goto retest;
1125         case IB_CM_TIMEWAIT:
1126                 /*
1127                  * The cm_acquire_id in cm_timewait_handler will stop working
1128                  * once we do xa_erase below, so just move to idle here for
1129                  * consistency.
1130                  */
1131                 cm_id->state = IB_CM_IDLE;
1132                 break;
1133         case IB_CM_IDLE:
1134                 break;
1135         }
1136         WARN_ON(cm_id->state != IB_CM_IDLE);
1137
1138         spin_lock(&cm.lock);
1139         /* Required for cleanup paths related cm_req_handler() */
1140         if (cm_id_priv->timewait_info) {
1141                 cm_remove_remote(cm_id_priv);
1142                 kfree(cm_id_priv->timewait_info);
1143                 cm_id_priv->timewait_info = NULL;
1144         }
1145
1146         WARN_ON(cm_id_priv->listen_sharecount);
1147         WARN_ON(!RB_EMPTY_NODE(&cm_id_priv->service_node));
1148         if (!RB_EMPTY_NODE(&cm_id_priv->sidr_id_node))
1149                 rb_erase(&cm_id_priv->sidr_id_node, &cm.remote_sidr_table);
1150         spin_unlock(&cm.lock);
1151         spin_unlock_irq(&cm_id_priv->lock);
1152
1153         xa_erase(&cm.local_id_table, cm_local_id(cm_id->local_id));
1154         cm_deref_id(cm_id_priv);
1155         do {
1156                 ret = wait_for_completion_timeout(&cm_id_priv->comp,
1157                                                   msecs_to_jiffies(
1158                                                   CM_DESTROY_ID_WAIT_TIMEOUT));
1159                 if (!ret) /* timeout happened */
1160                         cm_destroy_id_wait_timeout(cm_id, old_state);
1161         } while (!ret);
1162
1163         while ((work = cm_dequeue_work(cm_id_priv)) != NULL)
1164                 cm_free_work(work);
1165
1166         cm_destroy_av(&cm_id_priv->av);
1167         cm_destroy_av(&cm_id_priv->alt_av);
1168         kfree(cm_id_priv->private_data);
1169         kfree_rcu(cm_id_priv, rcu);
1170 }
1171
1172 void ib_destroy_cm_id(struct ib_cm_id *cm_id)
1173 {
1174         cm_destroy_id(cm_id, 0);
1175 }
1176 EXPORT_SYMBOL(ib_destroy_cm_id);
1177
1178 static int cm_init_listen(struct cm_id_private *cm_id_priv, __be64 service_id)
1179 {
1180         if ((service_id & IB_SERVICE_ID_AGN_MASK) == IB_CM_ASSIGN_SERVICE_ID &&
1181             (service_id != IB_CM_ASSIGN_SERVICE_ID))
1182                 return -EINVAL;
1183
1184         if (service_id == IB_CM_ASSIGN_SERVICE_ID)
1185                 cm_id_priv->id.service_id = cpu_to_be64(cm.listen_service_id++);
1186         else
1187                 cm_id_priv->id.service_id = service_id;
1188
1189         return 0;
1190 }
1191
1192 /**
1193  * ib_cm_listen - Initiates listening on the specified service ID for
1194  *   connection and service ID resolution requests.
1195  * @cm_id: Connection identifier associated with the listen request.
1196  * @service_id: Service identifier matched against incoming connection
1197  *   and service ID resolution requests.  The service ID should be specified
1198  *   network-byte order.  If set to IB_CM_ASSIGN_SERVICE_ID, the CM will
1199  *   assign a service ID to the caller.
1200  */
1201 int ib_cm_listen(struct ib_cm_id *cm_id, __be64 service_id)
1202 {
1203         struct cm_id_private *cm_id_priv =
1204                 container_of(cm_id, struct cm_id_private, id);
1205         unsigned long flags;
1206         int ret;
1207
1208         spin_lock_irqsave(&cm_id_priv->lock, flags);
1209         if (cm_id_priv->id.state != IB_CM_IDLE) {
1210                 ret = -EINVAL;
1211                 goto out;
1212         }
1213
1214         ret = cm_init_listen(cm_id_priv, service_id);
1215         if (ret)
1216                 goto out;
1217
1218         if (!cm_insert_listen(cm_id_priv, NULL)) {
1219                 ret = -EBUSY;
1220                 goto out;
1221         }
1222
1223         cm_id_priv->id.state = IB_CM_LISTEN;
1224         ret = 0;
1225
1226 out:
1227         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1228         return ret;
1229 }
1230 EXPORT_SYMBOL(ib_cm_listen);
1231
1232 /**
1233  * ib_cm_insert_listen - Create a new listening ib_cm_id and listen on
1234  *                       the given service ID.
1235  *
1236  * If there's an existing ID listening on that same device and service ID,
1237  * return it.
1238  *
1239  * @device: Device associated with the cm_id.  All related communication will
1240  * be associated with the specified device.
1241  * @cm_handler: Callback invoked to notify the user of CM events.
1242  * @service_id: Service identifier matched against incoming connection
1243  *   and service ID resolution requests.  The service ID should be specified
1244  *   network-byte order.  If set to IB_CM_ASSIGN_SERVICE_ID, the CM will
1245  *   assign a service ID to the caller.
1246  *
1247  * Callers should call ib_destroy_cm_id when done with the listener ID.
1248  */
1249 struct ib_cm_id *ib_cm_insert_listen(struct ib_device *device,
1250                                      ib_cm_handler cm_handler,
1251                                      __be64 service_id)
1252 {
1253         struct cm_id_private *listen_id_priv;
1254         struct cm_id_private *cm_id_priv;
1255         int err = 0;
1256
1257         /* Create an ID in advance, since the creation may sleep */
1258         cm_id_priv = cm_alloc_id_priv(device, cm_handler, NULL);
1259         if (IS_ERR(cm_id_priv))
1260                 return ERR_CAST(cm_id_priv);
1261
1262         err = cm_init_listen(cm_id_priv, service_id);
1263         if (err) {
1264                 ib_destroy_cm_id(&cm_id_priv->id);
1265                 return ERR_PTR(err);
1266         }
1267
1268         spin_lock_irq(&cm_id_priv->lock);
1269         listen_id_priv = cm_insert_listen(cm_id_priv, cm_handler);
1270         if (listen_id_priv != cm_id_priv) {
1271                 spin_unlock_irq(&cm_id_priv->lock);
1272                 ib_destroy_cm_id(&cm_id_priv->id);
1273                 if (!listen_id_priv)
1274                         return ERR_PTR(-EINVAL);
1275                 return &listen_id_priv->id;
1276         }
1277         cm_id_priv->id.state = IB_CM_LISTEN;
1278         spin_unlock_irq(&cm_id_priv->lock);
1279
1280         /*
1281          * A listen ID does not need to be in the xarray since it does not
1282          * receive mads, is not placed in the remote_id or remote_qpn rbtree,
1283          * and does not enter timewait.
1284          */
1285
1286         return &cm_id_priv->id;
1287 }
1288 EXPORT_SYMBOL(ib_cm_insert_listen);
1289
1290 static __be64 cm_form_tid(struct cm_id_private *cm_id_priv)
1291 {
1292         u64 hi_tid = 0, low_tid;
1293
1294         lockdep_assert_held(&cm_id_priv->lock);
1295
1296         low_tid = (u64)cm_id_priv->id.local_id;
1297         if (!cm_id_priv->av.port)
1298                 return cpu_to_be64(low_tid);
1299
1300         read_lock(&cm_id_priv->av.port->cm_dev->mad_agent_lock);
1301         if (cm_id_priv->av.port->mad_agent)
1302                 hi_tid = ((u64)cm_id_priv->av.port->mad_agent->hi_tid) << 32;
1303         read_unlock(&cm_id_priv->av.port->cm_dev->mad_agent_lock);
1304         return cpu_to_be64(hi_tid | low_tid);
1305 }
1306
1307 static void cm_format_mad_hdr(struct ib_mad_hdr *hdr,
1308                               __be16 attr_id, __be64 tid)
1309 {
1310         hdr->base_version  = IB_MGMT_BASE_VERSION;
1311         hdr->mgmt_class    = IB_MGMT_CLASS_CM;
1312         hdr->class_version = IB_CM_CLASS_VERSION;
1313         hdr->method        = IB_MGMT_METHOD_SEND;
1314         hdr->attr_id       = attr_id;
1315         hdr->tid           = tid;
1316 }
1317
1318 static void cm_format_mad_ece_hdr(struct ib_mad_hdr *hdr, __be16 attr_id,
1319                                   __be64 tid, u32 attr_mod)
1320 {
1321         cm_format_mad_hdr(hdr, attr_id, tid);
1322         hdr->attr_mod = cpu_to_be32(attr_mod);
1323 }
1324
1325 static void cm_format_req(struct cm_req_msg *req_msg,
1326                           struct cm_id_private *cm_id_priv,
1327                           struct ib_cm_req_param *param)
1328 {
1329         struct sa_path_rec *pri_path = param->primary_path;
1330         struct sa_path_rec *alt_path = param->alternate_path;
1331         bool pri_ext = false;
1332         __be16 lid;
1333
1334         if (pri_path->rec_type == SA_PATH_REC_TYPE_OPA)
1335                 pri_ext = opa_is_extended_lid(pri_path->opa.dlid,
1336                                               pri_path->opa.slid);
1337
1338         cm_format_mad_ece_hdr(&req_msg->hdr, CM_REQ_ATTR_ID,
1339                               cm_form_tid(cm_id_priv), param->ece.attr_mod);
1340
1341         IBA_SET(CM_REQ_LOCAL_COMM_ID, req_msg,
1342                 be32_to_cpu(cm_id_priv->id.local_id));
1343         IBA_SET(CM_REQ_SERVICE_ID, req_msg, be64_to_cpu(param->service_id));
1344         IBA_SET(CM_REQ_LOCAL_CA_GUID, req_msg,
1345                 be64_to_cpu(cm_id_priv->id.device->node_guid));
1346         IBA_SET(CM_REQ_LOCAL_QPN, req_msg, param->qp_num);
1347         IBA_SET(CM_REQ_INITIATOR_DEPTH, req_msg, param->initiator_depth);
1348         IBA_SET(CM_REQ_REMOTE_CM_RESPONSE_TIMEOUT, req_msg,
1349                 param->remote_cm_response_timeout);
1350         cm_req_set_qp_type(req_msg, param->qp_type);
1351         IBA_SET(CM_REQ_END_TO_END_FLOW_CONTROL, req_msg, param->flow_control);
1352         IBA_SET(CM_REQ_STARTING_PSN, req_msg, param->starting_psn);
1353         IBA_SET(CM_REQ_LOCAL_CM_RESPONSE_TIMEOUT, req_msg,
1354                 param->local_cm_response_timeout);
1355         IBA_SET(CM_REQ_PARTITION_KEY, req_msg,
1356                 be16_to_cpu(param->primary_path->pkey));
1357         IBA_SET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg,
1358                 param->primary_path->mtu);
1359         IBA_SET(CM_REQ_MAX_CM_RETRIES, req_msg, param->max_cm_retries);
1360
1361         if (param->qp_type != IB_QPT_XRC_INI) {
1362                 IBA_SET(CM_REQ_RESPONDER_RESOURCES, req_msg,
1363                         param->responder_resources);
1364                 IBA_SET(CM_REQ_RETRY_COUNT, req_msg, param->retry_count);
1365                 IBA_SET(CM_REQ_RNR_RETRY_COUNT, req_msg,
1366                         param->rnr_retry_count);
1367                 IBA_SET(CM_REQ_SRQ, req_msg, param->srq);
1368         }
1369
1370         *IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg) =
1371                 pri_path->sgid;
1372         *IBA_GET_MEM_PTR(CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg) =
1373                 pri_path->dgid;
1374         if (pri_ext) {
1375                 IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg)
1376                         ->global.interface_id =
1377                         OPA_MAKE_ID(be32_to_cpu(pri_path->opa.slid));
1378                 IBA_GET_MEM_PTR(CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg)
1379                         ->global.interface_id =
1380                         OPA_MAKE_ID(be32_to_cpu(pri_path->opa.dlid));
1381         }
1382         if (pri_path->hop_limit <= 1) {
1383                 IBA_SET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg,
1384                         be16_to_cpu(pri_ext ? 0 :
1385                                               htons(ntohl(sa_path_get_slid(
1386                                                       pri_path)))));
1387                 IBA_SET(CM_REQ_PRIMARY_REMOTE_PORT_LID, req_msg,
1388                         be16_to_cpu(pri_ext ? 0 :
1389                                               htons(ntohl(sa_path_get_dlid(
1390                                                       pri_path)))));
1391         } else {
1392
1393                 if (param->primary_path_inbound) {
1394                         lid = param->primary_path_inbound->ib.dlid;
1395                         IBA_SET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg,
1396                                 be16_to_cpu(lid));
1397                 } else
1398                         IBA_SET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg,
1399                                 be16_to_cpu(IB_LID_PERMISSIVE));
1400
1401                 /* Work-around until there's a way to obtain remote LID info */
1402                 IBA_SET(CM_REQ_PRIMARY_REMOTE_PORT_LID, req_msg,
1403                         be16_to_cpu(IB_LID_PERMISSIVE));
1404         }
1405         IBA_SET(CM_REQ_PRIMARY_FLOW_LABEL, req_msg,
1406                 be32_to_cpu(pri_path->flow_label));
1407         IBA_SET(CM_REQ_PRIMARY_PACKET_RATE, req_msg, pri_path->rate);
1408         IBA_SET(CM_REQ_PRIMARY_TRAFFIC_CLASS, req_msg, pri_path->traffic_class);
1409         IBA_SET(CM_REQ_PRIMARY_HOP_LIMIT, req_msg, pri_path->hop_limit);
1410         IBA_SET(CM_REQ_PRIMARY_SL, req_msg, pri_path->sl);
1411         IBA_SET(CM_REQ_PRIMARY_SUBNET_LOCAL, req_msg,
1412                 (pri_path->hop_limit <= 1));
1413         IBA_SET(CM_REQ_PRIMARY_LOCAL_ACK_TIMEOUT, req_msg,
1414                 cm_ack_timeout(cm_id_priv->av.port->cm_dev->ack_delay,
1415                                pri_path->packet_life_time));
1416
1417         if (alt_path) {
1418                 bool alt_ext = false;
1419
1420                 if (alt_path->rec_type == SA_PATH_REC_TYPE_OPA)
1421                         alt_ext = opa_is_extended_lid(alt_path->opa.dlid,
1422                                                       alt_path->opa.slid);
1423
1424                 *IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_LOCAL_PORT_GID, req_msg) =
1425                         alt_path->sgid;
1426                 *IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_REMOTE_PORT_GID, req_msg) =
1427                         alt_path->dgid;
1428                 if (alt_ext) {
1429                         IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_LOCAL_PORT_GID,
1430                                         req_msg)
1431                                 ->global.interface_id =
1432                                 OPA_MAKE_ID(be32_to_cpu(alt_path->opa.slid));
1433                         IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_REMOTE_PORT_GID,
1434                                         req_msg)
1435                                 ->global.interface_id =
1436                                 OPA_MAKE_ID(be32_to_cpu(alt_path->opa.dlid));
1437                 }
1438                 if (alt_path->hop_limit <= 1) {
1439                         IBA_SET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg,
1440                                 be16_to_cpu(
1441                                         alt_ext ? 0 :
1442                                                   htons(ntohl(sa_path_get_slid(
1443                                                           alt_path)))));
1444                         IBA_SET(CM_REQ_ALTERNATE_REMOTE_PORT_LID, req_msg,
1445                                 be16_to_cpu(
1446                                         alt_ext ? 0 :
1447                                                   htons(ntohl(sa_path_get_dlid(
1448                                                           alt_path)))));
1449                 } else {
1450                         IBA_SET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg,
1451                                 be16_to_cpu(IB_LID_PERMISSIVE));
1452                         IBA_SET(CM_REQ_ALTERNATE_REMOTE_PORT_LID, req_msg,
1453                                 be16_to_cpu(IB_LID_PERMISSIVE));
1454                 }
1455                 IBA_SET(CM_REQ_ALTERNATE_FLOW_LABEL, req_msg,
1456                         be32_to_cpu(alt_path->flow_label));
1457                 IBA_SET(CM_REQ_ALTERNATE_PACKET_RATE, req_msg, alt_path->rate);
1458                 IBA_SET(CM_REQ_ALTERNATE_TRAFFIC_CLASS, req_msg,
1459                         alt_path->traffic_class);
1460                 IBA_SET(CM_REQ_ALTERNATE_HOP_LIMIT, req_msg,
1461                         alt_path->hop_limit);
1462                 IBA_SET(CM_REQ_ALTERNATE_SL, req_msg, alt_path->sl);
1463                 IBA_SET(CM_REQ_ALTERNATE_SUBNET_LOCAL, req_msg,
1464                         (alt_path->hop_limit <= 1));
1465                 IBA_SET(CM_REQ_ALTERNATE_LOCAL_ACK_TIMEOUT, req_msg,
1466                         cm_ack_timeout(cm_id_priv->av.port->cm_dev->ack_delay,
1467                                        alt_path->packet_life_time));
1468         }
1469         IBA_SET(CM_REQ_VENDOR_ID, req_msg, param->ece.vendor_id);
1470
1471         if (param->private_data && param->private_data_len)
1472                 IBA_SET_MEM(CM_REQ_PRIVATE_DATA, req_msg, param->private_data,
1473                             param->private_data_len);
1474 }
1475
1476 static int cm_validate_req_param(struct ib_cm_req_param *param)
1477 {
1478         if (!param->primary_path)
1479                 return -EINVAL;
1480
1481         if (param->qp_type != IB_QPT_RC && param->qp_type != IB_QPT_UC &&
1482             param->qp_type != IB_QPT_XRC_INI)
1483                 return -EINVAL;
1484
1485         if (param->private_data &&
1486             param->private_data_len > IB_CM_REQ_PRIVATE_DATA_SIZE)
1487                 return -EINVAL;
1488
1489         if (param->alternate_path &&
1490             (param->alternate_path->pkey != param->primary_path->pkey ||
1491              param->alternate_path->mtu != param->primary_path->mtu))
1492                 return -EINVAL;
1493
1494         return 0;
1495 }
1496
1497 int ib_send_cm_req(struct ib_cm_id *cm_id,
1498                    struct ib_cm_req_param *param)
1499 {
1500         struct cm_av av = {}, alt_av = {};
1501         struct cm_id_private *cm_id_priv;
1502         struct ib_mad_send_buf *msg;
1503         struct cm_req_msg *req_msg;
1504         unsigned long flags;
1505         int ret;
1506
1507         ret = cm_validate_req_param(param);
1508         if (ret)
1509                 return ret;
1510
1511         /* Verify that we're not in timewait. */
1512         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
1513         spin_lock_irqsave(&cm_id_priv->lock, flags);
1514         if (cm_id->state != IB_CM_IDLE || WARN_ON(cm_id_priv->timewait_info)) {
1515                 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1516                 return -EINVAL;
1517         }
1518         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1519
1520         cm_id_priv->timewait_info = cm_create_timewait_info(cm_id_priv->
1521                                                             id.local_id);
1522         if (IS_ERR(cm_id_priv->timewait_info)) {
1523                 ret = PTR_ERR(cm_id_priv->timewait_info);
1524                 cm_id_priv->timewait_info = NULL;
1525                 return ret;
1526         }
1527
1528         ret = cm_init_av_by_path(param->primary_path,
1529                                  param->ppath_sgid_attr, &av);
1530         if (ret)
1531                 return ret;
1532         if (param->alternate_path) {
1533                 ret = cm_init_av_by_path(param->alternate_path, NULL,
1534                                          &alt_av);
1535                 if (ret) {
1536                         cm_destroy_av(&av);
1537                         return ret;
1538                 }
1539         }
1540         cm_id->service_id = param->service_id;
1541         cm_id_priv->timeout_ms = cm_convert_to_ms(
1542                                     param->primary_path->packet_life_time) * 2 +
1543                                  cm_convert_to_ms(
1544                                     param->remote_cm_response_timeout);
1545         cm_id_priv->max_cm_retries = param->max_cm_retries;
1546         cm_id_priv->initiator_depth = param->initiator_depth;
1547         cm_id_priv->responder_resources = param->responder_resources;
1548         cm_id_priv->retry_count = param->retry_count;
1549         cm_id_priv->path_mtu = param->primary_path->mtu;
1550         cm_id_priv->pkey = param->primary_path->pkey;
1551         cm_id_priv->qp_type = param->qp_type;
1552
1553         spin_lock_irqsave(&cm_id_priv->lock, flags);
1554
1555         cm_move_av_from_path(&cm_id_priv->av, &av);
1556         if (param->primary_path_outbound)
1557                 cm_id_priv->av.dlid_datapath =
1558                         be16_to_cpu(param->primary_path_outbound->ib.dlid);
1559
1560         if (param->alternate_path)
1561                 cm_move_av_from_path(&cm_id_priv->alt_av, &alt_av);
1562
1563         msg = cm_alloc_priv_msg(cm_id_priv, IB_CM_REQ_SENT);
1564         if (IS_ERR(msg)) {
1565                 ret = PTR_ERR(msg);
1566                 goto out_unlock;
1567         }
1568
1569         req_msg = (struct cm_req_msg *)msg->mad;
1570         cm_format_req(req_msg, cm_id_priv, param);
1571         cm_id_priv->tid = req_msg->hdr.tid;
1572
1573         cm_id_priv->local_qpn = cpu_to_be32(IBA_GET(CM_REQ_LOCAL_QPN, req_msg));
1574         cm_id_priv->rq_psn = cpu_to_be32(IBA_GET(CM_REQ_STARTING_PSN, req_msg));
1575
1576         trace_icm_send_req(&cm_id_priv->id);
1577         ret = ib_post_send_mad(msg, NULL);
1578         if (ret)
1579                 goto out_free;
1580         BUG_ON(cm_id->state != IB_CM_IDLE);
1581         cm_id->state = IB_CM_REQ_SENT;
1582         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1583         return 0;
1584 out_free:
1585         cm_free_priv_msg(msg);
1586 out_unlock:
1587         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1588         return ret;
1589 }
1590 EXPORT_SYMBOL(ib_send_cm_req);
1591
1592 static int cm_issue_rej(struct cm_port *port,
1593                         struct ib_mad_recv_wc *mad_recv_wc,
1594                         enum ib_cm_rej_reason reason,
1595                         enum cm_msg_response msg_rejected,
1596                         void *ari, u8 ari_length)
1597 {
1598         struct ib_mad_send_buf *msg = NULL;
1599         struct cm_rej_msg *rej_msg, *rcv_msg;
1600         int ret;
1601
1602         ret = cm_alloc_response_msg(port, mad_recv_wc, false, &msg);
1603         if (ret)
1604                 return ret;
1605
1606         /* We just need common CM header information.  Cast to any message. */
1607         rcv_msg = (struct cm_rej_msg *) mad_recv_wc->recv_buf.mad;
1608         rej_msg = (struct cm_rej_msg *) msg->mad;
1609
1610         cm_format_mad_hdr(&rej_msg->hdr, CM_REJ_ATTR_ID, rcv_msg->hdr.tid);
1611         IBA_SET(CM_REJ_REMOTE_COMM_ID, rej_msg,
1612                 IBA_GET(CM_REJ_LOCAL_COMM_ID, rcv_msg));
1613         IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg,
1614                 IBA_GET(CM_REJ_REMOTE_COMM_ID, rcv_msg));
1615         IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, msg_rejected);
1616         IBA_SET(CM_REJ_REASON, rej_msg, reason);
1617
1618         if (ari && ari_length) {
1619                 IBA_SET(CM_REJ_REJECTED_INFO_LENGTH, rej_msg, ari_length);
1620                 IBA_SET_MEM(CM_REJ_ARI, rej_msg, ari, ari_length);
1621         }
1622
1623         trace_icm_issue_rej(
1624                 IBA_GET(CM_REJ_LOCAL_COMM_ID, rcv_msg),
1625                 IBA_GET(CM_REJ_REMOTE_COMM_ID, rcv_msg));
1626         ret = ib_post_send_mad(msg, NULL);
1627         if (ret)
1628                 cm_free_msg(msg);
1629
1630         return ret;
1631 }
1632
1633 static bool cm_req_has_alt_path(struct cm_req_msg *req_msg)
1634 {
1635         return ((cpu_to_be16(
1636                         IBA_GET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg))) ||
1637                 (ib_is_opa_gid(IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_LOCAL_PORT_GID,
1638                                                req_msg))));
1639 }
1640
1641 static void cm_path_set_rec_type(struct ib_device *ib_device, u32 port_num,
1642                                  struct sa_path_rec *path, union ib_gid *gid)
1643 {
1644         if (ib_is_opa_gid(gid) && rdma_cap_opa_ah(ib_device, port_num))
1645                 path->rec_type = SA_PATH_REC_TYPE_OPA;
1646         else
1647                 path->rec_type = SA_PATH_REC_TYPE_IB;
1648 }
1649
1650 static void cm_format_path_lid_from_req(struct cm_req_msg *req_msg,
1651                                         struct sa_path_rec *primary_path,
1652                                         struct sa_path_rec *alt_path,
1653                                         struct ib_wc *wc)
1654 {
1655         u32 lid;
1656
1657         if (primary_path->rec_type != SA_PATH_REC_TYPE_OPA) {
1658                 sa_path_set_dlid(primary_path, wc->slid);
1659                 sa_path_set_slid(primary_path,
1660                                  IBA_GET(CM_REQ_PRIMARY_REMOTE_PORT_LID,
1661                                          req_msg));
1662         } else {
1663                 lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
1664                         CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg));
1665                 sa_path_set_dlid(primary_path, lid);
1666
1667                 lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
1668                         CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg));
1669                 sa_path_set_slid(primary_path, lid);
1670         }
1671
1672         if (!cm_req_has_alt_path(req_msg))
1673                 return;
1674
1675         if (alt_path->rec_type != SA_PATH_REC_TYPE_OPA) {
1676                 sa_path_set_dlid(alt_path,
1677                                  IBA_GET(CM_REQ_ALTERNATE_LOCAL_PORT_LID,
1678                                          req_msg));
1679                 sa_path_set_slid(alt_path,
1680                                  IBA_GET(CM_REQ_ALTERNATE_REMOTE_PORT_LID,
1681                                          req_msg));
1682         } else {
1683                 lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
1684                         CM_REQ_ALTERNATE_LOCAL_PORT_GID, req_msg));
1685                 sa_path_set_dlid(alt_path, lid);
1686
1687                 lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
1688                         CM_REQ_ALTERNATE_REMOTE_PORT_GID, req_msg));
1689                 sa_path_set_slid(alt_path, lid);
1690         }
1691 }
1692
1693 static void cm_format_paths_from_req(struct cm_req_msg *req_msg,
1694                                      struct sa_path_rec *primary_path,
1695                                      struct sa_path_rec *alt_path,
1696                                      struct ib_wc *wc)
1697 {
1698         primary_path->dgid =
1699                 *IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg);
1700         primary_path->sgid =
1701                 *IBA_GET_MEM_PTR(CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg);
1702         primary_path->flow_label =
1703                 cpu_to_be32(IBA_GET(CM_REQ_PRIMARY_FLOW_LABEL, req_msg));
1704         primary_path->hop_limit = IBA_GET(CM_REQ_PRIMARY_HOP_LIMIT, req_msg);
1705         primary_path->traffic_class =
1706                 IBA_GET(CM_REQ_PRIMARY_TRAFFIC_CLASS, req_msg);
1707         primary_path->reversible = 1;
1708         primary_path->pkey =
1709                 cpu_to_be16(IBA_GET(CM_REQ_PARTITION_KEY, req_msg));
1710         primary_path->sl = IBA_GET(CM_REQ_PRIMARY_SL, req_msg);
1711         primary_path->mtu_selector = IB_SA_EQ;
1712         primary_path->mtu = IBA_GET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg);
1713         primary_path->rate_selector = IB_SA_EQ;
1714         primary_path->rate = IBA_GET(CM_REQ_PRIMARY_PACKET_RATE, req_msg);
1715         primary_path->packet_life_time_selector = IB_SA_EQ;
1716         primary_path->packet_life_time =
1717                 IBA_GET(CM_REQ_PRIMARY_LOCAL_ACK_TIMEOUT, req_msg);
1718         primary_path->packet_life_time -= (primary_path->packet_life_time > 0);
1719         primary_path->service_id =
1720                 cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg));
1721         if (sa_path_is_roce(primary_path))
1722                 primary_path->roce.route_resolved = false;
1723
1724         if (cm_req_has_alt_path(req_msg)) {
1725                 alt_path->dgid = *IBA_GET_MEM_PTR(
1726                         CM_REQ_ALTERNATE_LOCAL_PORT_GID, req_msg);
1727                 alt_path->sgid = *IBA_GET_MEM_PTR(
1728                         CM_REQ_ALTERNATE_REMOTE_PORT_GID, req_msg);
1729                 alt_path->flow_label = cpu_to_be32(
1730                         IBA_GET(CM_REQ_ALTERNATE_FLOW_LABEL, req_msg));
1731                 alt_path->hop_limit =
1732                         IBA_GET(CM_REQ_ALTERNATE_HOP_LIMIT, req_msg);
1733                 alt_path->traffic_class =
1734                         IBA_GET(CM_REQ_ALTERNATE_TRAFFIC_CLASS, req_msg);
1735                 alt_path->reversible = 1;
1736                 alt_path->pkey =
1737                         cpu_to_be16(IBA_GET(CM_REQ_PARTITION_KEY, req_msg));
1738                 alt_path->sl = IBA_GET(CM_REQ_ALTERNATE_SL, req_msg);
1739                 alt_path->mtu_selector = IB_SA_EQ;
1740                 alt_path->mtu =
1741                         IBA_GET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg);
1742                 alt_path->rate_selector = IB_SA_EQ;
1743                 alt_path->rate = IBA_GET(CM_REQ_ALTERNATE_PACKET_RATE, req_msg);
1744                 alt_path->packet_life_time_selector = IB_SA_EQ;
1745                 alt_path->packet_life_time =
1746                         IBA_GET(CM_REQ_ALTERNATE_LOCAL_ACK_TIMEOUT, req_msg);
1747                 alt_path->packet_life_time -= (alt_path->packet_life_time > 0);
1748                 alt_path->service_id =
1749                         cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg));
1750
1751                 if (sa_path_is_roce(alt_path))
1752                         alt_path->roce.route_resolved = false;
1753         }
1754         cm_format_path_lid_from_req(req_msg, primary_path, alt_path, wc);
1755 }
1756
1757 static u16 cm_get_bth_pkey(struct cm_work *work)
1758 {
1759         struct ib_device *ib_dev = work->port->cm_dev->ib_device;
1760         u32 port_num = work->port->port_num;
1761         u16 pkey_index = work->mad_recv_wc->wc->pkey_index;
1762         u16 pkey;
1763         int ret;
1764
1765         ret = ib_get_cached_pkey(ib_dev, port_num, pkey_index, &pkey);
1766         if (ret) {
1767                 dev_warn_ratelimited(&ib_dev->dev, "ib_cm: Couldn't retrieve pkey for incoming request (port %u, pkey index %u). %d\n",
1768                                      port_num, pkey_index, ret);
1769                 return 0;
1770         }
1771
1772         return pkey;
1773 }
1774
1775 /**
1776  * cm_opa_to_ib_sgid - Convert OPA SGID to IB SGID
1777  * ULPs (such as IPoIB) do not understand OPA GIDs and will
1778  * reject them as the local_gid will not match the sgid. Therefore,
1779  * change the pathrec's SGID to an IB SGID.
1780  *
1781  * @work: Work completion
1782  * @path: Path record
1783  */
1784 static void cm_opa_to_ib_sgid(struct cm_work *work,
1785                               struct sa_path_rec *path)
1786 {
1787         struct ib_device *dev = work->port->cm_dev->ib_device;
1788         u32 port_num = work->port->port_num;
1789
1790         if (rdma_cap_opa_ah(dev, port_num) &&
1791             (ib_is_opa_gid(&path->sgid))) {
1792                 union ib_gid sgid;
1793
1794                 if (rdma_query_gid(dev, port_num, 0, &sgid)) {
1795                         dev_warn(&dev->dev,
1796                                  "Error updating sgid in CM request\n");
1797                         return;
1798                 }
1799
1800                 path->sgid = sgid;
1801         }
1802 }
1803
1804 static void cm_format_req_event(struct cm_work *work,
1805                                 struct cm_id_private *cm_id_priv,
1806                                 struct ib_cm_id *listen_id)
1807 {
1808         struct cm_req_msg *req_msg;
1809         struct ib_cm_req_event_param *param;
1810
1811         req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad;
1812         param = &work->cm_event.param.req_rcvd;
1813         param->listen_id = listen_id;
1814         param->bth_pkey = cm_get_bth_pkey(work);
1815         param->port = cm_id_priv->av.port->port_num;
1816         param->primary_path = &work->path[0];
1817         cm_opa_to_ib_sgid(work, param->primary_path);
1818         if (cm_req_has_alt_path(req_msg)) {
1819                 param->alternate_path = &work->path[1];
1820                 cm_opa_to_ib_sgid(work, param->alternate_path);
1821         } else {
1822                 param->alternate_path = NULL;
1823         }
1824         param->remote_ca_guid =
1825                 cpu_to_be64(IBA_GET(CM_REQ_LOCAL_CA_GUID, req_msg));
1826         param->remote_qkey = IBA_GET(CM_REQ_LOCAL_Q_KEY, req_msg);
1827         param->remote_qpn = IBA_GET(CM_REQ_LOCAL_QPN, req_msg);
1828         param->qp_type = cm_req_get_qp_type(req_msg);
1829         param->starting_psn = IBA_GET(CM_REQ_STARTING_PSN, req_msg);
1830         param->responder_resources = IBA_GET(CM_REQ_INITIATOR_DEPTH, req_msg);
1831         param->initiator_depth = IBA_GET(CM_REQ_RESPONDER_RESOURCES, req_msg);
1832         param->local_cm_response_timeout =
1833                 IBA_GET(CM_REQ_REMOTE_CM_RESPONSE_TIMEOUT, req_msg);
1834         param->flow_control = IBA_GET(CM_REQ_END_TO_END_FLOW_CONTROL, req_msg);
1835         param->remote_cm_response_timeout =
1836                 IBA_GET(CM_REQ_LOCAL_CM_RESPONSE_TIMEOUT, req_msg);
1837         param->retry_count = IBA_GET(CM_REQ_RETRY_COUNT, req_msg);
1838         param->rnr_retry_count = IBA_GET(CM_REQ_RNR_RETRY_COUNT, req_msg);
1839         param->srq = IBA_GET(CM_REQ_SRQ, req_msg);
1840         param->ppath_sgid_attr = cm_id_priv->av.ah_attr.grh.sgid_attr;
1841         param->ece.vendor_id = IBA_GET(CM_REQ_VENDOR_ID, req_msg);
1842         param->ece.attr_mod = be32_to_cpu(req_msg->hdr.attr_mod);
1843
1844         work->cm_event.private_data =
1845                 IBA_GET_MEM_PTR(CM_REQ_PRIVATE_DATA, req_msg);
1846 }
1847
1848 static void cm_process_work(struct cm_id_private *cm_id_priv,
1849                             struct cm_work *work)
1850 {
1851         int ret;
1852
1853         /* We will typically only have the current event to report. */
1854         ret = cm_id_priv->id.cm_handler(&cm_id_priv->id, &work->cm_event);
1855         cm_free_work(work);
1856
1857         while (!ret && !atomic_add_negative(-1, &cm_id_priv->work_count)) {
1858                 spin_lock_irq(&cm_id_priv->lock);
1859                 work = cm_dequeue_work(cm_id_priv);
1860                 spin_unlock_irq(&cm_id_priv->lock);
1861                 if (!work)
1862                         return;
1863
1864                 ret = cm_id_priv->id.cm_handler(&cm_id_priv->id,
1865                                                 &work->cm_event);
1866                 cm_free_work(work);
1867         }
1868         cm_deref_id(cm_id_priv);
1869         if (ret)
1870                 cm_destroy_id(&cm_id_priv->id, ret);
1871 }
1872
1873 static void cm_format_mra(struct cm_mra_msg *mra_msg,
1874                           struct cm_id_private *cm_id_priv,
1875                           enum cm_msg_response msg_mraed,
1876                           const void *private_data, u8 private_data_len)
1877 {
1878         cm_format_mad_hdr(&mra_msg->hdr, CM_MRA_ATTR_ID, cm_id_priv->tid);
1879         IBA_SET(CM_MRA_MESSAGE_MRAED, mra_msg, msg_mraed);
1880         IBA_SET(CM_MRA_LOCAL_COMM_ID, mra_msg,
1881                 be32_to_cpu(cm_id_priv->id.local_id));
1882         IBA_SET(CM_MRA_REMOTE_COMM_ID, mra_msg,
1883                 be32_to_cpu(cm_id_priv->id.remote_id));
1884         IBA_SET(CM_MRA_SERVICE_TIMEOUT, mra_msg, CM_MRA_SETTING);
1885
1886         if (private_data && private_data_len)
1887                 IBA_SET_MEM(CM_MRA_PRIVATE_DATA, mra_msg, private_data,
1888                             private_data_len);
1889 }
1890
1891 static void cm_format_rej(struct cm_rej_msg *rej_msg,
1892                           struct cm_id_private *cm_id_priv,
1893                           enum ib_cm_rej_reason reason, void *ari,
1894                           u8 ari_length, const void *private_data,
1895                           u8 private_data_len, enum ib_cm_state state)
1896 {
1897         lockdep_assert_held(&cm_id_priv->lock);
1898
1899         cm_format_mad_hdr(&rej_msg->hdr, CM_REJ_ATTR_ID, cm_id_priv->tid);
1900         IBA_SET(CM_REJ_REMOTE_COMM_ID, rej_msg,
1901                 be32_to_cpu(cm_id_priv->id.remote_id));
1902
1903         switch (state) {
1904         case IB_CM_REQ_RCVD:
1905                 IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg, be32_to_cpu(0));
1906                 IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, CM_MSG_RESPONSE_REQ);
1907                 break;
1908         case IB_CM_MRA_REQ_SENT:
1909                 IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg,
1910                         be32_to_cpu(cm_id_priv->id.local_id));
1911                 IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, CM_MSG_RESPONSE_REQ);
1912                 break;
1913         case IB_CM_REP_RCVD:
1914         case IB_CM_MRA_REP_SENT:
1915                 IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg,
1916                         be32_to_cpu(cm_id_priv->id.local_id));
1917                 IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, CM_MSG_RESPONSE_REP);
1918                 break;
1919         default:
1920                 IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg,
1921                         be32_to_cpu(cm_id_priv->id.local_id));
1922                 IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg,
1923                         CM_MSG_RESPONSE_OTHER);
1924                 break;
1925         }
1926
1927         IBA_SET(CM_REJ_REASON, rej_msg, reason);
1928         if (ari && ari_length) {
1929                 IBA_SET(CM_REJ_REJECTED_INFO_LENGTH, rej_msg, ari_length);
1930                 IBA_SET_MEM(CM_REJ_ARI, rej_msg, ari, ari_length);
1931         }
1932
1933         if (private_data && private_data_len)
1934                 IBA_SET_MEM(CM_REJ_PRIVATE_DATA, rej_msg, private_data,
1935                             private_data_len);
1936 }
1937
1938 static void cm_dup_req_handler(struct cm_work *work,
1939                                struct cm_id_private *cm_id_priv)
1940 {
1941         struct ib_mad_send_buf *msg = NULL;
1942         int ret;
1943
1944         atomic_long_inc(
1945                 &work->port->counters[CM_RECV_DUPLICATES][CM_REQ_COUNTER]);
1946
1947         /* Quick state check to discard duplicate REQs. */
1948         spin_lock_irq(&cm_id_priv->lock);
1949         if (cm_id_priv->id.state == IB_CM_REQ_RCVD) {
1950                 spin_unlock_irq(&cm_id_priv->lock);
1951                 return;
1952         }
1953         spin_unlock_irq(&cm_id_priv->lock);
1954
1955         ret = cm_alloc_response_msg(work->port, work->mad_recv_wc, true, &msg);
1956         if (ret)
1957                 return;
1958
1959         spin_lock_irq(&cm_id_priv->lock);
1960         switch (cm_id_priv->id.state) {
1961         case IB_CM_MRA_REQ_SENT:
1962                 cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
1963                               CM_MSG_RESPONSE_REQ,
1964                               cm_id_priv->private_data,
1965                               cm_id_priv->private_data_len);
1966                 break;
1967         case IB_CM_TIMEWAIT:
1968                 cm_format_rej((struct cm_rej_msg *)msg->mad, cm_id_priv,
1969                               IB_CM_REJ_STALE_CONN, NULL, 0, NULL, 0,
1970                               IB_CM_TIMEWAIT);
1971                 break;
1972         default:
1973                 goto unlock;
1974         }
1975         spin_unlock_irq(&cm_id_priv->lock);
1976
1977         trace_icm_send_dup_req(&cm_id_priv->id);
1978         ret = ib_post_send_mad(msg, NULL);
1979         if (ret)
1980                 goto free;
1981         return;
1982
1983 unlock: spin_unlock_irq(&cm_id_priv->lock);
1984 free:   cm_free_msg(msg);
1985 }
1986
1987 static struct cm_id_private *cm_match_req(struct cm_work *work,
1988                                           struct cm_id_private *cm_id_priv)
1989 {
1990         struct cm_id_private *listen_cm_id_priv, *cur_cm_id_priv;
1991         struct cm_timewait_info *timewait_info;
1992         struct cm_req_msg *req_msg;
1993
1994         req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad;
1995
1996         /* Check for possible duplicate REQ. */
1997         spin_lock_irq(&cm.lock);
1998         timewait_info = cm_insert_remote_id(cm_id_priv->timewait_info);
1999         if (timewait_info) {
2000                 cur_cm_id_priv = cm_acquire_id(timewait_info->work.local_id,
2001                                            timewait_info->work.remote_id);
2002                 spin_unlock_irq(&cm.lock);
2003                 if (cur_cm_id_priv) {
2004                         cm_dup_req_handler(work, cur_cm_id_priv);
2005                         cm_deref_id(cur_cm_id_priv);
2006                 }
2007                 return NULL;
2008         }
2009
2010         /* Check for stale connections. */
2011         timewait_info = cm_insert_remote_qpn(cm_id_priv->timewait_info);
2012         if (timewait_info) {
2013                 cm_remove_remote(cm_id_priv);
2014                 cur_cm_id_priv = cm_acquire_id(timewait_info->work.local_id,
2015                                            timewait_info->work.remote_id);
2016
2017                 spin_unlock_irq(&cm.lock);
2018                 cm_issue_rej(work->port, work->mad_recv_wc,
2019                              IB_CM_REJ_STALE_CONN, CM_MSG_RESPONSE_REQ,
2020                              NULL, 0);
2021                 if (cur_cm_id_priv) {
2022                         ib_send_cm_dreq(&cur_cm_id_priv->id, NULL, 0);
2023                         cm_deref_id(cur_cm_id_priv);
2024                 }
2025                 return NULL;
2026         }
2027
2028         /* Find matching listen request. */
2029         listen_cm_id_priv = cm_find_listen(
2030                 cm_id_priv->id.device,
2031                 cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg)));
2032         if (!listen_cm_id_priv) {
2033                 cm_remove_remote(cm_id_priv);
2034                 spin_unlock_irq(&cm.lock);
2035                 cm_issue_rej(work->port, work->mad_recv_wc,
2036                              IB_CM_REJ_INVALID_SERVICE_ID, CM_MSG_RESPONSE_REQ,
2037                              NULL, 0);
2038                 return NULL;
2039         }
2040         spin_unlock_irq(&cm.lock);
2041         return listen_cm_id_priv;
2042 }
2043
2044 /*
2045  * Work-around for inter-subnet connections.  If the LIDs are permissive,
2046  * we need to override the LID/SL data in the REQ with the LID information
2047  * in the work completion.
2048  */
2049 static void cm_process_routed_req(struct cm_req_msg *req_msg, struct ib_wc *wc)
2050 {
2051         if (!IBA_GET(CM_REQ_PRIMARY_SUBNET_LOCAL, req_msg)) {
2052                 if (cpu_to_be16(IBA_GET(CM_REQ_PRIMARY_LOCAL_PORT_LID,
2053                                         req_msg)) == IB_LID_PERMISSIVE) {
2054                         IBA_SET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg,
2055                                 be16_to_cpu(ib_lid_be16(wc->slid)));
2056                         IBA_SET(CM_REQ_PRIMARY_SL, req_msg, wc->sl);
2057                 }
2058
2059                 if (cpu_to_be16(IBA_GET(CM_REQ_PRIMARY_REMOTE_PORT_LID,
2060                                         req_msg)) == IB_LID_PERMISSIVE)
2061                         IBA_SET(CM_REQ_PRIMARY_REMOTE_PORT_LID, req_msg,
2062                                 wc->dlid_path_bits);
2063         }
2064
2065         if (!IBA_GET(CM_REQ_ALTERNATE_SUBNET_LOCAL, req_msg)) {
2066                 if (cpu_to_be16(IBA_GET(CM_REQ_ALTERNATE_LOCAL_PORT_LID,
2067                                         req_msg)) == IB_LID_PERMISSIVE) {
2068                         IBA_SET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg,
2069                                 be16_to_cpu(ib_lid_be16(wc->slid)));
2070                         IBA_SET(CM_REQ_ALTERNATE_SL, req_msg, wc->sl);
2071                 }
2072
2073                 if (cpu_to_be16(IBA_GET(CM_REQ_ALTERNATE_REMOTE_PORT_LID,
2074                                         req_msg)) == IB_LID_PERMISSIVE)
2075                         IBA_SET(CM_REQ_ALTERNATE_REMOTE_PORT_LID, req_msg,
2076                                 wc->dlid_path_bits);
2077         }
2078 }
2079
2080 static int cm_req_handler(struct cm_work *work)
2081 {
2082         struct cm_id_private *cm_id_priv, *listen_cm_id_priv;
2083         struct cm_req_msg *req_msg;
2084         const struct ib_global_route *grh;
2085         const struct ib_gid_attr *gid_attr;
2086         int ret;
2087
2088         req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad;
2089
2090         cm_id_priv =
2091                 cm_alloc_id_priv(work->port->cm_dev->ib_device, NULL, NULL);
2092         if (IS_ERR(cm_id_priv))
2093                 return PTR_ERR(cm_id_priv);
2094
2095         cm_id_priv->id.remote_id =
2096                 cpu_to_be32(IBA_GET(CM_REQ_LOCAL_COMM_ID, req_msg));
2097         cm_id_priv->id.service_id =
2098                 cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg));
2099         cm_id_priv->tid = req_msg->hdr.tid;
2100         cm_id_priv->timeout_ms = cm_convert_to_ms(
2101                 IBA_GET(CM_REQ_LOCAL_CM_RESPONSE_TIMEOUT, req_msg));
2102         cm_id_priv->max_cm_retries = IBA_GET(CM_REQ_MAX_CM_RETRIES, req_msg);
2103         cm_id_priv->remote_qpn =
2104                 cpu_to_be32(IBA_GET(CM_REQ_LOCAL_QPN, req_msg));
2105         cm_id_priv->initiator_depth =
2106                 IBA_GET(CM_REQ_RESPONDER_RESOURCES, req_msg);
2107         cm_id_priv->responder_resources =
2108                 IBA_GET(CM_REQ_INITIATOR_DEPTH, req_msg);
2109         cm_id_priv->path_mtu = IBA_GET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg);
2110         cm_id_priv->pkey = cpu_to_be16(IBA_GET(CM_REQ_PARTITION_KEY, req_msg));
2111         cm_id_priv->sq_psn = cpu_to_be32(IBA_GET(CM_REQ_STARTING_PSN, req_msg));
2112         cm_id_priv->retry_count = IBA_GET(CM_REQ_RETRY_COUNT, req_msg);
2113         cm_id_priv->rnr_retry_count = IBA_GET(CM_REQ_RNR_RETRY_COUNT, req_msg);
2114         cm_id_priv->qp_type = cm_req_get_qp_type(req_msg);
2115
2116         ret = cm_init_av_for_response(work->port, work->mad_recv_wc->wc,
2117                                       work->mad_recv_wc->recv_buf.grh,
2118                                       &cm_id_priv->av);
2119         if (ret)
2120                 goto destroy;
2121         cm_id_priv->timewait_info = cm_create_timewait_info(cm_id_priv->
2122                                                             id.local_id);
2123         if (IS_ERR(cm_id_priv->timewait_info)) {
2124                 ret = PTR_ERR(cm_id_priv->timewait_info);
2125                 cm_id_priv->timewait_info = NULL;
2126                 goto destroy;
2127         }
2128         cm_id_priv->timewait_info->work.remote_id = cm_id_priv->id.remote_id;
2129         cm_id_priv->timewait_info->remote_ca_guid =
2130                 cpu_to_be64(IBA_GET(CM_REQ_LOCAL_CA_GUID, req_msg));
2131         cm_id_priv->timewait_info->remote_qpn = cm_id_priv->remote_qpn;
2132
2133         /*
2134          * Note that the ID pointer is not in the xarray at this point,
2135          * so this set is only visible to the local thread.
2136          */
2137         cm_id_priv->id.state = IB_CM_REQ_RCVD;
2138
2139         listen_cm_id_priv = cm_match_req(work, cm_id_priv);
2140         if (!listen_cm_id_priv) {
2141                 trace_icm_no_listener_err(&cm_id_priv->id);
2142                 cm_id_priv->id.state = IB_CM_IDLE;
2143                 ret = -EINVAL;
2144                 goto destroy;
2145         }
2146
2147         memset(&work->path[0], 0, sizeof(work->path[0]));
2148         if (cm_req_has_alt_path(req_msg))
2149                 memset(&work->path[1], 0, sizeof(work->path[1]));
2150         grh = rdma_ah_read_grh(&cm_id_priv->av.ah_attr);
2151         gid_attr = grh->sgid_attr;
2152
2153         if (cm_id_priv->av.ah_attr.type == RDMA_AH_ATTR_TYPE_ROCE) {
2154                 work->path[0].rec_type =
2155                         sa_conv_gid_to_pathrec_type(gid_attr->gid_type);
2156         } else {
2157                 cm_process_routed_req(req_msg, work->mad_recv_wc->wc);
2158                 cm_path_set_rec_type(
2159                         work->port->cm_dev->ib_device, work->port->port_num,
2160                         &work->path[0],
2161                         IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID,
2162                                         req_msg));
2163         }
2164         if (cm_req_has_alt_path(req_msg))
2165                 work->path[1].rec_type = work->path[0].rec_type;
2166         cm_format_paths_from_req(req_msg, &work->path[0],
2167                                  &work->path[1], work->mad_recv_wc->wc);
2168         if (cm_id_priv->av.ah_attr.type == RDMA_AH_ATTR_TYPE_ROCE)
2169                 sa_path_set_dmac(&work->path[0],
2170                                  cm_id_priv->av.ah_attr.roce.dmac);
2171         work->path[0].hop_limit = grh->hop_limit;
2172
2173         /* This destroy call is needed to pair with cm_init_av_for_response */
2174         cm_destroy_av(&cm_id_priv->av);
2175         ret = cm_init_av_by_path(&work->path[0], gid_attr, &cm_id_priv->av);
2176         if (ret) {
2177                 int err;
2178
2179                 err = rdma_query_gid(work->port->cm_dev->ib_device,
2180                                      work->port->port_num, 0,
2181                                      &work->path[0].sgid);
2182                 if (err)
2183                         ib_send_cm_rej(&cm_id_priv->id, IB_CM_REJ_INVALID_GID,
2184                                        NULL, 0, NULL, 0);
2185                 else
2186                         ib_send_cm_rej(&cm_id_priv->id, IB_CM_REJ_INVALID_GID,
2187                                        &work->path[0].sgid,
2188                                        sizeof(work->path[0].sgid),
2189                                        NULL, 0);
2190                 goto rejected;
2191         }
2192         if (cm_id_priv->av.ah_attr.type == RDMA_AH_ATTR_TYPE_IB)
2193                 cm_id_priv->av.dlid_datapath =
2194                         IBA_GET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg);
2195
2196         if (cm_req_has_alt_path(req_msg)) {
2197                 ret = cm_init_av_by_path(&work->path[1], NULL,
2198                                          &cm_id_priv->alt_av);
2199                 if (ret) {
2200                         ib_send_cm_rej(&cm_id_priv->id,
2201                                        IB_CM_REJ_INVALID_ALT_GID,
2202                                        &work->path[0].sgid,
2203                                        sizeof(work->path[0].sgid), NULL, 0);
2204                         goto rejected;
2205                 }
2206         }
2207
2208         cm_id_priv->id.cm_handler = listen_cm_id_priv->id.cm_handler;
2209         cm_id_priv->id.context = listen_cm_id_priv->id.context;
2210         cm_format_req_event(work, cm_id_priv, &listen_cm_id_priv->id);
2211
2212         /* Now MAD handlers can see the new ID */
2213         spin_lock_irq(&cm_id_priv->lock);
2214         cm_finalize_id(cm_id_priv);
2215
2216         /* Refcount belongs to the event, pairs with cm_process_work() */
2217         refcount_inc(&cm_id_priv->refcount);
2218         cm_queue_work_unlock(cm_id_priv, work);
2219         /*
2220          * Since this ID was just created and was not made visible to other MAD
2221          * handlers until the cm_finalize_id() above we know that the
2222          * cm_process_work() will deliver the event and the listen_cm_id
2223          * embedded in the event can be derefed here.
2224          */
2225         cm_deref_id(listen_cm_id_priv);
2226         return 0;
2227
2228 rejected:
2229         cm_deref_id(listen_cm_id_priv);
2230 destroy:
2231         ib_destroy_cm_id(&cm_id_priv->id);
2232         return ret;
2233 }
2234
2235 static void cm_format_rep(struct cm_rep_msg *rep_msg,
2236                           struct cm_id_private *cm_id_priv,
2237                           struct ib_cm_rep_param *param)
2238 {
2239         cm_format_mad_ece_hdr(&rep_msg->hdr, CM_REP_ATTR_ID, cm_id_priv->tid,
2240                               param->ece.attr_mod);
2241         IBA_SET(CM_REP_LOCAL_COMM_ID, rep_msg,
2242                 be32_to_cpu(cm_id_priv->id.local_id));
2243         IBA_SET(CM_REP_REMOTE_COMM_ID, rep_msg,
2244                 be32_to_cpu(cm_id_priv->id.remote_id));
2245         IBA_SET(CM_REP_STARTING_PSN, rep_msg, param->starting_psn);
2246         IBA_SET(CM_REP_RESPONDER_RESOURCES, rep_msg,
2247                 param->responder_resources);
2248         IBA_SET(CM_REP_TARGET_ACK_DELAY, rep_msg,
2249                 cm_id_priv->av.port->cm_dev->ack_delay);
2250         IBA_SET(CM_REP_FAILOVER_ACCEPTED, rep_msg, param->failover_accepted);
2251         IBA_SET(CM_REP_RNR_RETRY_COUNT, rep_msg, param->rnr_retry_count);
2252         IBA_SET(CM_REP_LOCAL_CA_GUID, rep_msg,
2253                 be64_to_cpu(cm_id_priv->id.device->node_guid));
2254
2255         if (cm_id_priv->qp_type != IB_QPT_XRC_TGT) {
2256                 IBA_SET(CM_REP_INITIATOR_DEPTH, rep_msg,
2257                         param->initiator_depth);
2258                 IBA_SET(CM_REP_END_TO_END_FLOW_CONTROL, rep_msg,
2259                         param->flow_control);
2260                 IBA_SET(CM_REP_SRQ, rep_msg, param->srq);
2261                 IBA_SET(CM_REP_LOCAL_QPN, rep_msg, param->qp_num);
2262         } else {
2263                 IBA_SET(CM_REP_SRQ, rep_msg, 1);
2264                 IBA_SET(CM_REP_LOCAL_EE_CONTEXT_NUMBER, rep_msg, param->qp_num);
2265         }
2266
2267         IBA_SET(CM_REP_VENDOR_ID_L, rep_msg, param->ece.vendor_id);
2268         IBA_SET(CM_REP_VENDOR_ID_M, rep_msg, param->ece.vendor_id >> 8);
2269         IBA_SET(CM_REP_VENDOR_ID_H, rep_msg, param->ece.vendor_id >> 16);
2270
2271         if (param->private_data && param->private_data_len)
2272                 IBA_SET_MEM(CM_REP_PRIVATE_DATA, rep_msg, param->private_data,
2273                             param->private_data_len);
2274 }
2275
2276 int ib_send_cm_rep(struct ib_cm_id *cm_id,
2277                    struct ib_cm_rep_param *param)
2278 {
2279         struct cm_id_private *cm_id_priv;
2280         struct ib_mad_send_buf *msg;
2281         struct cm_rep_msg *rep_msg;
2282         unsigned long flags;
2283         int ret;
2284
2285         if (param->private_data &&
2286             param->private_data_len > IB_CM_REP_PRIVATE_DATA_SIZE)
2287                 return -EINVAL;
2288
2289         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
2290         spin_lock_irqsave(&cm_id_priv->lock, flags);
2291         if (cm_id->state != IB_CM_REQ_RCVD &&
2292             cm_id->state != IB_CM_MRA_REQ_SENT) {
2293                 trace_icm_send_rep_err(cm_id_priv->id.local_id, cm_id->state);
2294                 ret = -EINVAL;
2295                 goto out;
2296         }
2297
2298         msg = cm_alloc_priv_msg(cm_id_priv, IB_CM_REP_SENT);
2299         if (IS_ERR(msg)) {
2300                 ret = PTR_ERR(msg);
2301                 goto out;
2302         }
2303
2304         rep_msg = (struct cm_rep_msg *) msg->mad;
2305         cm_format_rep(rep_msg, cm_id_priv, param);
2306
2307         trace_icm_send_rep(cm_id);
2308         ret = ib_post_send_mad(msg, NULL);
2309         if (ret)
2310                 goto out_free;
2311
2312         cm_id->state = IB_CM_REP_SENT;
2313         cm_id_priv->initiator_depth = param->initiator_depth;
2314         cm_id_priv->responder_resources = param->responder_resources;
2315         cm_id_priv->rq_psn = cpu_to_be32(IBA_GET(CM_REP_STARTING_PSN, rep_msg));
2316         WARN_ONCE(param->qp_num & 0xFF000000,
2317                   "IBTA declares QPN to be 24 bits, but it is 0x%X\n",
2318                   param->qp_num);
2319         cm_id_priv->local_qpn = cpu_to_be32(param->qp_num & 0xFFFFFF);
2320         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2321         return 0;
2322
2323 out_free:
2324         cm_free_priv_msg(msg);
2325 out:
2326         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2327         return ret;
2328 }
2329 EXPORT_SYMBOL(ib_send_cm_rep);
2330
2331 static void cm_format_rtu(struct cm_rtu_msg *rtu_msg,
2332                           struct cm_id_private *cm_id_priv,
2333                           const void *private_data,
2334                           u8 private_data_len)
2335 {
2336         cm_format_mad_hdr(&rtu_msg->hdr, CM_RTU_ATTR_ID, cm_id_priv->tid);
2337         IBA_SET(CM_RTU_LOCAL_COMM_ID, rtu_msg,
2338                 be32_to_cpu(cm_id_priv->id.local_id));
2339         IBA_SET(CM_RTU_REMOTE_COMM_ID, rtu_msg,
2340                 be32_to_cpu(cm_id_priv->id.remote_id));
2341
2342         if (private_data && private_data_len)
2343                 IBA_SET_MEM(CM_RTU_PRIVATE_DATA, rtu_msg, private_data,
2344                             private_data_len);
2345 }
2346
2347 int ib_send_cm_rtu(struct ib_cm_id *cm_id,
2348                    const void *private_data,
2349                    u8 private_data_len)
2350 {
2351         struct cm_id_private *cm_id_priv;
2352         struct ib_mad_send_buf *msg;
2353         unsigned long flags;
2354         void *data;
2355         int ret;
2356
2357         if (private_data && private_data_len > IB_CM_RTU_PRIVATE_DATA_SIZE)
2358                 return -EINVAL;
2359
2360         data = cm_copy_private_data(private_data, private_data_len);
2361         if (IS_ERR(data))
2362                 return PTR_ERR(data);
2363
2364         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
2365         spin_lock_irqsave(&cm_id_priv->lock, flags);
2366         if (cm_id->state != IB_CM_REP_RCVD &&
2367             cm_id->state != IB_CM_MRA_REP_SENT) {
2368                 trace_icm_send_cm_rtu_err(cm_id);
2369                 ret = -EINVAL;
2370                 goto error;
2371         }
2372
2373         msg = cm_alloc_msg(cm_id_priv);
2374         if (IS_ERR(msg)) {
2375                 ret = PTR_ERR(msg);
2376                 goto error;
2377         }
2378
2379         cm_format_rtu((struct cm_rtu_msg *) msg->mad, cm_id_priv,
2380                       private_data, private_data_len);
2381
2382         trace_icm_send_rtu(cm_id);
2383         ret = ib_post_send_mad(msg, NULL);
2384         if (ret) {
2385                 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2386                 cm_free_msg(msg);
2387                 kfree(data);
2388                 return ret;
2389         }
2390
2391         cm_id->state = IB_CM_ESTABLISHED;
2392         cm_set_private_data(cm_id_priv, data, private_data_len);
2393         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2394         return 0;
2395
2396 error:  spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2397         kfree(data);
2398         return ret;
2399 }
2400 EXPORT_SYMBOL(ib_send_cm_rtu);
2401
2402 static void cm_format_rep_event(struct cm_work *work, enum ib_qp_type qp_type)
2403 {
2404         struct cm_rep_msg *rep_msg;
2405         struct ib_cm_rep_event_param *param;
2406
2407         rep_msg = (struct cm_rep_msg *)work->mad_recv_wc->recv_buf.mad;
2408         param = &work->cm_event.param.rep_rcvd;
2409         param->remote_ca_guid =
2410                 cpu_to_be64(IBA_GET(CM_REP_LOCAL_CA_GUID, rep_msg));
2411         param->remote_qkey = IBA_GET(CM_REP_LOCAL_Q_KEY, rep_msg);
2412         param->remote_qpn = be32_to_cpu(cm_rep_get_qpn(rep_msg, qp_type));
2413         param->starting_psn = IBA_GET(CM_REP_STARTING_PSN, rep_msg);
2414         param->responder_resources = IBA_GET(CM_REP_INITIATOR_DEPTH, rep_msg);
2415         param->initiator_depth = IBA_GET(CM_REP_RESPONDER_RESOURCES, rep_msg);
2416         param->target_ack_delay = IBA_GET(CM_REP_TARGET_ACK_DELAY, rep_msg);
2417         param->failover_accepted = IBA_GET(CM_REP_FAILOVER_ACCEPTED, rep_msg);
2418         param->flow_control = IBA_GET(CM_REP_END_TO_END_FLOW_CONTROL, rep_msg);
2419         param->rnr_retry_count = IBA_GET(CM_REP_RNR_RETRY_COUNT, rep_msg);
2420         param->srq = IBA_GET(CM_REP_SRQ, rep_msg);
2421         param->ece.vendor_id = IBA_GET(CM_REP_VENDOR_ID_H, rep_msg) << 16;
2422         param->ece.vendor_id |= IBA_GET(CM_REP_VENDOR_ID_M, rep_msg) << 8;
2423         param->ece.vendor_id |= IBA_GET(CM_REP_VENDOR_ID_L, rep_msg);
2424         param->ece.attr_mod = be32_to_cpu(rep_msg->hdr.attr_mod);
2425
2426         work->cm_event.private_data =
2427                 IBA_GET_MEM_PTR(CM_REP_PRIVATE_DATA, rep_msg);
2428 }
2429
2430 static void cm_dup_rep_handler(struct cm_work *work)
2431 {
2432         struct cm_id_private *cm_id_priv;
2433         struct cm_rep_msg *rep_msg;
2434         struct ib_mad_send_buf *msg = NULL;
2435         int ret;
2436
2437         rep_msg = (struct cm_rep_msg *) work->mad_recv_wc->recv_buf.mad;
2438         cm_id_priv = cm_acquire_id(
2439                 cpu_to_be32(IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg)),
2440                 cpu_to_be32(IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg)));
2441         if (!cm_id_priv)
2442                 return;
2443
2444         atomic_long_inc(
2445                 &work->port->counters[CM_RECV_DUPLICATES][CM_REP_COUNTER]);
2446         ret = cm_alloc_response_msg(work->port, work->mad_recv_wc, true, &msg);
2447         if (ret)
2448                 goto deref;
2449
2450         spin_lock_irq(&cm_id_priv->lock);
2451         if (cm_id_priv->id.state == IB_CM_ESTABLISHED)
2452                 cm_format_rtu((struct cm_rtu_msg *) msg->mad, cm_id_priv,
2453                               cm_id_priv->private_data,
2454                               cm_id_priv->private_data_len);
2455         else if (cm_id_priv->id.state == IB_CM_MRA_REP_SENT)
2456                 cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
2457                               CM_MSG_RESPONSE_REP,
2458                               cm_id_priv->private_data,
2459                               cm_id_priv->private_data_len);
2460         else
2461                 goto unlock;
2462         spin_unlock_irq(&cm_id_priv->lock);
2463
2464         trace_icm_send_dup_rep(&cm_id_priv->id);
2465         ret = ib_post_send_mad(msg, NULL);
2466         if (ret)
2467                 goto free;
2468         goto deref;
2469
2470 unlock: spin_unlock_irq(&cm_id_priv->lock);
2471 free:   cm_free_msg(msg);
2472 deref:  cm_deref_id(cm_id_priv);
2473 }
2474
2475 static int cm_rep_handler(struct cm_work *work)
2476 {
2477         struct cm_id_private *cm_id_priv;
2478         struct cm_rep_msg *rep_msg;
2479         int ret;
2480         struct cm_id_private *cur_cm_id_priv;
2481         struct cm_timewait_info *timewait_info;
2482
2483         rep_msg = (struct cm_rep_msg *)work->mad_recv_wc->recv_buf.mad;
2484         cm_id_priv = cm_acquire_id(
2485                 cpu_to_be32(IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg)), 0);
2486         if (!cm_id_priv) {
2487                 cm_dup_rep_handler(work);
2488                 trace_icm_remote_no_priv_err(
2489                          IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg));
2490                 return -EINVAL;
2491         }
2492
2493         cm_format_rep_event(work, cm_id_priv->qp_type);
2494
2495         spin_lock_irq(&cm_id_priv->lock);
2496         switch (cm_id_priv->id.state) {
2497         case IB_CM_REQ_SENT:
2498         case IB_CM_MRA_REQ_RCVD:
2499                 break;
2500         default:
2501                 ret = -EINVAL;
2502                 trace_icm_rep_unknown_err(
2503                         IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg),
2504                         IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg),
2505                         cm_id_priv->id.state);
2506                 spin_unlock_irq(&cm_id_priv->lock);
2507                 goto error;
2508         }
2509
2510         cm_id_priv->timewait_info->work.remote_id =
2511                 cpu_to_be32(IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg));
2512         cm_id_priv->timewait_info->remote_ca_guid =
2513                 cpu_to_be64(IBA_GET(CM_REP_LOCAL_CA_GUID, rep_msg));
2514         cm_id_priv->timewait_info->remote_qpn = cm_rep_get_qpn(rep_msg, cm_id_priv->qp_type);
2515
2516         spin_lock(&cm.lock);
2517         /* Check for duplicate REP. */
2518         if (cm_insert_remote_id(cm_id_priv->timewait_info)) {
2519                 spin_unlock(&cm.lock);
2520                 spin_unlock_irq(&cm_id_priv->lock);
2521                 ret = -EINVAL;
2522                 trace_icm_insert_failed_err(
2523                          IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg));
2524                 goto error;
2525         }
2526         /* Check for a stale connection. */
2527         timewait_info = cm_insert_remote_qpn(cm_id_priv->timewait_info);
2528         if (timewait_info) {
2529                 cm_remove_remote(cm_id_priv);
2530                 cur_cm_id_priv = cm_acquire_id(timewait_info->work.local_id,
2531                                            timewait_info->work.remote_id);
2532
2533                 spin_unlock(&cm.lock);
2534                 spin_unlock_irq(&cm_id_priv->lock);
2535                 cm_issue_rej(work->port, work->mad_recv_wc,
2536                              IB_CM_REJ_STALE_CONN, CM_MSG_RESPONSE_REP,
2537                              NULL, 0);
2538                 ret = -EINVAL;
2539                 trace_icm_staleconn_err(
2540                         IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg),
2541                         IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg));
2542
2543                 if (cur_cm_id_priv) {
2544                         ib_send_cm_dreq(&cur_cm_id_priv->id, NULL, 0);
2545                         cm_deref_id(cur_cm_id_priv);
2546                 }
2547
2548                 goto error;
2549         }
2550         spin_unlock(&cm.lock);
2551
2552         cm_id_priv->id.state = IB_CM_REP_RCVD;
2553         cm_id_priv->id.remote_id =
2554                 cpu_to_be32(IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg));
2555         cm_id_priv->remote_qpn = cm_rep_get_qpn(rep_msg, cm_id_priv->qp_type);
2556         cm_id_priv->initiator_depth =
2557                 IBA_GET(CM_REP_RESPONDER_RESOURCES, rep_msg);
2558         cm_id_priv->responder_resources =
2559                 IBA_GET(CM_REP_INITIATOR_DEPTH, rep_msg);
2560         cm_id_priv->sq_psn = cpu_to_be32(IBA_GET(CM_REP_STARTING_PSN, rep_msg));
2561         cm_id_priv->rnr_retry_count = IBA_GET(CM_REP_RNR_RETRY_COUNT, rep_msg);
2562         cm_id_priv->target_ack_delay =
2563                 IBA_GET(CM_REP_TARGET_ACK_DELAY, rep_msg);
2564         cm_id_priv->av.timeout =
2565                         cm_ack_timeout(cm_id_priv->target_ack_delay,
2566                                        cm_id_priv->av.timeout - 1);
2567         cm_id_priv->alt_av.timeout =
2568                         cm_ack_timeout(cm_id_priv->target_ack_delay,
2569                                        cm_id_priv->alt_av.timeout - 1);
2570
2571         ib_cancel_mad(cm_id_priv->msg);
2572         cm_queue_work_unlock(cm_id_priv, work);
2573         return 0;
2574
2575 error:
2576         cm_deref_id(cm_id_priv);
2577         return ret;
2578 }
2579
2580 static int cm_establish_handler(struct cm_work *work)
2581 {
2582         struct cm_id_private *cm_id_priv;
2583
2584         /* See comment in cm_establish about lookup. */
2585         cm_id_priv = cm_acquire_id(work->local_id, work->remote_id);
2586         if (!cm_id_priv)
2587                 return -EINVAL;
2588
2589         spin_lock_irq(&cm_id_priv->lock);
2590         if (cm_id_priv->id.state != IB_CM_ESTABLISHED) {
2591                 spin_unlock_irq(&cm_id_priv->lock);
2592                 goto out;
2593         }
2594
2595         ib_cancel_mad(cm_id_priv->msg);
2596         cm_queue_work_unlock(cm_id_priv, work);
2597         return 0;
2598 out:
2599         cm_deref_id(cm_id_priv);
2600         return -EINVAL;
2601 }
2602
2603 static int cm_rtu_handler(struct cm_work *work)
2604 {
2605         struct cm_id_private *cm_id_priv;
2606         struct cm_rtu_msg *rtu_msg;
2607
2608         rtu_msg = (struct cm_rtu_msg *)work->mad_recv_wc->recv_buf.mad;
2609         cm_id_priv = cm_acquire_id(
2610                 cpu_to_be32(IBA_GET(CM_RTU_REMOTE_COMM_ID, rtu_msg)),
2611                 cpu_to_be32(IBA_GET(CM_RTU_LOCAL_COMM_ID, rtu_msg)));
2612         if (!cm_id_priv)
2613                 return -EINVAL;
2614
2615         work->cm_event.private_data =
2616                 IBA_GET_MEM_PTR(CM_RTU_PRIVATE_DATA, rtu_msg);
2617
2618         spin_lock_irq(&cm_id_priv->lock);
2619         if (cm_id_priv->id.state != IB_CM_REP_SENT &&
2620             cm_id_priv->id.state != IB_CM_MRA_REP_RCVD) {
2621                 spin_unlock_irq(&cm_id_priv->lock);
2622                 atomic_long_inc(&work->port->counters[CM_RECV_DUPLICATES]
2623                                                      [CM_RTU_COUNTER]);
2624                 goto out;
2625         }
2626         cm_id_priv->id.state = IB_CM_ESTABLISHED;
2627
2628         ib_cancel_mad(cm_id_priv->msg);
2629         cm_queue_work_unlock(cm_id_priv, work);
2630         return 0;
2631 out:
2632         cm_deref_id(cm_id_priv);
2633         return -EINVAL;
2634 }
2635
2636 static void cm_format_dreq(struct cm_dreq_msg *dreq_msg,
2637                           struct cm_id_private *cm_id_priv,
2638                           const void *private_data,
2639                           u8 private_data_len)
2640 {
2641         cm_format_mad_hdr(&dreq_msg->hdr, CM_DREQ_ATTR_ID,
2642                           cm_form_tid(cm_id_priv));
2643         IBA_SET(CM_DREQ_LOCAL_COMM_ID, dreq_msg,
2644                 be32_to_cpu(cm_id_priv->id.local_id));
2645         IBA_SET(CM_DREQ_REMOTE_COMM_ID, dreq_msg,
2646                 be32_to_cpu(cm_id_priv->id.remote_id));
2647         IBA_SET(CM_DREQ_REMOTE_QPN_EECN, dreq_msg,
2648                 be32_to_cpu(cm_id_priv->remote_qpn));
2649
2650         if (private_data && private_data_len)
2651                 IBA_SET_MEM(CM_DREQ_PRIVATE_DATA, dreq_msg, private_data,
2652                             private_data_len);
2653 }
2654
2655 static void cm_issue_dreq(struct cm_id_private *cm_id_priv)
2656 {
2657         struct ib_mad_send_buf *msg;
2658         int ret;
2659
2660         lockdep_assert_held(&cm_id_priv->lock);
2661
2662         msg = cm_alloc_msg(cm_id_priv);
2663         if (IS_ERR(msg))
2664                 return;
2665
2666         cm_format_dreq((struct cm_dreq_msg *) msg->mad, cm_id_priv, NULL, 0);
2667
2668         trace_icm_send_dreq(&cm_id_priv->id);
2669         ret = ib_post_send_mad(msg, NULL);
2670         if (ret)
2671                 cm_free_msg(msg);
2672 }
2673
2674 int ib_send_cm_dreq(struct ib_cm_id *cm_id, const void *private_data,
2675                     u8 private_data_len)
2676 {
2677         struct cm_id_private *cm_id_priv =
2678                 container_of(cm_id, struct cm_id_private, id);
2679         struct ib_mad_send_buf *msg;
2680         unsigned long flags;
2681         int ret;
2682
2683         if (private_data && private_data_len > IB_CM_DREQ_PRIVATE_DATA_SIZE)
2684                 return -EINVAL;
2685
2686         spin_lock_irqsave(&cm_id_priv->lock, flags);
2687         if (cm_id_priv->id.state != IB_CM_ESTABLISHED) {
2688                 trace_icm_dreq_skipped(&cm_id_priv->id);
2689                 ret = -EINVAL;
2690                 goto unlock;
2691         }
2692
2693         if (cm_id_priv->id.lap_state == IB_CM_LAP_SENT ||
2694             cm_id_priv->id.lap_state == IB_CM_MRA_LAP_RCVD)
2695                 ib_cancel_mad(cm_id_priv->msg);
2696
2697         msg = cm_alloc_priv_msg(cm_id_priv, IB_CM_DREQ_SENT);
2698         if (IS_ERR(msg)) {
2699                 cm_enter_timewait(cm_id_priv);
2700                 ret = PTR_ERR(msg);
2701                 goto unlock;
2702         }
2703
2704         cm_format_dreq((struct cm_dreq_msg *) msg->mad, cm_id_priv,
2705                        private_data, private_data_len);
2706
2707         trace_icm_send_dreq(&cm_id_priv->id);
2708         ret = ib_post_send_mad(msg, NULL);
2709         if (ret) {
2710                 cm_enter_timewait(cm_id_priv);
2711                 cm_free_priv_msg(msg);
2712                 goto unlock;
2713         }
2714
2715         cm_id_priv->id.state = IB_CM_DREQ_SENT;
2716 unlock:
2717         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2718         return ret;
2719 }
2720 EXPORT_SYMBOL(ib_send_cm_dreq);
2721
2722 static void cm_format_drep(struct cm_drep_msg *drep_msg,
2723                           struct cm_id_private *cm_id_priv,
2724                           const void *private_data,
2725                           u8 private_data_len)
2726 {
2727         cm_format_mad_hdr(&drep_msg->hdr, CM_DREP_ATTR_ID, cm_id_priv->tid);
2728         IBA_SET(CM_DREP_LOCAL_COMM_ID, drep_msg,
2729                 be32_to_cpu(cm_id_priv->id.local_id));
2730         IBA_SET(CM_DREP_REMOTE_COMM_ID, drep_msg,
2731                 be32_to_cpu(cm_id_priv->id.remote_id));
2732
2733         if (private_data && private_data_len)
2734                 IBA_SET_MEM(CM_DREP_PRIVATE_DATA, drep_msg, private_data,
2735                             private_data_len);
2736 }
2737
2738 static int cm_send_drep_locked(struct cm_id_private *cm_id_priv,
2739                                void *private_data, u8 private_data_len)
2740 {
2741         struct ib_mad_send_buf *msg;
2742         int ret;
2743
2744         lockdep_assert_held(&cm_id_priv->lock);
2745
2746         if (private_data && private_data_len > IB_CM_DREP_PRIVATE_DATA_SIZE)
2747                 return -EINVAL;
2748
2749         if (cm_id_priv->id.state != IB_CM_DREQ_RCVD) {
2750                 trace_icm_send_drep_err(&cm_id_priv->id);
2751                 kfree(private_data);
2752                 return -EINVAL;
2753         }
2754
2755         cm_set_private_data(cm_id_priv, private_data, private_data_len);
2756         cm_enter_timewait(cm_id_priv);
2757
2758         msg = cm_alloc_msg(cm_id_priv);
2759         if (IS_ERR(msg))
2760                 return PTR_ERR(msg);
2761
2762         cm_format_drep((struct cm_drep_msg *) msg->mad, cm_id_priv,
2763                        private_data, private_data_len);
2764
2765         trace_icm_send_drep(&cm_id_priv->id);
2766         ret = ib_post_send_mad(msg, NULL);
2767         if (ret) {
2768                 cm_free_msg(msg);
2769                 return ret;
2770         }
2771         return 0;
2772 }
2773
2774 int ib_send_cm_drep(struct ib_cm_id *cm_id, const void *private_data,
2775                     u8 private_data_len)
2776 {
2777         struct cm_id_private *cm_id_priv =
2778                 container_of(cm_id, struct cm_id_private, id);
2779         unsigned long flags;
2780         void *data;
2781         int ret;
2782
2783         data = cm_copy_private_data(private_data, private_data_len);
2784         if (IS_ERR(data))
2785                 return PTR_ERR(data);
2786
2787         spin_lock_irqsave(&cm_id_priv->lock, flags);
2788         ret = cm_send_drep_locked(cm_id_priv, data, private_data_len);
2789         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2790         return ret;
2791 }
2792 EXPORT_SYMBOL(ib_send_cm_drep);
2793
2794 static int cm_issue_drep(struct cm_port *port,
2795                          struct ib_mad_recv_wc *mad_recv_wc)
2796 {
2797         struct ib_mad_send_buf *msg = NULL;
2798         struct cm_dreq_msg *dreq_msg;
2799         struct cm_drep_msg *drep_msg;
2800         int ret;
2801
2802         ret = cm_alloc_response_msg(port, mad_recv_wc, true, &msg);
2803         if (ret)
2804                 return ret;
2805
2806         dreq_msg = (struct cm_dreq_msg *) mad_recv_wc->recv_buf.mad;
2807         drep_msg = (struct cm_drep_msg *) msg->mad;
2808
2809         cm_format_mad_hdr(&drep_msg->hdr, CM_DREP_ATTR_ID, dreq_msg->hdr.tid);
2810         IBA_SET(CM_DREP_REMOTE_COMM_ID, drep_msg,
2811                 IBA_GET(CM_DREQ_LOCAL_COMM_ID, dreq_msg));
2812         IBA_SET(CM_DREP_LOCAL_COMM_ID, drep_msg,
2813                 IBA_GET(CM_DREQ_REMOTE_COMM_ID, dreq_msg));
2814
2815         trace_icm_issue_drep(
2816                 IBA_GET(CM_DREQ_LOCAL_COMM_ID, dreq_msg),
2817                 IBA_GET(CM_DREQ_REMOTE_COMM_ID, dreq_msg));
2818         ret = ib_post_send_mad(msg, NULL);
2819         if (ret)
2820                 cm_free_msg(msg);
2821
2822         return ret;
2823 }
2824
2825 static int cm_dreq_handler(struct cm_work *work)
2826 {
2827         struct cm_id_private *cm_id_priv;
2828         struct cm_dreq_msg *dreq_msg;
2829         struct ib_mad_send_buf *msg = NULL;
2830
2831         dreq_msg = (struct cm_dreq_msg *)work->mad_recv_wc->recv_buf.mad;
2832         cm_id_priv = cm_acquire_id(
2833                 cpu_to_be32(IBA_GET(CM_DREQ_REMOTE_COMM_ID, dreq_msg)),
2834                 cpu_to_be32(IBA_GET(CM_DREQ_LOCAL_COMM_ID, dreq_msg)));
2835         if (!cm_id_priv) {
2836                 atomic_long_inc(&work->port->counters[CM_RECV_DUPLICATES]
2837                                                      [CM_DREQ_COUNTER]);
2838                 cm_issue_drep(work->port, work->mad_recv_wc);
2839                 trace_icm_no_priv_err(
2840                         IBA_GET(CM_DREQ_LOCAL_COMM_ID, dreq_msg),
2841                         IBA_GET(CM_DREQ_REMOTE_COMM_ID, dreq_msg));
2842                 return -EINVAL;
2843         }
2844
2845         work->cm_event.private_data =
2846                 IBA_GET_MEM_PTR(CM_DREQ_PRIVATE_DATA, dreq_msg);
2847
2848         spin_lock_irq(&cm_id_priv->lock);
2849         if (cm_id_priv->local_qpn !=
2850             cpu_to_be32(IBA_GET(CM_DREQ_REMOTE_QPN_EECN, dreq_msg)))
2851                 goto unlock;
2852
2853         switch (cm_id_priv->id.state) {
2854         case IB_CM_REP_SENT:
2855         case IB_CM_DREQ_SENT:
2856         case IB_CM_MRA_REP_RCVD:
2857                 ib_cancel_mad(cm_id_priv->msg);
2858                 break;
2859         case IB_CM_ESTABLISHED:
2860                 if (cm_id_priv->id.lap_state == IB_CM_LAP_SENT ||
2861                     cm_id_priv->id.lap_state == IB_CM_MRA_LAP_RCVD)
2862                         ib_cancel_mad(cm_id_priv->msg);
2863                 break;
2864         case IB_CM_TIMEWAIT:
2865                 atomic_long_inc(&work->port->counters[CM_RECV_DUPLICATES]
2866                                                      [CM_DREQ_COUNTER]);
2867                 msg = cm_alloc_response_msg_no_ah(work->port, work->mad_recv_wc,
2868                                                   true);
2869                 if (IS_ERR(msg))
2870                         goto unlock;
2871
2872                 cm_format_drep((struct cm_drep_msg *) msg->mad, cm_id_priv,
2873                                cm_id_priv->private_data,
2874                                cm_id_priv->private_data_len);
2875                 spin_unlock_irq(&cm_id_priv->lock);
2876
2877                 if (cm_create_response_msg_ah(work->port, work->mad_recv_wc, msg) ||
2878                     ib_post_send_mad(msg, NULL))
2879                         cm_free_msg(msg);
2880                 goto deref;
2881         case IB_CM_DREQ_RCVD:
2882                 atomic_long_inc(&work->port->counters[CM_RECV_DUPLICATES]
2883                                                      [CM_DREQ_COUNTER]);
2884                 goto unlock;
2885         default:
2886                 trace_icm_dreq_unknown_err(&cm_id_priv->id);
2887                 goto unlock;
2888         }
2889         cm_id_priv->id.state = IB_CM_DREQ_RCVD;
2890         cm_id_priv->tid = dreq_msg->hdr.tid;
2891         cm_queue_work_unlock(cm_id_priv, work);
2892         return 0;
2893
2894 unlock: spin_unlock_irq(&cm_id_priv->lock);
2895 deref:  cm_deref_id(cm_id_priv);
2896         return -EINVAL;
2897 }
2898
2899 static int cm_drep_handler(struct cm_work *work)
2900 {
2901         struct cm_id_private *cm_id_priv;
2902         struct cm_drep_msg *drep_msg;
2903
2904         drep_msg = (struct cm_drep_msg *)work->mad_recv_wc->recv_buf.mad;
2905         cm_id_priv = cm_acquire_id(
2906                 cpu_to_be32(IBA_GET(CM_DREP_REMOTE_COMM_ID, drep_msg)),
2907                 cpu_to_be32(IBA_GET(CM_DREP_LOCAL_COMM_ID, drep_msg)));
2908         if (!cm_id_priv)
2909                 return -EINVAL;
2910
2911         work->cm_event.private_data =
2912                 IBA_GET_MEM_PTR(CM_DREP_PRIVATE_DATA, drep_msg);
2913
2914         spin_lock_irq(&cm_id_priv->lock);
2915         if (cm_id_priv->id.state != IB_CM_DREQ_SENT &&
2916             cm_id_priv->id.state != IB_CM_DREQ_RCVD) {
2917                 spin_unlock_irq(&cm_id_priv->lock);
2918                 goto out;
2919         }
2920         cm_enter_timewait(cm_id_priv);
2921
2922         ib_cancel_mad(cm_id_priv->msg);
2923         cm_queue_work_unlock(cm_id_priv, work);
2924         return 0;
2925 out:
2926         cm_deref_id(cm_id_priv);
2927         return -EINVAL;
2928 }
2929
2930 static int cm_send_rej_locked(struct cm_id_private *cm_id_priv,
2931                               enum ib_cm_rej_reason reason, void *ari,
2932                               u8 ari_length, const void *private_data,
2933                               u8 private_data_len)
2934 {
2935         enum ib_cm_state state = cm_id_priv->id.state;
2936         struct ib_mad_send_buf *msg;
2937         int ret;
2938
2939         lockdep_assert_held(&cm_id_priv->lock);
2940
2941         if ((private_data && private_data_len > IB_CM_REJ_PRIVATE_DATA_SIZE) ||
2942             (ari && ari_length > IB_CM_REJ_ARI_LENGTH))
2943                 return -EINVAL;
2944
2945         trace_icm_send_rej(&cm_id_priv->id, reason);
2946
2947         switch (state) {
2948         case IB_CM_REQ_SENT:
2949         case IB_CM_MRA_REQ_RCVD:
2950         case IB_CM_REQ_RCVD:
2951         case IB_CM_MRA_REQ_SENT:
2952         case IB_CM_REP_RCVD:
2953         case IB_CM_MRA_REP_SENT:
2954                 cm_reset_to_idle(cm_id_priv);
2955                 msg = cm_alloc_msg(cm_id_priv);
2956                 if (IS_ERR(msg))
2957                         return PTR_ERR(msg);
2958                 cm_format_rej((struct cm_rej_msg *)msg->mad, cm_id_priv, reason,
2959                               ari, ari_length, private_data, private_data_len,
2960                               state);
2961                 break;
2962         case IB_CM_REP_SENT:
2963         case IB_CM_MRA_REP_RCVD:
2964                 cm_enter_timewait(cm_id_priv);
2965                 msg = cm_alloc_msg(cm_id_priv);
2966                 if (IS_ERR(msg))
2967                         return PTR_ERR(msg);
2968                 cm_format_rej((struct cm_rej_msg *)msg->mad, cm_id_priv, reason,
2969                               ari, ari_length, private_data, private_data_len,
2970                               state);
2971                 break;
2972         default:
2973                 trace_icm_send_unknown_rej_err(&cm_id_priv->id);
2974                 return -EINVAL;
2975         }
2976
2977         ret = ib_post_send_mad(msg, NULL);
2978         if (ret) {
2979                 cm_free_msg(msg);
2980                 return ret;
2981         }
2982
2983         return 0;
2984 }
2985
2986 int ib_send_cm_rej(struct ib_cm_id *cm_id, enum ib_cm_rej_reason reason,
2987                    void *ari, u8 ari_length, const void *private_data,
2988                    u8 private_data_len)
2989 {
2990         struct cm_id_private *cm_id_priv =
2991                 container_of(cm_id, struct cm_id_private, id);
2992         unsigned long flags;
2993         int ret;
2994
2995         spin_lock_irqsave(&cm_id_priv->lock, flags);
2996         ret = cm_send_rej_locked(cm_id_priv, reason, ari, ari_length,
2997                                  private_data, private_data_len);
2998         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2999         return ret;
3000 }
3001 EXPORT_SYMBOL(ib_send_cm_rej);
3002
3003 static void cm_format_rej_event(struct cm_work *work)
3004 {
3005         struct cm_rej_msg *rej_msg;
3006         struct ib_cm_rej_event_param *param;
3007
3008         rej_msg = (struct cm_rej_msg *)work->mad_recv_wc->recv_buf.mad;
3009         param = &work->cm_event.param.rej_rcvd;
3010         param->ari = IBA_GET_MEM_PTR(CM_REJ_ARI, rej_msg);
3011         param->ari_length = IBA_GET(CM_REJ_REJECTED_INFO_LENGTH, rej_msg);
3012         param->reason = IBA_GET(CM_REJ_REASON, rej_msg);
3013         work->cm_event.private_data =
3014                 IBA_GET_MEM_PTR(CM_REJ_PRIVATE_DATA, rej_msg);
3015 }
3016
3017 static struct cm_id_private *cm_acquire_rejected_id(struct cm_rej_msg *rej_msg)
3018 {
3019         struct cm_id_private *cm_id_priv;
3020         __be32 remote_id;
3021
3022         remote_id = cpu_to_be32(IBA_GET(CM_REJ_LOCAL_COMM_ID, rej_msg));
3023
3024         if (IBA_GET(CM_REJ_REASON, rej_msg) == IB_CM_REJ_TIMEOUT) {
3025                 cm_id_priv = cm_find_remote_id(
3026                         *((__be64 *)IBA_GET_MEM_PTR(CM_REJ_ARI, rej_msg)),
3027                         remote_id);
3028         } else if (IBA_GET(CM_REJ_MESSAGE_REJECTED, rej_msg) ==
3029                    CM_MSG_RESPONSE_REQ)
3030                 cm_id_priv = cm_acquire_id(
3031                         cpu_to_be32(IBA_GET(CM_REJ_REMOTE_COMM_ID, rej_msg)),
3032                         0);
3033         else
3034                 cm_id_priv = cm_acquire_id(
3035                         cpu_to_be32(IBA_GET(CM_REJ_REMOTE_COMM_ID, rej_msg)),
3036                         remote_id);
3037
3038         return cm_id_priv;
3039 }
3040
3041 static int cm_rej_handler(struct cm_work *work)
3042 {
3043         struct cm_id_private *cm_id_priv;
3044         struct cm_rej_msg *rej_msg;
3045
3046         rej_msg = (struct cm_rej_msg *)work->mad_recv_wc->recv_buf.mad;
3047         cm_id_priv = cm_acquire_rejected_id(rej_msg);
3048         if (!cm_id_priv)
3049                 return -EINVAL;
3050
3051         cm_format_rej_event(work);
3052
3053         spin_lock_irq(&cm_id_priv->lock);
3054         switch (cm_id_priv->id.state) {
3055         case IB_CM_REQ_SENT:
3056         case IB_CM_MRA_REQ_RCVD:
3057         case IB_CM_REP_SENT:
3058         case IB_CM_MRA_REP_RCVD:
3059                 ib_cancel_mad(cm_id_priv->msg);
3060                 fallthrough;
3061         case IB_CM_REQ_RCVD:
3062         case IB_CM_MRA_REQ_SENT:
3063                 if (IBA_GET(CM_REJ_REASON, rej_msg) == IB_CM_REJ_STALE_CONN)
3064                         cm_enter_timewait(cm_id_priv);
3065                 else
3066                         cm_reset_to_idle(cm_id_priv);
3067                 break;
3068         case IB_CM_DREQ_SENT:
3069                 ib_cancel_mad(cm_id_priv->msg);
3070                 fallthrough;
3071         case IB_CM_REP_RCVD:
3072         case IB_CM_MRA_REP_SENT:
3073                 cm_enter_timewait(cm_id_priv);
3074                 break;
3075         case IB_CM_ESTABLISHED:
3076                 if (cm_id_priv->id.lap_state == IB_CM_LAP_UNINIT ||
3077                     cm_id_priv->id.lap_state == IB_CM_LAP_SENT) {
3078                         if (cm_id_priv->id.lap_state == IB_CM_LAP_SENT)
3079                                 ib_cancel_mad(cm_id_priv->msg);
3080                         cm_enter_timewait(cm_id_priv);
3081                         break;
3082                 }
3083                 fallthrough;
3084         default:
3085                 trace_icm_rej_unknown_err(&cm_id_priv->id);
3086                 spin_unlock_irq(&cm_id_priv->lock);
3087                 goto out;
3088         }
3089
3090         cm_queue_work_unlock(cm_id_priv, work);
3091         return 0;
3092 out:
3093         cm_deref_id(cm_id_priv);
3094         return -EINVAL;
3095 }
3096
3097 int ib_prepare_cm_mra(struct ib_cm_id *cm_id)
3098 {
3099         struct cm_id_private *cm_id_priv;
3100         enum ib_cm_state cm_state;
3101         enum ib_cm_lap_state lap_state;
3102         unsigned long flags;
3103         int ret = 0;
3104
3105         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3106
3107         spin_lock_irqsave(&cm_id_priv->lock, flags);
3108         switch (cm_id_priv->id.state) {
3109         case IB_CM_REQ_RCVD:
3110                 cm_state = IB_CM_MRA_REQ_SENT;
3111                 lap_state = cm_id->lap_state;
3112                 break;
3113         case IB_CM_REP_RCVD:
3114                 cm_state = IB_CM_MRA_REP_SENT;
3115                 lap_state = cm_id->lap_state;
3116                 break;
3117         case IB_CM_ESTABLISHED:
3118                 if (cm_id->lap_state == IB_CM_LAP_RCVD) {
3119                         cm_state = cm_id->state;
3120                         lap_state = IB_CM_MRA_LAP_SENT;
3121                         break;
3122                 }
3123                 fallthrough;
3124         default:
3125                 trace_icm_prepare_mra_unknown_err(&cm_id_priv->id);
3126                 ret = -EINVAL;
3127                 goto error_unlock;
3128         }
3129
3130         cm_id->state = cm_state;
3131         cm_id->lap_state = lap_state;
3132         cm_set_private_data(cm_id_priv, NULL, 0);
3133
3134 error_unlock:
3135         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3136         return ret;
3137 }
3138 EXPORT_SYMBOL(ib_prepare_cm_mra);
3139
3140 static struct cm_id_private *cm_acquire_mraed_id(struct cm_mra_msg *mra_msg)
3141 {
3142         switch (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg)) {
3143         case CM_MSG_RESPONSE_REQ:
3144                 return cm_acquire_id(
3145                         cpu_to_be32(IBA_GET(CM_MRA_REMOTE_COMM_ID, mra_msg)),
3146                         0);
3147         case CM_MSG_RESPONSE_REP:
3148         case CM_MSG_RESPONSE_OTHER:
3149                 return cm_acquire_id(
3150                         cpu_to_be32(IBA_GET(CM_MRA_REMOTE_COMM_ID, mra_msg)),
3151                         cpu_to_be32(IBA_GET(CM_MRA_LOCAL_COMM_ID, mra_msg)));
3152         default:
3153                 return NULL;
3154         }
3155 }
3156
3157 static int cm_mra_handler(struct cm_work *work)
3158 {
3159         struct cm_id_private *cm_id_priv;
3160         struct cm_mra_msg *mra_msg;
3161         int timeout;
3162
3163         mra_msg = (struct cm_mra_msg *)work->mad_recv_wc->recv_buf.mad;
3164         cm_id_priv = cm_acquire_mraed_id(mra_msg);
3165         if (!cm_id_priv)
3166                 return -EINVAL;
3167
3168         work->cm_event.private_data =
3169                 IBA_GET_MEM_PTR(CM_MRA_PRIVATE_DATA, mra_msg);
3170         work->cm_event.param.mra_rcvd.service_timeout =
3171                 IBA_GET(CM_MRA_SERVICE_TIMEOUT, mra_msg);
3172         timeout = cm_convert_to_ms(IBA_GET(CM_MRA_SERVICE_TIMEOUT, mra_msg)) +
3173                   cm_convert_to_ms(cm_id_priv->av.timeout);
3174
3175         spin_lock_irq(&cm_id_priv->lock);
3176         switch (cm_id_priv->id.state) {
3177         case IB_CM_REQ_SENT:
3178                 if (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg) !=
3179                             CM_MSG_RESPONSE_REQ ||
3180                     ib_modify_mad(cm_id_priv->msg, timeout))
3181                         goto out;
3182                 cm_id_priv->id.state = IB_CM_MRA_REQ_RCVD;
3183                 break;
3184         case IB_CM_REP_SENT:
3185                 if (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg) !=
3186                             CM_MSG_RESPONSE_REP ||
3187                     ib_modify_mad(cm_id_priv->msg, timeout))
3188                         goto out;
3189                 cm_id_priv->id.state = IB_CM_MRA_REP_RCVD;
3190                 break;
3191         case IB_CM_ESTABLISHED:
3192                 if (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg) !=
3193                             CM_MSG_RESPONSE_OTHER ||
3194                     cm_id_priv->id.lap_state != IB_CM_LAP_SENT ||
3195                     ib_modify_mad(cm_id_priv->msg, timeout)) {
3196                         if (cm_id_priv->id.lap_state == IB_CM_MRA_LAP_RCVD)
3197                                 atomic_long_inc(
3198                                         &work->port->counters[CM_RECV_DUPLICATES]
3199                                                              [CM_MRA_COUNTER]);
3200                         goto out;
3201                 }
3202                 cm_id_priv->id.lap_state = IB_CM_MRA_LAP_RCVD;
3203                 break;
3204         case IB_CM_MRA_REQ_RCVD:
3205         case IB_CM_MRA_REP_RCVD:
3206                 atomic_long_inc(&work->port->counters[CM_RECV_DUPLICATES]
3207                                                      [CM_MRA_COUNTER]);
3208                 fallthrough;
3209         default:
3210                 trace_icm_mra_unknown_err(&cm_id_priv->id);
3211                 goto out;
3212         }
3213
3214         cm_id_priv->msg->context[1] = (void *) (unsigned long)
3215                                       cm_id_priv->id.state;
3216         cm_queue_work_unlock(cm_id_priv, work);
3217         return 0;
3218 out:
3219         spin_unlock_irq(&cm_id_priv->lock);
3220         cm_deref_id(cm_id_priv);
3221         return -EINVAL;
3222 }
3223
3224 static void cm_format_path_lid_from_lap(struct cm_lap_msg *lap_msg,
3225                                         struct sa_path_rec *path)
3226 {
3227         u32 lid;
3228
3229         if (path->rec_type != SA_PATH_REC_TYPE_OPA) {
3230                 sa_path_set_dlid(path, IBA_GET(CM_LAP_ALTERNATE_LOCAL_PORT_LID,
3231                                                lap_msg));
3232                 sa_path_set_slid(path, IBA_GET(CM_LAP_ALTERNATE_REMOTE_PORT_LID,
3233                                                lap_msg));
3234         } else {
3235                 lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
3236                         CM_LAP_ALTERNATE_LOCAL_PORT_GID, lap_msg));
3237                 sa_path_set_dlid(path, lid);
3238
3239                 lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
3240                         CM_LAP_ALTERNATE_REMOTE_PORT_GID, lap_msg));
3241                 sa_path_set_slid(path, lid);
3242         }
3243 }
3244
3245 static void cm_format_path_from_lap(struct cm_id_private *cm_id_priv,
3246                                     struct sa_path_rec *path,
3247                                     struct cm_lap_msg *lap_msg)
3248 {
3249         path->dgid = *IBA_GET_MEM_PTR(CM_LAP_ALTERNATE_LOCAL_PORT_GID, lap_msg);
3250         path->sgid =
3251                 *IBA_GET_MEM_PTR(CM_LAP_ALTERNATE_REMOTE_PORT_GID, lap_msg);
3252         path->flow_label =
3253                 cpu_to_be32(IBA_GET(CM_LAP_ALTERNATE_FLOW_LABEL, lap_msg));
3254         path->hop_limit = IBA_GET(CM_LAP_ALTERNATE_HOP_LIMIT, lap_msg);
3255         path->traffic_class = IBA_GET(CM_LAP_ALTERNATE_TRAFFIC_CLASS, lap_msg);
3256         path->reversible = 1;
3257         path->pkey = cm_id_priv->pkey;
3258         path->sl = IBA_GET(CM_LAP_ALTERNATE_SL, lap_msg);
3259         path->mtu_selector = IB_SA_EQ;
3260         path->mtu = cm_id_priv->path_mtu;
3261         path->rate_selector = IB_SA_EQ;
3262         path->rate = IBA_GET(CM_LAP_ALTERNATE_PACKET_RATE, lap_msg);
3263         path->packet_life_time_selector = IB_SA_EQ;
3264         path->packet_life_time =
3265                 IBA_GET(CM_LAP_ALTERNATE_LOCAL_ACK_TIMEOUT, lap_msg);
3266         path->packet_life_time -= (path->packet_life_time > 0);
3267         cm_format_path_lid_from_lap(lap_msg, path);
3268 }
3269
3270 static int cm_lap_handler(struct cm_work *work)
3271 {
3272         struct cm_id_private *cm_id_priv;
3273         struct cm_lap_msg *lap_msg;
3274         struct ib_cm_lap_event_param *param;
3275         struct ib_mad_send_buf *msg = NULL;
3276         struct rdma_ah_attr ah_attr;
3277         struct cm_av alt_av = {};
3278         int ret;
3279
3280         /* Currently Alternate path messages are not supported for
3281          * RoCE link layer.
3282          */
3283         if (rdma_protocol_roce(work->port->cm_dev->ib_device,
3284                                work->port->port_num))
3285                 return -EINVAL;
3286
3287         /* todo: verify LAP request and send reject APR if invalid. */
3288         lap_msg = (struct cm_lap_msg *)work->mad_recv_wc->recv_buf.mad;
3289         cm_id_priv = cm_acquire_id(
3290                 cpu_to_be32(IBA_GET(CM_LAP_REMOTE_COMM_ID, lap_msg)),
3291                 cpu_to_be32(IBA_GET(CM_LAP_LOCAL_COMM_ID, lap_msg)));
3292         if (!cm_id_priv)
3293                 return -EINVAL;
3294
3295         param = &work->cm_event.param.lap_rcvd;
3296         memset(&work->path[0], 0, sizeof(work->path[1]));
3297         cm_path_set_rec_type(work->port->cm_dev->ib_device,
3298                              work->port->port_num, &work->path[0],
3299                              IBA_GET_MEM_PTR(CM_LAP_ALTERNATE_LOCAL_PORT_GID,
3300                                              lap_msg));
3301         param->alternate_path = &work->path[0];
3302         cm_format_path_from_lap(cm_id_priv, param->alternate_path, lap_msg);
3303         work->cm_event.private_data =
3304                 IBA_GET_MEM_PTR(CM_LAP_PRIVATE_DATA, lap_msg);
3305
3306         ret = ib_init_ah_attr_from_wc(work->port->cm_dev->ib_device,
3307                                       work->port->port_num,
3308                                       work->mad_recv_wc->wc,
3309                                       work->mad_recv_wc->recv_buf.grh,
3310                                       &ah_attr);
3311         if (ret)
3312                 goto deref;
3313
3314         ret = cm_init_av_by_path(param->alternate_path, NULL, &alt_av);
3315         if (ret) {
3316                 rdma_destroy_ah_attr(&ah_attr);
3317                 goto deref;
3318         }
3319
3320         spin_lock_irq(&cm_id_priv->lock);
3321         cm_init_av_for_lap(work->port, work->mad_recv_wc->wc,
3322                            &ah_attr, &cm_id_priv->av);
3323         cm_move_av_from_path(&cm_id_priv->alt_av, &alt_av);
3324
3325         if (cm_id_priv->id.state != IB_CM_ESTABLISHED)
3326                 goto unlock;
3327
3328         switch (cm_id_priv->id.lap_state) {
3329         case IB_CM_LAP_UNINIT:
3330         case IB_CM_LAP_IDLE:
3331                 break;
3332         case IB_CM_MRA_LAP_SENT:
3333                 atomic_long_inc(&work->port->counters[CM_RECV_DUPLICATES]
3334                                                      [CM_LAP_COUNTER]);
3335                 msg = cm_alloc_response_msg_no_ah(work->port, work->mad_recv_wc,
3336                                                   true);
3337                 if (IS_ERR(msg))
3338                         goto unlock;
3339
3340                 cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
3341                               CM_MSG_RESPONSE_OTHER,
3342                               cm_id_priv->private_data,
3343                               cm_id_priv->private_data_len);
3344                 spin_unlock_irq(&cm_id_priv->lock);
3345
3346                 if (cm_create_response_msg_ah(work->port, work->mad_recv_wc, msg) ||
3347                     ib_post_send_mad(msg, NULL))
3348                         cm_free_msg(msg);
3349                 goto deref;
3350         case IB_CM_LAP_RCVD:
3351                 atomic_long_inc(&work->port->counters[CM_RECV_DUPLICATES]
3352                                                      [CM_LAP_COUNTER]);
3353                 goto unlock;
3354         default:
3355                 goto unlock;
3356         }
3357
3358         cm_id_priv->id.lap_state = IB_CM_LAP_RCVD;
3359         cm_id_priv->tid = lap_msg->hdr.tid;
3360         cm_queue_work_unlock(cm_id_priv, work);
3361         return 0;
3362
3363 unlock: spin_unlock_irq(&cm_id_priv->lock);
3364 deref:  cm_deref_id(cm_id_priv);
3365         return -EINVAL;
3366 }
3367
3368 static int cm_apr_handler(struct cm_work *work)
3369 {
3370         struct cm_id_private *cm_id_priv;
3371         struct cm_apr_msg *apr_msg;
3372
3373         /* Currently Alternate path messages are not supported for
3374          * RoCE link layer.
3375          */
3376         if (rdma_protocol_roce(work->port->cm_dev->ib_device,
3377                                work->port->port_num))
3378                 return -EINVAL;
3379
3380         apr_msg = (struct cm_apr_msg *)work->mad_recv_wc->recv_buf.mad;
3381         cm_id_priv = cm_acquire_id(
3382                 cpu_to_be32(IBA_GET(CM_APR_REMOTE_COMM_ID, apr_msg)),
3383                 cpu_to_be32(IBA_GET(CM_APR_LOCAL_COMM_ID, apr_msg)));
3384         if (!cm_id_priv)
3385                 return -EINVAL; /* Unmatched reply. */
3386
3387         work->cm_event.param.apr_rcvd.ap_status =
3388                 IBA_GET(CM_APR_AR_STATUS, apr_msg);
3389         work->cm_event.param.apr_rcvd.apr_info =
3390                 IBA_GET_MEM_PTR(CM_APR_ADDITIONAL_INFORMATION, apr_msg);
3391         work->cm_event.param.apr_rcvd.info_len =
3392                 IBA_GET(CM_APR_ADDITIONAL_INFORMATION_LENGTH, apr_msg);
3393         work->cm_event.private_data =
3394                 IBA_GET_MEM_PTR(CM_APR_PRIVATE_DATA, apr_msg);
3395
3396         spin_lock_irq(&cm_id_priv->lock);
3397         if (cm_id_priv->id.state != IB_CM_ESTABLISHED ||
3398             (cm_id_priv->id.lap_state != IB_CM_LAP_SENT &&
3399              cm_id_priv->id.lap_state != IB_CM_MRA_LAP_RCVD)) {
3400                 spin_unlock_irq(&cm_id_priv->lock);
3401                 goto out;
3402         }
3403         cm_id_priv->id.lap_state = IB_CM_LAP_IDLE;
3404         ib_cancel_mad(cm_id_priv->msg);
3405         cm_queue_work_unlock(cm_id_priv, work);
3406         return 0;
3407 out:
3408         cm_deref_id(cm_id_priv);
3409         return -EINVAL;
3410 }
3411
3412 static int cm_timewait_handler(struct cm_work *work)
3413 {
3414         struct cm_timewait_info *timewait_info;
3415         struct cm_id_private *cm_id_priv;
3416
3417         timewait_info = container_of(work, struct cm_timewait_info, work);
3418         spin_lock_irq(&cm.lock);
3419         list_del(&timewait_info->list);
3420         spin_unlock_irq(&cm.lock);
3421
3422         cm_id_priv = cm_acquire_id(timewait_info->work.local_id,
3423                                    timewait_info->work.remote_id);
3424         if (!cm_id_priv)
3425                 return -EINVAL;
3426
3427         spin_lock_irq(&cm_id_priv->lock);
3428         if (cm_id_priv->id.state != IB_CM_TIMEWAIT ||
3429             cm_id_priv->remote_qpn != timewait_info->remote_qpn) {
3430                 spin_unlock_irq(&cm_id_priv->lock);
3431                 goto out;
3432         }
3433         cm_id_priv->id.state = IB_CM_IDLE;
3434         cm_queue_work_unlock(cm_id_priv, work);
3435         return 0;
3436 out:
3437         cm_deref_id(cm_id_priv);
3438         return -EINVAL;
3439 }
3440
3441 static void cm_format_sidr_req(struct cm_sidr_req_msg *sidr_req_msg,
3442                                struct cm_id_private *cm_id_priv,
3443                                struct ib_cm_sidr_req_param *param)
3444 {
3445         cm_format_mad_hdr(&sidr_req_msg->hdr, CM_SIDR_REQ_ATTR_ID,
3446                           cm_form_tid(cm_id_priv));
3447         IBA_SET(CM_SIDR_REQ_REQUESTID, sidr_req_msg,
3448                 be32_to_cpu(cm_id_priv->id.local_id));
3449         IBA_SET(CM_SIDR_REQ_PARTITION_KEY, sidr_req_msg,
3450                 be16_to_cpu(param->path->pkey));
3451         IBA_SET(CM_SIDR_REQ_SERVICEID, sidr_req_msg,
3452                 be64_to_cpu(param->service_id));
3453
3454         if (param->private_data && param->private_data_len)
3455                 IBA_SET_MEM(CM_SIDR_REQ_PRIVATE_DATA, sidr_req_msg,
3456                             param->private_data, param->private_data_len);
3457 }
3458
3459 int ib_send_cm_sidr_req(struct ib_cm_id *cm_id,
3460                         struct ib_cm_sidr_req_param *param)
3461 {
3462         struct cm_id_private *cm_id_priv;
3463         struct ib_mad_send_buf *msg;
3464         struct cm_av av = {};
3465         unsigned long flags;
3466         int ret;
3467
3468         if (!param->path || (param->private_data &&
3469              param->private_data_len > IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE))
3470                 return -EINVAL;
3471
3472         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3473         ret = cm_init_av_by_path(param->path, param->sgid_attr, &av);
3474         if (ret)
3475                 return ret;
3476
3477         spin_lock_irqsave(&cm_id_priv->lock, flags);
3478         cm_move_av_from_path(&cm_id_priv->av, &av);
3479         cm_id->service_id = param->service_id;
3480         cm_id_priv->timeout_ms = param->timeout_ms;
3481         cm_id_priv->max_cm_retries = param->max_cm_retries;
3482         if (cm_id->state != IB_CM_IDLE) {
3483                 ret = -EINVAL;
3484                 goto out_unlock;
3485         }
3486
3487         msg = cm_alloc_priv_msg(cm_id_priv, IB_CM_SIDR_REQ_SENT);
3488         if (IS_ERR(msg)) {
3489                 ret = PTR_ERR(msg);
3490                 goto out_unlock;
3491         }
3492
3493         cm_format_sidr_req((struct cm_sidr_req_msg *)msg->mad, cm_id_priv,
3494                            param);
3495
3496         trace_icm_send_sidr_req(&cm_id_priv->id);
3497         ret = ib_post_send_mad(msg, NULL);
3498         if (ret)
3499                 goto out_free;
3500         cm_id->state = IB_CM_SIDR_REQ_SENT;
3501         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3502         return 0;
3503 out_free:
3504         cm_free_priv_msg(msg);
3505 out_unlock:
3506         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3507         return ret;
3508 }
3509 EXPORT_SYMBOL(ib_send_cm_sidr_req);
3510
3511 static void cm_format_sidr_req_event(struct cm_work *work,
3512                                      const struct cm_id_private *rx_cm_id,
3513                                      struct ib_cm_id *listen_id)
3514 {
3515         struct cm_sidr_req_msg *sidr_req_msg;
3516         struct ib_cm_sidr_req_event_param *param;
3517
3518         sidr_req_msg = (struct cm_sidr_req_msg *)
3519                                 work->mad_recv_wc->recv_buf.mad;
3520         param = &work->cm_event.param.sidr_req_rcvd;
3521         param->pkey = IBA_GET(CM_SIDR_REQ_PARTITION_KEY, sidr_req_msg);
3522         param->listen_id = listen_id;
3523         param->service_id =
3524                 cpu_to_be64(IBA_GET(CM_SIDR_REQ_SERVICEID, sidr_req_msg));
3525         param->bth_pkey = cm_get_bth_pkey(work);
3526         param->port = work->port->port_num;
3527         param->sgid_attr = rx_cm_id->av.ah_attr.grh.sgid_attr;
3528         work->cm_event.private_data =
3529                 IBA_GET_MEM_PTR(CM_SIDR_REQ_PRIVATE_DATA, sidr_req_msg);
3530 }
3531
3532 static int cm_sidr_req_handler(struct cm_work *work)
3533 {
3534         struct cm_id_private *cm_id_priv, *listen_cm_id_priv;
3535         struct cm_sidr_req_msg *sidr_req_msg;
3536         struct ib_wc *wc;
3537         int ret;
3538
3539         cm_id_priv =
3540                 cm_alloc_id_priv(work->port->cm_dev->ib_device, NULL, NULL);
3541         if (IS_ERR(cm_id_priv))
3542                 return PTR_ERR(cm_id_priv);
3543
3544         /* Record SGID/SLID and request ID for lookup. */
3545         sidr_req_msg = (struct cm_sidr_req_msg *)
3546                                 work->mad_recv_wc->recv_buf.mad;
3547
3548         cm_id_priv->id.remote_id =
3549                 cpu_to_be32(IBA_GET(CM_SIDR_REQ_REQUESTID, sidr_req_msg));
3550         cm_id_priv->id.service_id =
3551                 cpu_to_be64(IBA_GET(CM_SIDR_REQ_SERVICEID, sidr_req_msg));
3552         cm_id_priv->tid = sidr_req_msg->hdr.tid;
3553
3554         wc = work->mad_recv_wc->wc;
3555         cm_id_priv->sidr_slid = wc->slid;
3556         ret = cm_init_av_for_response(work->port, work->mad_recv_wc->wc,
3557                                       work->mad_recv_wc->recv_buf.grh,
3558                                       &cm_id_priv->av);
3559         if (ret)
3560                 goto out;
3561
3562         spin_lock_irq(&cm.lock);
3563         listen_cm_id_priv = cm_insert_remote_sidr(cm_id_priv);
3564         if (listen_cm_id_priv) {
3565                 spin_unlock_irq(&cm.lock);
3566                 atomic_long_inc(&work->port->counters[CM_RECV_DUPLICATES]
3567                                                      [CM_SIDR_REQ_COUNTER]);
3568                 goto out; /* Duplicate message. */
3569         }
3570         cm_id_priv->id.state = IB_CM_SIDR_REQ_RCVD;
3571         listen_cm_id_priv = cm_find_listen(cm_id_priv->id.device,
3572                                            cm_id_priv->id.service_id);
3573         if (!listen_cm_id_priv) {
3574                 spin_unlock_irq(&cm.lock);
3575                 ib_send_cm_sidr_rep(&cm_id_priv->id,
3576                                     &(struct ib_cm_sidr_rep_param){
3577                                             .status = IB_SIDR_UNSUPPORTED });
3578                 goto out; /* No match. */
3579         }
3580         spin_unlock_irq(&cm.lock);
3581
3582         cm_id_priv->id.cm_handler = listen_cm_id_priv->id.cm_handler;
3583         cm_id_priv->id.context = listen_cm_id_priv->id.context;
3584
3585         /*
3586          * A SIDR ID does not need to be in the xarray since it does not receive
3587          * mads, is not placed in the remote_id or remote_qpn rbtree, and does
3588          * not enter timewait.
3589          */
3590
3591         cm_format_sidr_req_event(work, cm_id_priv, &listen_cm_id_priv->id);
3592         ret = cm_id_priv->id.cm_handler(&cm_id_priv->id, &work->cm_event);
3593         cm_free_work(work);
3594         /*
3595          * A pointer to the listen_cm_id is held in the event, so this deref
3596          * must be after the event is delivered above.
3597          */
3598         cm_deref_id(listen_cm_id_priv);
3599         if (ret)
3600                 cm_destroy_id(&cm_id_priv->id, ret);
3601         return 0;
3602 out:
3603         ib_destroy_cm_id(&cm_id_priv->id);
3604         return -EINVAL;
3605 }
3606
3607 static void cm_format_sidr_rep(struct cm_sidr_rep_msg *sidr_rep_msg,
3608                                struct cm_id_private *cm_id_priv,
3609                                struct ib_cm_sidr_rep_param *param)
3610 {
3611         cm_format_mad_ece_hdr(&sidr_rep_msg->hdr, CM_SIDR_REP_ATTR_ID,
3612                               cm_id_priv->tid, param->ece.attr_mod);
3613         IBA_SET(CM_SIDR_REP_REQUESTID, sidr_rep_msg,
3614                 be32_to_cpu(cm_id_priv->id.remote_id));
3615         IBA_SET(CM_SIDR_REP_STATUS, sidr_rep_msg, param->status);
3616         IBA_SET(CM_SIDR_REP_QPN, sidr_rep_msg, param->qp_num);
3617         IBA_SET(CM_SIDR_REP_SERVICEID, sidr_rep_msg,
3618                 be64_to_cpu(cm_id_priv->id.service_id));
3619         IBA_SET(CM_SIDR_REP_Q_KEY, sidr_rep_msg, param->qkey);
3620         IBA_SET(CM_SIDR_REP_VENDOR_ID_L, sidr_rep_msg,
3621                 param->ece.vendor_id & 0xFF);
3622         IBA_SET(CM_SIDR_REP_VENDOR_ID_H, sidr_rep_msg,
3623                 (param->ece.vendor_id >> 8) & 0xFF);
3624
3625         if (param->info && param->info_length)
3626                 IBA_SET_MEM(CM_SIDR_REP_ADDITIONAL_INFORMATION, sidr_rep_msg,
3627                             param->info, param->info_length);
3628
3629         if (param->private_data && param->private_data_len)
3630                 IBA_SET_MEM(CM_SIDR_REP_PRIVATE_DATA, sidr_rep_msg,
3631                             param->private_data, param->private_data_len);
3632 }
3633
3634 static int cm_send_sidr_rep_locked(struct cm_id_private *cm_id_priv,
3635                                    struct ib_cm_sidr_rep_param *param)
3636 {
3637         struct ib_mad_send_buf *msg;
3638         unsigned long flags;
3639         int ret;
3640
3641         lockdep_assert_held(&cm_id_priv->lock);
3642
3643         if ((param->info && param->info_length > IB_CM_SIDR_REP_INFO_LENGTH) ||
3644             (param->private_data &&
3645              param->private_data_len > IB_CM_SIDR_REP_PRIVATE_DATA_SIZE))
3646                 return -EINVAL;
3647
3648         if (cm_id_priv->id.state != IB_CM_SIDR_REQ_RCVD)
3649                 return -EINVAL;
3650
3651         msg = cm_alloc_msg(cm_id_priv);
3652         if (IS_ERR(msg))
3653                 return PTR_ERR(msg);
3654
3655         cm_format_sidr_rep((struct cm_sidr_rep_msg *) msg->mad, cm_id_priv,
3656                            param);
3657         trace_icm_send_sidr_rep(&cm_id_priv->id);
3658         ret = ib_post_send_mad(msg, NULL);
3659         if (ret) {
3660                 cm_free_msg(msg);
3661                 return ret;
3662         }
3663         cm_id_priv->id.state = IB_CM_IDLE;
3664         spin_lock_irqsave(&cm.lock, flags);
3665         if (!RB_EMPTY_NODE(&cm_id_priv->sidr_id_node)) {
3666                 rb_erase(&cm_id_priv->sidr_id_node, &cm.remote_sidr_table);
3667                 RB_CLEAR_NODE(&cm_id_priv->sidr_id_node);
3668         }
3669         spin_unlock_irqrestore(&cm.lock, flags);
3670         return 0;
3671 }
3672
3673 int ib_send_cm_sidr_rep(struct ib_cm_id *cm_id,
3674                         struct ib_cm_sidr_rep_param *param)
3675 {
3676         struct cm_id_private *cm_id_priv =
3677                 container_of(cm_id, struct cm_id_private, id);
3678         unsigned long flags;
3679         int ret;
3680
3681         spin_lock_irqsave(&cm_id_priv->lock, flags);
3682         ret = cm_send_sidr_rep_locked(cm_id_priv, param);
3683         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3684         return ret;
3685 }
3686 EXPORT_SYMBOL(ib_send_cm_sidr_rep);
3687
3688 static void cm_format_sidr_rep_event(struct cm_work *work,
3689                                      const struct cm_id_private *cm_id_priv)
3690 {
3691         struct cm_sidr_rep_msg *sidr_rep_msg;
3692         struct ib_cm_sidr_rep_event_param *param;
3693
3694         sidr_rep_msg = (struct cm_sidr_rep_msg *)
3695                                 work->mad_recv_wc->recv_buf.mad;
3696         param = &work->cm_event.param.sidr_rep_rcvd;
3697         param->status = IBA_GET(CM_SIDR_REP_STATUS, sidr_rep_msg);
3698         param->qkey = IBA_GET(CM_SIDR_REP_Q_KEY, sidr_rep_msg);
3699         param->qpn = IBA_GET(CM_SIDR_REP_QPN, sidr_rep_msg);
3700         param->info = IBA_GET_MEM_PTR(CM_SIDR_REP_ADDITIONAL_INFORMATION,
3701                                       sidr_rep_msg);
3702         param->info_len = IBA_GET(CM_SIDR_REP_ADDITIONAL_INFORMATION_LENGTH,
3703                                   sidr_rep_msg);
3704         param->sgid_attr = cm_id_priv->av.ah_attr.grh.sgid_attr;
3705         work->cm_event.private_data =
3706                 IBA_GET_MEM_PTR(CM_SIDR_REP_PRIVATE_DATA, sidr_rep_msg);
3707 }
3708
3709 static int cm_sidr_rep_handler(struct cm_work *work)
3710 {
3711         struct cm_sidr_rep_msg *sidr_rep_msg;
3712         struct cm_id_private *cm_id_priv;
3713
3714         sidr_rep_msg = (struct cm_sidr_rep_msg *)
3715                                 work->mad_recv_wc->recv_buf.mad;
3716         cm_id_priv = cm_acquire_id(
3717                 cpu_to_be32(IBA_GET(CM_SIDR_REP_REQUESTID, sidr_rep_msg)), 0);
3718         if (!cm_id_priv)
3719                 return -EINVAL; /* Unmatched reply. */
3720
3721         spin_lock_irq(&cm_id_priv->lock);
3722         if (cm_id_priv->id.state != IB_CM_SIDR_REQ_SENT) {
3723                 spin_unlock_irq(&cm_id_priv->lock);
3724                 goto out;
3725         }
3726         cm_id_priv->id.state = IB_CM_IDLE;
3727         ib_cancel_mad(cm_id_priv->msg);
3728         spin_unlock_irq(&cm_id_priv->lock);
3729
3730         cm_format_sidr_rep_event(work, cm_id_priv);
3731         cm_process_work(cm_id_priv, work);
3732         return 0;
3733 out:
3734         cm_deref_id(cm_id_priv);
3735         return -EINVAL;
3736 }
3737
3738 static void cm_process_send_error(struct cm_id_private *cm_id_priv,
3739                                   struct ib_mad_send_buf *msg,
3740                                   enum ib_wc_status wc_status)
3741 {
3742         enum ib_cm_state state = (unsigned long) msg->context[1];
3743         struct ib_cm_event cm_event = {};
3744         int ret;
3745
3746         /* Discard old sends. */
3747         spin_lock_irq(&cm_id_priv->lock);
3748         if (msg != cm_id_priv->msg) {
3749                 spin_unlock_irq(&cm_id_priv->lock);
3750                 cm_free_msg(msg);
3751                 cm_deref_id(cm_id_priv);
3752                 return;
3753         }
3754         cm_free_priv_msg(msg);
3755
3756         if (state != cm_id_priv->id.state || wc_status == IB_WC_SUCCESS ||
3757             wc_status == IB_WC_WR_FLUSH_ERR)
3758                 goto out_unlock;
3759
3760         trace_icm_mad_send_err(state, wc_status);
3761         switch (state) {
3762         case IB_CM_REQ_SENT:
3763         case IB_CM_MRA_REQ_RCVD:
3764                 cm_reset_to_idle(cm_id_priv);
3765                 cm_event.event = IB_CM_REQ_ERROR;
3766                 break;
3767         case IB_CM_REP_SENT:
3768         case IB_CM_MRA_REP_RCVD:
3769                 cm_reset_to_idle(cm_id_priv);
3770                 cm_event.event = IB_CM_REP_ERROR;
3771                 break;
3772         case IB_CM_DREQ_SENT:
3773                 cm_enter_timewait(cm_id_priv);
3774                 cm_event.event = IB_CM_DREQ_ERROR;
3775                 break;
3776         case IB_CM_SIDR_REQ_SENT:
3777                 cm_id_priv->id.state = IB_CM_IDLE;
3778                 cm_event.event = IB_CM_SIDR_REQ_ERROR;
3779                 break;
3780         default:
3781                 goto out_unlock;
3782         }
3783         spin_unlock_irq(&cm_id_priv->lock);
3784         cm_event.param.send_status = wc_status;
3785
3786         /* No other events can occur on the cm_id at this point. */
3787         ret = cm_id_priv->id.cm_handler(&cm_id_priv->id, &cm_event);
3788         if (ret)
3789                 ib_destroy_cm_id(&cm_id_priv->id);
3790         return;
3791 out_unlock:
3792         spin_unlock_irq(&cm_id_priv->lock);
3793 }
3794
3795 static void cm_send_handler(struct ib_mad_agent *mad_agent,
3796                             struct ib_mad_send_wc *mad_send_wc)
3797 {
3798         struct ib_mad_send_buf *msg = mad_send_wc->send_buf;
3799         struct cm_id_private *cm_id_priv;
3800         struct cm_port *port;
3801         u16 attr_index;
3802
3803         port = mad_agent->context;
3804         attr_index = be16_to_cpu(((struct ib_mad_hdr *)
3805                                   msg->mad)->attr_id) - CM_ATTR_ID_OFFSET;
3806
3807         if (msg->context[0] == CM_DIRECT_RETRY_CTX) {
3808                 msg->retries = 1;
3809                 cm_id_priv = NULL;
3810         } else {
3811                 cm_id_priv = msg->context[0];
3812         }
3813
3814         atomic_long_add(1 + msg->retries, &port->counters[CM_XMIT][attr_index]);
3815         if (msg->retries)
3816                 atomic_long_add(msg->retries,
3817                                 &port->counters[CM_XMIT_RETRIES][attr_index]);
3818
3819         if (cm_id_priv)
3820                 cm_process_send_error(cm_id_priv, msg, mad_send_wc->status);
3821         else
3822                 cm_free_msg(msg);
3823 }
3824
3825 static void cm_work_handler(struct work_struct *_work)
3826 {
3827         struct cm_work *work = container_of(_work, struct cm_work, work.work);
3828         int ret;
3829
3830         switch (work->cm_event.event) {
3831         case IB_CM_REQ_RECEIVED:
3832                 ret = cm_req_handler(work);
3833                 break;
3834         case IB_CM_MRA_RECEIVED:
3835                 ret = cm_mra_handler(work);
3836                 break;
3837         case IB_CM_REJ_RECEIVED:
3838                 ret = cm_rej_handler(work);
3839                 break;
3840         case IB_CM_REP_RECEIVED:
3841                 ret = cm_rep_handler(work);
3842                 break;
3843         case IB_CM_RTU_RECEIVED:
3844                 ret = cm_rtu_handler(work);
3845                 break;
3846         case IB_CM_USER_ESTABLISHED:
3847                 ret = cm_establish_handler(work);
3848                 break;
3849         case IB_CM_DREQ_RECEIVED:
3850                 ret = cm_dreq_handler(work);
3851                 break;
3852         case IB_CM_DREP_RECEIVED:
3853                 ret = cm_drep_handler(work);
3854                 break;
3855         case IB_CM_SIDR_REQ_RECEIVED:
3856                 ret = cm_sidr_req_handler(work);
3857                 break;
3858         case IB_CM_SIDR_REP_RECEIVED:
3859                 ret = cm_sidr_rep_handler(work);
3860                 break;
3861         case IB_CM_LAP_RECEIVED:
3862                 ret = cm_lap_handler(work);
3863                 break;
3864         case IB_CM_APR_RECEIVED:
3865                 ret = cm_apr_handler(work);
3866                 break;
3867         case IB_CM_TIMEWAIT_EXIT:
3868                 ret = cm_timewait_handler(work);
3869                 break;
3870         default:
3871                 trace_icm_handler_err(work->cm_event.event);
3872                 ret = -EINVAL;
3873                 break;
3874         }
3875         if (ret)
3876                 cm_free_work(work);
3877 }
3878
3879 static int cm_establish(struct ib_cm_id *cm_id)
3880 {
3881         struct cm_id_private *cm_id_priv;
3882         struct cm_work *work;
3883         unsigned long flags;
3884         int ret = 0;
3885         struct cm_device *cm_dev;
3886
3887         cm_dev = ib_get_client_data(cm_id->device, &cm_client);
3888         if (!cm_dev)
3889                 return -ENODEV;
3890
3891         work = kmalloc(sizeof *work, GFP_ATOMIC);
3892         if (!work)
3893                 return -ENOMEM;
3894
3895         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3896         spin_lock_irqsave(&cm_id_priv->lock, flags);
3897         switch (cm_id->state) {
3898         case IB_CM_REP_SENT:
3899         case IB_CM_MRA_REP_RCVD:
3900                 cm_id->state = IB_CM_ESTABLISHED;
3901                 break;
3902         case IB_CM_ESTABLISHED:
3903                 ret = -EISCONN;
3904                 break;
3905         default:
3906                 trace_icm_establish_err(cm_id);
3907                 ret = -EINVAL;
3908                 break;
3909         }
3910         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3911
3912         if (ret) {
3913                 kfree(work);
3914                 goto out;
3915         }
3916
3917         /*
3918          * The CM worker thread may try to destroy the cm_id before it
3919          * can execute this work item.  To prevent potential deadlock,
3920          * we need to find the cm_id once we're in the context of the
3921          * worker thread, rather than holding a reference on it.
3922          */
3923         INIT_DELAYED_WORK(&work->work, cm_work_handler);
3924         work->local_id = cm_id->local_id;
3925         work->remote_id = cm_id->remote_id;
3926         work->mad_recv_wc = NULL;
3927         work->cm_event.event = IB_CM_USER_ESTABLISHED;
3928
3929         /* Check if the device started its remove_one */
3930         spin_lock_irqsave(&cm.lock, flags);
3931         if (!cm_dev->going_down) {
3932                 queue_delayed_work(cm.wq, &work->work, 0);
3933         } else {
3934                 kfree(work);
3935                 ret = -ENODEV;
3936         }
3937         spin_unlock_irqrestore(&cm.lock, flags);
3938
3939 out:
3940         return ret;
3941 }
3942
3943 static int cm_migrate(struct ib_cm_id *cm_id)
3944 {
3945         struct cm_id_private *cm_id_priv;
3946         unsigned long flags;
3947         int ret = 0;
3948
3949         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3950         spin_lock_irqsave(&cm_id_priv->lock, flags);
3951         if (cm_id->state == IB_CM_ESTABLISHED &&
3952             (cm_id->lap_state == IB_CM_LAP_UNINIT ||
3953              cm_id->lap_state == IB_CM_LAP_IDLE)) {
3954                 cm_id->lap_state = IB_CM_LAP_IDLE;
3955                 cm_id_priv->av = cm_id_priv->alt_av;
3956         } else
3957                 ret = -EINVAL;
3958         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3959
3960         return ret;
3961 }
3962
3963 int ib_cm_notify(struct ib_cm_id *cm_id, enum ib_event_type event)
3964 {
3965         int ret;
3966
3967         switch (event) {
3968         case IB_EVENT_COMM_EST:
3969                 ret = cm_establish(cm_id);
3970                 break;
3971         case IB_EVENT_PATH_MIG:
3972                 ret = cm_migrate(cm_id);
3973                 break;
3974         default:
3975                 ret = -EINVAL;
3976         }
3977         return ret;
3978 }
3979 EXPORT_SYMBOL(ib_cm_notify);
3980
3981 static void cm_recv_handler(struct ib_mad_agent *mad_agent,
3982                             struct ib_mad_send_buf *send_buf,
3983                             struct ib_mad_recv_wc *mad_recv_wc)
3984 {
3985         struct cm_port *port = mad_agent->context;
3986         struct cm_work *work;
3987         enum ib_cm_event_type event;
3988         bool alt_path = false;
3989         u16 attr_id;
3990         int paths = 0;
3991         int going_down = 0;
3992
3993         switch (mad_recv_wc->recv_buf.mad->mad_hdr.attr_id) {
3994         case CM_REQ_ATTR_ID:
3995                 alt_path = cm_req_has_alt_path((struct cm_req_msg *)
3996                                                 mad_recv_wc->recv_buf.mad);
3997                 paths = 1 + (alt_path != 0);
3998                 event = IB_CM_REQ_RECEIVED;
3999                 break;
4000         case CM_MRA_ATTR_ID:
4001                 event = IB_CM_MRA_RECEIVED;
4002                 break;
4003         case CM_REJ_ATTR_ID:
4004                 event = IB_CM_REJ_RECEIVED;
4005                 break;
4006         case CM_REP_ATTR_ID:
4007                 event = IB_CM_REP_RECEIVED;
4008                 break;
4009         case CM_RTU_ATTR_ID:
4010                 event = IB_CM_RTU_RECEIVED;
4011                 break;
4012         case CM_DREQ_ATTR_ID:
4013                 event = IB_CM_DREQ_RECEIVED;
4014                 break;
4015         case CM_DREP_ATTR_ID:
4016                 event = IB_CM_DREP_RECEIVED;
4017                 break;
4018         case CM_SIDR_REQ_ATTR_ID:
4019                 event = IB_CM_SIDR_REQ_RECEIVED;
4020                 break;
4021         case CM_SIDR_REP_ATTR_ID:
4022                 event = IB_CM_SIDR_REP_RECEIVED;
4023                 break;
4024         case CM_LAP_ATTR_ID:
4025                 paths = 1;
4026                 event = IB_CM_LAP_RECEIVED;
4027                 break;
4028         case CM_APR_ATTR_ID:
4029                 event = IB_CM_APR_RECEIVED;
4030                 break;
4031         default:
4032                 ib_free_recv_mad(mad_recv_wc);
4033                 return;
4034         }
4035
4036         attr_id = be16_to_cpu(mad_recv_wc->recv_buf.mad->mad_hdr.attr_id);
4037         atomic_long_inc(&port->counters[CM_RECV][attr_id - CM_ATTR_ID_OFFSET]);
4038
4039         work = kmalloc(struct_size(work, path, paths), GFP_KERNEL);
4040         if (!work) {
4041                 ib_free_recv_mad(mad_recv_wc);
4042                 return;
4043         }
4044
4045         INIT_DELAYED_WORK(&work->work, cm_work_handler);
4046         work->cm_event.event = event;
4047         work->mad_recv_wc = mad_recv_wc;
4048         work->port = port;
4049
4050         /* Check if the device started its remove_one */
4051         spin_lock_irq(&cm.lock);
4052         if (!port->cm_dev->going_down)
4053                 queue_delayed_work(cm.wq, &work->work, 0);
4054         else
4055                 going_down = 1;
4056         spin_unlock_irq(&cm.lock);
4057
4058         if (going_down) {
4059                 kfree(work);
4060                 ib_free_recv_mad(mad_recv_wc);
4061         }
4062 }
4063
4064 static int cm_init_qp_init_attr(struct cm_id_private *cm_id_priv,
4065                                 struct ib_qp_attr *qp_attr,
4066                                 int *qp_attr_mask)
4067 {
4068         unsigned long flags;
4069         int ret;
4070
4071         spin_lock_irqsave(&cm_id_priv->lock, flags);
4072         switch (cm_id_priv->id.state) {
4073         case IB_CM_REQ_SENT:
4074         case IB_CM_MRA_REQ_RCVD:
4075         case IB_CM_REQ_RCVD:
4076         case IB_CM_MRA_REQ_SENT:
4077         case IB_CM_REP_RCVD:
4078         case IB_CM_MRA_REP_SENT:
4079         case IB_CM_REP_SENT:
4080         case IB_CM_MRA_REP_RCVD:
4081         case IB_CM_ESTABLISHED:
4082                 *qp_attr_mask = IB_QP_STATE | IB_QP_ACCESS_FLAGS |
4083                                 IB_QP_PKEY_INDEX | IB_QP_PORT;
4084                 qp_attr->qp_access_flags = IB_ACCESS_REMOTE_WRITE;
4085                 if (cm_id_priv->responder_resources) {
4086                         struct ib_device *ib_dev = cm_id_priv->id.device;
4087                         u64 support_flush = ib_dev->attrs.device_cap_flags &
4088                           (IB_DEVICE_FLUSH_GLOBAL | IB_DEVICE_FLUSH_PERSISTENT);
4089                         u32 flushable = support_flush ?
4090                                         (IB_ACCESS_FLUSH_GLOBAL |
4091                                          IB_ACCESS_FLUSH_PERSISTENT) : 0;
4092
4093                         qp_attr->qp_access_flags |= IB_ACCESS_REMOTE_READ |
4094                                                     IB_ACCESS_REMOTE_ATOMIC |
4095                                                     flushable;
4096                 }
4097                 qp_attr->pkey_index = cm_id_priv->av.pkey_index;
4098                 if (cm_id_priv->av.port)
4099                         qp_attr->port_num = cm_id_priv->av.port->port_num;
4100                 ret = 0;
4101                 break;
4102         default:
4103                 trace_icm_qp_init_err(&cm_id_priv->id);
4104                 ret = -EINVAL;
4105                 break;
4106         }
4107         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
4108         return ret;
4109 }
4110
4111 static int cm_init_qp_rtr_attr(struct cm_id_private *cm_id_priv,
4112                                struct ib_qp_attr *qp_attr,
4113                                int *qp_attr_mask)
4114 {
4115         unsigned long flags;
4116         int ret;
4117
4118         spin_lock_irqsave(&cm_id_priv->lock, flags);
4119         switch (cm_id_priv->id.state) {
4120         case IB_CM_REQ_RCVD:
4121         case IB_CM_MRA_REQ_SENT:
4122         case IB_CM_REP_RCVD:
4123         case IB_CM_MRA_REP_SENT:
4124         case IB_CM_REP_SENT:
4125         case IB_CM_MRA_REP_RCVD:
4126         case IB_CM_ESTABLISHED:
4127                 *qp_attr_mask = IB_QP_STATE | IB_QP_AV | IB_QP_PATH_MTU |
4128                                 IB_QP_DEST_QPN | IB_QP_RQ_PSN;
4129                 qp_attr->ah_attr = cm_id_priv->av.ah_attr;
4130                 if ((qp_attr->ah_attr.type == RDMA_AH_ATTR_TYPE_IB) &&
4131                     cm_id_priv->av.dlid_datapath &&
4132                     (cm_id_priv->av.dlid_datapath != 0xffff))
4133                         qp_attr->ah_attr.ib.dlid = cm_id_priv->av.dlid_datapath;
4134                 qp_attr->path_mtu = cm_id_priv->path_mtu;
4135                 qp_attr->dest_qp_num = be32_to_cpu(cm_id_priv->remote_qpn);
4136                 qp_attr->rq_psn = be32_to_cpu(cm_id_priv->rq_psn);
4137                 if (cm_id_priv->qp_type == IB_QPT_RC ||
4138                     cm_id_priv->qp_type == IB_QPT_XRC_TGT) {
4139                         *qp_attr_mask |= IB_QP_MAX_DEST_RD_ATOMIC |
4140                                          IB_QP_MIN_RNR_TIMER;
4141                         qp_attr->max_dest_rd_atomic =
4142                                         cm_id_priv->responder_resources;
4143                         qp_attr->min_rnr_timer = 0;
4144                 }
4145                 if (rdma_ah_get_dlid(&cm_id_priv->alt_av.ah_attr) &&
4146                     cm_id_priv->alt_av.port) {
4147                         *qp_attr_mask |= IB_QP_ALT_PATH;
4148                         qp_attr->alt_port_num = cm_id_priv->alt_av.port->port_num;
4149                         qp_attr->alt_pkey_index = cm_id_priv->alt_av.pkey_index;
4150                         qp_attr->alt_timeout = cm_id_priv->alt_av.timeout;
4151                         qp_attr->alt_ah_attr = cm_id_priv->alt_av.ah_attr;
4152                 }
4153                 ret = 0;
4154                 break;
4155         default:
4156                 trace_icm_qp_rtr_err(&cm_id_priv->id);
4157                 ret = -EINVAL;
4158                 break;
4159         }
4160         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
4161         return ret;
4162 }
4163
4164 static int cm_init_qp_rts_attr(struct cm_id_private *cm_id_priv,
4165                                struct ib_qp_attr *qp_attr,
4166                                int *qp_attr_mask)
4167 {
4168         unsigned long flags;
4169         int ret;
4170
4171         spin_lock_irqsave(&cm_id_priv->lock, flags);
4172         switch (cm_id_priv->id.state) {
4173         /* Allow transition to RTS before sending REP */
4174         case IB_CM_REQ_RCVD:
4175         case IB_CM_MRA_REQ_SENT:
4176
4177         case IB_CM_REP_RCVD:
4178         case IB_CM_MRA_REP_SENT:
4179         case IB_CM_REP_SENT:
4180         case IB_CM_MRA_REP_RCVD:
4181         case IB_CM_ESTABLISHED:
4182                 if (cm_id_priv->id.lap_state == IB_CM_LAP_UNINIT) {
4183                         *qp_attr_mask = IB_QP_STATE | IB_QP_SQ_PSN;
4184                         qp_attr->sq_psn = be32_to_cpu(cm_id_priv->sq_psn);
4185                         switch (cm_id_priv->qp_type) {
4186                         case IB_QPT_RC:
4187                         case IB_QPT_XRC_INI:
4188                                 *qp_attr_mask |= IB_QP_RETRY_CNT | IB_QP_RNR_RETRY |
4189                                                  IB_QP_MAX_QP_RD_ATOMIC;
4190                                 qp_attr->retry_cnt = cm_id_priv->retry_count;
4191                                 qp_attr->rnr_retry = cm_id_priv->rnr_retry_count;
4192                                 qp_attr->max_rd_atomic = cm_id_priv->initiator_depth;
4193                                 fallthrough;
4194                         case IB_QPT_XRC_TGT:
4195                                 *qp_attr_mask |= IB_QP_TIMEOUT;
4196                                 qp_attr->timeout = cm_id_priv->av.timeout;
4197                                 break;
4198                         default:
4199                                 break;
4200                         }
4201                         if (rdma_ah_get_dlid(&cm_id_priv->alt_av.ah_attr)) {
4202                                 *qp_attr_mask |= IB_QP_PATH_MIG_STATE;
4203                                 qp_attr->path_mig_state = IB_MIG_REARM;
4204                         }
4205                 } else {
4206                         *qp_attr_mask = IB_QP_ALT_PATH | IB_QP_PATH_MIG_STATE;
4207                         if (cm_id_priv->alt_av.port)
4208                                 qp_attr->alt_port_num =
4209                                         cm_id_priv->alt_av.port->port_num;
4210                         qp_attr->alt_pkey_index = cm_id_priv->alt_av.pkey_index;
4211                         qp_attr->alt_timeout = cm_id_priv->alt_av.timeout;
4212                         qp_attr->alt_ah_attr = cm_id_priv->alt_av.ah_attr;
4213                         qp_attr->path_mig_state = IB_MIG_REARM;
4214                 }
4215                 ret = 0;
4216                 break;
4217         default:
4218                 trace_icm_qp_rts_err(&cm_id_priv->id);
4219                 ret = -EINVAL;
4220                 break;
4221         }
4222         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
4223         return ret;
4224 }
4225
4226 int ib_cm_init_qp_attr(struct ib_cm_id *cm_id,
4227                        struct ib_qp_attr *qp_attr,
4228                        int *qp_attr_mask)
4229 {
4230         struct cm_id_private *cm_id_priv;
4231         int ret;
4232
4233         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
4234         switch (qp_attr->qp_state) {
4235         case IB_QPS_INIT:
4236                 ret = cm_init_qp_init_attr(cm_id_priv, qp_attr, qp_attr_mask);
4237                 break;
4238         case IB_QPS_RTR:
4239                 ret = cm_init_qp_rtr_attr(cm_id_priv, qp_attr, qp_attr_mask);
4240                 break;
4241         case IB_QPS_RTS:
4242                 ret = cm_init_qp_rts_attr(cm_id_priv, qp_attr, qp_attr_mask);
4243                 break;
4244         default:
4245                 ret = -EINVAL;
4246                 break;
4247         }
4248         return ret;
4249 }
4250 EXPORT_SYMBOL(ib_cm_init_qp_attr);
4251
4252 static ssize_t cm_show_counter(struct ib_device *ibdev, u32 port_num,
4253                                struct ib_port_attribute *attr, char *buf)
4254 {
4255         struct cm_counter_attribute *cm_attr =
4256                 container_of(attr, struct cm_counter_attribute, attr);
4257         struct cm_device *cm_dev = ib_get_client_data(ibdev, &cm_client);
4258
4259         if (WARN_ON(!cm_dev))
4260                 return -EINVAL;
4261
4262         return sysfs_emit(
4263                 buf, "%ld\n",
4264                 atomic_long_read(
4265                         &cm_dev->port[port_num - 1]
4266                                  ->counters[cm_attr->group][cm_attr->index]));
4267 }
4268
4269 #define CM_COUNTER_ATTR(_name, _group, _index)                                 \
4270         {                                                                      \
4271                 .attr = __ATTR(_name, 0444, cm_show_counter, NULL),            \
4272                 .group = _group, .index = _index                               \
4273         }
4274
4275 #define CM_COUNTER_GROUP(_group, _name)                                        \
4276         static struct cm_counter_attribute cm_counter_attr_##_group[] = {      \
4277                 CM_COUNTER_ATTR(req, _group, CM_REQ_COUNTER),                  \
4278                 CM_COUNTER_ATTR(mra, _group, CM_MRA_COUNTER),                  \
4279                 CM_COUNTER_ATTR(rej, _group, CM_REJ_COUNTER),                  \
4280                 CM_COUNTER_ATTR(rep, _group, CM_REP_COUNTER),                  \
4281                 CM_COUNTER_ATTR(rtu, _group, CM_RTU_COUNTER),                  \
4282                 CM_COUNTER_ATTR(dreq, _group, CM_DREQ_COUNTER),                \
4283                 CM_COUNTER_ATTR(drep, _group, CM_DREP_COUNTER),                \
4284                 CM_COUNTER_ATTR(sidr_req, _group, CM_SIDR_REQ_COUNTER),        \
4285                 CM_COUNTER_ATTR(sidr_rep, _group, CM_SIDR_REP_COUNTER),        \
4286                 CM_COUNTER_ATTR(lap, _group, CM_LAP_COUNTER),                  \
4287                 CM_COUNTER_ATTR(apr, _group, CM_APR_COUNTER),                  \
4288         };                                                                     \
4289         static struct attribute *cm_counter_attrs_##_group[] = {               \
4290                 &cm_counter_attr_##_group[0].attr.attr,                        \
4291                 &cm_counter_attr_##_group[1].attr.attr,                        \
4292                 &cm_counter_attr_##_group[2].attr.attr,                        \
4293                 &cm_counter_attr_##_group[3].attr.attr,                        \
4294                 &cm_counter_attr_##_group[4].attr.attr,                        \
4295                 &cm_counter_attr_##_group[5].attr.attr,                        \
4296                 &cm_counter_attr_##_group[6].attr.attr,                        \
4297                 &cm_counter_attr_##_group[7].attr.attr,                        \
4298                 &cm_counter_attr_##_group[8].attr.attr,                        \
4299                 &cm_counter_attr_##_group[9].attr.attr,                        \
4300                 &cm_counter_attr_##_group[10].attr.attr,                       \
4301                 NULL,                                                          \
4302         };                                                                     \
4303         static const struct attribute_group cm_counter_group_##_group = {      \
4304                 .name = _name,                                                 \
4305                 .attrs = cm_counter_attrs_##_group,                            \
4306         };
4307
4308 CM_COUNTER_GROUP(CM_XMIT, "cm_tx_msgs")
4309 CM_COUNTER_GROUP(CM_XMIT_RETRIES, "cm_tx_retries")
4310 CM_COUNTER_GROUP(CM_RECV, "cm_rx_msgs")
4311 CM_COUNTER_GROUP(CM_RECV_DUPLICATES, "cm_rx_duplicates")
4312
4313 static const struct attribute_group *cm_counter_groups[] = {
4314         &cm_counter_group_CM_XMIT,
4315         &cm_counter_group_CM_XMIT_RETRIES,
4316         &cm_counter_group_CM_RECV,
4317         &cm_counter_group_CM_RECV_DUPLICATES,
4318         NULL,
4319 };
4320
4321 static int cm_add_one(struct ib_device *ib_device)
4322 {
4323         struct cm_device *cm_dev;
4324         struct cm_port *port;
4325         struct ib_mad_reg_req reg_req = {
4326                 .mgmt_class = IB_MGMT_CLASS_CM,
4327                 .mgmt_class_version = IB_CM_CLASS_VERSION,
4328         };
4329         struct ib_port_modify port_modify = {
4330                 .set_port_cap_mask = IB_PORT_CM_SUP
4331         };
4332         unsigned long flags;
4333         int ret;
4334         int count = 0;
4335         u32 i;
4336
4337         cm_dev = kzalloc(struct_size(cm_dev, port, ib_device->phys_port_cnt),
4338                          GFP_KERNEL);
4339         if (!cm_dev)
4340                 return -ENOMEM;
4341
4342         kref_init(&cm_dev->kref);
4343         rwlock_init(&cm_dev->mad_agent_lock);
4344         cm_dev->ib_device = ib_device;
4345         cm_dev->ack_delay = ib_device->attrs.local_ca_ack_delay;
4346         cm_dev->going_down = 0;
4347
4348         ib_set_client_data(ib_device, &cm_client, cm_dev);
4349
4350         set_bit(IB_MGMT_METHOD_SEND, reg_req.method_mask);
4351         rdma_for_each_port (ib_device, i) {
4352                 if (!rdma_cap_ib_cm(ib_device, i))
4353                         continue;
4354
4355                 port = kzalloc(sizeof *port, GFP_KERNEL);
4356                 if (!port) {
4357                         ret = -ENOMEM;
4358                         goto error1;
4359                 }
4360
4361                 cm_dev->port[i-1] = port;
4362                 port->cm_dev = cm_dev;
4363                 port->port_num = i;
4364
4365                 ret = ib_port_register_client_groups(ib_device, i,
4366                                                      cm_counter_groups);
4367                 if (ret)
4368                         goto error1;
4369
4370                 port->mad_agent = ib_register_mad_agent(ib_device, i,
4371                                                         IB_QPT_GSI,
4372                                                         &reg_req,
4373                                                         0,
4374                                                         cm_send_handler,
4375                                                         cm_recv_handler,
4376                                                         port,
4377                                                         0);
4378                 if (IS_ERR(port->mad_agent)) {
4379                         ret = PTR_ERR(port->mad_agent);
4380                         goto error2;
4381                 }
4382
4383                 ret = ib_modify_port(ib_device, i, 0, &port_modify);
4384                 if (ret)
4385                         goto error3;
4386
4387                 count++;
4388         }
4389
4390         if (!count) {
4391                 ret = -EOPNOTSUPP;
4392                 goto free;
4393         }
4394
4395         write_lock_irqsave(&cm.device_lock, flags);
4396         list_add_tail(&cm_dev->list, &cm.device_list);
4397         write_unlock_irqrestore(&cm.device_lock, flags);
4398         return 0;
4399
4400 error3:
4401         ib_unregister_mad_agent(port->mad_agent);
4402 error2:
4403         ib_port_unregister_client_groups(ib_device, i, cm_counter_groups);
4404 error1:
4405         port_modify.set_port_cap_mask = 0;
4406         port_modify.clr_port_cap_mask = IB_PORT_CM_SUP;
4407         while (--i) {
4408                 if (!rdma_cap_ib_cm(ib_device, i))
4409                         continue;
4410
4411                 port = cm_dev->port[i-1];
4412                 ib_modify_port(ib_device, port->port_num, 0, &port_modify);
4413                 ib_unregister_mad_agent(port->mad_agent);
4414                 ib_port_unregister_client_groups(ib_device, i,
4415                                                  cm_counter_groups);
4416         }
4417 free:
4418         cm_device_put(cm_dev);
4419         return ret;
4420 }
4421
4422 static void cm_remove_one(struct ib_device *ib_device, void *client_data)
4423 {
4424         struct cm_device *cm_dev = client_data;
4425         struct cm_port *port;
4426         struct ib_port_modify port_modify = {
4427                 .clr_port_cap_mask = IB_PORT_CM_SUP
4428         };
4429         unsigned long flags;
4430         u32 i;
4431
4432         write_lock_irqsave(&cm.device_lock, flags);
4433         list_del(&cm_dev->list);
4434         write_unlock_irqrestore(&cm.device_lock, flags);
4435
4436         spin_lock_irq(&cm.lock);
4437         cm_dev->going_down = 1;
4438         spin_unlock_irq(&cm.lock);
4439
4440         rdma_for_each_port (ib_device, i) {
4441                 struct ib_mad_agent *mad_agent;
4442
4443                 if (!rdma_cap_ib_cm(ib_device, i))
4444                         continue;
4445
4446                 port = cm_dev->port[i-1];
4447                 mad_agent = port->mad_agent;
4448                 ib_modify_port(ib_device, port->port_num, 0, &port_modify);
4449                 /*
4450                  * We flush the queue here after the going_down set, this
4451                  * verify that no new works will be queued in the recv handler,
4452                  * after that we can call the unregister_mad_agent
4453                  */
4454                 flush_workqueue(cm.wq);
4455                 /*
4456                  * The above ensures no call paths from the work are running,
4457                  * the remaining paths all take the mad_agent_lock.
4458                  */
4459                 write_lock(&cm_dev->mad_agent_lock);
4460                 port->mad_agent = NULL;
4461                 write_unlock(&cm_dev->mad_agent_lock);
4462                 ib_unregister_mad_agent(mad_agent);
4463                 ib_port_unregister_client_groups(ib_device, i,
4464                                                  cm_counter_groups);
4465         }
4466
4467         cm_device_put(cm_dev);
4468 }
4469
4470 static int __init ib_cm_init(void)
4471 {
4472         int ret;
4473
4474         INIT_LIST_HEAD(&cm.device_list);
4475         rwlock_init(&cm.device_lock);
4476         spin_lock_init(&cm.lock);
4477         cm.listen_service_table = RB_ROOT;
4478         cm.listen_service_id = be64_to_cpu(IB_CM_ASSIGN_SERVICE_ID);
4479         cm.remote_id_table = RB_ROOT;
4480         cm.remote_qp_table = RB_ROOT;
4481         cm.remote_sidr_table = RB_ROOT;
4482         xa_init_flags(&cm.local_id_table, XA_FLAGS_ALLOC);
4483         get_random_bytes(&cm.random_id_operand, sizeof cm.random_id_operand);
4484         INIT_LIST_HEAD(&cm.timewait_list);
4485
4486         cm.wq = alloc_workqueue("ib_cm", 0, 1);
4487         if (!cm.wq) {
4488                 ret = -ENOMEM;
4489                 goto error2;
4490         }
4491
4492         ret = ib_register_client(&cm_client);
4493         if (ret)
4494                 goto error3;
4495
4496         return 0;
4497 error3:
4498         destroy_workqueue(cm.wq);
4499 error2:
4500         return ret;
4501 }
4502
4503 static void __exit ib_cm_cleanup(void)
4504 {
4505         struct cm_timewait_info *timewait_info, *tmp;
4506
4507         spin_lock_irq(&cm.lock);
4508         list_for_each_entry(timewait_info, &cm.timewait_list, list)
4509                 cancel_delayed_work(&timewait_info->work.work);
4510         spin_unlock_irq(&cm.lock);
4511
4512         ib_unregister_client(&cm_client);
4513         destroy_workqueue(cm.wq);
4514
4515         list_for_each_entry_safe(timewait_info, tmp, &cm.timewait_list, list) {
4516                 list_del(&timewait_info->list);
4517                 kfree(timewait_info);
4518         }
4519
4520         WARN_ON(!xa_empty(&cm.local_id_table));
4521 }
4522
4523 module_init(ib_cm_init);
4524 module_exit(ib_cm_cleanup);