drivers/infiniband/core/cm.c

   1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
   2 /*
   3  * Copyright (c) 2004-2007 Intel Corporation.  All rights reserved.
   4  * Copyright (c) 2004 Topspin Corporation.  All rights reserved.
   5  * Copyright (c) 2004, 2005 Voltaire Corporation.  All rights reserved.
   6  * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
   7  * Copyright (c) 2019, Mellanox Technologies inc.  All rights reserved.
   8  */
   9
  10 #include <linux/completion.h>
  11 #include <linux/dma-mapping.h>
  12 #include <linux/device.h>
  13 #include <linux/module.h>
  14 #include <linux/err.h>
  15 #include <linux/idr.h>
  16 #include <linux/interrupt.h>
  17 #include <linux/random.h>
  18 #include <linux/rbtree.h>
  19 #include <linux/spinlock.h>
  20 #include <linux/slab.h>
  21 #include <linux/sysfs.h>
  22 #include <linux/workqueue.h>
  23 #include <linux/kdev_t.h>
  24 #include <linux/etherdevice.h>
  25
  26 #include <rdma/ib_cache.h>
  27 #include <rdma/ib_cm.h>
  28 #include <rdma/ib_sysfs.h>
  29 #include "cm_msgs.h"
  30 #include "core_priv.h"
  31 #include "cm_trace.h"
  32
  33 MODULE_AUTHOR("Sean Hefty");
  34 MODULE_DESCRIPTION("InfiniBand CM");
  35 MODULE_LICENSE("Dual BSD/GPL");
  36
  37 #define CM_DESTROY_ID_WAIT_TIMEOUT 10000 /* msecs */
  38 static const char * const ibcm_rej_reason_strs[] = {
  39         [IB_CM_REJ_NO_QP]                       = "no QP",
  40         [IB_CM_REJ_NO_EEC]                      = "no EEC",
  41         [IB_CM_REJ_NO_RESOURCES]                = "no resources",
  42         [IB_CM_REJ_TIMEOUT]                     = "timeout",
  43         [IB_CM_REJ_UNSUPPORTED]                 = "unsupported",
  44         [IB_CM_REJ_INVALID_COMM_ID]             = "invalid comm ID",
  45         [IB_CM_REJ_INVALID_COMM_INSTANCE]       = "invalid comm instance",
  46         [IB_CM_REJ_INVALID_SERVICE_ID]          = "invalid service ID",
  47         [IB_CM_REJ_INVALID_TRANSPORT_TYPE]      = "invalid transport type",
  48         [IB_CM_REJ_STALE_CONN]                  = "stale conn",
  49         [IB_CM_REJ_RDC_NOT_EXIST]               = "RDC not exist",
  50         [IB_CM_REJ_INVALID_GID]                 = "invalid GID",
  51         [IB_CM_REJ_INVALID_LID]                 = "invalid LID",
  52         [IB_CM_REJ_INVALID_SL]                  = "invalid SL",
  53         [IB_CM_REJ_INVALID_TRAFFIC_CLASS]       = "invalid traffic class",
  54         [IB_CM_REJ_INVALID_HOP_LIMIT]           = "invalid hop limit",
  55         [IB_CM_REJ_INVALID_PACKET_RATE]         = "invalid packet rate",
  56         [IB_CM_REJ_INVALID_ALT_GID]             = "invalid alt GID",
  57         [IB_CM_REJ_INVALID_ALT_LID]             = "invalid alt LID",
  58         [IB_CM_REJ_INVALID_ALT_SL]              = "invalid alt SL",
  59         [IB_CM_REJ_INVALID_ALT_TRAFFIC_CLASS]   = "invalid alt traffic class",
  60         [IB_CM_REJ_INVALID_ALT_HOP_LIMIT]       = "invalid alt hop limit",
  61         [IB_CM_REJ_INVALID_ALT_PACKET_RATE]     = "invalid alt packet rate",
  62         [IB_CM_REJ_PORT_CM_REDIRECT]            = "port CM redirect",
  63         [IB_CM_REJ_PORT_REDIRECT]               = "port redirect",
  64         [IB_CM_REJ_INVALID_MTU]                 = "invalid MTU",
  65         [IB_CM_REJ_INSUFFICIENT_RESP_RESOURCES] = "insufficient resp resources",
  66         [IB_CM_REJ_CONSUMER_DEFINED]            = "consumer defined",
  67         [IB_CM_REJ_INVALID_RNR_RETRY]           = "invalid RNR retry",
  68         [IB_CM_REJ_DUPLICATE_LOCAL_COMM_ID]     = "duplicate local comm ID",
  69         [IB_CM_REJ_INVALID_CLASS_VERSION]       = "invalid class version",
  70         [IB_CM_REJ_INVALID_FLOW_LABEL]          = "invalid flow label",
  71         [IB_CM_REJ_INVALID_ALT_FLOW_LABEL]      = "invalid alt flow label",
  72         [IB_CM_REJ_VENDOR_OPTION_NOT_SUPPORTED] =
  73                 "vendor option is not supported",
  74 };
  75
  76 const char *__attribute_const__ ibcm_reject_msg(int reason)
  77 {
  78         size_t index = reason;
  79
  80         if (index < ARRAY_SIZE(ibcm_rej_reason_strs) &&
  81             ibcm_rej_reason_strs[index])
  82                 return ibcm_rej_reason_strs[index];
  83         else
  84                 return "unrecognized reason";
  85 }
  86 EXPORT_SYMBOL(ibcm_reject_msg);
  87
  88 struct cm_id_private;
  89 struct cm_work;
  90 static int cm_add_one(struct ib_device *device);
  91 static void cm_remove_one(struct ib_device *device, void *client_data);
  92 static void cm_process_work(struct cm_id_private *cm_id_priv,
  93                             struct cm_work *work);
  94 static int cm_send_sidr_rep_locked(struct cm_id_private *cm_id_priv,
  95                                    struct ib_cm_sidr_rep_param *param);
  96 static int cm_send_dreq_locked(struct cm_id_private *cm_id_priv,
  97                                const void *private_data, u8 private_data_len);
  98 static int cm_send_drep_locked(struct cm_id_private *cm_id_priv,
  99                                void *private_data, u8 private_data_len);
 100 static int cm_send_rej_locked(struct cm_id_private *cm_id_priv,
 101                               enum ib_cm_rej_reason reason, void *ari,
 102                               u8 ari_length, const void *private_data,
 103                               u8 private_data_len);
 104
 105 static struct ib_client cm_client = {
 106         .name   = "cm",
 107         .add    = cm_add_one,
 108         .remove = cm_remove_one
 109 };
 110
 111 static struct ib_cm {
 112         spinlock_t lock;
 113         struct list_head device_list;
 114         rwlock_t device_lock;
 115         struct rb_root listen_service_table;
 116         u64 listen_service_id;
 117         /* struct rb_root peer_service_table; todo: fix peer to peer */
 118         struct rb_root remote_qp_table;
 119         struct rb_root remote_id_table;
 120         struct rb_root remote_sidr_table;
 121         struct xarray local_id_table;
 122         u32 local_id_next;
 123         __be32 random_id_operand;
 124         struct list_head timewait_list;
 125         struct workqueue_struct *wq;
 126 } cm;
 127
 128 /* Counter indexes ordered by attribute ID */
 129 enum {
 130         CM_REQ_COUNTER,
 131         CM_MRA_COUNTER,
 132         CM_REJ_COUNTER,
 133         CM_REP_COUNTER,
 134         CM_RTU_COUNTER,
 135         CM_DREQ_COUNTER,
 136         CM_DREP_COUNTER,
 137         CM_SIDR_REQ_COUNTER,
 138         CM_SIDR_REP_COUNTER,
 139         CM_LAP_COUNTER,
 140         CM_APR_COUNTER,
 141         CM_ATTR_COUNT,
 142         CM_ATTR_ID_OFFSET = 0x0010,
 143 };
 144
 145 enum {
 146         CM_XMIT,
 147         CM_XMIT_RETRIES,
 148         CM_RECV,
 149         CM_RECV_DUPLICATES,
 150         CM_COUNTER_GROUPS
 151 };
 152
 153 struct cm_counter_attribute {
 154         struct ib_port_attribute attr;
 155         unsigned short group;
 156         unsigned short index;
 157 };
 158
 159 struct cm_port {
 160         struct cm_device *cm_dev;
 161         struct ib_mad_agent *mad_agent;
 162         u32 port_num;
 163         atomic_long_t counters[CM_COUNTER_GROUPS][CM_ATTR_COUNT];
 164 };
 165
 166 struct cm_device {
 167         struct kref kref;
 168         struct list_head list;
 169         spinlock_t mad_agent_lock;
 170         struct ib_device *ib_device;
 171         u8 ack_delay;
 172         int going_down;
 173         struct cm_port *port[];
 174 };
 175
 176 struct cm_av {
 177         struct cm_port *port;
 178         struct rdma_ah_attr ah_attr;
 179         u16 dlid_datapath;
 180         u16 pkey_index;
 181         u8 timeout;
 182 };
 183
 184 struct cm_work {
 185         struct delayed_work work;
 186         struct list_head list;
 187         struct cm_port *port;
 188         struct ib_mad_recv_wc *mad_recv_wc;     /* Received MADs */
 189         __be32 local_id;                        /* Established / timewait */
 190         __be32 remote_id;
 191         struct ib_cm_event cm_event;
 192         struct sa_path_rec path[];
 193 };
 194
 195 struct cm_timewait_info {
 196         struct cm_work work;
 197         struct list_head list;
 198         struct rb_node remote_qp_node;
 199         struct rb_node remote_id_node;
 200         __be64 remote_ca_guid;
 201         __be32 remote_qpn;
 202         u8 inserted_remote_qp;
 203         u8 inserted_remote_id;
 204 };
 205
 206 struct cm_id_private {
 207         struct ib_cm_id id;
 208
 209         struct rb_node service_node;
 210         struct rb_node sidr_id_node;
 211         u32 sidr_slid;
 212         spinlock_t lock;        /* Do not acquire inside cm.lock */
 213         struct completion comp;
 214         refcount_t refcount;
 215         /* Number of clients sharing this ib_cm_id. Only valid for listeners.
 216          * Protected by the cm.lock spinlock.
 217          */
 218         int listen_sharecount;
 219         struct rcu_head rcu;
 220
 221         struct ib_mad_send_buf *msg;
 222         struct cm_timewait_info *timewait_info;
 223         /* todo: use alternate port on send failure */
 224         struct cm_av av;
 225         struct cm_av alt_av;
 226
 227         void *private_data;
 228         __be64 tid;
 229         __be32 local_qpn;
 230         __be32 remote_qpn;
 231         enum ib_qp_type qp_type;
 232         __be32 sq_psn;
 233         __be32 rq_psn;
 234         int timeout_ms;
 235         enum ib_mtu path_mtu;
 236         __be16 pkey;
 237         u8 private_data_len;
 238         u8 max_cm_retries;
 239         u8 responder_resources;
 240         u8 initiator_depth;
 241         u8 retry_count;
 242         u8 rnr_retry_count;
 243         u8 service_timeout;
 244         u8 target_ack_delay;
 245
 246         struct list_head work_list;
 247         atomic_t work_count;
 248
 249         struct rdma_ucm_ece ece;
 250 };
 251
 252 static void cm_dev_release(struct kref *kref)
 253 {
 254         struct cm_device *cm_dev = container_of(kref, struct cm_device, kref);
 255         u32 i;
 256
 257         rdma_for_each_port(cm_dev->ib_device, i)
 258                 kfree(cm_dev->port[i - 1]);
 259
 260         kfree(cm_dev);
 261 }
 262
 263 static void cm_device_put(struct cm_device *cm_dev)
 264 {
 265         kref_put(&cm_dev->kref, cm_dev_release);
 266 }
 267
 268 static void cm_work_handler(struct work_struct *work);
 269
 270 static inline void cm_deref_id(struct cm_id_private *cm_id_priv)
 271 {
 272         if (refcount_dec_and_test(&cm_id_priv->refcount))
 273                 complete(&cm_id_priv->comp);
 274 }
 275
 276 static struct ib_mad_send_buf *cm_alloc_msg(struct cm_id_private *cm_id_priv)
 277 {
 278         struct ib_mad_agent *mad_agent;
 279         struct ib_mad_send_buf *m;
 280         struct ib_ah *ah;
 281
 282         lockdep_assert_held(&cm_id_priv->lock);
 283
 284         if (!cm_id_priv->av.port)
 285                 return ERR_PTR(-EINVAL);
 286
 287         spin_lock(&cm_id_priv->av.port->cm_dev->mad_agent_lock);
 288         mad_agent = cm_id_priv->av.port->mad_agent;
 289         if (!mad_agent) {
 290                 m = ERR_PTR(-EINVAL);
 291                 goto out;
 292         }
 293
 294         ah = rdma_create_ah(mad_agent->qp->pd, &cm_id_priv->av.ah_attr, 0);
 295         if (IS_ERR(ah)) {
 296                 m = ERR_CAST(ah);
 297                 goto out;
 298         }
 299
 300         m = ib_create_send_mad(mad_agent, cm_id_priv->id.remote_cm_qpn,
 301                                cm_id_priv->av.pkey_index,
 302                                0, IB_MGMT_MAD_HDR, IB_MGMT_MAD_DATA,
 303                                GFP_ATOMIC,
 304                                IB_MGMT_BASE_VERSION);
 305         if (IS_ERR(m)) {
 306                 rdma_destroy_ah(ah, 0);
 307                 goto out;
 308         }
 309
 310         /* Timeout set by caller if response is expected. */
 311         m->ah = ah;
 312         m->retries = cm_id_priv->max_cm_retries;
 313
 314         refcount_inc(&cm_id_priv->refcount);
 315         m->context[0] = cm_id_priv;
 316
 317 out:
 318         spin_unlock(&cm_id_priv->av.port->cm_dev->mad_agent_lock);
 319         return m;
 320 }
 321
 322 static void cm_free_msg(struct ib_mad_send_buf *msg)
 323 {
 324         struct cm_id_private *cm_id_priv = msg->context[0];
 325
 326         if (msg->ah)
 327                 rdma_destroy_ah(msg->ah, 0);
 328         cm_deref_id(cm_id_priv);
 329         ib_free_send_mad(msg);
 330 }
 331
 332 static struct ib_mad_send_buf *
 333 cm_alloc_priv_msg(struct cm_id_private *cm_id_priv)
 334 {
 335         struct ib_mad_send_buf *msg;
 336
 337         lockdep_assert_held(&cm_id_priv->lock);
 338
 339         msg = cm_alloc_msg(cm_id_priv);
 340         if (IS_ERR(msg))
 341                 return msg;
 342         cm_id_priv->msg = msg;
 343         return msg;
 344 }
 345
 346 static void cm_free_priv_msg(struct ib_mad_send_buf *msg)
 347 {
 348         struct cm_id_private *cm_id_priv = msg->context[0];
 349
 350         lockdep_assert_held(&cm_id_priv->lock);
 351
 352         if (!WARN_ON(cm_id_priv->msg != msg))
 353                 cm_id_priv->msg = NULL;
 354
 355         if (msg->ah)
 356                 rdma_destroy_ah(msg->ah, 0);
 357         cm_deref_id(cm_id_priv);
 358         ib_free_send_mad(msg);
 359 }
 360
 361 static struct ib_mad_send_buf *cm_alloc_response_msg_no_ah(struct cm_port *port,
 362                                                            struct ib_mad_recv_wc *mad_recv_wc)
 363 {
 364         return ib_create_send_mad(port->mad_agent, 1, mad_recv_wc->wc->pkey_index,
 365                                   0, IB_MGMT_MAD_HDR, IB_MGMT_MAD_DATA,
 366                                   GFP_ATOMIC,
 367                                   IB_MGMT_BASE_VERSION);
 368 }
 369
 370 static int cm_create_response_msg_ah(struct cm_port *port,
 371                                      struct ib_mad_recv_wc *mad_recv_wc,
 372                                      struct ib_mad_send_buf *msg)
 373 {
 374         struct ib_ah *ah;
 375
 376         ah = ib_create_ah_from_wc(port->mad_agent->qp->pd, mad_recv_wc->wc,
 377                                   mad_recv_wc->recv_buf.grh, port->port_num);
 378         if (IS_ERR(ah))
 379                 return PTR_ERR(ah);
 380
 381         msg->ah = ah;
 382         return 0;
 383 }
 384
 385 static int cm_alloc_response_msg(struct cm_port *port,
 386                                  struct ib_mad_recv_wc *mad_recv_wc,
 387                                  struct ib_mad_send_buf **msg)
 388 {
 389         struct ib_mad_send_buf *m;
 390         int ret;
 391
 392         m = cm_alloc_response_msg_no_ah(port, mad_recv_wc);
 393         if (IS_ERR(m))
 394                 return PTR_ERR(m);
 395
 396         ret = cm_create_response_msg_ah(port, mad_recv_wc, m);
 397         if (ret) {
 398                 ib_free_send_mad(m);
 399                 return ret;
 400         }
 401
 402         *msg = m;
 403         return 0;
 404 }
 405
 406 static void cm_free_response_msg(struct ib_mad_send_buf *msg)
 407 {
 408         if (msg->ah)
 409                 rdma_destroy_ah(msg->ah, 0);
 410         ib_free_send_mad(msg);
 411 }
 412
 413 static void *cm_copy_private_data(const void *private_data, u8 private_data_len)
 414 {
 415         void *data;
 416
 417         if (!private_data || !private_data_len)
 418                 return NULL;
 419
 420         data = kmemdup(private_data, private_data_len, GFP_KERNEL);
 421         if (!data)
 422                 return ERR_PTR(-ENOMEM);
 423
 424         return data;
 425 }
 426
 427 static void cm_set_private_data(struct cm_id_private *cm_id_priv,
 428                                  void *private_data, u8 private_data_len)
 429 {
 430         if (cm_id_priv->private_data && cm_id_priv->private_data_len)
 431                 kfree(cm_id_priv->private_data);
 432
 433         cm_id_priv->private_data = private_data;
 434         cm_id_priv->private_data_len = private_data_len;
 435 }
 436
 437 static void cm_set_av_port(struct cm_av *av, struct cm_port *port)
 438 {
 439         struct cm_port *old_port = av->port;
 440
 441         if (old_port == port)
 442                 return;
 443
 444         av->port = port;
 445         if (old_port)
 446                 cm_device_put(old_port->cm_dev);
 447         if (port)
 448                 kref_get(&port->cm_dev->kref);
 449 }
 450
 451 static void cm_init_av_for_lap(struct cm_port *port, struct ib_wc *wc,
 452                                struct rdma_ah_attr *ah_attr, struct cm_av *av)
 453 {
 454         cm_set_av_port(av, port);
 455         av->pkey_index = wc->pkey_index;
 456         rdma_move_ah_attr(&av->ah_attr, ah_attr);
 457 }
 458
 459 static int cm_init_av_for_response(struct cm_port *port, struct ib_wc *wc,
 460                                    struct ib_grh *grh, struct cm_av *av)
 461 {
 462         cm_set_av_port(av, port);
 463         av->pkey_index = wc->pkey_index;
 464         return ib_init_ah_attr_from_wc(port->cm_dev->ib_device,
 465                                        port->port_num, wc,
 466                                        grh, &av->ah_attr);
 467 }
 468
 469 static struct cm_port *
 470 get_cm_port_from_path(struct sa_path_rec *path, const struct ib_gid_attr *attr)
 471 {
 472         struct cm_device *cm_dev;
 473         struct cm_port *port = NULL;
 474         unsigned long flags;
 475
 476         if (attr) {
 477                 read_lock_irqsave(&cm.device_lock, flags);
 478                 list_for_each_entry(cm_dev, &cm.device_list, list) {
 479                         if (cm_dev->ib_device == attr->device) {
 480                                 port = cm_dev->port[attr->port_num - 1];
 481                                 break;
 482                         }
 483                 }
 484                 read_unlock_irqrestore(&cm.device_lock, flags);
 485         } else {
 486                 /* SGID attribute can be NULL in following
 487                  * conditions.
 488                  * (a) Alternative path
 489                  * (b) IB link layer without GRH
 490                  * (c) LAP send messages
 491                  */
 492                 read_lock_irqsave(&cm.device_lock, flags);
 493                 list_for_each_entry(cm_dev, &cm.device_list, list) {
 494                         attr = rdma_find_gid(cm_dev->ib_device,
 495                                              &path->sgid,
 496                                              sa_conv_pathrec_to_gid_type(path),
 497                                              NULL);
 498                         if (!IS_ERR(attr)) {
 499                                 port = cm_dev->port[attr->port_num - 1];
 500                                 break;
 501                         }
 502                 }
 503                 read_unlock_irqrestore(&cm.device_lock, flags);
 504                 if (port)
 505                         rdma_put_gid_attr(attr);
 506         }
 507         return port;
 508 }
 509
 510 static int cm_init_av_by_path(struct sa_path_rec *path,
 511                               const struct ib_gid_attr *sgid_attr,
 512                               struct cm_av *av)
 513 {
 514         struct rdma_ah_attr new_ah_attr;
 515         struct cm_device *cm_dev;
 516         struct cm_port *port;
 517         int ret;
 518
 519         port = get_cm_port_from_path(path, sgid_attr);
 520         if (!port)
 521                 return -EINVAL;
 522         cm_dev = port->cm_dev;
 523
 524         ret = ib_find_cached_pkey(cm_dev->ib_device, port->port_num,
 525                                   be16_to_cpu(path->pkey), &av->pkey_index);
 526         if (ret)
 527                 return ret;
 528
 529         cm_set_av_port(av, port);
 530
 531         /*
 532          * av->ah_attr might be initialized based on wc or during
 533          * request processing time which might have reference to sgid_attr.
 534          * So initialize a new ah_attr on stack.
 535          * If initialization fails, old ah_attr is used for sending any
 536          * responses. If initialization is successful, than new ah_attr
 537          * is used by overwriting the old one. So that right ah_attr
 538          * can be used to return an error response.
 539          */
 540         ret = ib_init_ah_attr_from_path(cm_dev->ib_device, port->port_num, path,
 541                                         &new_ah_attr, sgid_attr);
 542         if (ret)
 543                 return ret;
 544
 545         av->timeout = path->packet_life_time + 1;
 546         rdma_move_ah_attr(&av->ah_attr, &new_ah_attr);
 547         return 0;
 548 }
 549
 550 /* Move av created by cm_init_av_by_path(), so av.dgid is not moved */
 551 static void cm_move_av_from_path(struct cm_av *dest, struct cm_av *src)
 552 {
 553         cm_set_av_port(dest, src->port);
 554         cm_set_av_port(src, NULL);
 555         dest->pkey_index = src->pkey_index;
 556         rdma_move_ah_attr(&dest->ah_attr, &src->ah_attr);
 557         dest->timeout = src->timeout;
 558 }
 559
 560 static void cm_destroy_av(struct cm_av *av)
 561 {
 562         rdma_destroy_ah_attr(&av->ah_attr);
 563         cm_set_av_port(av, NULL);
 564 }
 565
 566 static u32 cm_local_id(__be32 local_id)
 567 {
 568         return (__force u32) (local_id ^ cm.random_id_operand);
 569 }
 570
 571 static struct cm_id_private *cm_acquire_id(__be32 local_id, __be32 remote_id)
 572 {
 573         struct cm_id_private *cm_id_priv;
 574
 575         rcu_read_lock();
 576         cm_id_priv = xa_load(&cm.local_id_table, cm_local_id(local_id));
 577         if (!cm_id_priv || cm_id_priv->id.remote_id != remote_id ||
 578             !refcount_inc_not_zero(&cm_id_priv->refcount))
 579                 cm_id_priv = NULL;
 580         rcu_read_unlock();
 581
 582         return cm_id_priv;
 583 }
 584
 585 /*
 586  * Trivial helpers to strip endian annotation and compare; the
 587  * endianness doesn't actually matter since we just need a stable
 588  * order for the RB tree.
 589  */
 590 static int be32_lt(__be32 a, __be32 b)
 591 {
 592         return (__force u32) a < (__force u32) b;
 593 }
 594
 595 static int be32_gt(__be32 a, __be32 b)
 596 {
 597         return (__force u32) a > (__force u32) b;
 598 }
 599
 600 static int be64_lt(__be64 a, __be64 b)
 601 {
 602         return (__force u64) a < (__force u64) b;
 603 }
 604
 605 static int be64_gt(__be64 a, __be64 b)
 606 {
 607         return (__force u64) a > (__force u64) b;
 608 }
 609
 610 /*
 611  * Inserts a new cm_id_priv into the listen_service_table. Returns cm_id_priv
 612  * if the new ID was inserted, NULL if it could not be inserted due to a
 613  * collision, or the existing cm_id_priv ready for shared usage.
 614  */
 615 static struct cm_id_private *cm_insert_listen(struct cm_id_private *cm_id_priv,
 616                                               ib_cm_handler shared_handler)
 617 {
 618         struct rb_node **link = &cm.listen_service_table.rb_node;
 619         struct rb_node *parent = NULL;
 620         struct cm_id_private *cur_cm_id_priv;
 621         __be64 service_id = cm_id_priv->id.service_id;
 622         unsigned long flags;
 623
 624         spin_lock_irqsave(&cm.lock, flags);
 625         while (*link) {
 626                 parent = *link;
 627                 cur_cm_id_priv = rb_entry(parent, struct cm_id_private,
 628                                           service_node);
 629
 630                 if (cm_id_priv->id.device < cur_cm_id_priv->id.device)
 631                         link = &(*link)->rb_left;
 632                 else if (cm_id_priv->id.device > cur_cm_id_priv->id.device)
 633                         link = &(*link)->rb_right;
 634                 else if (be64_lt(service_id, cur_cm_id_priv->id.service_id))
 635                         link = &(*link)->rb_left;
 636                 else if (be64_gt(service_id, cur_cm_id_priv->id.service_id))
 637                         link = &(*link)->rb_right;
 638                 else {
 639                         /*
 640                          * Sharing an ib_cm_id with different handlers is not
 641                          * supported
 642                          */
 643                         if (cur_cm_id_priv->id.cm_handler != shared_handler ||
 644                             cur_cm_id_priv->id.context ||
 645                             WARN_ON(!cur_cm_id_priv->id.cm_handler)) {
 646                                 spin_unlock_irqrestore(&cm.lock, flags);
 647                                 return NULL;
 648                         }
 649                         refcount_inc(&cur_cm_id_priv->refcount);
 650                         cur_cm_id_priv->listen_sharecount++;
 651                         spin_unlock_irqrestore(&cm.lock, flags);
 652                         return cur_cm_id_priv;
 653                 }
 654         }
 655         cm_id_priv->listen_sharecount++;
 656         rb_link_node(&cm_id_priv->service_node, parent, link);
 657         rb_insert_color(&cm_id_priv->service_node, &cm.listen_service_table);
 658         spin_unlock_irqrestore(&cm.lock, flags);
 659         return cm_id_priv;
 660 }
 661
 662 static struct cm_id_private *cm_find_listen(struct ib_device *device,
 663                                             __be64 service_id)
 664 {
 665         struct rb_node *node = cm.listen_service_table.rb_node;
 666         struct cm_id_private *cm_id_priv;
 667
 668         while (node) {
 669                 cm_id_priv = rb_entry(node, struct cm_id_private, service_node);
 670
 671                 if (device < cm_id_priv->id.device)
 672                         node = node->rb_left;
 673                 else if (device > cm_id_priv->id.device)
 674                         node = node->rb_right;
 675                 else if (be64_lt(service_id, cm_id_priv->id.service_id))
 676                         node = node->rb_left;
 677                 else if (be64_gt(service_id, cm_id_priv->id.service_id))
 678                         node = node->rb_right;
 679                 else {
 680                         refcount_inc(&cm_id_priv->refcount);
 681                         return cm_id_priv;
 682                 }
 683         }
 684         return NULL;
 685 }
 686
 687 static struct cm_timewait_info *
 688 cm_insert_remote_id(struct cm_timewait_info *timewait_info)
 689 {
 690         struct rb_node **link = &cm.remote_id_table.rb_node;
 691         struct rb_node *parent = NULL;
 692         struct cm_timewait_info *cur_timewait_info;
 693         __be64 remote_ca_guid = timewait_info->remote_ca_guid;
 694         __be32 remote_id = timewait_info->work.remote_id;
 695
 696         while (*link) {
 697                 parent = *link;
 698                 cur_timewait_info = rb_entry(parent, struct cm_timewait_info,
 699                                              remote_id_node);
 700                 if (be32_lt(remote_id, cur_timewait_info->work.remote_id))
 701                         link = &(*link)->rb_left;
 702                 else if (be32_gt(remote_id, cur_timewait_info->work.remote_id))
 703                         link = &(*link)->rb_right;
 704                 else if (be64_lt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
 705                         link = &(*link)->rb_left;
 706                 else if (be64_gt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
 707                         link = &(*link)->rb_right;
 708                 else
 709                         return cur_timewait_info;
 710         }
 711         timewait_info->inserted_remote_id = 1;
 712         rb_link_node(&timewait_info->remote_id_node, parent, link);
 713         rb_insert_color(&timewait_info->remote_id_node, &cm.remote_id_table);
 714         return NULL;
 715 }
 716
 717 static struct cm_id_private *cm_find_remote_id(__be64 remote_ca_guid,
 718                                                __be32 remote_id)
 719 {
 720         struct rb_node *node = cm.remote_id_table.rb_node;
 721         struct cm_timewait_info *timewait_info;
 722         struct cm_id_private *res = NULL;
 723
 724         spin_lock_irq(&cm.lock);
 725         while (node) {
 726                 timewait_info = rb_entry(node, struct cm_timewait_info,
 727                                          remote_id_node);
 728                 if (be32_lt(remote_id, timewait_info->work.remote_id))
 729                         node = node->rb_left;
 730                 else if (be32_gt(remote_id, timewait_info->work.remote_id))
 731                         node = node->rb_right;
 732                 else if (be64_lt(remote_ca_guid, timewait_info->remote_ca_guid))
 733                         node = node->rb_left;
 734                 else if (be64_gt(remote_ca_guid, timewait_info->remote_ca_guid))
 735                         node = node->rb_right;
 736                 else {
 737                         res = cm_acquire_id(timewait_info->work.local_id,
 738                                              timewait_info->work.remote_id);
 739                         break;
 740                 }
 741         }
 742         spin_unlock_irq(&cm.lock);
 743         return res;
 744 }
 745
 746 static struct cm_timewait_info *
 747 cm_insert_remote_qpn(struct cm_timewait_info *timewait_info)
 748 {
 749         struct rb_node **link = &cm.remote_qp_table.rb_node;
 750         struct rb_node *parent = NULL;
 751         struct cm_timewait_info *cur_timewait_info;
 752         __be64 remote_ca_guid = timewait_info->remote_ca_guid;
 753         __be32 remote_qpn = timewait_info->remote_qpn;
 754
 755         while (*link) {
 756                 parent = *link;
 757                 cur_timewait_info = rb_entry(parent, struct cm_timewait_info,
 758                                              remote_qp_node);
 759                 if (be32_lt(remote_qpn, cur_timewait_info->remote_qpn))
 760                         link = &(*link)->rb_left;
 761                 else if (be32_gt(remote_qpn, cur_timewait_info->remote_qpn))
 762                         link = &(*link)->rb_right;
 763                 else if (be64_lt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
 764                         link = &(*link)->rb_left;
 765                 else if (be64_gt(remote_ca_guid, cur_timewait_info->remote_ca_guid))
 766                         link = &(*link)->rb_right;
 767                 else
 768                         return cur_timewait_info;
 769         }
 770         timewait_info->inserted_remote_qp = 1;
 771         rb_link_node(&timewait_info->remote_qp_node, parent, link);
 772         rb_insert_color(&timewait_info->remote_qp_node, &cm.remote_qp_table);
 773         return NULL;
 774 }
 775
 776 static struct cm_id_private *
 777 cm_insert_remote_sidr(struct cm_id_private *cm_id_priv)
 778 {
 779         struct rb_node **link = &cm.remote_sidr_table.rb_node;
 780         struct rb_node *parent = NULL;
 781         struct cm_id_private *cur_cm_id_priv;
 782         __be32 remote_id = cm_id_priv->id.remote_id;
 783
 784         while (*link) {
 785                 parent = *link;
 786                 cur_cm_id_priv = rb_entry(parent, struct cm_id_private,
 787                                           sidr_id_node);
 788                 if (be32_lt(remote_id, cur_cm_id_priv->id.remote_id))
 789                         link = &(*link)->rb_left;
 790                 else if (be32_gt(remote_id, cur_cm_id_priv->id.remote_id))
 791                         link = &(*link)->rb_right;
 792                 else {
 793                         if (cur_cm_id_priv->sidr_slid < cm_id_priv->sidr_slid)
 794                                 link = &(*link)->rb_left;
 795                         else if (cur_cm_id_priv->sidr_slid > cm_id_priv->sidr_slid)
 796                                 link = &(*link)->rb_right;
 797                         else
 798                                 return cur_cm_id_priv;
 799                 }
 800         }
 801         rb_link_node(&cm_id_priv->sidr_id_node, parent, link);
 802         rb_insert_color(&cm_id_priv->sidr_id_node, &cm.remote_sidr_table);
 803         return NULL;
 804 }
 805
 806 static struct cm_id_private *cm_alloc_id_priv(struct ib_device *device,
 807                                               ib_cm_handler cm_handler,
 808                                               void *context)
 809 {
 810         struct cm_id_private *cm_id_priv;
 811         u32 id;
 812         int ret;
 813
 814         cm_id_priv = kzalloc(sizeof *cm_id_priv, GFP_KERNEL);
 815         if (!cm_id_priv)
 816                 return ERR_PTR(-ENOMEM);
 817
 818         cm_id_priv->id.state = IB_CM_IDLE;
 819         cm_id_priv->id.device = device;
 820         cm_id_priv->id.cm_handler = cm_handler;
 821         cm_id_priv->id.context = context;
 822         cm_id_priv->id.remote_cm_qpn = 1;
 823
 824         RB_CLEAR_NODE(&cm_id_priv->service_node);
 825         RB_CLEAR_NODE(&cm_id_priv->sidr_id_node);
 826         spin_lock_init(&cm_id_priv->lock);
 827         init_completion(&cm_id_priv->comp);
 828         INIT_LIST_HEAD(&cm_id_priv->work_list);
 829         atomic_set(&cm_id_priv->work_count, -1);
 830         refcount_set(&cm_id_priv->refcount, 1);
 831
 832         ret = xa_alloc_cyclic(&cm.local_id_table, &id, NULL, xa_limit_32b,
 833                               &cm.local_id_next, GFP_KERNEL);
 834         if (ret < 0)
 835                 goto error;
 836         cm_id_priv->id.local_id = (__force __be32)id ^ cm.random_id_operand;
 837
 838         return cm_id_priv;
 839
 840 error:
 841         kfree(cm_id_priv);
 842         return ERR_PTR(ret);
 843 }
 844
 845 /*
 846  * Make the ID visible to the MAD handlers and other threads that use the
 847  * xarray.
 848  */
 849 static void cm_finalize_id(struct cm_id_private *cm_id_priv)
 850 {
 851         xa_store(&cm.local_id_table, cm_local_id(cm_id_priv->id.local_id),
 852                  cm_id_priv, GFP_ATOMIC);
 853 }
 854
 855 struct ib_cm_id *ib_create_cm_id(struct ib_device *device,
 856                                  ib_cm_handler cm_handler,
 857                                  void *context)
 858 {
 859         struct cm_id_private *cm_id_priv;
 860
 861         cm_id_priv = cm_alloc_id_priv(device, cm_handler, context);
 862         if (IS_ERR(cm_id_priv))
 863                 return ERR_CAST(cm_id_priv);
 864
 865         cm_finalize_id(cm_id_priv);
 866         return &cm_id_priv->id;
 867 }
 868 EXPORT_SYMBOL(ib_create_cm_id);
 869
 870 static struct cm_work *cm_dequeue_work(struct cm_id_private *cm_id_priv)
 871 {
 872         struct cm_work *work;
 873
 874         if (list_empty(&cm_id_priv->work_list))
 875                 return NULL;
 876
 877         work = list_entry(cm_id_priv->work_list.next, struct cm_work, list);
 878         list_del(&work->list);
 879         return work;
 880 }
 881
 882 static void cm_free_work(struct cm_work *work)
 883 {
 884         if (work->mad_recv_wc)
 885                 ib_free_recv_mad(work->mad_recv_wc);
 886         kfree(work);
 887 }
 888
 889 static void cm_queue_work_unlock(struct cm_id_private *cm_id_priv,
 890                                  struct cm_work *work)
 891         __releases(&cm_id_priv->lock)
 892 {
 893         bool immediate;
 894
 895         /*
 896          * To deliver the event to the user callback we have the drop the
 897          * spinlock, however, we need to ensure that the user callback is single
 898          * threaded and receives events in the temporal order. If there are
 899          * already events being processed then thread new events onto a list,
 900          * the thread currently processing will pick them up.
 901          */
 902         immediate = atomic_inc_and_test(&cm_id_priv->work_count);
 903         if (!immediate) {
 904                 list_add_tail(&work->list, &cm_id_priv->work_list);
 905                 /*
 906                  * This routine always consumes incoming reference. Once queued
 907                  * to the work_list then a reference is held by the thread
 908                  * currently running cm_process_work() and this reference is not
 909                  * needed.
 910                  */
 911                 cm_deref_id(cm_id_priv);
 912         }
 913         spin_unlock_irq(&cm_id_priv->lock);
 914
 915         if (immediate)
 916                 cm_process_work(cm_id_priv, work);
 917 }
 918
 919 static inline int cm_convert_to_ms(int iba_time)
 920 {
 921         /* approximate conversion to ms from 4.096us x 2^iba_time */
 922         return 1 << max(iba_time - 8, 0);
 923 }
 924
 925 /*
 926  * calculate: 4.096x2^ack_timeout = 4.096x2^ack_delay + 2x4.096x2^life_time
 927  * Because of how ack_timeout is stored, adding one doubles the timeout.
 928  * To avoid large timeouts, select the max(ack_delay, life_time + 1), and
 929  * increment it (round up) only if the other is within 50%.
 930  */
 931 static u8 cm_ack_timeout(u8 ca_ack_delay, u8 packet_life_time)
 932 {
 933         int ack_timeout = packet_life_time + 1;
 934
 935         if (ack_timeout >= ca_ack_delay)
 936                 ack_timeout += (ca_ack_delay >= (ack_timeout - 1));
 937         else
 938                 ack_timeout = ca_ack_delay +
 939                               (ack_timeout >= (ca_ack_delay - 1));
 940
 941         return min(31, ack_timeout);
 942 }
 943
 944 static void cm_remove_remote(struct cm_id_private *cm_id_priv)
 945 {
 946         struct cm_timewait_info *timewait_info = cm_id_priv->timewait_info;
 947
 948         if (timewait_info->inserted_remote_id) {
 949                 rb_erase(&timewait_info->remote_id_node, &cm.remote_id_table);
 950                 timewait_info->inserted_remote_id = 0;
 951         }
 952
 953         if (timewait_info->inserted_remote_qp) {
 954                 rb_erase(&timewait_info->remote_qp_node, &cm.remote_qp_table);
 955                 timewait_info->inserted_remote_qp = 0;
 956         }
 957 }
 958
 959 static struct cm_timewait_info *cm_create_timewait_info(__be32 local_id)
 960 {
 961         struct cm_timewait_info *timewait_info;
 962
 963         timewait_info = kzalloc(sizeof *timewait_info, GFP_KERNEL);
 964         if (!timewait_info)
 965                 return ERR_PTR(-ENOMEM);
 966
 967         timewait_info->work.local_id = local_id;
 968         INIT_DELAYED_WORK(&timewait_info->work.work, cm_work_handler);
 969         timewait_info->work.cm_event.event = IB_CM_TIMEWAIT_EXIT;
 970         return timewait_info;
 971 }
 972
 973 static void cm_enter_timewait(struct cm_id_private *cm_id_priv)
 974 {
 975         int wait_time;
 976         unsigned long flags;
 977         struct cm_device *cm_dev;
 978
 979         lockdep_assert_held(&cm_id_priv->lock);
 980
 981         cm_dev = ib_get_client_data(cm_id_priv->id.device, &cm_client);
 982         if (!cm_dev)
 983                 return;
 984
 985         spin_lock_irqsave(&cm.lock, flags);
 986         cm_remove_remote(cm_id_priv);
 987         list_add_tail(&cm_id_priv->timewait_info->list, &cm.timewait_list);
 988         spin_unlock_irqrestore(&cm.lock, flags);
 989
 990         /*
 991          * The cm_id could be destroyed by the user before we exit timewait.
 992          * To protect against this, we search for the cm_id after exiting
 993          * timewait before notifying the user that we've exited timewait.
 994          */
 995         cm_id_priv->id.state = IB_CM_TIMEWAIT;
 996         wait_time = cm_convert_to_ms(cm_id_priv->av.timeout);
 997
 998         /* Check if the device started its remove_one */
 999         spin_lock_irqsave(&cm.lock, flags);
1000         if (!cm_dev->going_down)
1001                 queue_delayed_work(cm.wq, &cm_id_priv->timewait_info->work.work,
1002                                    msecs_to_jiffies(wait_time));
1003         spin_unlock_irqrestore(&cm.lock, flags);
1004
1005         /*
1006          * The timewait_info is converted into a work and gets freed during
1007          * cm_free_work() in cm_timewait_handler().
1008          */
1009         BUILD_BUG_ON(offsetof(struct cm_timewait_info, work) != 0);
1010         cm_id_priv->timewait_info = NULL;
1011 }
1012
1013 static void cm_reset_to_idle(struct cm_id_private *cm_id_priv)
1014 {
1015         unsigned long flags;
1016
1017         lockdep_assert_held(&cm_id_priv->lock);
1018
1019         cm_id_priv->id.state = IB_CM_IDLE;
1020         if (cm_id_priv->timewait_info) {
1021                 spin_lock_irqsave(&cm.lock, flags);
1022                 cm_remove_remote(cm_id_priv);
1023                 spin_unlock_irqrestore(&cm.lock, flags);
1024                 kfree(cm_id_priv->timewait_info);
1025                 cm_id_priv->timewait_info = NULL;
1026         }
1027 }
1028
1029 static noinline void cm_destroy_id_wait_timeout(struct ib_cm_id *cm_id,
1030                                                 enum ib_cm_state old_state)
1031 {
1032         struct cm_id_private *cm_id_priv;
1033
1034         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
1035         pr_err("%s: cm_id=%p timed out. state %d -> %d, refcnt=%d\n", __func__,
1036                cm_id, old_state, cm_id->state, refcount_read(&cm_id_priv->refcount));
1037 }
1038
1039 static void cm_destroy_id(struct ib_cm_id *cm_id, int err)
1040 {
1041         struct cm_id_private *cm_id_priv;
1042         enum ib_cm_state old_state;
1043         struct cm_work *work;
1044         int ret;
1045
1046         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
1047         spin_lock_irq(&cm_id_priv->lock);
1048         old_state = cm_id->state;
1049 retest:
1050         switch (cm_id->state) {
1051         case IB_CM_LISTEN:
1052                 spin_lock(&cm.lock);
1053                 if (--cm_id_priv->listen_sharecount > 0) {
1054                         /* The id is still shared. */
1055                         WARN_ON(refcount_read(&cm_id_priv->refcount) == 1);
1056                         spin_unlock(&cm.lock);
1057                         spin_unlock_irq(&cm_id_priv->lock);
1058                         cm_deref_id(cm_id_priv);
1059                         return;
1060                 }
1061                 cm_id->state = IB_CM_IDLE;
1062                 rb_erase(&cm_id_priv->service_node, &cm.listen_service_table);
1063                 RB_CLEAR_NODE(&cm_id_priv->service_node);
1064                 spin_unlock(&cm.lock);
1065                 break;
1066         case IB_CM_SIDR_REQ_SENT:
1067                 cm_id->state = IB_CM_IDLE;
1068                 ib_cancel_mad(cm_id_priv->msg);
1069                 break;
1070         case IB_CM_SIDR_REQ_RCVD:
1071                 cm_send_sidr_rep_locked(cm_id_priv,
1072                                         &(struct ib_cm_sidr_rep_param){
1073                                                 .status = IB_SIDR_REJECT });
1074                 /* cm_send_sidr_rep_locked will not move to IDLE if it fails */
1075                 cm_id->state = IB_CM_IDLE;
1076                 break;
1077         case IB_CM_REQ_SENT:
1078         case IB_CM_MRA_REQ_RCVD:
1079                 ib_cancel_mad(cm_id_priv->msg);
1080                 cm_send_rej_locked(cm_id_priv, IB_CM_REJ_TIMEOUT,
1081                                    &cm_id_priv->id.device->node_guid,
1082                                    sizeof(cm_id_priv->id.device->node_guid),
1083                                    NULL, 0);
1084                 break;
1085         case IB_CM_REQ_RCVD:
1086                 if (err == -ENOMEM) {
1087                         /* Do not reject to allow future retries. */
1088                         cm_reset_to_idle(cm_id_priv);
1089                 } else {
1090                         cm_send_rej_locked(cm_id_priv,
1091                                            IB_CM_REJ_CONSUMER_DEFINED, NULL, 0,
1092                                            NULL, 0);
1093                 }
1094                 break;
1095         case IB_CM_REP_SENT:
1096         case IB_CM_MRA_REP_RCVD:
1097                 ib_cancel_mad(cm_id_priv->msg);
1098                 cm_send_rej_locked(cm_id_priv, IB_CM_REJ_CONSUMER_DEFINED, NULL,
1099                                    0, NULL, 0);
1100                 goto retest;
1101         case IB_CM_MRA_REQ_SENT:
1102         case IB_CM_REP_RCVD:
1103         case IB_CM_MRA_REP_SENT:
1104                 cm_send_rej_locked(cm_id_priv, IB_CM_REJ_CONSUMER_DEFINED, NULL,
1105                                    0, NULL, 0);
1106                 break;
1107         case IB_CM_ESTABLISHED:
1108                 if (cm_id_priv->qp_type == IB_QPT_XRC_TGT) {
1109                         cm_id->state = IB_CM_IDLE;
1110                         break;
1111                 }
1112                 cm_send_dreq_locked(cm_id_priv, NULL, 0);
1113                 goto retest;
1114         case IB_CM_DREQ_SENT:
1115                 ib_cancel_mad(cm_id_priv->msg);
1116                 cm_enter_timewait(cm_id_priv);
1117                 goto retest;
1118         case IB_CM_DREQ_RCVD:
1119                 cm_send_drep_locked(cm_id_priv, NULL, 0);
1120                 WARN_ON(cm_id->state != IB_CM_TIMEWAIT);
1121                 goto retest;
1122         case IB_CM_TIMEWAIT:
1123                 /*
1124                  * The cm_acquire_id in cm_timewait_handler will stop working
1125                  * once we do xa_erase below, so just move to idle here for
1126                  * consistency.
1127                  */
1128                 cm_id->state = IB_CM_IDLE;
1129                 break;
1130         case IB_CM_IDLE:
1131                 break;
1132         }
1133         WARN_ON(cm_id->state != IB_CM_IDLE);
1134
1135         spin_lock(&cm.lock);
1136         /* Required for cleanup paths related cm_req_handler() */
1137         if (cm_id_priv->timewait_info) {
1138                 cm_remove_remote(cm_id_priv);
1139                 kfree(cm_id_priv->timewait_info);
1140                 cm_id_priv->timewait_info = NULL;
1141         }
1142
1143         WARN_ON(cm_id_priv->listen_sharecount);
1144         WARN_ON(!RB_EMPTY_NODE(&cm_id_priv->service_node));
1145         if (!RB_EMPTY_NODE(&cm_id_priv->sidr_id_node))
1146                 rb_erase(&cm_id_priv->sidr_id_node, &cm.remote_sidr_table);
1147         spin_unlock(&cm.lock);
1148         spin_unlock_irq(&cm_id_priv->lock);
1149
1150         xa_erase(&cm.local_id_table, cm_local_id(cm_id->local_id));
1151         cm_deref_id(cm_id_priv);
1152         do {
1153                 ret = wait_for_completion_timeout(&cm_id_priv->comp,
1154                                                   msecs_to_jiffies(
1155                                                   CM_DESTROY_ID_WAIT_TIMEOUT));
1156                 if (!ret) /* timeout happened */
1157                         cm_destroy_id_wait_timeout(cm_id, old_state);
1158         } while (!ret);
1159
1160         while ((work = cm_dequeue_work(cm_id_priv)) != NULL)
1161                 cm_free_work(work);
1162
1163         cm_destroy_av(&cm_id_priv->av);
1164         cm_destroy_av(&cm_id_priv->alt_av);
1165         kfree(cm_id_priv->private_data);
1166         kfree_rcu(cm_id_priv, rcu);
1167 }
1168
1169 void ib_destroy_cm_id(struct ib_cm_id *cm_id)
1170 {
1171         cm_destroy_id(cm_id, 0);
1172 }
1173 EXPORT_SYMBOL(ib_destroy_cm_id);
1174
1175 static int cm_init_listen(struct cm_id_private *cm_id_priv, __be64 service_id)
1176 {
1177         if ((service_id & IB_SERVICE_ID_AGN_MASK) == IB_CM_ASSIGN_SERVICE_ID &&
1178             (service_id != IB_CM_ASSIGN_SERVICE_ID))
1179                 return -EINVAL;
1180
1181         if (service_id == IB_CM_ASSIGN_SERVICE_ID)
1182                 cm_id_priv->id.service_id = cpu_to_be64(cm.listen_service_id++);
1183         else
1184                 cm_id_priv->id.service_id = service_id;
1185
1186         return 0;
1187 }
1188
1189 /**
1190  * ib_cm_listen - Initiates listening on the specified service ID for
1191  *   connection and service ID resolution requests.
1192  * @cm_id: Connection identifier associated with the listen request.
1193  * @service_id: Service identifier matched against incoming connection
1194  *   and service ID resolution requests.  The service ID should be specified
1195  *   network-byte order.  If set to IB_CM_ASSIGN_SERVICE_ID, the CM will
1196  *   assign a service ID to the caller.
1197  */
1198 int ib_cm_listen(struct ib_cm_id *cm_id, __be64 service_id)
1199 {
1200         struct cm_id_private *cm_id_priv =
1201                 container_of(cm_id, struct cm_id_private, id);
1202         unsigned long flags;
1203         int ret;
1204
1205         spin_lock_irqsave(&cm_id_priv->lock, flags);
1206         if (cm_id_priv->id.state != IB_CM_IDLE) {
1207                 ret = -EINVAL;
1208                 goto out;
1209         }
1210
1211         ret = cm_init_listen(cm_id_priv, service_id);
1212         if (ret)
1213                 goto out;
1214
1215         if (!cm_insert_listen(cm_id_priv, NULL)) {
1216                 ret = -EBUSY;
1217                 goto out;
1218         }
1219
1220         cm_id_priv->id.state = IB_CM_LISTEN;
1221         ret = 0;
1222
1223 out:
1224         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1225         return ret;
1226 }
1227 EXPORT_SYMBOL(ib_cm_listen);
1228
1229 /**
1230  * ib_cm_insert_listen - Create a new listening ib_cm_id and listen on
1231  *                       the given service ID.
1232  *
1233  * If there's an existing ID listening on that same device and service ID,
1234  * return it.
1235  *
1236  * @device: Device associated with the cm_id.  All related communication will
1237  * be associated with the specified device.
1238  * @cm_handler: Callback invoked to notify the user of CM events.
1239  * @service_id: Service identifier matched against incoming connection
1240  *   and service ID resolution requests.  The service ID should be specified
1241  *   network-byte order.  If set to IB_CM_ASSIGN_SERVICE_ID, the CM will
1242  *   assign a service ID to the caller.
1243  *
1244  * Callers should call ib_destroy_cm_id when done with the listener ID.
1245  */
1246 struct ib_cm_id *ib_cm_insert_listen(struct ib_device *device,
1247                                      ib_cm_handler cm_handler,
1248                                      __be64 service_id)
1249 {
1250         struct cm_id_private *listen_id_priv;
1251         struct cm_id_private *cm_id_priv;
1252         int err = 0;
1253
1254         /* Create an ID in advance, since the creation may sleep */
1255         cm_id_priv = cm_alloc_id_priv(device, cm_handler, NULL);
1256         if (IS_ERR(cm_id_priv))
1257                 return ERR_CAST(cm_id_priv);
1258
1259         err = cm_init_listen(cm_id_priv, service_id);
1260         if (err) {
1261                 ib_destroy_cm_id(&cm_id_priv->id);
1262                 return ERR_PTR(err);
1263         }
1264
1265         spin_lock_irq(&cm_id_priv->lock);
1266         listen_id_priv = cm_insert_listen(cm_id_priv, cm_handler);
1267         if (listen_id_priv != cm_id_priv) {
1268                 spin_unlock_irq(&cm_id_priv->lock);
1269                 ib_destroy_cm_id(&cm_id_priv->id);
1270                 if (!listen_id_priv)
1271                         return ERR_PTR(-EINVAL);
1272                 return &listen_id_priv->id;
1273         }
1274         cm_id_priv->id.state = IB_CM_LISTEN;
1275         spin_unlock_irq(&cm_id_priv->lock);
1276
1277         /*
1278          * A listen ID does not need to be in the xarray since it does not
1279          * receive mads, is not placed in the remote_id or remote_qpn rbtree,
1280          * and does not enter timewait.
1281          */
1282
1283         return &cm_id_priv->id;
1284 }
1285 EXPORT_SYMBOL(ib_cm_insert_listen);
1286
1287 static __be64 cm_form_tid(struct cm_id_private *cm_id_priv)
1288 {
1289         u64 hi_tid = 0, low_tid;
1290
1291         lockdep_assert_held(&cm_id_priv->lock);
1292
1293         low_tid = (u64)cm_id_priv->id.local_id;
1294         if (!cm_id_priv->av.port)
1295                 return cpu_to_be64(low_tid);
1296
1297         spin_lock(&cm_id_priv->av.port->cm_dev->mad_agent_lock);
1298         if (cm_id_priv->av.port->mad_agent)
1299                 hi_tid = ((u64)cm_id_priv->av.port->mad_agent->hi_tid) << 32;
1300         spin_unlock(&cm_id_priv->av.port->cm_dev->mad_agent_lock);
1301         return cpu_to_be64(hi_tid | low_tid);
1302 }
1303
1304 static void cm_format_mad_hdr(struct ib_mad_hdr *hdr,
1305                               __be16 attr_id, __be64 tid)
1306 {
1307         hdr->base_version  = IB_MGMT_BASE_VERSION;
1308         hdr->mgmt_class    = IB_MGMT_CLASS_CM;
1309         hdr->class_version = IB_CM_CLASS_VERSION;
1310         hdr->method        = IB_MGMT_METHOD_SEND;
1311         hdr->attr_id       = attr_id;
1312         hdr->tid           = tid;
1313 }
1314
1315 static void cm_format_mad_ece_hdr(struct ib_mad_hdr *hdr, __be16 attr_id,
1316                                   __be64 tid, u32 attr_mod)
1317 {
1318         cm_format_mad_hdr(hdr, attr_id, tid);
1319         hdr->attr_mod = cpu_to_be32(attr_mod);
1320 }
1321
1322 static void cm_format_req(struct cm_req_msg *req_msg,
1323                           struct cm_id_private *cm_id_priv,
1324                           struct ib_cm_req_param *param)
1325 {
1326         struct sa_path_rec *pri_path = param->primary_path;
1327         struct sa_path_rec *alt_path = param->alternate_path;
1328         bool pri_ext = false;
1329         __be16 lid;
1330
1331         if (pri_path->rec_type == SA_PATH_REC_TYPE_OPA)
1332                 pri_ext = opa_is_extended_lid(pri_path->opa.dlid,
1333                                               pri_path->opa.slid);
1334
1335         cm_format_mad_ece_hdr(&req_msg->hdr, CM_REQ_ATTR_ID,
1336                               cm_form_tid(cm_id_priv), param->ece.attr_mod);
1337
1338         IBA_SET(CM_REQ_LOCAL_COMM_ID, req_msg,
1339                 be32_to_cpu(cm_id_priv->id.local_id));
1340         IBA_SET(CM_REQ_SERVICE_ID, req_msg, be64_to_cpu(param->service_id));
1341         IBA_SET(CM_REQ_LOCAL_CA_GUID, req_msg,
1342                 be64_to_cpu(cm_id_priv->id.device->node_guid));
1343         IBA_SET(CM_REQ_LOCAL_QPN, req_msg, param->qp_num);
1344         IBA_SET(CM_REQ_INITIATOR_DEPTH, req_msg, param->initiator_depth);
1345         IBA_SET(CM_REQ_REMOTE_CM_RESPONSE_TIMEOUT, req_msg,
1346                 param->remote_cm_response_timeout);
1347         cm_req_set_qp_type(req_msg, param->qp_type);
1348         IBA_SET(CM_REQ_END_TO_END_FLOW_CONTROL, req_msg, param->flow_control);
1349         IBA_SET(CM_REQ_STARTING_PSN, req_msg, param->starting_psn);
1350         IBA_SET(CM_REQ_LOCAL_CM_RESPONSE_TIMEOUT, req_msg,
1351                 param->local_cm_response_timeout);
1352         IBA_SET(CM_REQ_PARTITION_KEY, req_msg,
1353                 be16_to_cpu(param->primary_path->pkey));
1354         IBA_SET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg,
1355                 param->primary_path->mtu);
1356         IBA_SET(CM_REQ_MAX_CM_RETRIES, req_msg, param->max_cm_retries);
1357
1358         if (param->qp_type != IB_QPT_XRC_INI) {
1359                 IBA_SET(CM_REQ_RESPONDER_RESOURCES, req_msg,
1360                         param->responder_resources);
1361                 IBA_SET(CM_REQ_RETRY_COUNT, req_msg, param->retry_count);
1362                 IBA_SET(CM_REQ_RNR_RETRY_COUNT, req_msg,
1363                         param->rnr_retry_count);
1364                 IBA_SET(CM_REQ_SRQ, req_msg, param->srq);
1365         }
1366
1367         *IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg) =
1368                 pri_path->sgid;
1369         *IBA_GET_MEM_PTR(CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg) =
1370                 pri_path->dgid;
1371         if (pri_ext) {
1372                 IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg)
1373                         ->global.interface_id =
1374                         OPA_MAKE_ID(be32_to_cpu(pri_path->opa.slid));
1375                 IBA_GET_MEM_PTR(CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg)
1376                         ->global.interface_id =
1377                         OPA_MAKE_ID(be32_to_cpu(pri_path->opa.dlid));
1378         }
1379         if (pri_path->hop_limit <= 1) {
1380                 IBA_SET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg,
1381                         be16_to_cpu(pri_ext ? 0 :
1382                                               htons(ntohl(sa_path_get_slid(
1383                                                       pri_path)))));
1384                 IBA_SET(CM_REQ_PRIMARY_REMOTE_PORT_LID, req_msg,
1385                         be16_to_cpu(pri_ext ? 0 :
1386                                               htons(ntohl(sa_path_get_dlid(
1387                                                       pri_path)))));
1388         } else {
1389
1390                 if (param->primary_path_inbound) {
1391                         lid = param->primary_path_inbound->ib.dlid;
1392                         IBA_SET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg,
1393                                 be16_to_cpu(lid));
1394                 } else
1395                         IBA_SET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg,
1396                                 be16_to_cpu(IB_LID_PERMISSIVE));
1397
1398                 /* Work-around until there's a way to obtain remote LID info */
1399                 IBA_SET(CM_REQ_PRIMARY_REMOTE_PORT_LID, req_msg,
1400                         be16_to_cpu(IB_LID_PERMISSIVE));
1401         }
1402         IBA_SET(CM_REQ_PRIMARY_FLOW_LABEL, req_msg,
1403                 be32_to_cpu(pri_path->flow_label));
1404         IBA_SET(CM_REQ_PRIMARY_PACKET_RATE, req_msg, pri_path->rate);
1405         IBA_SET(CM_REQ_PRIMARY_TRAFFIC_CLASS, req_msg, pri_path->traffic_class);
1406         IBA_SET(CM_REQ_PRIMARY_HOP_LIMIT, req_msg, pri_path->hop_limit);
1407         IBA_SET(CM_REQ_PRIMARY_SL, req_msg, pri_path->sl);
1408         IBA_SET(CM_REQ_PRIMARY_SUBNET_LOCAL, req_msg,
1409                 (pri_path->hop_limit <= 1));
1410         IBA_SET(CM_REQ_PRIMARY_LOCAL_ACK_TIMEOUT, req_msg,
1411                 cm_ack_timeout(cm_id_priv->av.port->cm_dev->ack_delay,
1412                                pri_path->packet_life_time));
1413
1414         if (alt_path) {
1415                 bool alt_ext = false;
1416
1417                 if (alt_path->rec_type == SA_PATH_REC_TYPE_OPA)
1418                         alt_ext = opa_is_extended_lid(alt_path->opa.dlid,
1419                                                       alt_path->opa.slid);
1420
1421                 *IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_LOCAL_PORT_GID, req_msg) =
1422                         alt_path->sgid;
1423                 *IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_REMOTE_PORT_GID, req_msg) =
1424                         alt_path->dgid;
1425                 if (alt_ext) {
1426                         IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_LOCAL_PORT_GID,
1427                                         req_msg)
1428                                 ->global.interface_id =
1429                                 OPA_MAKE_ID(be32_to_cpu(alt_path->opa.slid));
1430                         IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_REMOTE_PORT_GID,
1431                                         req_msg)
1432                                 ->global.interface_id =
1433                                 OPA_MAKE_ID(be32_to_cpu(alt_path->opa.dlid));
1434                 }
1435                 if (alt_path->hop_limit <= 1) {
1436                         IBA_SET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg,
1437                                 be16_to_cpu(
1438                                         alt_ext ? 0 :
1439                                                   htons(ntohl(sa_path_get_slid(
1440                                                           alt_path)))));
1441                         IBA_SET(CM_REQ_ALTERNATE_REMOTE_PORT_LID, req_msg,
1442                                 be16_to_cpu(
1443                                         alt_ext ? 0 :
1444                                                   htons(ntohl(sa_path_get_dlid(
1445                                                           alt_path)))));
1446                 } else {
1447                         IBA_SET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg,
1448                                 be16_to_cpu(IB_LID_PERMISSIVE));
1449                         IBA_SET(CM_REQ_ALTERNATE_REMOTE_PORT_LID, req_msg,
1450                                 be16_to_cpu(IB_LID_PERMISSIVE));
1451                 }
1452                 IBA_SET(CM_REQ_ALTERNATE_FLOW_LABEL, req_msg,
1453                         be32_to_cpu(alt_path->flow_label));
1454                 IBA_SET(CM_REQ_ALTERNATE_PACKET_RATE, req_msg, alt_path->rate);
1455                 IBA_SET(CM_REQ_ALTERNATE_TRAFFIC_CLASS, req_msg,
1456                         alt_path->traffic_class);
1457                 IBA_SET(CM_REQ_ALTERNATE_HOP_LIMIT, req_msg,
1458                         alt_path->hop_limit);
1459                 IBA_SET(CM_REQ_ALTERNATE_SL, req_msg, alt_path->sl);
1460                 IBA_SET(CM_REQ_ALTERNATE_SUBNET_LOCAL, req_msg,
1461                         (alt_path->hop_limit <= 1));
1462                 IBA_SET(CM_REQ_ALTERNATE_LOCAL_ACK_TIMEOUT, req_msg,
1463                         cm_ack_timeout(cm_id_priv->av.port->cm_dev->ack_delay,
1464                                        alt_path->packet_life_time));
1465         }
1466         IBA_SET(CM_REQ_VENDOR_ID, req_msg, param->ece.vendor_id);
1467
1468         if (param->private_data && param->private_data_len)
1469                 IBA_SET_MEM(CM_REQ_PRIVATE_DATA, req_msg, param->private_data,
1470                             param->private_data_len);
1471 }
1472
1473 static int cm_validate_req_param(struct ib_cm_req_param *param)
1474 {
1475         if (!param->primary_path)
1476                 return -EINVAL;
1477
1478         if (param->qp_type != IB_QPT_RC && param->qp_type != IB_QPT_UC &&
1479             param->qp_type != IB_QPT_XRC_INI)
1480                 return -EINVAL;
1481
1482         if (param->private_data &&
1483             param->private_data_len > IB_CM_REQ_PRIVATE_DATA_SIZE)
1484                 return -EINVAL;
1485
1486         if (param->alternate_path &&
1487             (param->alternate_path->pkey != param->primary_path->pkey ||
1488              param->alternate_path->mtu != param->primary_path->mtu))
1489                 return -EINVAL;
1490
1491         return 0;
1492 }
1493
1494 int ib_send_cm_req(struct ib_cm_id *cm_id,
1495                    struct ib_cm_req_param *param)
1496 {
1497         struct cm_av av = {}, alt_av = {};
1498         struct cm_id_private *cm_id_priv;
1499         struct ib_mad_send_buf *msg;
1500         struct cm_req_msg *req_msg;
1501         unsigned long flags;
1502         int ret;
1503
1504         ret = cm_validate_req_param(param);
1505         if (ret)
1506                 return ret;
1507
1508         /* Verify that we're not in timewait. */
1509         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
1510         spin_lock_irqsave(&cm_id_priv->lock, flags);
1511         if (cm_id->state != IB_CM_IDLE || WARN_ON(cm_id_priv->timewait_info)) {
1512                 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1513                 return -EINVAL;
1514         }
1515         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1516
1517         cm_id_priv->timewait_info = cm_create_timewait_info(cm_id_priv->
1518                                                             id.local_id);
1519         if (IS_ERR(cm_id_priv->timewait_info)) {
1520                 ret = PTR_ERR(cm_id_priv->timewait_info);
1521                 cm_id_priv->timewait_info = NULL;
1522                 return ret;
1523         }
1524
1525         ret = cm_init_av_by_path(param->primary_path,
1526                                  param->ppath_sgid_attr, &av);
1527         if (ret)
1528                 return ret;
1529         if (param->alternate_path) {
1530                 ret = cm_init_av_by_path(param->alternate_path, NULL,
1531                                          &alt_av);
1532                 if (ret) {
1533                         cm_destroy_av(&av);
1534                         return ret;
1535                 }
1536         }
1537         cm_id->service_id = param->service_id;
1538         cm_id_priv->timeout_ms = cm_convert_to_ms(
1539                                     param->primary_path->packet_life_time) * 2 +
1540                                  cm_convert_to_ms(
1541                                     param->remote_cm_response_timeout);
1542         cm_id_priv->max_cm_retries = param->max_cm_retries;
1543         cm_id_priv->initiator_depth = param->initiator_depth;
1544         cm_id_priv->responder_resources = param->responder_resources;
1545         cm_id_priv->retry_count = param->retry_count;
1546         cm_id_priv->path_mtu = param->primary_path->mtu;
1547         cm_id_priv->pkey = param->primary_path->pkey;
1548         cm_id_priv->qp_type = param->qp_type;
1549
1550         spin_lock_irqsave(&cm_id_priv->lock, flags);
1551
1552         cm_move_av_from_path(&cm_id_priv->av, &av);
1553         if (param->primary_path_outbound)
1554                 cm_id_priv->av.dlid_datapath =
1555                         be16_to_cpu(param->primary_path_outbound->ib.dlid);
1556
1557         if (param->alternate_path)
1558                 cm_move_av_from_path(&cm_id_priv->alt_av, &alt_av);
1559
1560         msg = cm_alloc_priv_msg(cm_id_priv);
1561         if (IS_ERR(msg)) {
1562                 ret = PTR_ERR(msg);
1563                 goto out_unlock;
1564         }
1565
1566         req_msg = (struct cm_req_msg *)msg->mad;
1567         cm_format_req(req_msg, cm_id_priv, param);
1568         cm_id_priv->tid = req_msg->hdr.tid;
1569         msg->timeout_ms = cm_id_priv->timeout_ms;
1570         msg->context[1] = (void *)(unsigned long)IB_CM_REQ_SENT;
1571
1572         cm_id_priv->local_qpn = cpu_to_be32(IBA_GET(CM_REQ_LOCAL_QPN, req_msg));
1573         cm_id_priv->rq_psn = cpu_to_be32(IBA_GET(CM_REQ_STARTING_PSN, req_msg));
1574
1575         trace_icm_send_req(&cm_id_priv->id);
1576         ret = ib_post_send_mad(msg, NULL);
1577         if (ret)
1578                 goto out_free;
1579         BUG_ON(cm_id->state != IB_CM_IDLE);
1580         cm_id->state = IB_CM_REQ_SENT;
1581         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1582         return 0;
1583 out_free:
1584         cm_free_priv_msg(msg);
1585 out_unlock:
1586         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
1587         return ret;
1588 }
1589 EXPORT_SYMBOL(ib_send_cm_req);
1590
1591 static int cm_issue_rej(struct cm_port *port,
1592                         struct ib_mad_recv_wc *mad_recv_wc,
1593                         enum ib_cm_rej_reason reason,
1594                         enum cm_msg_response msg_rejected,
1595                         void *ari, u8 ari_length)
1596 {
1597         struct ib_mad_send_buf *msg = NULL;
1598         struct cm_rej_msg *rej_msg, *rcv_msg;
1599         int ret;
1600
1601         ret = cm_alloc_response_msg(port, mad_recv_wc, &msg);
1602         if (ret)
1603                 return ret;
1604
1605         /* We just need common CM header information.  Cast to any message. */
1606         rcv_msg = (struct cm_rej_msg *) mad_recv_wc->recv_buf.mad;
1607         rej_msg = (struct cm_rej_msg *) msg->mad;
1608
1609         cm_format_mad_hdr(&rej_msg->hdr, CM_REJ_ATTR_ID, rcv_msg->hdr.tid);
1610         IBA_SET(CM_REJ_REMOTE_COMM_ID, rej_msg,
1611                 IBA_GET(CM_REJ_LOCAL_COMM_ID, rcv_msg));
1612         IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg,
1613                 IBA_GET(CM_REJ_REMOTE_COMM_ID, rcv_msg));
1614         IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, msg_rejected);
1615         IBA_SET(CM_REJ_REASON, rej_msg, reason);
1616
1617         if (ari && ari_length) {
1618                 IBA_SET(CM_REJ_REJECTED_INFO_LENGTH, rej_msg, ari_length);
1619                 IBA_SET_MEM(CM_REJ_ARI, rej_msg, ari, ari_length);
1620         }
1621
1622         trace_icm_issue_rej(
1623                 IBA_GET(CM_REJ_LOCAL_COMM_ID, rcv_msg),
1624                 IBA_GET(CM_REJ_REMOTE_COMM_ID, rcv_msg));
1625         ret = ib_post_send_mad(msg, NULL);
1626         if (ret)
1627                 cm_free_response_msg(msg);
1628
1629         return ret;
1630 }
1631
1632 static bool cm_req_has_alt_path(struct cm_req_msg *req_msg)
1633 {
1634         return ((cpu_to_be16(
1635                         IBA_GET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg))) ||
1636                 (ib_is_opa_gid(IBA_GET_MEM_PTR(CM_REQ_ALTERNATE_LOCAL_PORT_GID,
1637                                                req_msg))));
1638 }
1639
1640 static void cm_path_set_rec_type(struct ib_device *ib_device, u32 port_num,
1641                                  struct sa_path_rec *path, union ib_gid *gid)
1642 {
1643         if (ib_is_opa_gid(gid) && rdma_cap_opa_ah(ib_device, port_num))
1644                 path->rec_type = SA_PATH_REC_TYPE_OPA;
1645         else
1646                 path->rec_type = SA_PATH_REC_TYPE_IB;
1647 }
1648
1649 static void cm_format_path_lid_from_req(struct cm_req_msg *req_msg,
1650                                         struct sa_path_rec *primary_path,
1651                                         struct sa_path_rec *alt_path,
1652                                         struct ib_wc *wc)
1653 {
1654         u32 lid;
1655
1656         if (primary_path->rec_type != SA_PATH_REC_TYPE_OPA) {
1657                 sa_path_set_dlid(primary_path, wc->slid);
1658                 sa_path_set_slid(primary_path,
1659                                  IBA_GET(CM_REQ_PRIMARY_REMOTE_PORT_LID,
1660                                          req_msg));
1661         } else {
1662                 lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
1663                         CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg));
1664                 sa_path_set_dlid(primary_path, lid);
1665
1666                 lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
1667                         CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg));
1668                 sa_path_set_slid(primary_path, lid);
1669         }
1670
1671         if (!cm_req_has_alt_path(req_msg))
1672                 return;
1673
1674         if (alt_path->rec_type != SA_PATH_REC_TYPE_OPA) {
1675                 sa_path_set_dlid(alt_path,
1676                                  IBA_GET(CM_REQ_ALTERNATE_LOCAL_PORT_LID,
1677                                          req_msg));
1678                 sa_path_set_slid(alt_path,
1679                                  IBA_GET(CM_REQ_ALTERNATE_REMOTE_PORT_LID,
1680                                          req_msg));
1681         } else {
1682                 lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
1683                         CM_REQ_ALTERNATE_LOCAL_PORT_GID, req_msg));
1684                 sa_path_set_dlid(alt_path, lid);
1685
1686                 lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
1687                         CM_REQ_ALTERNATE_REMOTE_PORT_GID, req_msg));
1688                 sa_path_set_slid(alt_path, lid);
1689         }
1690 }
1691
1692 static void cm_format_paths_from_req(struct cm_req_msg *req_msg,
1693                                      struct sa_path_rec *primary_path,
1694                                      struct sa_path_rec *alt_path,
1695                                      struct ib_wc *wc)
1696 {
1697         primary_path->dgid =
1698                 *IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID, req_msg);
1699         primary_path->sgid =
1700                 *IBA_GET_MEM_PTR(CM_REQ_PRIMARY_REMOTE_PORT_GID, req_msg);
1701         primary_path->flow_label =
1702                 cpu_to_be32(IBA_GET(CM_REQ_PRIMARY_FLOW_LABEL, req_msg));
1703         primary_path->hop_limit = IBA_GET(CM_REQ_PRIMARY_HOP_LIMIT, req_msg);
1704         primary_path->traffic_class =
1705                 IBA_GET(CM_REQ_PRIMARY_TRAFFIC_CLASS, req_msg);
1706         primary_path->reversible = 1;
1707         primary_path->pkey =
1708                 cpu_to_be16(IBA_GET(CM_REQ_PARTITION_KEY, req_msg));
1709         primary_path->sl = IBA_GET(CM_REQ_PRIMARY_SL, req_msg);
1710         primary_path->mtu_selector = IB_SA_EQ;
1711         primary_path->mtu = IBA_GET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg);
1712         primary_path->rate_selector = IB_SA_EQ;
1713         primary_path->rate = IBA_GET(CM_REQ_PRIMARY_PACKET_RATE, req_msg);
1714         primary_path->packet_life_time_selector = IB_SA_EQ;
1715         primary_path->packet_life_time =
1716                 IBA_GET(CM_REQ_PRIMARY_LOCAL_ACK_TIMEOUT, req_msg);
1717         primary_path->packet_life_time -= (primary_path->packet_life_time > 0);
1718         primary_path->service_id =
1719                 cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg));
1720         if (sa_path_is_roce(primary_path))
1721                 primary_path->roce.route_resolved = false;
1722
1723         if (cm_req_has_alt_path(req_msg)) {
1724                 alt_path->dgid = *IBA_GET_MEM_PTR(
1725                         CM_REQ_ALTERNATE_LOCAL_PORT_GID, req_msg);
1726                 alt_path->sgid = *IBA_GET_MEM_PTR(
1727                         CM_REQ_ALTERNATE_REMOTE_PORT_GID, req_msg);
1728                 alt_path->flow_label = cpu_to_be32(
1729                         IBA_GET(CM_REQ_ALTERNATE_FLOW_LABEL, req_msg));
1730                 alt_path->hop_limit =
1731                         IBA_GET(CM_REQ_ALTERNATE_HOP_LIMIT, req_msg);
1732                 alt_path->traffic_class =
1733                         IBA_GET(CM_REQ_ALTERNATE_TRAFFIC_CLASS, req_msg);
1734                 alt_path->reversible = 1;
1735                 alt_path->pkey =
1736                         cpu_to_be16(IBA_GET(CM_REQ_PARTITION_KEY, req_msg));
1737                 alt_path->sl = IBA_GET(CM_REQ_ALTERNATE_SL, req_msg);
1738                 alt_path->mtu_selector = IB_SA_EQ;
1739                 alt_path->mtu =
1740                         IBA_GET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg);
1741                 alt_path->rate_selector = IB_SA_EQ;
1742                 alt_path->rate = IBA_GET(CM_REQ_ALTERNATE_PACKET_RATE, req_msg);
1743                 alt_path->packet_life_time_selector = IB_SA_EQ;
1744                 alt_path->packet_life_time =
1745                         IBA_GET(CM_REQ_ALTERNATE_LOCAL_ACK_TIMEOUT, req_msg);
1746                 alt_path->packet_life_time -= (alt_path->packet_life_time > 0);
1747                 alt_path->service_id =
1748                         cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg));
1749
1750                 if (sa_path_is_roce(alt_path))
1751                         alt_path->roce.route_resolved = false;
1752         }
1753         cm_format_path_lid_from_req(req_msg, primary_path, alt_path, wc);
1754 }
1755
1756 static u16 cm_get_bth_pkey(struct cm_work *work)
1757 {
1758         struct ib_device *ib_dev = work->port->cm_dev->ib_device;
1759         u32 port_num = work->port->port_num;
1760         u16 pkey_index = work->mad_recv_wc->wc->pkey_index;
1761         u16 pkey;
1762         int ret;
1763
1764         ret = ib_get_cached_pkey(ib_dev, port_num, pkey_index, &pkey);
1765         if (ret) {
1766                 dev_warn_ratelimited(&ib_dev->dev, "ib_cm: Couldn't retrieve pkey for incoming request (port %u, pkey index %u). %d\n",
1767                                      port_num, pkey_index, ret);
1768                 return 0;
1769         }
1770
1771         return pkey;
1772 }
1773
1774 /**
1775  * cm_opa_to_ib_sgid - Convert OPA SGID to IB SGID
1776  * ULPs (such as IPoIB) do not understand OPA GIDs and will
1777  * reject them as the local_gid will not match the sgid. Therefore,
1778  * change the pathrec's SGID to an IB SGID.
1779  *
1780  * @work: Work completion
1781  * @path: Path record
1782  */
1783 static void cm_opa_to_ib_sgid(struct cm_work *work,
1784                               struct sa_path_rec *path)
1785 {
1786         struct ib_device *dev = work->port->cm_dev->ib_device;
1787         u32 port_num = work->port->port_num;
1788
1789         if (rdma_cap_opa_ah(dev, port_num) &&
1790             (ib_is_opa_gid(&path->sgid))) {
1791                 union ib_gid sgid;
1792
1793                 if (rdma_query_gid(dev, port_num, 0, &sgid)) {
1794                         dev_warn(&dev->dev,
1795                                  "Error updating sgid in CM request\n");
1796                         return;
1797                 }
1798
1799                 path->sgid = sgid;
1800         }
1801 }
1802
1803 static void cm_format_req_event(struct cm_work *work,
1804                                 struct cm_id_private *cm_id_priv,
1805                                 struct ib_cm_id *listen_id)
1806 {
1807         struct cm_req_msg *req_msg;
1808         struct ib_cm_req_event_param *param;
1809
1810         req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad;
1811         param = &work->cm_event.param.req_rcvd;
1812         param->listen_id = listen_id;
1813         param->bth_pkey = cm_get_bth_pkey(work);
1814         param->port = cm_id_priv->av.port->port_num;
1815         param->primary_path = &work->path[0];
1816         cm_opa_to_ib_sgid(work, param->primary_path);
1817         if (cm_req_has_alt_path(req_msg)) {
1818                 param->alternate_path = &work->path[1];
1819                 cm_opa_to_ib_sgid(work, param->alternate_path);
1820         } else {
1821                 param->alternate_path = NULL;
1822         }
1823         param->remote_ca_guid =
1824                 cpu_to_be64(IBA_GET(CM_REQ_LOCAL_CA_GUID, req_msg));
1825         param->remote_qkey = IBA_GET(CM_REQ_LOCAL_Q_KEY, req_msg);
1826         param->remote_qpn = IBA_GET(CM_REQ_LOCAL_QPN, req_msg);
1827         param->qp_type = cm_req_get_qp_type(req_msg);
1828         param->starting_psn = IBA_GET(CM_REQ_STARTING_PSN, req_msg);
1829         param->responder_resources = IBA_GET(CM_REQ_INITIATOR_DEPTH, req_msg);
1830         param->initiator_depth = IBA_GET(CM_REQ_RESPONDER_RESOURCES, req_msg);
1831         param->local_cm_response_timeout =
1832                 IBA_GET(CM_REQ_REMOTE_CM_RESPONSE_TIMEOUT, req_msg);
1833         param->flow_control = IBA_GET(CM_REQ_END_TO_END_FLOW_CONTROL, req_msg);
1834         param->remote_cm_response_timeout =
1835                 IBA_GET(CM_REQ_LOCAL_CM_RESPONSE_TIMEOUT, req_msg);
1836         param->retry_count = IBA_GET(CM_REQ_RETRY_COUNT, req_msg);
1837         param->rnr_retry_count = IBA_GET(CM_REQ_RNR_RETRY_COUNT, req_msg);
1838         param->srq = IBA_GET(CM_REQ_SRQ, req_msg);
1839         param->ppath_sgid_attr = cm_id_priv->av.ah_attr.grh.sgid_attr;
1840         param->ece.vendor_id = IBA_GET(CM_REQ_VENDOR_ID, req_msg);
1841         param->ece.attr_mod = be32_to_cpu(req_msg->hdr.attr_mod);
1842
1843         work->cm_event.private_data =
1844                 IBA_GET_MEM_PTR(CM_REQ_PRIVATE_DATA, req_msg);
1845 }
1846
1847 static void cm_process_work(struct cm_id_private *cm_id_priv,
1848                             struct cm_work *work)
1849 {
1850         int ret;
1851
1852         /* We will typically only have the current event to report. */
1853         ret = cm_id_priv->id.cm_handler(&cm_id_priv->id, &work->cm_event);
1854         cm_free_work(work);
1855
1856         while (!ret && !atomic_add_negative(-1, &cm_id_priv->work_count)) {
1857                 spin_lock_irq(&cm_id_priv->lock);
1858                 work = cm_dequeue_work(cm_id_priv);
1859                 spin_unlock_irq(&cm_id_priv->lock);
1860                 if (!work)
1861                         return;
1862
1863                 ret = cm_id_priv->id.cm_handler(&cm_id_priv->id,
1864                                                 &work->cm_event);
1865                 cm_free_work(work);
1866         }
1867         cm_deref_id(cm_id_priv);
1868         if (ret)
1869                 cm_destroy_id(&cm_id_priv->id, ret);
1870 }
1871
1872 static void cm_format_mra(struct cm_mra_msg *mra_msg,
1873                           struct cm_id_private *cm_id_priv,
1874                           enum cm_msg_response msg_mraed, u8 service_timeout,
1875                           const void *private_data, u8 private_data_len)
1876 {
1877         cm_format_mad_hdr(&mra_msg->hdr, CM_MRA_ATTR_ID, cm_id_priv->tid);
1878         IBA_SET(CM_MRA_MESSAGE_MRAED, mra_msg, msg_mraed);
1879         IBA_SET(CM_MRA_LOCAL_COMM_ID, mra_msg,
1880                 be32_to_cpu(cm_id_priv->id.local_id));
1881         IBA_SET(CM_MRA_REMOTE_COMM_ID, mra_msg,
1882                 be32_to_cpu(cm_id_priv->id.remote_id));
1883         IBA_SET(CM_MRA_SERVICE_TIMEOUT, mra_msg, service_timeout);
1884
1885         if (private_data && private_data_len)
1886                 IBA_SET_MEM(CM_MRA_PRIVATE_DATA, mra_msg, private_data,
1887                             private_data_len);
1888 }
1889
1890 static void cm_format_rej(struct cm_rej_msg *rej_msg,
1891                           struct cm_id_private *cm_id_priv,
1892                           enum ib_cm_rej_reason reason, void *ari,
1893                           u8 ari_length, const void *private_data,
1894                           u8 private_data_len, enum ib_cm_state state)
1895 {
1896         lockdep_assert_held(&cm_id_priv->lock);
1897
1898         cm_format_mad_hdr(&rej_msg->hdr, CM_REJ_ATTR_ID, cm_id_priv->tid);
1899         IBA_SET(CM_REJ_REMOTE_COMM_ID, rej_msg,
1900                 be32_to_cpu(cm_id_priv->id.remote_id));
1901
1902         switch (state) {
1903         case IB_CM_REQ_RCVD:
1904                 IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg, be32_to_cpu(0));
1905                 IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, CM_MSG_RESPONSE_REQ);
1906                 break;
1907         case IB_CM_MRA_REQ_SENT:
1908                 IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg,
1909                         be32_to_cpu(cm_id_priv->id.local_id));
1910                 IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, CM_MSG_RESPONSE_REQ);
1911                 break;
1912         case IB_CM_REP_RCVD:
1913         case IB_CM_MRA_REP_SENT:
1914                 IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg,
1915                         be32_to_cpu(cm_id_priv->id.local_id));
1916                 IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg, CM_MSG_RESPONSE_REP);
1917                 break;
1918         default:
1919                 IBA_SET(CM_REJ_LOCAL_COMM_ID, rej_msg,
1920                         be32_to_cpu(cm_id_priv->id.local_id));
1921                 IBA_SET(CM_REJ_MESSAGE_REJECTED, rej_msg,
1922                         CM_MSG_RESPONSE_OTHER);
1923                 break;
1924         }
1925
1926         IBA_SET(CM_REJ_REASON, rej_msg, reason);
1927         if (ari && ari_length) {
1928                 IBA_SET(CM_REJ_REJECTED_INFO_LENGTH, rej_msg, ari_length);
1929                 IBA_SET_MEM(CM_REJ_ARI, rej_msg, ari, ari_length);
1930         }
1931
1932         if (private_data && private_data_len)
1933                 IBA_SET_MEM(CM_REJ_PRIVATE_DATA, rej_msg, private_data,
1934                             private_data_len);
1935 }
1936
1937 static void cm_dup_req_handler(struct cm_work *work,
1938                                struct cm_id_private *cm_id_priv)
1939 {
1940         struct ib_mad_send_buf *msg = NULL;
1941         int ret;
1942
1943         atomic_long_inc(
1944                 &work->port->counters[CM_RECV_DUPLICATES][CM_REQ_COUNTER]);
1945
1946         /* Quick state check to discard duplicate REQs. */
1947         spin_lock_irq(&cm_id_priv->lock);
1948         if (cm_id_priv->id.state == IB_CM_REQ_RCVD) {
1949                 spin_unlock_irq(&cm_id_priv->lock);
1950                 return;
1951         }
1952         spin_unlock_irq(&cm_id_priv->lock);
1953
1954         ret = cm_alloc_response_msg(work->port, work->mad_recv_wc, &msg);
1955         if (ret)
1956                 return;
1957
1958         spin_lock_irq(&cm_id_priv->lock);
1959         switch (cm_id_priv->id.state) {
1960         case IB_CM_MRA_REQ_SENT:
1961                 cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
1962                               CM_MSG_RESPONSE_REQ, cm_id_priv->service_timeout,
1963                               cm_id_priv->private_data,
1964                               cm_id_priv->private_data_len);
1965                 break;
1966         case IB_CM_TIMEWAIT:
1967                 cm_format_rej((struct cm_rej_msg *)msg->mad, cm_id_priv,
1968                               IB_CM_REJ_STALE_CONN, NULL, 0, NULL, 0,
1969                               IB_CM_TIMEWAIT);
1970                 break;
1971         default:
1972                 goto unlock;
1973         }
1974         spin_unlock_irq(&cm_id_priv->lock);
1975
1976         trace_icm_send_dup_req(&cm_id_priv->id);
1977         ret = ib_post_send_mad(msg, NULL);
1978         if (ret)
1979                 goto free;
1980         return;
1981
1982 unlock: spin_unlock_irq(&cm_id_priv->lock);
1983 free:   cm_free_response_msg(msg);
1984 }
1985
1986 static struct cm_id_private *cm_match_req(struct cm_work *work,
1987                                           struct cm_id_private *cm_id_priv)
1988 {
1989         struct cm_id_private *listen_cm_id_priv, *cur_cm_id_priv;
1990         struct cm_timewait_info *timewait_info;
1991         struct cm_req_msg *req_msg;
1992
1993         req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad;
1994
1995         /* Check for possible duplicate REQ. */
1996         spin_lock_irq(&cm.lock);
1997         timewait_info = cm_insert_remote_id(cm_id_priv->timewait_info);
1998         if (timewait_info) {
1999                 cur_cm_id_priv = cm_acquire_id(timewait_info->work.local_id,
2000                                            timewait_info->work.remote_id);
2001                 spin_unlock_irq(&cm.lock);
2002                 if (cur_cm_id_priv) {
2003                         cm_dup_req_handler(work, cur_cm_id_priv);
2004                         cm_deref_id(cur_cm_id_priv);
2005                 }
2006                 return NULL;
2007         }
2008
2009         /* Check for stale connections. */
2010         timewait_info = cm_insert_remote_qpn(cm_id_priv->timewait_info);
2011         if (timewait_info) {
2012                 cm_remove_remote(cm_id_priv);
2013                 cur_cm_id_priv = cm_acquire_id(timewait_info->work.local_id,
2014                                            timewait_info->work.remote_id);
2015
2016                 spin_unlock_irq(&cm.lock);
2017                 cm_issue_rej(work->port, work->mad_recv_wc,
2018                              IB_CM_REJ_STALE_CONN, CM_MSG_RESPONSE_REQ,
2019                              NULL, 0);
2020                 if (cur_cm_id_priv) {
2021                         ib_send_cm_dreq(&cur_cm_id_priv->id, NULL, 0);
2022                         cm_deref_id(cur_cm_id_priv);
2023                 }
2024                 return NULL;
2025         }
2026
2027         /* Find matching listen request. */
2028         listen_cm_id_priv = cm_find_listen(
2029                 cm_id_priv->id.device,
2030                 cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg)));
2031         if (!listen_cm_id_priv) {
2032                 cm_remove_remote(cm_id_priv);
2033                 spin_unlock_irq(&cm.lock);
2034                 cm_issue_rej(work->port, work->mad_recv_wc,
2035                              IB_CM_REJ_INVALID_SERVICE_ID, CM_MSG_RESPONSE_REQ,
2036                              NULL, 0);
2037                 return NULL;
2038         }
2039         spin_unlock_irq(&cm.lock);
2040         return listen_cm_id_priv;
2041 }
2042
2043 /*
2044  * Work-around for inter-subnet connections.  If the LIDs are permissive,
2045  * we need to override the LID/SL data in the REQ with the LID information
2046  * in the work completion.
2047  */
2048 static void cm_process_routed_req(struct cm_req_msg *req_msg, struct ib_wc *wc)
2049 {
2050         if (!IBA_GET(CM_REQ_PRIMARY_SUBNET_LOCAL, req_msg)) {
2051                 if (cpu_to_be16(IBA_GET(CM_REQ_PRIMARY_LOCAL_PORT_LID,
2052                                         req_msg)) == IB_LID_PERMISSIVE) {
2053                         IBA_SET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg,
2054                                 be16_to_cpu(ib_lid_be16(wc->slid)));
2055                         IBA_SET(CM_REQ_PRIMARY_SL, req_msg, wc->sl);
2056                 }
2057
2058                 if (cpu_to_be16(IBA_GET(CM_REQ_PRIMARY_REMOTE_PORT_LID,
2059                                         req_msg)) == IB_LID_PERMISSIVE)
2060                         IBA_SET(CM_REQ_PRIMARY_REMOTE_PORT_LID, req_msg,
2061                                 wc->dlid_path_bits);
2062         }
2063
2064         if (!IBA_GET(CM_REQ_ALTERNATE_SUBNET_LOCAL, req_msg)) {
2065                 if (cpu_to_be16(IBA_GET(CM_REQ_ALTERNATE_LOCAL_PORT_LID,
2066                                         req_msg)) == IB_LID_PERMISSIVE) {
2067                         IBA_SET(CM_REQ_ALTERNATE_LOCAL_PORT_LID, req_msg,
2068                                 be16_to_cpu(ib_lid_be16(wc->slid)));
2069                         IBA_SET(CM_REQ_ALTERNATE_SL, req_msg, wc->sl);
2070                 }
2071
2072                 if (cpu_to_be16(IBA_GET(CM_REQ_ALTERNATE_REMOTE_PORT_LID,
2073                                         req_msg)) == IB_LID_PERMISSIVE)
2074                         IBA_SET(CM_REQ_ALTERNATE_REMOTE_PORT_LID, req_msg,
2075                                 wc->dlid_path_bits);
2076         }
2077 }
2078
2079 static int cm_req_handler(struct cm_work *work)
2080 {
2081         struct cm_id_private *cm_id_priv, *listen_cm_id_priv;
2082         struct cm_req_msg *req_msg;
2083         const struct ib_global_route *grh;
2084         const struct ib_gid_attr *gid_attr;
2085         int ret;
2086
2087         req_msg = (struct cm_req_msg *)work->mad_recv_wc->recv_buf.mad;
2088
2089         cm_id_priv =
2090                 cm_alloc_id_priv(work->port->cm_dev->ib_device, NULL, NULL);
2091         if (IS_ERR(cm_id_priv))
2092                 return PTR_ERR(cm_id_priv);
2093
2094         cm_id_priv->id.remote_id =
2095                 cpu_to_be32(IBA_GET(CM_REQ_LOCAL_COMM_ID, req_msg));
2096         cm_id_priv->id.service_id =
2097                 cpu_to_be64(IBA_GET(CM_REQ_SERVICE_ID, req_msg));
2098         cm_id_priv->tid = req_msg->hdr.tid;
2099         cm_id_priv->timeout_ms = cm_convert_to_ms(
2100                 IBA_GET(CM_REQ_LOCAL_CM_RESPONSE_TIMEOUT, req_msg));
2101         cm_id_priv->max_cm_retries = IBA_GET(CM_REQ_MAX_CM_RETRIES, req_msg);
2102         cm_id_priv->remote_qpn =
2103                 cpu_to_be32(IBA_GET(CM_REQ_LOCAL_QPN, req_msg));
2104         cm_id_priv->initiator_depth =
2105                 IBA_GET(CM_REQ_RESPONDER_RESOURCES, req_msg);
2106         cm_id_priv->responder_resources =
2107                 IBA_GET(CM_REQ_INITIATOR_DEPTH, req_msg);
2108         cm_id_priv->path_mtu = IBA_GET(CM_REQ_PATH_PACKET_PAYLOAD_MTU, req_msg);
2109         cm_id_priv->pkey = cpu_to_be16(IBA_GET(CM_REQ_PARTITION_KEY, req_msg));
2110         cm_id_priv->sq_psn = cpu_to_be32(IBA_GET(CM_REQ_STARTING_PSN, req_msg));
2111         cm_id_priv->retry_count = IBA_GET(CM_REQ_RETRY_COUNT, req_msg);
2112         cm_id_priv->rnr_retry_count = IBA_GET(CM_REQ_RNR_RETRY_COUNT, req_msg);
2113         cm_id_priv->qp_type = cm_req_get_qp_type(req_msg);
2114
2115         ret = cm_init_av_for_response(work->port, work->mad_recv_wc->wc,
2116                                       work->mad_recv_wc->recv_buf.grh,
2117                                       &cm_id_priv->av);
2118         if (ret)
2119                 goto destroy;
2120         cm_id_priv->timewait_info = cm_create_timewait_info(cm_id_priv->
2121                                                             id.local_id);
2122         if (IS_ERR(cm_id_priv->timewait_info)) {
2123                 ret = PTR_ERR(cm_id_priv->timewait_info);
2124                 cm_id_priv->timewait_info = NULL;
2125                 goto destroy;
2126         }
2127         cm_id_priv->timewait_info->work.remote_id = cm_id_priv->id.remote_id;
2128         cm_id_priv->timewait_info->remote_ca_guid =
2129                 cpu_to_be64(IBA_GET(CM_REQ_LOCAL_CA_GUID, req_msg));
2130         cm_id_priv->timewait_info->remote_qpn = cm_id_priv->remote_qpn;
2131
2132         /*
2133          * Note that the ID pointer is not in the xarray at this point,
2134          * so this set is only visible to the local thread.
2135          */
2136         cm_id_priv->id.state = IB_CM_REQ_RCVD;
2137
2138         listen_cm_id_priv = cm_match_req(work, cm_id_priv);
2139         if (!listen_cm_id_priv) {
2140                 trace_icm_no_listener_err(&cm_id_priv->id);
2141                 cm_id_priv->id.state = IB_CM_IDLE;
2142                 ret = -EINVAL;
2143                 goto destroy;
2144         }
2145
2146         memset(&work->path[0], 0, sizeof(work->path[0]));
2147         if (cm_req_has_alt_path(req_msg))
2148                 memset(&work->path[1], 0, sizeof(work->path[1]));
2149         grh = rdma_ah_read_grh(&cm_id_priv->av.ah_attr);
2150         gid_attr = grh->sgid_attr;
2151
2152         if (cm_id_priv->av.ah_attr.type == RDMA_AH_ATTR_TYPE_ROCE) {
2153                 work->path[0].rec_type =
2154                         sa_conv_gid_to_pathrec_type(gid_attr->gid_type);
2155         } else {
2156                 cm_process_routed_req(req_msg, work->mad_recv_wc->wc);
2157                 cm_path_set_rec_type(
2158                         work->port->cm_dev->ib_device, work->port->port_num,
2159                         &work->path[0],
2160                         IBA_GET_MEM_PTR(CM_REQ_PRIMARY_LOCAL_PORT_GID,
2161                                         req_msg));
2162         }
2163         if (cm_req_has_alt_path(req_msg))
2164                 work->path[1].rec_type = work->path[0].rec_type;
2165         cm_format_paths_from_req(req_msg, &work->path[0],
2166                                  &work->path[1], work->mad_recv_wc->wc);
2167         if (cm_id_priv->av.ah_attr.type == RDMA_AH_ATTR_TYPE_ROCE)
2168                 sa_path_set_dmac(&work->path[0],
2169                                  cm_id_priv->av.ah_attr.roce.dmac);
2170         work->path[0].hop_limit = grh->hop_limit;
2171
2172         /* This destroy call is needed to pair with cm_init_av_for_response */
2173         cm_destroy_av(&cm_id_priv->av);
2174         ret = cm_init_av_by_path(&work->path[0], gid_attr, &cm_id_priv->av);
2175         if (ret) {
2176                 int err;
2177
2178                 err = rdma_query_gid(work->port->cm_dev->ib_device,
2179                                      work->port->port_num, 0,
2180                                      &work->path[0].sgid);
2181                 if (err)
2182                         ib_send_cm_rej(&cm_id_priv->id, IB_CM_REJ_INVALID_GID,
2183                                        NULL, 0, NULL, 0);
2184                 else
2185                         ib_send_cm_rej(&cm_id_priv->id, IB_CM_REJ_INVALID_GID,
2186                                        &work->path[0].sgid,
2187                                        sizeof(work->path[0].sgid),
2188                                        NULL, 0);
2189                 goto rejected;
2190         }
2191         if (cm_id_priv->av.ah_attr.type == RDMA_AH_ATTR_TYPE_IB)
2192                 cm_id_priv->av.dlid_datapath =
2193                         IBA_GET(CM_REQ_PRIMARY_LOCAL_PORT_LID, req_msg);
2194
2195         if (cm_req_has_alt_path(req_msg)) {
2196                 ret = cm_init_av_by_path(&work->path[1], NULL,
2197                                          &cm_id_priv->alt_av);
2198                 if (ret) {
2199                         ib_send_cm_rej(&cm_id_priv->id,
2200                                        IB_CM_REJ_INVALID_ALT_GID,
2201                                        &work->path[0].sgid,
2202                                        sizeof(work->path[0].sgid), NULL, 0);
2203                         goto rejected;
2204                 }
2205         }
2206
2207         cm_id_priv->id.cm_handler = listen_cm_id_priv->id.cm_handler;
2208         cm_id_priv->id.context = listen_cm_id_priv->id.context;
2209         cm_format_req_event(work, cm_id_priv, &listen_cm_id_priv->id);
2210
2211         /* Now MAD handlers can see the new ID */
2212         spin_lock_irq(&cm_id_priv->lock);
2213         cm_finalize_id(cm_id_priv);
2214
2215         /* Refcount belongs to the event, pairs with cm_process_work() */
2216         refcount_inc(&cm_id_priv->refcount);
2217         cm_queue_work_unlock(cm_id_priv, work);
2218         /*
2219          * Since this ID was just created and was not made visible to other MAD
2220          * handlers until the cm_finalize_id() above we know that the
2221          * cm_process_work() will deliver the event and the listen_cm_id
2222          * embedded in the event can be derefed here.
2223          */
2224         cm_deref_id(listen_cm_id_priv);
2225         return 0;
2226
2227 rejected:
2228         cm_deref_id(listen_cm_id_priv);
2229 destroy:
2230         ib_destroy_cm_id(&cm_id_priv->id);
2231         return ret;
2232 }
2233
2234 static void cm_format_rep(struct cm_rep_msg *rep_msg,
2235                           struct cm_id_private *cm_id_priv,
2236                           struct ib_cm_rep_param *param)
2237 {
2238         cm_format_mad_ece_hdr(&rep_msg->hdr, CM_REP_ATTR_ID, cm_id_priv->tid,
2239                               param->ece.attr_mod);
2240         IBA_SET(CM_REP_LOCAL_COMM_ID, rep_msg,
2241                 be32_to_cpu(cm_id_priv->id.local_id));
2242         IBA_SET(CM_REP_REMOTE_COMM_ID, rep_msg,
2243                 be32_to_cpu(cm_id_priv->id.remote_id));
2244         IBA_SET(CM_REP_STARTING_PSN, rep_msg, param->starting_psn);
2245         IBA_SET(CM_REP_RESPONDER_RESOURCES, rep_msg,
2246                 param->responder_resources);
2247         IBA_SET(CM_REP_TARGET_ACK_DELAY, rep_msg,
2248                 cm_id_priv->av.port->cm_dev->ack_delay);
2249         IBA_SET(CM_REP_FAILOVER_ACCEPTED, rep_msg, param->failover_accepted);
2250         IBA_SET(CM_REP_RNR_RETRY_COUNT, rep_msg, param->rnr_retry_count);
2251         IBA_SET(CM_REP_LOCAL_CA_GUID, rep_msg,
2252                 be64_to_cpu(cm_id_priv->id.device->node_guid));
2253
2254         if (cm_id_priv->qp_type != IB_QPT_XRC_TGT) {
2255                 IBA_SET(CM_REP_INITIATOR_DEPTH, rep_msg,
2256                         param->initiator_depth);
2257                 IBA_SET(CM_REP_END_TO_END_FLOW_CONTROL, rep_msg,
2258                         param->flow_control);
2259                 IBA_SET(CM_REP_SRQ, rep_msg, param->srq);
2260                 IBA_SET(CM_REP_LOCAL_QPN, rep_msg, param->qp_num);
2261         } else {
2262                 IBA_SET(CM_REP_SRQ, rep_msg, 1);
2263                 IBA_SET(CM_REP_LOCAL_EE_CONTEXT_NUMBER, rep_msg, param->qp_num);
2264         }
2265
2266         IBA_SET(CM_REP_VENDOR_ID_L, rep_msg, param->ece.vendor_id);
2267         IBA_SET(CM_REP_VENDOR_ID_M, rep_msg, param->ece.vendor_id >> 8);
2268         IBA_SET(CM_REP_VENDOR_ID_H, rep_msg, param->ece.vendor_id >> 16);
2269
2270         if (param->private_data && param->private_data_len)
2271                 IBA_SET_MEM(CM_REP_PRIVATE_DATA, rep_msg, param->private_data,
2272                             param->private_data_len);
2273 }
2274
2275 int ib_send_cm_rep(struct ib_cm_id *cm_id,
2276                    struct ib_cm_rep_param *param)
2277 {
2278         struct cm_id_private *cm_id_priv;
2279         struct ib_mad_send_buf *msg;
2280         struct cm_rep_msg *rep_msg;
2281         unsigned long flags;
2282         int ret;
2283
2284         if (param->private_data &&
2285             param->private_data_len > IB_CM_REP_PRIVATE_DATA_SIZE)
2286                 return -EINVAL;
2287
2288         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
2289         spin_lock_irqsave(&cm_id_priv->lock, flags);
2290         if (cm_id->state != IB_CM_REQ_RCVD &&
2291             cm_id->state != IB_CM_MRA_REQ_SENT) {
2292                 trace_icm_send_rep_err(cm_id_priv->id.local_id, cm_id->state);
2293                 ret = -EINVAL;
2294                 goto out;
2295         }
2296
2297         msg = cm_alloc_priv_msg(cm_id_priv);
2298         if (IS_ERR(msg)) {
2299                 ret = PTR_ERR(msg);
2300                 goto out;
2301         }
2302
2303         rep_msg = (struct cm_rep_msg *) msg->mad;
2304         cm_format_rep(rep_msg, cm_id_priv, param);
2305         msg->timeout_ms = cm_id_priv->timeout_ms;
2306         msg->context[1] = (void *) (unsigned long) IB_CM_REP_SENT;
2307
2308         trace_icm_send_rep(cm_id);
2309         ret = ib_post_send_mad(msg, NULL);
2310         if (ret)
2311                 goto out_free;
2312
2313         cm_id->state = IB_CM_REP_SENT;
2314         cm_id_priv->initiator_depth = param->initiator_depth;
2315         cm_id_priv->responder_resources = param->responder_resources;
2316         cm_id_priv->rq_psn = cpu_to_be32(IBA_GET(CM_REP_STARTING_PSN, rep_msg));
2317         WARN_ONCE(param->qp_num & 0xFF000000,
2318                   "IBTA declares QPN to be 24 bits, but it is 0x%X\n",
2319                   param->qp_num);
2320         cm_id_priv->local_qpn = cpu_to_be32(param->qp_num & 0xFFFFFF);
2321         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2322         return 0;
2323
2324 out_free:
2325         cm_free_priv_msg(msg);
2326 out:
2327         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2328         return ret;
2329 }
2330 EXPORT_SYMBOL(ib_send_cm_rep);
2331
2332 static void cm_format_rtu(struct cm_rtu_msg *rtu_msg,
2333                           struct cm_id_private *cm_id_priv,
2334                           const void *private_data,
2335                           u8 private_data_len)
2336 {
2337         cm_format_mad_hdr(&rtu_msg->hdr, CM_RTU_ATTR_ID, cm_id_priv->tid);
2338         IBA_SET(CM_RTU_LOCAL_COMM_ID, rtu_msg,
2339                 be32_to_cpu(cm_id_priv->id.local_id));
2340         IBA_SET(CM_RTU_REMOTE_COMM_ID, rtu_msg,
2341                 be32_to_cpu(cm_id_priv->id.remote_id));
2342
2343         if (private_data && private_data_len)
2344                 IBA_SET_MEM(CM_RTU_PRIVATE_DATA, rtu_msg, private_data,
2345                             private_data_len);
2346 }
2347
2348 int ib_send_cm_rtu(struct ib_cm_id *cm_id,
2349                    const void *private_data,
2350                    u8 private_data_len)
2351 {
2352         struct cm_id_private *cm_id_priv;
2353         struct ib_mad_send_buf *msg;
2354         unsigned long flags;
2355         void *data;
2356         int ret;
2357
2358         if (private_data && private_data_len > IB_CM_RTU_PRIVATE_DATA_SIZE)
2359                 return -EINVAL;
2360
2361         data = cm_copy_private_data(private_data, private_data_len);
2362         if (IS_ERR(data))
2363                 return PTR_ERR(data);
2364
2365         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
2366         spin_lock_irqsave(&cm_id_priv->lock, flags);
2367         if (cm_id->state != IB_CM_REP_RCVD &&
2368             cm_id->state != IB_CM_MRA_REP_SENT) {
2369                 trace_icm_send_cm_rtu_err(cm_id);
2370                 ret = -EINVAL;
2371                 goto error;
2372         }
2373
2374         msg = cm_alloc_msg(cm_id_priv);
2375         if (IS_ERR(msg)) {
2376                 ret = PTR_ERR(msg);
2377                 goto error;
2378         }
2379
2380         cm_format_rtu((struct cm_rtu_msg *) msg->mad, cm_id_priv,
2381                       private_data, private_data_len);
2382
2383         trace_icm_send_rtu(cm_id);
2384         ret = ib_post_send_mad(msg, NULL);
2385         if (ret) {
2386                 spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2387                 cm_free_msg(msg);
2388                 kfree(data);
2389                 return ret;
2390         }
2391
2392         cm_id->state = IB_CM_ESTABLISHED;
2393         cm_set_private_data(cm_id_priv, data, private_data_len);
2394         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2395         return 0;
2396
2397 error:  spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2398         kfree(data);
2399         return ret;
2400 }
2401 EXPORT_SYMBOL(ib_send_cm_rtu);
2402
2403 static void cm_format_rep_event(struct cm_work *work, enum ib_qp_type qp_type)
2404 {
2405         struct cm_rep_msg *rep_msg;
2406         struct ib_cm_rep_event_param *param;
2407
2408         rep_msg = (struct cm_rep_msg *)work->mad_recv_wc->recv_buf.mad;
2409         param = &work->cm_event.param.rep_rcvd;
2410         param->remote_ca_guid =
2411                 cpu_to_be64(IBA_GET(CM_REP_LOCAL_CA_GUID, rep_msg));
2412         param->remote_qkey = IBA_GET(CM_REP_LOCAL_Q_KEY, rep_msg);
2413         param->remote_qpn = be32_to_cpu(cm_rep_get_qpn(rep_msg, qp_type));
2414         param->starting_psn = IBA_GET(CM_REP_STARTING_PSN, rep_msg);
2415         param->responder_resources = IBA_GET(CM_REP_INITIATOR_DEPTH, rep_msg);
2416         param->initiator_depth = IBA_GET(CM_REP_RESPONDER_RESOURCES, rep_msg);
2417         param->target_ack_delay = IBA_GET(CM_REP_TARGET_ACK_DELAY, rep_msg);
2418         param->failover_accepted = IBA_GET(CM_REP_FAILOVER_ACCEPTED, rep_msg);
2419         param->flow_control = IBA_GET(CM_REP_END_TO_END_FLOW_CONTROL, rep_msg);
2420         param->rnr_retry_count = IBA_GET(CM_REP_RNR_RETRY_COUNT, rep_msg);
2421         param->srq = IBA_GET(CM_REP_SRQ, rep_msg);
2422         param->ece.vendor_id = IBA_GET(CM_REP_VENDOR_ID_H, rep_msg) << 16;
2423         param->ece.vendor_id |= IBA_GET(CM_REP_VENDOR_ID_M, rep_msg) << 8;
2424         param->ece.vendor_id |= IBA_GET(CM_REP_VENDOR_ID_L, rep_msg);
2425         param->ece.attr_mod = be32_to_cpu(rep_msg->hdr.attr_mod);
2426
2427         work->cm_event.private_data =
2428                 IBA_GET_MEM_PTR(CM_REP_PRIVATE_DATA, rep_msg);
2429 }
2430
2431 static void cm_dup_rep_handler(struct cm_work *work)
2432 {
2433         struct cm_id_private *cm_id_priv;
2434         struct cm_rep_msg *rep_msg;
2435         struct ib_mad_send_buf *msg = NULL;
2436         int ret;
2437
2438         rep_msg = (struct cm_rep_msg *) work->mad_recv_wc->recv_buf.mad;
2439         cm_id_priv = cm_acquire_id(
2440                 cpu_to_be32(IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg)),
2441                 cpu_to_be32(IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg)));
2442         if (!cm_id_priv)
2443                 return;
2444
2445         atomic_long_inc(
2446                 &work->port->counters[CM_RECV_DUPLICATES][CM_REP_COUNTER]);
2447         ret = cm_alloc_response_msg(work->port, work->mad_recv_wc, &msg);
2448         if (ret)
2449                 goto deref;
2450
2451         spin_lock_irq(&cm_id_priv->lock);
2452         if (cm_id_priv->id.state == IB_CM_ESTABLISHED)
2453                 cm_format_rtu((struct cm_rtu_msg *) msg->mad, cm_id_priv,
2454                               cm_id_priv->private_data,
2455                               cm_id_priv->private_data_len);
2456         else if (cm_id_priv->id.state == IB_CM_MRA_REP_SENT)
2457                 cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
2458                               CM_MSG_RESPONSE_REP, cm_id_priv->service_timeout,
2459                               cm_id_priv->private_data,
2460                               cm_id_priv->private_data_len);
2461         else
2462                 goto unlock;
2463         spin_unlock_irq(&cm_id_priv->lock);
2464
2465         trace_icm_send_dup_rep(&cm_id_priv->id);
2466         ret = ib_post_send_mad(msg, NULL);
2467         if (ret)
2468                 goto free;
2469         goto deref;
2470
2471 unlock: spin_unlock_irq(&cm_id_priv->lock);
2472 free:   cm_free_response_msg(msg);
2473 deref:  cm_deref_id(cm_id_priv);
2474 }
2475
2476 static int cm_rep_handler(struct cm_work *work)
2477 {
2478         struct cm_id_private *cm_id_priv;
2479         struct cm_rep_msg *rep_msg;
2480         int ret;
2481         struct cm_id_private *cur_cm_id_priv;
2482         struct cm_timewait_info *timewait_info;
2483
2484         rep_msg = (struct cm_rep_msg *)work->mad_recv_wc->recv_buf.mad;
2485         cm_id_priv = cm_acquire_id(
2486                 cpu_to_be32(IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg)), 0);
2487         if (!cm_id_priv) {
2488                 cm_dup_rep_handler(work);
2489                 trace_icm_remote_no_priv_err(
2490                          IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg));
2491                 return -EINVAL;
2492         }
2493
2494         cm_format_rep_event(work, cm_id_priv->qp_type);
2495
2496         spin_lock_irq(&cm_id_priv->lock);
2497         switch (cm_id_priv->id.state) {
2498         case IB_CM_REQ_SENT:
2499         case IB_CM_MRA_REQ_RCVD:
2500                 break;
2501         default:
2502                 ret = -EINVAL;
2503                 trace_icm_rep_unknown_err(
2504                         IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg),
2505                         IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg),
2506                         cm_id_priv->id.state);
2507                 spin_unlock_irq(&cm_id_priv->lock);
2508                 goto error;
2509         }
2510
2511         cm_id_priv->timewait_info->work.remote_id =
2512                 cpu_to_be32(IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg));
2513         cm_id_priv->timewait_info->remote_ca_guid =
2514                 cpu_to_be64(IBA_GET(CM_REP_LOCAL_CA_GUID, rep_msg));
2515         cm_id_priv->timewait_info->remote_qpn = cm_rep_get_qpn(rep_msg, cm_id_priv->qp_type);
2516
2517         spin_lock(&cm.lock);
2518         /* Check for duplicate REP. */
2519         if (cm_insert_remote_id(cm_id_priv->timewait_info)) {
2520                 spin_unlock(&cm.lock);
2521                 spin_unlock_irq(&cm_id_priv->lock);
2522                 ret = -EINVAL;
2523                 trace_icm_insert_failed_err(
2524                          IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg));
2525                 goto error;
2526         }
2527         /* Check for a stale connection. */
2528         timewait_info = cm_insert_remote_qpn(cm_id_priv->timewait_info);
2529         if (timewait_info) {
2530                 cm_remove_remote(cm_id_priv);
2531                 cur_cm_id_priv = cm_acquire_id(timewait_info->work.local_id,
2532                                            timewait_info->work.remote_id);
2533
2534                 spin_unlock(&cm.lock);
2535                 spin_unlock_irq(&cm_id_priv->lock);
2536                 cm_issue_rej(work->port, work->mad_recv_wc,
2537                              IB_CM_REJ_STALE_CONN, CM_MSG_RESPONSE_REP,
2538                              NULL, 0);
2539                 ret = -EINVAL;
2540                 trace_icm_staleconn_err(
2541                         IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg),
2542                         IBA_GET(CM_REP_REMOTE_COMM_ID, rep_msg));
2543
2544                 if (cur_cm_id_priv) {
2545                         ib_send_cm_dreq(&cur_cm_id_priv->id, NULL, 0);
2546                         cm_deref_id(cur_cm_id_priv);
2547                 }
2548
2549                 goto error;
2550         }
2551         spin_unlock(&cm.lock);
2552
2553         cm_id_priv->id.state = IB_CM_REP_RCVD;
2554         cm_id_priv->id.remote_id =
2555                 cpu_to_be32(IBA_GET(CM_REP_LOCAL_COMM_ID, rep_msg));
2556         cm_id_priv->remote_qpn = cm_rep_get_qpn(rep_msg, cm_id_priv->qp_type);
2557         cm_id_priv->initiator_depth =
2558                 IBA_GET(CM_REP_RESPONDER_RESOURCES, rep_msg);
2559         cm_id_priv->responder_resources =
2560                 IBA_GET(CM_REP_INITIATOR_DEPTH, rep_msg);
2561         cm_id_priv->sq_psn = cpu_to_be32(IBA_GET(CM_REP_STARTING_PSN, rep_msg));
2562         cm_id_priv->rnr_retry_count = IBA_GET(CM_REP_RNR_RETRY_COUNT, rep_msg);
2563         cm_id_priv->target_ack_delay =
2564                 IBA_GET(CM_REP_TARGET_ACK_DELAY, rep_msg);
2565         cm_id_priv->av.timeout =
2566                         cm_ack_timeout(cm_id_priv->target_ack_delay,
2567                                        cm_id_priv->av.timeout - 1);
2568         cm_id_priv->alt_av.timeout =
2569                         cm_ack_timeout(cm_id_priv->target_ack_delay,
2570                                        cm_id_priv->alt_av.timeout - 1);
2571
2572         ib_cancel_mad(cm_id_priv->msg);
2573         cm_queue_work_unlock(cm_id_priv, work);
2574         return 0;
2575
2576 error:
2577         cm_deref_id(cm_id_priv);
2578         return ret;
2579 }
2580
2581 static int cm_establish_handler(struct cm_work *work)
2582 {
2583         struct cm_id_private *cm_id_priv;
2584
2585         /* See comment in cm_establish about lookup. */
2586         cm_id_priv = cm_acquire_id(work->local_id, work->remote_id);
2587         if (!cm_id_priv)
2588                 return -EINVAL;
2589
2590         spin_lock_irq(&cm_id_priv->lock);
2591         if (cm_id_priv->id.state != IB_CM_ESTABLISHED) {
2592                 spin_unlock_irq(&cm_id_priv->lock);
2593                 goto out;
2594         }
2595
2596         ib_cancel_mad(cm_id_priv->msg);
2597         cm_queue_work_unlock(cm_id_priv, work);
2598         return 0;
2599 out:
2600         cm_deref_id(cm_id_priv);
2601         return -EINVAL;
2602 }
2603
2604 static int cm_rtu_handler(struct cm_work *work)
2605 {
2606         struct cm_id_private *cm_id_priv;
2607         struct cm_rtu_msg *rtu_msg;
2608
2609         rtu_msg = (struct cm_rtu_msg *)work->mad_recv_wc->recv_buf.mad;
2610         cm_id_priv = cm_acquire_id(
2611                 cpu_to_be32(IBA_GET(CM_RTU_REMOTE_COMM_ID, rtu_msg)),
2612                 cpu_to_be32(IBA_GET(CM_RTU_LOCAL_COMM_ID, rtu_msg)));
2613         if (!cm_id_priv)
2614                 return -EINVAL;
2615
2616         work->cm_event.private_data =
2617                 IBA_GET_MEM_PTR(CM_RTU_PRIVATE_DATA, rtu_msg);
2618
2619         spin_lock_irq(&cm_id_priv->lock);
2620         if (cm_id_priv->id.state != IB_CM_REP_SENT &&
2621             cm_id_priv->id.state != IB_CM_MRA_REP_RCVD) {
2622                 spin_unlock_irq(&cm_id_priv->lock);
2623                 atomic_long_inc(&work->port->counters[CM_RECV_DUPLICATES]
2624                                                      [CM_RTU_COUNTER]);
2625                 goto out;
2626         }
2627         cm_id_priv->id.state = IB_CM_ESTABLISHED;
2628
2629         ib_cancel_mad(cm_id_priv->msg);
2630         cm_queue_work_unlock(cm_id_priv, work);
2631         return 0;
2632 out:
2633         cm_deref_id(cm_id_priv);
2634         return -EINVAL;
2635 }
2636
2637 static void cm_format_dreq(struct cm_dreq_msg *dreq_msg,
2638                           struct cm_id_private *cm_id_priv,
2639                           const void *private_data,
2640                           u8 private_data_len)
2641 {
2642         cm_format_mad_hdr(&dreq_msg->hdr, CM_DREQ_ATTR_ID,
2643                           cm_form_tid(cm_id_priv));
2644         IBA_SET(CM_DREQ_LOCAL_COMM_ID, dreq_msg,
2645                 be32_to_cpu(cm_id_priv->id.local_id));
2646         IBA_SET(CM_DREQ_REMOTE_COMM_ID, dreq_msg,
2647                 be32_to_cpu(cm_id_priv->id.remote_id));
2648         IBA_SET(CM_DREQ_REMOTE_QPN_EECN, dreq_msg,
2649                 be32_to_cpu(cm_id_priv->remote_qpn));
2650
2651         if (private_data && private_data_len)
2652                 IBA_SET_MEM(CM_DREQ_PRIVATE_DATA, dreq_msg, private_data,
2653                             private_data_len);
2654 }
2655
2656 static int cm_send_dreq_locked(struct cm_id_private *cm_id_priv,
2657                                const void *private_data, u8 private_data_len)
2658 {
2659         struct ib_mad_send_buf *msg;
2660         int ret;
2661
2662         lockdep_assert_held(&cm_id_priv->lock);
2663
2664         if (private_data && private_data_len > IB_CM_DREQ_PRIVATE_DATA_SIZE)
2665                 return -EINVAL;
2666
2667         if (cm_id_priv->id.state != IB_CM_ESTABLISHED) {
2668                 trace_icm_dreq_skipped(&cm_id_priv->id);
2669                 return -EINVAL;
2670         }
2671
2672         if (cm_id_priv->id.lap_state == IB_CM_LAP_SENT ||
2673             cm_id_priv->id.lap_state == IB_CM_MRA_LAP_RCVD)
2674                 ib_cancel_mad(cm_id_priv->msg);
2675
2676         msg = cm_alloc_priv_msg(cm_id_priv);
2677         if (IS_ERR(msg)) {
2678                 cm_enter_timewait(cm_id_priv);
2679                 return PTR_ERR(msg);
2680         }
2681
2682         cm_format_dreq((struct cm_dreq_msg *) msg->mad, cm_id_priv,
2683                        private_data, private_data_len);
2684         msg->timeout_ms = cm_id_priv->timeout_ms;
2685         msg->context[1] = (void *) (unsigned long) IB_CM_DREQ_SENT;
2686
2687         trace_icm_send_dreq(&cm_id_priv->id);
2688         ret = ib_post_send_mad(msg, NULL);
2689         if (ret) {
2690                 cm_enter_timewait(cm_id_priv);
2691                 cm_free_priv_msg(msg);
2692                 return ret;
2693         }
2694
2695         cm_id_priv->id.state = IB_CM_DREQ_SENT;
2696         return 0;
2697 }
2698
2699 int ib_send_cm_dreq(struct ib_cm_id *cm_id, const void *private_data,
2700                     u8 private_data_len)
2701 {
2702         struct cm_id_private *cm_id_priv =
2703                 container_of(cm_id, struct cm_id_private, id);
2704         unsigned long flags;
2705         int ret;
2706
2707         spin_lock_irqsave(&cm_id_priv->lock, flags);
2708         ret = cm_send_dreq_locked(cm_id_priv, private_data, private_data_len);
2709         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2710         return ret;
2711 }
2712 EXPORT_SYMBOL(ib_send_cm_dreq);
2713
2714 static void cm_format_drep(struct cm_drep_msg *drep_msg,
2715                           struct cm_id_private *cm_id_priv,
2716                           const void *private_data,
2717                           u8 private_data_len)
2718 {
2719         cm_format_mad_hdr(&drep_msg->hdr, CM_DREP_ATTR_ID, cm_id_priv->tid);
2720         IBA_SET(CM_DREP_LOCAL_COMM_ID, drep_msg,
2721                 be32_to_cpu(cm_id_priv->id.local_id));
2722         IBA_SET(CM_DREP_REMOTE_COMM_ID, drep_msg,
2723                 be32_to_cpu(cm_id_priv->id.remote_id));
2724
2725         if (private_data && private_data_len)
2726                 IBA_SET_MEM(CM_DREP_PRIVATE_DATA, drep_msg, private_data,
2727                             private_data_len);
2728 }
2729
2730 static int cm_send_drep_locked(struct cm_id_private *cm_id_priv,
2731                                void *private_data, u8 private_data_len)
2732 {
2733         struct ib_mad_send_buf *msg;
2734         int ret;
2735
2736         lockdep_assert_held(&cm_id_priv->lock);
2737
2738         if (private_data && private_data_len > IB_CM_DREP_PRIVATE_DATA_SIZE)
2739                 return -EINVAL;
2740
2741         if (cm_id_priv->id.state != IB_CM_DREQ_RCVD) {
2742                 trace_icm_send_drep_err(&cm_id_priv->id);
2743                 kfree(private_data);
2744                 return -EINVAL;
2745         }
2746
2747         cm_set_private_data(cm_id_priv, private_data, private_data_len);
2748         cm_enter_timewait(cm_id_priv);
2749
2750         msg = cm_alloc_msg(cm_id_priv);
2751         if (IS_ERR(msg))
2752                 return PTR_ERR(msg);
2753
2754         cm_format_drep((struct cm_drep_msg *) msg->mad, cm_id_priv,
2755                        private_data, private_data_len);
2756
2757         trace_icm_send_drep(&cm_id_priv->id);
2758         ret = ib_post_send_mad(msg, NULL);
2759         if (ret) {
2760                 cm_free_msg(msg);
2761                 return ret;
2762         }
2763         return 0;
2764 }
2765
2766 int ib_send_cm_drep(struct ib_cm_id *cm_id, const void *private_data,
2767                     u8 private_data_len)
2768 {
2769         struct cm_id_private *cm_id_priv =
2770                 container_of(cm_id, struct cm_id_private, id);
2771         unsigned long flags;
2772         void *data;
2773         int ret;
2774
2775         data = cm_copy_private_data(private_data, private_data_len);
2776         if (IS_ERR(data))
2777                 return PTR_ERR(data);
2778
2779         spin_lock_irqsave(&cm_id_priv->lock, flags);
2780         ret = cm_send_drep_locked(cm_id_priv, data, private_data_len);
2781         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2782         return ret;
2783 }
2784 EXPORT_SYMBOL(ib_send_cm_drep);
2785
2786 static int cm_issue_drep(struct cm_port *port,
2787                          struct ib_mad_recv_wc *mad_recv_wc)
2788 {
2789         struct ib_mad_send_buf *msg = NULL;
2790         struct cm_dreq_msg *dreq_msg;
2791         struct cm_drep_msg *drep_msg;
2792         int ret;
2793
2794         ret = cm_alloc_response_msg(port, mad_recv_wc, &msg);
2795         if (ret)
2796                 return ret;
2797
2798         dreq_msg = (struct cm_dreq_msg *) mad_recv_wc->recv_buf.mad;
2799         drep_msg = (struct cm_drep_msg *) msg->mad;
2800
2801         cm_format_mad_hdr(&drep_msg->hdr, CM_DREP_ATTR_ID, dreq_msg->hdr.tid);
2802         IBA_SET(CM_DREP_REMOTE_COMM_ID, drep_msg,
2803                 IBA_GET(CM_DREQ_LOCAL_COMM_ID, dreq_msg));
2804         IBA_SET(CM_DREP_LOCAL_COMM_ID, drep_msg,
2805                 IBA_GET(CM_DREQ_REMOTE_COMM_ID, dreq_msg));
2806
2807         trace_icm_issue_drep(
2808                 IBA_GET(CM_DREQ_LOCAL_COMM_ID, dreq_msg),
2809                 IBA_GET(CM_DREQ_REMOTE_COMM_ID, dreq_msg));
2810         ret = ib_post_send_mad(msg, NULL);
2811         if (ret)
2812                 cm_free_response_msg(msg);
2813
2814         return ret;
2815 }
2816
2817 static int cm_dreq_handler(struct cm_work *work)
2818 {
2819         struct cm_id_private *cm_id_priv;
2820         struct cm_dreq_msg *dreq_msg;
2821         struct ib_mad_send_buf *msg = NULL;
2822
2823         dreq_msg = (struct cm_dreq_msg *)work->mad_recv_wc->recv_buf.mad;
2824         cm_id_priv = cm_acquire_id(
2825                 cpu_to_be32(IBA_GET(CM_DREQ_REMOTE_COMM_ID, dreq_msg)),
2826                 cpu_to_be32(IBA_GET(CM_DREQ_LOCAL_COMM_ID, dreq_msg)));
2827         if (!cm_id_priv) {
2828                 atomic_long_inc(&work->port->counters[CM_RECV_DUPLICATES]
2829                                                      [CM_DREQ_COUNTER]);
2830                 cm_issue_drep(work->port, work->mad_recv_wc);
2831                 trace_icm_no_priv_err(
2832                         IBA_GET(CM_DREQ_LOCAL_COMM_ID, dreq_msg),
2833                         IBA_GET(CM_DREQ_REMOTE_COMM_ID, dreq_msg));
2834                 return -EINVAL;
2835         }
2836
2837         work->cm_event.private_data =
2838                 IBA_GET_MEM_PTR(CM_DREQ_PRIVATE_DATA, dreq_msg);
2839
2840         spin_lock_irq(&cm_id_priv->lock);
2841         if (cm_id_priv->local_qpn !=
2842             cpu_to_be32(IBA_GET(CM_DREQ_REMOTE_QPN_EECN, dreq_msg)))
2843                 goto unlock;
2844
2845         switch (cm_id_priv->id.state) {
2846         case IB_CM_REP_SENT:
2847         case IB_CM_DREQ_SENT:
2848         case IB_CM_MRA_REP_RCVD:
2849                 ib_cancel_mad(cm_id_priv->msg);
2850                 break;
2851         case IB_CM_ESTABLISHED:
2852                 if (cm_id_priv->id.lap_state == IB_CM_LAP_SENT ||
2853                     cm_id_priv->id.lap_state == IB_CM_MRA_LAP_RCVD)
2854                         ib_cancel_mad(cm_id_priv->msg);
2855                 break;
2856         case IB_CM_TIMEWAIT:
2857                 atomic_long_inc(&work->port->counters[CM_RECV_DUPLICATES]
2858                                                      [CM_DREQ_COUNTER]);
2859                 msg = cm_alloc_response_msg_no_ah(work->port, work->mad_recv_wc);
2860                 if (IS_ERR(msg))
2861                         goto unlock;
2862
2863                 cm_format_drep((struct cm_drep_msg *) msg->mad, cm_id_priv,
2864                                cm_id_priv->private_data,
2865                                cm_id_priv->private_data_len);
2866                 spin_unlock_irq(&cm_id_priv->lock);
2867
2868                 if (cm_create_response_msg_ah(work->port, work->mad_recv_wc, msg) ||
2869                     ib_post_send_mad(msg, NULL))
2870                         cm_free_response_msg(msg);
2871                 goto deref;
2872         case IB_CM_DREQ_RCVD:
2873                 atomic_long_inc(&work->port->counters[CM_RECV_DUPLICATES]
2874                                                      [CM_DREQ_COUNTER]);
2875                 goto unlock;
2876         default:
2877                 trace_icm_dreq_unknown_err(&cm_id_priv->id);
2878                 goto unlock;
2879         }
2880         cm_id_priv->id.state = IB_CM_DREQ_RCVD;
2881         cm_id_priv->tid = dreq_msg->hdr.tid;
2882         cm_queue_work_unlock(cm_id_priv, work);
2883         return 0;
2884
2885 unlock: spin_unlock_irq(&cm_id_priv->lock);
2886 deref:  cm_deref_id(cm_id_priv);
2887         return -EINVAL;
2888 }
2889
2890 static int cm_drep_handler(struct cm_work *work)
2891 {
2892         struct cm_id_private *cm_id_priv;
2893         struct cm_drep_msg *drep_msg;
2894
2895         drep_msg = (struct cm_drep_msg *)work->mad_recv_wc->recv_buf.mad;
2896         cm_id_priv = cm_acquire_id(
2897                 cpu_to_be32(IBA_GET(CM_DREP_REMOTE_COMM_ID, drep_msg)),
2898                 cpu_to_be32(IBA_GET(CM_DREP_LOCAL_COMM_ID, drep_msg)));
2899         if (!cm_id_priv)
2900                 return -EINVAL;
2901
2902         work->cm_event.private_data =
2903                 IBA_GET_MEM_PTR(CM_DREP_PRIVATE_DATA, drep_msg);
2904
2905         spin_lock_irq(&cm_id_priv->lock);
2906         if (cm_id_priv->id.state != IB_CM_DREQ_SENT &&
2907             cm_id_priv->id.state != IB_CM_DREQ_RCVD) {
2908                 spin_unlock_irq(&cm_id_priv->lock);
2909                 goto out;
2910         }
2911         cm_enter_timewait(cm_id_priv);
2912
2913         ib_cancel_mad(cm_id_priv->msg);
2914         cm_queue_work_unlock(cm_id_priv, work);
2915         return 0;
2916 out:
2917         cm_deref_id(cm_id_priv);
2918         return -EINVAL;
2919 }
2920
2921 static int cm_send_rej_locked(struct cm_id_private *cm_id_priv,
2922                               enum ib_cm_rej_reason reason, void *ari,
2923                               u8 ari_length, const void *private_data,
2924                               u8 private_data_len)
2925 {
2926         enum ib_cm_state state = cm_id_priv->id.state;
2927         struct ib_mad_send_buf *msg;
2928         int ret;
2929
2930         lockdep_assert_held(&cm_id_priv->lock);
2931
2932         if ((private_data && private_data_len > IB_CM_REJ_PRIVATE_DATA_SIZE) ||
2933             (ari && ari_length > IB_CM_REJ_ARI_LENGTH))
2934                 return -EINVAL;
2935
2936         trace_icm_send_rej(&cm_id_priv->id, reason);
2937
2938         switch (state) {
2939         case IB_CM_REQ_SENT:
2940         case IB_CM_MRA_REQ_RCVD:
2941         case IB_CM_REQ_RCVD:
2942         case IB_CM_MRA_REQ_SENT:
2943         case IB_CM_REP_RCVD:
2944         case IB_CM_MRA_REP_SENT:
2945                 cm_reset_to_idle(cm_id_priv);
2946                 msg = cm_alloc_msg(cm_id_priv);
2947                 if (IS_ERR(msg))
2948                         return PTR_ERR(msg);
2949                 cm_format_rej((struct cm_rej_msg *)msg->mad, cm_id_priv, reason,
2950                               ari, ari_length, private_data, private_data_len,
2951                               state);
2952                 break;
2953         case IB_CM_REP_SENT:
2954         case IB_CM_MRA_REP_RCVD:
2955                 cm_enter_timewait(cm_id_priv);
2956                 msg = cm_alloc_msg(cm_id_priv);
2957                 if (IS_ERR(msg))
2958                         return PTR_ERR(msg);
2959                 cm_format_rej((struct cm_rej_msg *)msg->mad, cm_id_priv, reason,
2960                               ari, ari_length, private_data, private_data_len,
2961                               state);
2962                 break;
2963         default:
2964                 trace_icm_send_unknown_rej_err(&cm_id_priv->id);
2965                 return -EINVAL;
2966         }
2967
2968         ret = ib_post_send_mad(msg, NULL);
2969         if (ret) {
2970                 cm_free_msg(msg);
2971                 return ret;
2972         }
2973
2974         return 0;
2975 }
2976
2977 int ib_send_cm_rej(struct ib_cm_id *cm_id, enum ib_cm_rej_reason reason,
2978                    void *ari, u8 ari_length, const void *private_data,
2979                    u8 private_data_len)
2980 {
2981         struct cm_id_private *cm_id_priv =
2982                 container_of(cm_id, struct cm_id_private, id);
2983         unsigned long flags;
2984         int ret;
2985
2986         spin_lock_irqsave(&cm_id_priv->lock, flags);
2987         ret = cm_send_rej_locked(cm_id_priv, reason, ari, ari_length,
2988                                  private_data, private_data_len);
2989         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
2990         return ret;
2991 }
2992 EXPORT_SYMBOL(ib_send_cm_rej);
2993
2994 static void cm_format_rej_event(struct cm_work *work)
2995 {
2996         struct cm_rej_msg *rej_msg;
2997         struct ib_cm_rej_event_param *param;
2998
2999         rej_msg = (struct cm_rej_msg *)work->mad_recv_wc->recv_buf.mad;
3000         param = &work->cm_event.param.rej_rcvd;
3001         param->ari = IBA_GET_MEM_PTR(CM_REJ_ARI, rej_msg);
3002         param->ari_length = IBA_GET(CM_REJ_REJECTED_INFO_LENGTH, rej_msg);
3003         param->reason = IBA_GET(CM_REJ_REASON, rej_msg);
3004         work->cm_event.private_data =
3005                 IBA_GET_MEM_PTR(CM_REJ_PRIVATE_DATA, rej_msg);
3006 }
3007
3008 static struct cm_id_private *cm_acquire_rejected_id(struct cm_rej_msg *rej_msg)
3009 {
3010         struct cm_id_private *cm_id_priv;
3011         __be32 remote_id;
3012
3013         remote_id = cpu_to_be32(IBA_GET(CM_REJ_LOCAL_COMM_ID, rej_msg));
3014
3015         if (IBA_GET(CM_REJ_REASON, rej_msg) == IB_CM_REJ_TIMEOUT) {
3016                 cm_id_priv = cm_find_remote_id(
3017                         *((__be64 *)IBA_GET_MEM_PTR(CM_REJ_ARI, rej_msg)),
3018                         remote_id);
3019         } else if (IBA_GET(CM_REJ_MESSAGE_REJECTED, rej_msg) ==
3020                    CM_MSG_RESPONSE_REQ)
3021                 cm_id_priv = cm_acquire_id(
3022                         cpu_to_be32(IBA_GET(CM_REJ_REMOTE_COMM_ID, rej_msg)),
3023                         0);
3024         else
3025                 cm_id_priv = cm_acquire_id(
3026                         cpu_to_be32(IBA_GET(CM_REJ_REMOTE_COMM_ID, rej_msg)),
3027                         remote_id);
3028
3029         return cm_id_priv;
3030 }
3031
3032 static int cm_rej_handler(struct cm_work *work)
3033 {
3034         struct cm_id_private *cm_id_priv;
3035         struct cm_rej_msg *rej_msg;
3036
3037         rej_msg = (struct cm_rej_msg *)work->mad_recv_wc->recv_buf.mad;
3038         cm_id_priv = cm_acquire_rejected_id(rej_msg);
3039         if (!cm_id_priv)
3040                 return -EINVAL;
3041
3042         cm_format_rej_event(work);
3043
3044         spin_lock_irq(&cm_id_priv->lock);
3045         switch (cm_id_priv->id.state) {
3046         case IB_CM_REQ_SENT:
3047         case IB_CM_MRA_REQ_RCVD:
3048         case IB_CM_REP_SENT:
3049         case IB_CM_MRA_REP_RCVD:
3050                 ib_cancel_mad(cm_id_priv->msg);
3051                 fallthrough;
3052         case IB_CM_REQ_RCVD:
3053         case IB_CM_MRA_REQ_SENT:
3054                 if (IBA_GET(CM_REJ_REASON, rej_msg) == IB_CM_REJ_STALE_CONN)
3055                         cm_enter_timewait(cm_id_priv);
3056                 else
3057                         cm_reset_to_idle(cm_id_priv);
3058                 break;
3059         case IB_CM_DREQ_SENT:
3060                 ib_cancel_mad(cm_id_priv->msg);
3061                 fallthrough;
3062         case IB_CM_REP_RCVD:
3063         case IB_CM_MRA_REP_SENT:
3064                 cm_enter_timewait(cm_id_priv);
3065                 break;
3066         case IB_CM_ESTABLISHED:
3067                 if (cm_id_priv->id.lap_state == IB_CM_LAP_UNINIT ||
3068                     cm_id_priv->id.lap_state == IB_CM_LAP_SENT) {
3069                         if (cm_id_priv->id.lap_state == IB_CM_LAP_SENT)
3070                                 ib_cancel_mad(cm_id_priv->msg);
3071                         cm_enter_timewait(cm_id_priv);
3072                         break;
3073                 }
3074                 fallthrough;
3075         default:
3076                 trace_icm_rej_unknown_err(&cm_id_priv->id);
3077                 spin_unlock_irq(&cm_id_priv->lock);
3078                 goto out;
3079         }
3080
3081         cm_queue_work_unlock(cm_id_priv, work);
3082         return 0;
3083 out:
3084         cm_deref_id(cm_id_priv);
3085         return -EINVAL;
3086 }
3087
3088 int ib_send_cm_mra(struct ib_cm_id *cm_id,
3089                    u8 service_timeout,
3090                    const void *private_data,
3091                    u8 private_data_len)
3092 {
3093         struct cm_id_private *cm_id_priv;
3094         struct ib_mad_send_buf *msg;
3095         enum ib_cm_state cm_state;
3096         enum ib_cm_lap_state lap_state;
3097         enum cm_msg_response msg_response;
3098         void *data;
3099         unsigned long flags;
3100         int ret;
3101
3102         if (private_data && private_data_len > IB_CM_MRA_PRIVATE_DATA_SIZE)
3103                 return -EINVAL;
3104
3105         data = cm_copy_private_data(private_data, private_data_len);
3106         if (IS_ERR(data))
3107                 return PTR_ERR(data);
3108
3109         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3110
3111         spin_lock_irqsave(&cm_id_priv->lock, flags);
3112         switch (cm_id_priv->id.state) {
3113         case IB_CM_REQ_RCVD:
3114                 cm_state = IB_CM_MRA_REQ_SENT;
3115                 lap_state = cm_id->lap_state;
3116                 msg_response = CM_MSG_RESPONSE_REQ;
3117                 break;
3118         case IB_CM_REP_RCVD:
3119                 cm_state = IB_CM_MRA_REP_SENT;
3120                 lap_state = cm_id->lap_state;
3121                 msg_response = CM_MSG_RESPONSE_REP;
3122                 break;
3123         case IB_CM_ESTABLISHED:
3124                 if (cm_id->lap_state == IB_CM_LAP_RCVD) {
3125                         cm_state = cm_id->state;
3126                         lap_state = IB_CM_MRA_LAP_SENT;
3127                         msg_response = CM_MSG_RESPONSE_OTHER;
3128                         break;
3129                 }
3130                 fallthrough;
3131         default:
3132                 trace_icm_send_mra_unknown_err(&cm_id_priv->id);
3133                 ret = -EINVAL;
3134                 goto error_unlock;
3135         }
3136
3137         if (!(service_timeout & IB_CM_MRA_FLAG_DELAY)) {
3138                 msg = cm_alloc_msg(cm_id_priv);
3139                 if (IS_ERR(msg)) {
3140                         ret = PTR_ERR(msg);
3141                         goto error_unlock;
3142                 }
3143
3144                 cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
3145                               msg_response, service_timeout,
3146                               private_data, private_data_len);
3147                 trace_icm_send_mra(cm_id);
3148                 ret = ib_post_send_mad(msg, NULL);
3149                 if (ret)
3150                         goto error_free_msg;
3151         }
3152
3153         cm_id->state = cm_state;
3154         cm_id->lap_state = lap_state;
3155         cm_id_priv->service_timeout = service_timeout;
3156         cm_set_private_data(cm_id_priv, data, private_data_len);
3157         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3158         return 0;
3159
3160 error_free_msg:
3161         cm_free_msg(msg);
3162 error_unlock:
3163         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3164         kfree(data);
3165         return ret;
3166 }
3167 EXPORT_SYMBOL(ib_send_cm_mra);
3168
3169 static struct cm_id_private *cm_acquire_mraed_id(struct cm_mra_msg *mra_msg)
3170 {
3171         switch (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg)) {
3172         case CM_MSG_RESPONSE_REQ:
3173                 return cm_acquire_id(
3174                         cpu_to_be32(IBA_GET(CM_MRA_REMOTE_COMM_ID, mra_msg)),
3175                         0);
3176         case CM_MSG_RESPONSE_REP:
3177         case CM_MSG_RESPONSE_OTHER:
3178                 return cm_acquire_id(
3179                         cpu_to_be32(IBA_GET(CM_MRA_REMOTE_COMM_ID, mra_msg)),
3180                         cpu_to_be32(IBA_GET(CM_MRA_LOCAL_COMM_ID, mra_msg)));
3181         default:
3182                 return NULL;
3183         }
3184 }
3185
3186 static int cm_mra_handler(struct cm_work *work)
3187 {
3188         struct cm_id_private *cm_id_priv;
3189         struct cm_mra_msg *mra_msg;
3190         int timeout;
3191
3192         mra_msg = (struct cm_mra_msg *)work->mad_recv_wc->recv_buf.mad;
3193         cm_id_priv = cm_acquire_mraed_id(mra_msg);
3194         if (!cm_id_priv)
3195                 return -EINVAL;
3196
3197         work->cm_event.private_data =
3198                 IBA_GET_MEM_PTR(CM_MRA_PRIVATE_DATA, mra_msg);
3199         work->cm_event.param.mra_rcvd.service_timeout =
3200                 IBA_GET(CM_MRA_SERVICE_TIMEOUT, mra_msg);
3201         timeout = cm_convert_to_ms(IBA_GET(CM_MRA_SERVICE_TIMEOUT, mra_msg)) +
3202                   cm_convert_to_ms(cm_id_priv->av.timeout);
3203
3204         spin_lock_irq(&cm_id_priv->lock);
3205         switch (cm_id_priv->id.state) {
3206         case IB_CM_REQ_SENT:
3207                 if (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg) !=
3208                             CM_MSG_RESPONSE_REQ ||
3209                     ib_modify_mad(cm_id_priv->msg, timeout))
3210                         goto out;
3211                 cm_id_priv->id.state = IB_CM_MRA_REQ_RCVD;
3212                 break;
3213         case IB_CM_REP_SENT:
3214                 if (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg) !=
3215                             CM_MSG_RESPONSE_REP ||
3216                     ib_modify_mad(cm_id_priv->msg, timeout))
3217                         goto out;
3218                 cm_id_priv->id.state = IB_CM_MRA_REP_RCVD;
3219                 break;
3220         case IB_CM_ESTABLISHED:
3221                 if (IBA_GET(CM_MRA_MESSAGE_MRAED, mra_msg) !=
3222                             CM_MSG_RESPONSE_OTHER ||
3223                     cm_id_priv->id.lap_state != IB_CM_LAP_SENT ||
3224                     ib_modify_mad(cm_id_priv->msg, timeout)) {
3225                         if (cm_id_priv->id.lap_state == IB_CM_MRA_LAP_RCVD)
3226                                 atomic_long_inc(
3227                                         &work->port->counters[CM_RECV_DUPLICATES]
3228                                                              [CM_MRA_COUNTER]);
3229                         goto out;
3230                 }
3231                 cm_id_priv->id.lap_state = IB_CM_MRA_LAP_RCVD;
3232                 break;
3233         case IB_CM_MRA_REQ_RCVD:
3234         case IB_CM_MRA_REP_RCVD:
3235                 atomic_long_inc(&work->port->counters[CM_RECV_DUPLICATES]
3236                                                      [CM_MRA_COUNTER]);
3237                 fallthrough;
3238         default:
3239                 trace_icm_mra_unknown_err(&cm_id_priv->id);
3240                 goto out;
3241         }
3242
3243         cm_id_priv->msg->context[1] = (void *) (unsigned long)
3244                                       cm_id_priv->id.state;
3245         cm_queue_work_unlock(cm_id_priv, work);
3246         return 0;
3247 out:
3248         spin_unlock_irq(&cm_id_priv->lock);
3249         cm_deref_id(cm_id_priv);
3250         return -EINVAL;
3251 }
3252
3253 static void cm_format_path_lid_from_lap(struct cm_lap_msg *lap_msg,
3254                                         struct sa_path_rec *path)
3255 {
3256         u32 lid;
3257
3258         if (path->rec_type != SA_PATH_REC_TYPE_OPA) {
3259                 sa_path_set_dlid(path, IBA_GET(CM_LAP_ALTERNATE_LOCAL_PORT_LID,
3260                                                lap_msg));
3261                 sa_path_set_slid(path, IBA_GET(CM_LAP_ALTERNATE_REMOTE_PORT_LID,
3262                                                lap_msg));
3263         } else {
3264                 lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
3265                         CM_LAP_ALTERNATE_LOCAL_PORT_GID, lap_msg));
3266                 sa_path_set_dlid(path, lid);
3267
3268                 lid = opa_get_lid_from_gid(IBA_GET_MEM_PTR(
3269                         CM_LAP_ALTERNATE_REMOTE_PORT_GID, lap_msg));
3270                 sa_path_set_slid(path, lid);
3271         }
3272 }
3273
3274 static void cm_format_path_from_lap(struct cm_id_private *cm_id_priv,
3275                                     struct sa_path_rec *path,
3276                                     struct cm_lap_msg *lap_msg)
3277 {
3278         path->dgid = *IBA_GET_MEM_PTR(CM_LAP_ALTERNATE_LOCAL_PORT_GID, lap_msg);
3279         path->sgid =
3280                 *IBA_GET_MEM_PTR(CM_LAP_ALTERNATE_REMOTE_PORT_GID, lap_msg);
3281         path->flow_label =
3282                 cpu_to_be32(IBA_GET(CM_LAP_ALTERNATE_FLOW_LABEL, lap_msg));
3283         path->hop_limit = IBA_GET(CM_LAP_ALTERNATE_HOP_LIMIT, lap_msg);
3284         path->traffic_class = IBA_GET(CM_LAP_ALTERNATE_TRAFFIC_CLASS, lap_msg);
3285         path->reversible = 1;
3286         path->pkey = cm_id_priv->pkey;
3287         path->sl = IBA_GET(CM_LAP_ALTERNATE_SL, lap_msg);
3288         path->mtu_selector = IB_SA_EQ;
3289         path->mtu = cm_id_priv->path_mtu;
3290         path->rate_selector = IB_SA_EQ;
3291         path->rate = IBA_GET(CM_LAP_ALTERNATE_PACKET_RATE, lap_msg);
3292         path->packet_life_time_selector = IB_SA_EQ;
3293         path->packet_life_time =
3294                 IBA_GET(CM_LAP_ALTERNATE_LOCAL_ACK_TIMEOUT, lap_msg);
3295         path->packet_life_time -= (path->packet_life_time > 0);
3296         cm_format_path_lid_from_lap(lap_msg, path);
3297 }
3298
3299 static int cm_lap_handler(struct cm_work *work)
3300 {
3301         struct cm_id_private *cm_id_priv;
3302         struct cm_lap_msg *lap_msg;
3303         struct ib_cm_lap_event_param *param;
3304         struct ib_mad_send_buf *msg = NULL;
3305         struct rdma_ah_attr ah_attr;
3306         struct cm_av alt_av = {};
3307         int ret;
3308
3309         /* Currently Alternate path messages are not supported for
3310          * RoCE link layer.
3311          */
3312         if (rdma_protocol_roce(work->port->cm_dev->ib_device,
3313                                work->port->port_num))
3314                 return -EINVAL;
3315
3316         /* todo: verify LAP request and send reject APR if invalid. */
3317         lap_msg = (struct cm_lap_msg *)work->mad_recv_wc->recv_buf.mad;
3318         cm_id_priv = cm_acquire_id(
3319                 cpu_to_be32(IBA_GET(CM_LAP_REMOTE_COMM_ID, lap_msg)),
3320                 cpu_to_be32(IBA_GET(CM_LAP_LOCAL_COMM_ID, lap_msg)));
3321         if (!cm_id_priv)
3322                 return -EINVAL;
3323
3324         param = &work->cm_event.param.lap_rcvd;
3325         memset(&work->path[0], 0, sizeof(work->path[1]));
3326         cm_path_set_rec_type(work->port->cm_dev->ib_device,
3327                              work->port->port_num, &work->path[0],
3328                              IBA_GET_MEM_PTR(CM_LAP_ALTERNATE_LOCAL_PORT_GID,
3329                                              lap_msg));
3330         param->alternate_path = &work->path[0];
3331         cm_format_path_from_lap(cm_id_priv, param->alternate_path, lap_msg);
3332         work->cm_event.private_data =
3333                 IBA_GET_MEM_PTR(CM_LAP_PRIVATE_DATA, lap_msg);
3334
3335         ret = ib_init_ah_attr_from_wc(work->port->cm_dev->ib_device,
3336                                       work->port->port_num,
3337                                       work->mad_recv_wc->wc,
3338                                       work->mad_recv_wc->recv_buf.grh,
3339                                       &ah_attr);
3340         if (ret)
3341                 goto deref;
3342
3343         ret = cm_init_av_by_path(param->alternate_path, NULL, &alt_av);
3344         if (ret) {
3345                 rdma_destroy_ah_attr(&ah_attr);
3346                 goto deref;
3347         }
3348
3349         spin_lock_irq(&cm_id_priv->lock);
3350         cm_init_av_for_lap(work->port, work->mad_recv_wc->wc,
3351                            &ah_attr, &cm_id_priv->av);
3352         cm_move_av_from_path(&cm_id_priv->alt_av, &alt_av);
3353
3354         if (cm_id_priv->id.state != IB_CM_ESTABLISHED)
3355                 goto unlock;
3356
3357         switch (cm_id_priv->id.lap_state) {
3358         case IB_CM_LAP_UNINIT:
3359         case IB_CM_LAP_IDLE:
3360                 break;
3361         case IB_CM_MRA_LAP_SENT:
3362                 atomic_long_inc(&work->port->counters[CM_RECV_DUPLICATES]
3363                                                      [CM_LAP_COUNTER]);
3364                 msg = cm_alloc_response_msg_no_ah(work->port, work->mad_recv_wc);
3365                 if (IS_ERR(msg))
3366                         goto unlock;
3367
3368                 cm_format_mra((struct cm_mra_msg *) msg->mad, cm_id_priv,
3369                               CM_MSG_RESPONSE_OTHER,
3370                               cm_id_priv->service_timeout,
3371                               cm_id_priv->private_data,
3372                               cm_id_priv->private_data_len);
3373                 spin_unlock_irq(&cm_id_priv->lock);
3374
3375                 if (cm_create_response_msg_ah(work->port, work->mad_recv_wc, msg) ||
3376                     ib_post_send_mad(msg, NULL))
3377                         cm_free_response_msg(msg);
3378                 goto deref;
3379         case IB_CM_LAP_RCVD:
3380                 atomic_long_inc(&work->port->counters[CM_RECV_DUPLICATES]
3381                                                      [CM_LAP_COUNTER]);
3382                 goto unlock;
3383         default:
3384                 goto unlock;
3385         }
3386
3387         cm_id_priv->id.lap_state = IB_CM_LAP_RCVD;
3388         cm_id_priv->tid = lap_msg->hdr.tid;
3389         cm_queue_work_unlock(cm_id_priv, work);
3390         return 0;
3391
3392 unlock: spin_unlock_irq(&cm_id_priv->lock);
3393 deref:  cm_deref_id(cm_id_priv);
3394         return -EINVAL;
3395 }
3396
3397 static int cm_apr_handler(struct cm_work *work)
3398 {
3399         struct cm_id_private *cm_id_priv;
3400         struct cm_apr_msg *apr_msg;
3401
3402         /* Currently Alternate path messages are not supported for
3403          * RoCE link layer.
3404          */
3405         if (rdma_protocol_roce(work->port->cm_dev->ib_device,
3406                                work->port->port_num))
3407                 return -EINVAL;
3408
3409         apr_msg = (struct cm_apr_msg *)work->mad_recv_wc->recv_buf.mad;
3410         cm_id_priv = cm_acquire_id(
3411                 cpu_to_be32(IBA_GET(CM_APR_REMOTE_COMM_ID, apr_msg)),
3412                 cpu_to_be32(IBA_GET(CM_APR_LOCAL_COMM_ID, apr_msg)));
3413         if (!cm_id_priv)
3414                 return -EINVAL; /* Unmatched reply. */
3415
3416         work->cm_event.param.apr_rcvd.ap_status =
3417                 IBA_GET(CM_APR_AR_STATUS, apr_msg);
3418         work->cm_event.param.apr_rcvd.apr_info =
3419                 IBA_GET_MEM_PTR(CM_APR_ADDITIONAL_INFORMATION, apr_msg);
3420         work->cm_event.param.apr_rcvd.info_len =
3421                 IBA_GET(CM_APR_ADDITIONAL_INFORMATION_LENGTH, apr_msg);
3422         work->cm_event.private_data =
3423                 IBA_GET_MEM_PTR(CM_APR_PRIVATE_DATA, apr_msg);
3424
3425         spin_lock_irq(&cm_id_priv->lock);
3426         if (cm_id_priv->id.state != IB_CM_ESTABLISHED ||
3427             (cm_id_priv->id.lap_state != IB_CM_LAP_SENT &&
3428              cm_id_priv->id.lap_state != IB_CM_MRA_LAP_RCVD)) {
3429                 spin_unlock_irq(&cm_id_priv->lock);
3430                 goto out;
3431         }
3432         cm_id_priv->id.lap_state = IB_CM_LAP_IDLE;
3433         ib_cancel_mad(cm_id_priv->msg);
3434         cm_queue_work_unlock(cm_id_priv, work);
3435         return 0;
3436 out:
3437         cm_deref_id(cm_id_priv);
3438         return -EINVAL;
3439 }
3440
3441 static int cm_timewait_handler(struct cm_work *work)
3442 {
3443         struct cm_timewait_info *timewait_info;
3444         struct cm_id_private *cm_id_priv;
3445
3446         timewait_info = container_of(work, struct cm_timewait_info, work);
3447         spin_lock_irq(&cm.lock);
3448         list_del(&timewait_info->list);
3449         spin_unlock_irq(&cm.lock);
3450
3451         cm_id_priv = cm_acquire_id(timewait_info->work.local_id,
3452                                    timewait_info->work.remote_id);
3453         if (!cm_id_priv)
3454                 return -EINVAL;
3455
3456         spin_lock_irq(&cm_id_priv->lock);
3457         if (cm_id_priv->id.state != IB_CM_TIMEWAIT ||
3458             cm_id_priv->remote_qpn != timewait_info->remote_qpn) {
3459                 spin_unlock_irq(&cm_id_priv->lock);
3460                 goto out;
3461         }
3462         cm_id_priv->id.state = IB_CM_IDLE;
3463         cm_queue_work_unlock(cm_id_priv, work);
3464         return 0;
3465 out:
3466         cm_deref_id(cm_id_priv);
3467         return -EINVAL;
3468 }
3469
3470 static void cm_format_sidr_req(struct cm_sidr_req_msg *sidr_req_msg,
3471                                struct cm_id_private *cm_id_priv,
3472                                struct ib_cm_sidr_req_param *param)
3473 {
3474         cm_format_mad_hdr(&sidr_req_msg->hdr, CM_SIDR_REQ_ATTR_ID,
3475                           cm_form_tid(cm_id_priv));
3476         IBA_SET(CM_SIDR_REQ_REQUESTID, sidr_req_msg,
3477                 be32_to_cpu(cm_id_priv->id.local_id));
3478         IBA_SET(CM_SIDR_REQ_PARTITION_KEY, sidr_req_msg,
3479                 be16_to_cpu(param->path->pkey));
3480         IBA_SET(CM_SIDR_REQ_SERVICEID, sidr_req_msg,
3481                 be64_to_cpu(param->service_id));
3482
3483         if (param->private_data && param->private_data_len)
3484                 IBA_SET_MEM(CM_SIDR_REQ_PRIVATE_DATA, sidr_req_msg,
3485                             param->private_data, param->private_data_len);
3486 }
3487
3488 int ib_send_cm_sidr_req(struct ib_cm_id *cm_id,
3489                         struct ib_cm_sidr_req_param *param)
3490 {
3491         struct cm_id_private *cm_id_priv;
3492         struct ib_mad_send_buf *msg;
3493         struct cm_av av = {};
3494         unsigned long flags;
3495         int ret;
3496
3497         if (!param->path || (param->private_data &&
3498              param->private_data_len > IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE))
3499                 return -EINVAL;
3500
3501         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3502         ret = cm_init_av_by_path(param->path, param->sgid_attr, &av);
3503         if (ret)
3504                 return ret;
3505
3506         spin_lock_irqsave(&cm_id_priv->lock, flags);
3507         cm_move_av_from_path(&cm_id_priv->av, &av);
3508         cm_id->service_id = param->service_id;
3509         cm_id_priv->timeout_ms = param->timeout_ms;
3510         cm_id_priv->max_cm_retries = param->max_cm_retries;
3511         if (cm_id->state != IB_CM_IDLE) {
3512                 ret = -EINVAL;
3513                 goto out_unlock;
3514         }
3515
3516         msg = cm_alloc_priv_msg(cm_id_priv);
3517         if (IS_ERR(msg)) {
3518                 ret = PTR_ERR(msg);
3519                 goto out_unlock;
3520         }
3521
3522         cm_format_sidr_req((struct cm_sidr_req_msg *)msg->mad, cm_id_priv,
3523                            param);
3524         msg->timeout_ms = cm_id_priv->timeout_ms;
3525         msg->context[1] = (void *)(unsigned long)IB_CM_SIDR_REQ_SENT;
3526
3527         trace_icm_send_sidr_req(&cm_id_priv->id);
3528         ret = ib_post_send_mad(msg, NULL);
3529         if (ret)
3530                 goto out_free;
3531         cm_id->state = IB_CM_SIDR_REQ_SENT;
3532         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3533         return 0;
3534 out_free:
3535         cm_free_priv_msg(msg);
3536 out_unlock:
3537         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3538         return ret;
3539 }
3540 EXPORT_SYMBOL(ib_send_cm_sidr_req);
3541
3542 static void cm_format_sidr_req_event(struct cm_work *work,
3543                                      const struct cm_id_private *rx_cm_id,
3544                                      struct ib_cm_id *listen_id)
3545 {
3546         struct cm_sidr_req_msg *sidr_req_msg;
3547         struct ib_cm_sidr_req_event_param *param;
3548
3549         sidr_req_msg = (struct cm_sidr_req_msg *)
3550                                 work->mad_recv_wc->recv_buf.mad;
3551         param = &work->cm_event.param.sidr_req_rcvd;
3552         param->pkey = IBA_GET(CM_SIDR_REQ_PARTITION_KEY, sidr_req_msg);
3553         param->listen_id = listen_id;
3554         param->service_id =
3555                 cpu_to_be64(IBA_GET(CM_SIDR_REQ_SERVICEID, sidr_req_msg));
3556         param->bth_pkey = cm_get_bth_pkey(work);
3557         param->port = work->port->port_num;
3558         param->sgid_attr = rx_cm_id->av.ah_attr.grh.sgid_attr;
3559         work->cm_event.private_data =
3560                 IBA_GET_MEM_PTR(CM_SIDR_REQ_PRIVATE_DATA, sidr_req_msg);
3561 }
3562
3563 static int cm_sidr_req_handler(struct cm_work *work)
3564 {
3565         struct cm_id_private *cm_id_priv, *listen_cm_id_priv;
3566         struct cm_sidr_req_msg *sidr_req_msg;
3567         struct ib_wc *wc;
3568         int ret;
3569
3570         cm_id_priv =
3571                 cm_alloc_id_priv(work->port->cm_dev->ib_device, NULL, NULL);
3572         if (IS_ERR(cm_id_priv))
3573                 return PTR_ERR(cm_id_priv);
3574
3575         /* Record SGID/SLID and request ID for lookup. */
3576         sidr_req_msg = (struct cm_sidr_req_msg *)
3577                                 work->mad_recv_wc->recv_buf.mad;
3578
3579         cm_id_priv->id.remote_id =
3580                 cpu_to_be32(IBA_GET(CM_SIDR_REQ_REQUESTID, sidr_req_msg));
3581         cm_id_priv->id.service_id =
3582                 cpu_to_be64(IBA_GET(CM_SIDR_REQ_SERVICEID, sidr_req_msg));
3583         cm_id_priv->tid = sidr_req_msg->hdr.tid;
3584
3585         wc = work->mad_recv_wc->wc;
3586         cm_id_priv->sidr_slid = wc->slid;
3587         ret = cm_init_av_for_response(work->port, work->mad_recv_wc->wc,
3588                                       work->mad_recv_wc->recv_buf.grh,
3589                                       &cm_id_priv->av);
3590         if (ret)
3591                 goto out;
3592
3593         spin_lock_irq(&cm.lock);
3594         listen_cm_id_priv = cm_insert_remote_sidr(cm_id_priv);
3595         if (listen_cm_id_priv) {
3596                 spin_unlock_irq(&cm.lock);
3597                 atomic_long_inc(&work->port->counters[CM_RECV_DUPLICATES]
3598                                                      [CM_SIDR_REQ_COUNTER]);
3599                 goto out; /* Duplicate message. */
3600         }
3601         cm_id_priv->id.state = IB_CM_SIDR_REQ_RCVD;
3602         listen_cm_id_priv = cm_find_listen(cm_id_priv->id.device,
3603                                            cm_id_priv->id.service_id);
3604         if (!listen_cm_id_priv) {
3605                 spin_unlock_irq(&cm.lock);
3606                 ib_send_cm_sidr_rep(&cm_id_priv->id,
3607                                     &(struct ib_cm_sidr_rep_param){
3608                                             .status = IB_SIDR_UNSUPPORTED });
3609                 goto out; /* No match. */
3610         }
3611         spin_unlock_irq(&cm.lock);
3612
3613         cm_id_priv->id.cm_handler = listen_cm_id_priv->id.cm_handler;
3614         cm_id_priv->id.context = listen_cm_id_priv->id.context;
3615
3616         /*
3617          * A SIDR ID does not need to be in the xarray since it does not receive
3618          * mads, is not placed in the remote_id or remote_qpn rbtree, and does
3619          * not enter timewait.
3620          */
3621
3622         cm_format_sidr_req_event(work, cm_id_priv, &listen_cm_id_priv->id);
3623         ret = cm_id_priv->id.cm_handler(&cm_id_priv->id, &work->cm_event);
3624         cm_free_work(work);
3625         /*
3626          * A pointer to the listen_cm_id is held in the event, so this deref
3627          * must be after the event is delivered above.
3628          */
3629         cm_deref_id(listen_cm_id_priv);
3630         if (ret)
3631                 cm_destroy_id(&cm_id_priv->id, ret);
3632         return 0;
3633 out:
3634         ib_destroy_cm_id(&cm_id_priv->id);
3635         return -EINVAL;
3636 }
3637
3638 static void cm_format_sidr_rep(struct cm_sidr_rep_msg *sidr_rep_msg,
3639                                struct cm_id_private *cm_id_priv,
3640                                struct ib_cm_sidr_rep_param *param)
3641 {
3642         cm_format_mad_ece_hdr(&sidr_rep_msg->hdr, CM_SIDR_REP_ATTR_ID,
3643                               cm_id_priv->tid, param->ece.attr_mod);
3644         IBA_SET(CM_SIDR_REP_REQUESTID, sidr_rep_msg,
3645                 be32_to_cpu(cm_id_priv->id.remote_id));
3646         IBA_SET(CM_SIDR_REP_STATUS, sidr_rep_msg, param->status);
3647         IBA_SET(CM_SIDR_REP_QPN, sidr_rep_msg, param->qp_num);
3648         IBA_SET(CM_SIDR_REP_SERVICEID, sidr_rep_msg,
3649                 be64_to_cpu(cm_id_priv->id.service_id));
3650         IBA_SET(CM_SIDR_REP_Q_KEY, sidr_rep_msg, param->qkey);
3651         IBA_SET(CM_SIDR_REP_VENDOR_ID_L, sidr_rep_msg,
3652                 param->ece.vendor_id & 0xFF);
3653         IBA_SET(CM_SIDR_REP_VENDOR_ID_H, sidr_rep_msg,
3654                 (param->ece.vendor_id >> 8) & 0xFF);
3655
3656         if (param->info && param->info_length)
3657                 IBA_SET_MEM(CM_SIDR_REP_ADDITIONAL_INFORMATION, sidr_rep_msg,
3658                             param->info, param->info_length);
3659
3660         if (param->private_data && param->private_data_len)
3661                 IBA_SET_MEM(CM_SIDR_REP_PRIVATE_DATA, sidr_rep_msg,
3662                             param->private_data, param->private_data_len);
3663 }
3664
3665 static int cm_send_sidr_rep_locked(struct cm_id_private *cm_id_priv,
3666                                    struct ib_cm_sidr_rep_param *param)
3667 {
3668         struct ib_mad_send_buf *msg;
3669         unsigned long flags;
3670         int ret;
3671
3672         lockdep_assert_held(&cm_id_priv->lock);
3673
3674         if ((param->info && param->info_length > IB_CM_SIDR_REP_INFO_LENGTH) ||
3675             (param->private_data &&
3676              param->private_data_len > IB_CM_SIDR_REP_PRIVATE_DATA_SIZE))
3677                 return -EINVAL;
3678
3679         if (cm_id_priv->id.state != IB_CM_SIDR_REQ_RCVD)
3680                 return -EINVAL;
3681
3682         msg = cm_alloc_msg(cm_id_priv);
3683         if (IS_ERR(msg))
3684                 return PTR_ERR(msg);
3685
3686         cm_format_sidr_rep((struct cm_sidr_rep_msg *) msg->mad, cm_id_priv,
3687                            param);
3688         trace_icm_send_sidr_rep(&cm_id_priv->id);
3689         ret = ib_post_send_mad(msg, NULL);
3690         if (ret) {
3691                 cm_free_msg(msg);
3692                 return ret;
3693         }
3694         cm_id_priv->id.state = IB_CM_IDLE;
3695         spin_lock_irqsave(&cm.lock, flags);
3696         if (!RB_EMPTY_NODE(&cm_id_priv->sidr_id_node)) {
3697                 rb_erase(&cm_id_priv->sidr_id_node, &cm.remote_sidr_table);
3698                 RB_CLEAR_NODE(&cm_id_priv->sidr_id_node);
3699         }
3700         spin_unlock_irqrestore(&cm.lock, flags);
3701         return 0;
3702 }
3703
3704 int ib_send_cm_sidr_rep(struct ib_cm_id *cm_id,
3705                         struct ib_cm_sidr_rep_param *param)
3706 {
3707         struct cm_id_private *cm_id_priv =
3708                 container_of(cm_id, struct cm_id_private, id);
3709         unsigned long flags;
3710         int ret;
3711
3712         spin_lock_irqsave(&cm_id_priv->lock, flags);
3713         ret = cm_send_sidr_rep_locked(cm_id_priv, param);
3714         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3715         return ret;
3716 }
3717 EXPORT_SYMBOL(ib_send_cm_sidr_rep);
3718
3719 static void cm_format_sidr_rep_event(struct cm_work *work,
3720                                      const struct cm_id_private *cm_id_priv)
3721 {
3722         struct cm_sidr_rep_msg *sidr_rep_msg;
3723         struct ib_cm_sidr_rep_event_param *param;
3724
3725         sidr_rep_msg = (struct cm_sidr_rep_msg *)
3726                                 work->mad_recv_wc->recv_buf.mad;
3727         param = &work->cm_event.param.sidr_rep_rcvd;
3728         param->status = IBA_GET(CM_SIDR_REP_STATUS, sidr_rep_msg);
3729         param->qkey = IBA_GET(CM_SIDR_REP_Q_KEY, sidr_rep_msg);
3730         param->qpn = IBA_GET(CM_SIDR_REP_QPN, sidr_rep_msg);
3731         param->info = IBA_GET_MEM_PTR(CM_SIDR_REP_ADDITIONAL_INFORMATION,
3732                                       sidr_rep_msg);
3733         param->info_len = IBA_GET(CM_SIDR_REP_ADDITIONAL_INFORMATION_LENGTH,
3734                                   sidr_rep_msg);
3735         param->sgid_attr = cm_id_priv->av.ah_attr.grh.sgid_attr;
3736         work->cm_event.private_data =
3737                 IBA_GET_MEM_PTR(CM_SIDR_REP_PRIVATE_DATA, sidr_rep_msg);
3738 }
3739
3740 static int cm_sidr_rep_handler(struct cm_work *work)
3741 {
3742         struct cm_sidr_rep_msg *sidr_rep_msg;
3743         struct cm_id_private *cm_id_priv;
3744
3745         sidr_rep_msg = (struct cm_sidr_rep_msg *)
3746                                 work->mad_recv_wc->recv_buf.mad;
3747         cm_id_priv = cm_acquire_id(
3748                 cpu_to_be32(IBA_GET(CM_SIDR_REP_REQUESTID, sidr_rep_msg)), 0);
3749         if (!cm_id_priv)
3750                 return -EINVAL; /* Unmatched reply. */
3751
3752         spin_lock_irq(&cm_id_priv->lock);
3753         if (cm_id_priv->id.state != IB_CM_SIDR_REQ_SENT) {
3754                 spin_unlock_irq(&cm_id_priv->lock);
3755                 goto out;
3756         }
3757         cm_id_priv->id.state = IB_CM_IDLE;
3758         ib_cancel_mad(cm_id_priv->msg);
3759         spin_unlock_irq(&cm_id_priv->lock);
3760
3761         cm_format_sidr_rep_event(work, cm_id_priv);
3762         cm_process_work(cm_id_priv, work);
3763         return 0;
3764 out:
3765         cm_deref_id(cm_id_priv);
3766         return -EINVAL;
3767 }
3768
3769 static void cm_process_send_error(struct cm_id_private *cm_id_priv,
3770                                   struct ib_mad_send_buf *msg,
3771                                   enum ib_cm_state state,
3772                                   enum ib_wc_status wc_status)
3773 {
3774         struct ib_cm_event cm_event = {};
3775         int ret;
3776
3777         /* Discard old sends or ones without a response. */
3778         spin_lock_irq(&cm_id_priv->lock);
3779         if (msg != cm_id_priv->msg) {
3780                 spin_unlock_irq(&cm_id_priv->lock);
3781                 cm_free_msg(msg);
3782                 return;
3783         }
3784         cm_free_priv_msg(msg);
3785
3786         if (state != cm_id_priv->id.state || wc_status == IB_WC_SUCCESS ||
3787             wc_status == IB_WC_WR_FLUSH_ERR)
3788                 goto out_unlock;
3789
3790         trace_icm_mad_send_err(state, wc_status);
3791         switch (state) {
3792         case IB_CM_REQ_SENT:
3793         case IB_CM_MRA_REQ_RCVD:
3794                 cm_reset_to_idle(cm_id_priv);
3795                 cm_event.event = IB_CM_REQ_ERROR;
3796                 break;
3797         case IB_CM_REP_SENT:
3798         case IB_CM_MRA_REP_RCVD:
3799                 cm_reset_to_idle(cm_id_priv);
3800                 cm_event.event = IB_CM_REP_ERROR;
3801                 break;
3802         case IB_CM_DREQ_SENT:
3803                 cm_enter_timewait(cm_id_priv);
3804                 cm_event.event = IB_CM_DREQ_ERROR;
3805                 break;
3806         case IB_CM_SIDR_REQ_SENT:
3807                 cm_id_priv->id.state = IB_CM_IDLE;
3808                 cm_event.event = IB_CM_SIDR_REQ_ERROR;
3809                 break;
3810         default:
3811                 goto out_unlock;
3812         }
3813         spin_unlock_irq(&cm_id_priv->lock);
3814         cm_event.param.send_status = wc_status;
3815
3816         /* No other events can occur on the cm_id at this point. */
3817         ret = cm_id_priv->id.cm_handler(&cm_id_priv->id, &cm_event);
3818         if (ret)
3819                 ib_destroy_cm_id(&cm_id_priv->id);
3820         return;
3821 out_unlock:
3822         spin_unlock_irq(&cm_id_priv->lock);
3823 }
3824
3825 static void cm_send_handler(struct ib_mad_agent *mad_agent,
3826                             struct ib_mad_send_wc *mad_send_wc)
3827 {
3828         struct ib_mad_send_buf *msg = mad_send_wc->send_buf;
3829         struct cm_id_private *cm_id_priv = msg->context[0];
3830         enum ib_cm_state state =
3831                 (enum ib_cm_state)(unsigned long)msg->context[1];
3832         struct cm_port *port;
3833         u16 attr_index;
3834
3835         port = mad_agent->context;
3836         attr_index = be16_to_cpu(((struct ib_mad_hdr *)
3837                                   msg->mad)->attr_id) - CM_ATTR_ID_OFFSET;
3838
3839         /*
3840          * If the send was in response to a received message (context[0] is not
3841          * set to a cm_id), and is not a REJ, then it is a send that was
3842          * manually retried.
3843          */
3844         if (!cm_id_priv && (attr_index != CM_REJ_COUNTER))
3845                 msg->retries = 1;
3846
3847         atomic_long_add(1 + msg->retries, &port->counters[CM_XMIT][attr_index]);
3848         if (msg->retries)
3849                 atomic_long_add(msg->retries,
3850                                 &port->counters[CM_XMIT_RETRIES][attr_index]);
3851
3852         if (cm_id_priv)
3853                 cm_process_send_error(cm_id_priv, msg, state,
3854                                       mad_send_wc->status);
3855         else
3856                 cm_free_response_msg(msg);
3857 }
3858
3859 static void cm_work_handler(struct work_struct *_work)
3860 {
3861         struct cm_work *work = container_of(_work, struct cm_work, work.work);
3862         int ret;
3863
3864         switch (work->cm_event.event) {
3865         case IB_CM_REQ_RECEIVED:
3866                 ret = cm_req_handler(work);
3867                 break;
3868         case IB_CM_MRA_RECEIVED:
3869                 ret = cm_mra_handler(work);
3870                 break;
3871         case IB_CM_REJ_RECEIVED:
3872                 ret = cm_rej_handler(work);
3873                 break;
3874         case IB_CM_REP_RECEIVED:
3875                 ret = cm_rep_handler(work);
3876                 break;
3877         case IB_CM_RTU_RECEIVED:
3878                 ret = cm_rtu_handler(work);
3879                 break;
3880         case IB_CM_USER_ESTABLISHED:
3881                 ret = cm_establish_handler(work);
3882                 break;
3883         case IB_CM_DREQ_RECEIVED:
3884                 ret = cm_dreq_handler(work);
3885                 break;
3886         case IB_CM_DREP_RECEIVED:
3887                 ret = cm_drep_handler(work);
3888                 break;
3889         case IB_CM_SIDR_REQ_RECEIVED:
3890                 ret = cm_sidr_req_handler(work);
3891                 break;
3892         case IB_CM_SIDR_REP_RECEIVED:
3893                 ret = cm_sidr_rep_handler(work);
3894                 break;
3895         case IB_CM_LAP_RECEIVED:
3896                 ret = cm_lap_handler(work);
3897                 break;
3898         case IB_CM_APR_RECEIVED:
3899                 ret = cm_apr_handler(work);
3900                 break;
3901         case IB_CM_TIMEWAIT_EXIT:
3902                 ret = cm_timewait_handler(work);
3903                 break;
3904         default:
3905                 trace_icm_handler_err(work->cm_event.event);
3906                 ret = -EINVAL;
3907                 break;
3908         }
3909         if (ret)
3910                 cm_free_work(work);
3911 }
3912
3913 static int cm_establish(struct ib_cm_id *cm_id)
3914 {
3915         struct cm_id_private *cm_id_priv;
3916         struct cm_work *work;
3917         unsigned long flags;
3918         int ret = 0;
3919         struct cm_device *cm_dev;
3920
3921         cm_dev = ib_get_client_data(cm_id->device, &cm_client);
3922         if (!cm_dev)
3923                 return -ENODEV;
3924
3925         work = kmalloc(sizeof *work, GFP_ATOMIC);
3926         if (!work)
3927                 return -ENOMEM;
3928
3929         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3930         spin_lock_irqsave(&cm_id_priv->lock, flags);
3931         switch (cm_id->state) {
3932         case IB_CM_REP_SENT:
3933         case IB_CM_MRA_REP_RCVD:
3934                 cm_id->state = IB_CM_ESTABLISHED;
3935                 break;
3936         case IB_CM_ESTABLISHED:
3937                 ret = -EISCONN;
3938                 break;
3939         default:
3940                 trace_icm_establish_err(cm_id);
3941                 ret = -EINVAL;
3942                 break;
3943         }
3944         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3945
3946         if (ret) {
3947                 kfree(work);
3948                 goto out;
3949         }
3950
3951         /*
3952          * The CM worker thread may try to destroy the cm_id before it
3953          * can execute this work item.  To prevent potential deadlock,
3954          * we need to find the cm_id once we're in the context of the
3955          * worker thread, rather than holding a reference on it.
3956          */
3957         INIT_DELAYED_WORK(&work->work, cm_work_handler);
3958         work->local_id = cm_id->local_id;
3959         work->remote_id = cm_id->remote_id;
3960         work->mad_recv_wc = NULL;
3961         work->cm_event.event = IB_CM_USER_ESTABLISHED;
3962
3963         /* Check if the device started its remove_one */
3964         spin_lock_irqsave(&cm.lock, flags);
3965         if (!cm_dev->going_down) {
3966                 queue_delayed_work(cm.wq, &work->work, 0);
3967         } else {
3968                 kfree(work);
3969                 ret = -ENODEV;
3970         }
3971         spin_unlock_irqrestore(&cm.lock, flags);
3972
3973 out:
3974         return ret;
3975 }
3976
3977 static int cm_migrate(struct ib_cm_id *cm_id)
3978 {
3979         struct cm_id_private *cm_id_priv;
3980         unsigned long flags;
3981         int ret = 0;
3982
3983         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
3984         spin_lock_irqsave(&cm_id_priv->lock, flags);
3985         if (cm_id->state == IB_CM_ESTABLISHED &&
3986             (cm_id->lap_state == IB_CM_LAP_UNINIT ||
3987              cm_id->lap_state == IB_CM_LAP_IDLE)) {
3988                 cm_id->lap_state = IB_CM_LAP_IDLE;
3989                 cm_id_priv->av = cm_id_priv->alt_av;
3990         } else
3991                 ret = -EINVAL;
3992         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
3993
3994         return ret;
3995 }
3996
3997 int ib_cm_notify(struct ib_cm_id *cm_id, enum ib_event_type event)
3998 {
3999         int ret;
4000
4001         switch (event) {
4002         case IB_EVENT_COMM_EST:
4003                 ret = cm_establish(cm_id);
4004                 break;
4005         case IB_EVENT_PATH_MIG:
4006                 ret = cm_migrate(cm_id);
4007                 break;
4008         default:
4009                 ret = -EINVAL;
4010         }
4011         return ret;
4012 }
4013 EXPORT_SYMBOL(ib_cm_notify);
4014
4015 static void cm_recv_handler(struct ib_mad_agent *mad_agent,
4016                             struct ib_mad_send_buf *send_buf,
4017                             struct ib_mad_recv_wc *mad_recv_wc)
4018 {
4019         struct cm_port *port = mad_agent->context;
4020         struct cm_work *work;
4021         enum ib_cm_event_type event;
4022         bool alt_path = false;
4023         u16 attr_id;
4024         int paths = 0;
4025         int going_down = 0;
4026
4027         switch (mad_recv_wc->recv_buf.mad->mad_hdr.attr_id) {
4028         case CM_REQ_ATTR_ID:
4029                 alt_path = cm_req_has_alt_path((struct cm_req_msg *)
4030                                                 mad_recv_wc->recv_buf.mad);
4031                 paths = 1 + (alt_path != 0);
4032                 event = IB_CM_REQ_RECEIVED;
4033                 break;
4034         case CM_MRA_ATTR_ID:
4035                 event = IB_CM_MRA_RECEIVED;
4036                 break;
4037         case CM_REJ_ATTR_ID:
4038                 event = IB_CM_REJ_RECEIVED;
4039                 break;
4040         case CM_REP_ATTR_ID:
4041                 event = IB_CM_REP_RECEIVED;
4042                 break;
4043         case CM_RTU_ATTR_ID:
4044                 event = IB_CM_RTU_RECEIVED;
4045                 break;
4046         case CM_DREQ_ATTR_ID:
4047                 event = IB_CM_DREQ_RECEIVED;
4048                 break;
4049         case CM_DREP_ATTR_ID:
4050                 event = IB_CM_DREP_RECEIVED;
4051                 break;
4052         case CM_SIDR_REQ_ATTR_ID:
4053                 event = IB_CM_SIDR_REQ_RECEIVED;
4054                 break;
4055         case CM_SIDR_REP_ATTR_ID:
4056                 event = IB_CM_SIDR_REP_RECEIVED;
4057                 break;
4058         case CM_LAP_ATTR_ID:
4059                 paths = 1;
4060                 event = IB_CM_LAP_RECEIVED;
4061                 break;
4062         case CM_APR_ATTR_ID:
4063                 event = IB_CM_APR_RECEIVED;
4064                 break;
4065         default:
4066                 ib_free_recv_mad(mad_recv_wc);
4067                 return;
4068         }
4069
4070         attr_id = be16_to_cpu(mad_recv_wc->recv_buf.mad->mad_hdr.attr_id);
4071         atomic_long_inc(&port->counters[CM_RECV][attr_id - CM_ATTR_ID_OFFSET]);
4072
4073         work = kmalloc(struct_size(work, path, paths), GFP_KERNEL);
4074         if (!work) {
4075                 ib_free_recv_mad(mad_recv_wc);
4076                 return;
4077         }
4078
4079         INIT_DELAYED_WORK(&work->work, cm_work_handler);
4080         work->cm_event.event = event;
4081         work->mad_recv_wc = mad_recv_wc;
4082         work->port = port;
4083
4084         /* Check if the device started its remove_one */
4085         spin_lock_irq(&cm.lock);
4086         if (!port->cm_dev->going_down)
4087                 queue_delayed_work(cm.wq, &work->work, 0);
4088         else
4089                 going_down = 1;
4090         spin_unlock_irq(&cm.lock);
4091
4092         if (going_down) {
4093                 kfree(work);
4094                 ib_free_recv_mad(mad_recv_wc);
4095         }
4096 }
4097
4098 static int cm_init_qp_init_attr(struct cm_id_private *cm_id_priv,
4099                                 struct ib_qp_attr *qp_attr,
4100                                 int *qp_attr_mask)
4101 {
4102         unsigned long flags;
4103         int ret;
4104
4105         spin_lock_irqsave(&cm_id_priv->lock, flags);
4106         switch (cm_id_priv->id.state) {
4107         case IB_CM_REQ_SENT:
4108         case IB_CM_MRA_REQ_RCVD:
4109         case IB_CM_REQ_RCVD:
4110         case IB_CM_MRA_REQ_SENT:
4111         case IB_CM_REP_RCVD:
4112         case IB_CM_MRA_REP_SENT:
4113         case IB_CM_REP_SENT:
4114         case IB_CM_MRA_REP_RCVD:
4115         case IB_CM_ESTABLISHED:
4116                 *qp_attr_mask = IB_QP_STATE | IB_QP_ACCESS_FLAGS |
4117                                 IB_QP_PKEY_INDEX | IB_QP_PORT;
4118                 qp_attr->qp_access_flags = IB_ACCESS_REMOTE_WRITE;
4119                 if (cm_id_priv->responder_resources) {
4120                         struct ib_device *ib_dev = cm_id_priv->id.device;
4121                         u64 support_flush = ib_dev->attrs.device_cap_flags &
4122                           (IB_DEVICE_FLUSH_GLOBAL | IB_DEVICE_FLUSH_PERSISTENT);
4123                         u32 flushable = support_flush ?
4124                                         (IB_ACCESS_FLUSH_GLOBAL |
4125                                          IB_ACCESS_FLUSH_PERSISTENT) : 0;
4126
4127                         qp_attr->qp_access_flags |= IB_ACCESS_REMOTE_READ |
4128                                                     IB_ACCESS_REMOTE_ATOMIC |
4129                                                     flushable;
4130                 }
4131                 qp_attr->pkey_index = cm_id_priv->av.pkey_index;
4132                 if (cm_id_priv->av.port)
4133                         qp_attr->port_num = cm_id_priv->av.port->port_num;
4134                 ret = 0;
4135                 break;
4136         default:
4137                 trace_icm_qp_init_err(&cm_id_priv->id);
4138                 ret = -EINVAL;
4139                 break;
4140         }
4141         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
4142         return ret;
4143 }
4144
4145 static int cm_init_qp_rtr_attr(struct cm_id_private *cm_id_priv,
4146                                struct ib_qp_attr *qp_attr,
4147                                int *qp_attr_mask)
4148 {
4149         unsigned long flags;
4150         int ret;
4151
4152         spin_lock_irqsave(&cm_id_priv->lock, flags);
4153         switch (cm_id_priv->id.state) {
4154         case IB_CM_REQ_RCVD:
4155         case IB_CM_MRA_REQ_SENT:
4156         case IB_CM_REP_RCVD:
4157         case IB_CM_MRA_REP_SENT:
4158         case IB_CM_REP_SENT:
4159         case IB_CM_MRA_REP_RCVD:
4160         case IB_CM_ESTABLISHED:
4161                 *qp_attr_mask = IB_QP_STATE | IB_QP_AV | IB_QP_PATH_MTU |
4162                                 IB_QP_DEST_QPN | IB_QP_RQ_PSN;
4163                 qp_attr->ah_attr = cm_id_priv->av.ah_attr;
4164                 if ((qp_attr->ah_attr.type == RDMA_AH_ATTR_TYPE_IB) &&
4165                     cm_id_priv->av.dlid_datapath &&
4166                     (cm_id_priv->av.dlid_datapath != 0xffff))
4167                         qp_attr->ah_attr.ib.dlid = cm_id_priv->av.dlid_datapath;
4168                 qp_attr->path_mtu = cm_id_priv->path_mtu;
4169                 qp_attr->dest_qp_num = be32_to_cpu(cm_id_priv->remote_qpn);
4170                 qp_attr->rq_psn = be32_to_cpu(cm_id_priv->rq_psn);
4171                 if (cm_id_priv->qp_type == IB_QPT_RC ||
4172                     cm_id_priv->qp_type == IB_QPT_XRC_TGT) {
4173                         *qp_attr_mask |= IB_QP_MAX_DEST_RD_ATOMIC |
4174                                          IB_QP_MIN_RNR_TIMER;
4175                         qp_attr->max_dest_rd_atomic =
4176                                         cm_id_priv->responder_resources;
4177                         qp_attr->min_rnr_timer = 0;
4178                 }
4179                 if (rdma_ah_get_dlid(&cm_id_priv->alt_av.ah_attr) &&
4180                     cm_id_priv->alt_av.port) {
4181                         *qp_attr_mask |= IB_QP_ALT_PATH;
4182                         qp_attr->alt_port_num = cm_id_priv->alt_av.port->port_num;
4183                         qp_attr->alt_pkey_index = cm_id_priv->alt_av.pkey_index;
4184                         qp_attr->alt_timeout = cm_id_priv->alt_av.timeout;
4185                         qp_attr->alt_ah_attr = cm_id_priv->alt_av.ah_attr;
4186                 }
4187                 ret = 0;
4188                 break;
4189         default:
4190                 trace_icm_qp_rtr_err(&cm_id_priv->id);
4191                 ret = -EINVAL;
4192                 break;
4193         }
4194         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
4195         return ret;
4196 }
4197
4198 static int cm_init_qp_rts_attr(struct cm_id_private *cm_id_priv,
4199                                struct ib_qp_attr *qp_attr,
4200                                int *qp_attr_mask)
4201 {
4202         unsigned long flags;
4203         int ret;
4204
4205         spin_lock_irqsave(&cm_id_priv->lock, flags);
4206         switch (cm_id_priv->id.state) {
4207         /* Allow transition to RTS before sending REP */
4208         case IB_CM_REQ_RCVD:
4209         case IB_CM_MRA_REQ_SENT:
4210
4211         case IB_CM_REP_RCVD:
4212         case IB_CM_MRA_REP_SENT:
4213         case IB_CM_REP_SENT:
4214         case IB_CM_MRA_REP_RCVD:
4215         case IB_CM_ESTABLISHED:
4216                 if (cm_id_priv->id.lap_state == IB_CM_LAP_UNINIT) {
4217                         *qp_attr_mask = IB_QP_STATE | IB_QP_SQ_PSN;
4218                         qp_attr->sq_psn = be32_to_cpu(cm_id_priv->sq_psn);
4219                         switch (cm_id_priv->qp_type) {
4220                         case IB_QPT_RC:
4221                         case IB_QPT_XRC_INI:
4222                                 *qp_attr_mask |= IB_QP_RETRY_CNT | IB_QP_RNR_RETRY |
4223                                                  IB_QP_MAX_QP_RD_ATOMIC;
4224                                 qp_attr->retry_cnt = cm_id_priv->retry_count;
4225                                 qp_attr->rnr_retry = cm_id_priv->rnr_retry_count;
4226                                 qp_attr->max_rd_atomic = cm_id_priv->initiator_depth;
4227                                 fallthrough;
4228                         case IB_QPT_XRC_TGT:
4229                                 *qp_attr_mask |= IB_QP_TIMEOUT;
4230                                 qp_attr->timeout = cm_id_priv->av.timeout;
4231                                 break;
4232                         default:
4233                                 break;
4234                         }
4235                         if (rdma_ah_get_dlid(&cm_id_priv->alt_av.ah_attr)) {
4236                                 *qp_attr_mask |= IB_QP_PATH_MIG_STATE;
4237                                 qp_attr->path_mig_state = IB_MIG_REARM;
4238                         }
4239                 } else {
4240                         *qp_attr_mask = IB_QP_ALT_PATH | IB_QP_PATH_MIG_STATE;
4241                         if (cm_id_priv->alt_av.port)
4242                                 qp_attr->alt_port_num =
4243                                         cm_id_priv->alt_av.port->port_num;
4244                         qp_attr->alt_pkey_index = cm_id_priv->alt_av.pkey_index;
4245                         qp_attr->alt_timeout = cm_id_priv->alt_av.timeout;
4246                         qp_attr->alt_ah_attr = cm_id_priv->alt_av.ah_attr;
4247                         qp_attr->path_mig_state = IB_MIG_REARM;
4248                 }
4249                 ret = 0;
4250                 break;
4251         default:
4252                 trace_icm_qp_rts_err(&cm_id_priv->id);
4253                 ret = -EINVAL;
4254                 break;
4255         }
4256         spin_unlock_irqrestore(&cm_id_priv->lock, flags);
4257         return ret;
4258 }
4259
4260 int ib_cm_init_qp_attr(struct ib_cm_id *cm_id,
4261                        struct ib_qp_attr *qp_attr,
4262                        int *qp_attr_mask)
4263 {
4264         struct cm_id_private *cm_id_priv;
4265         int ret;
4266
4267         cm_id_priv = container_of(cm_id, struct cm_id_private, id);
4268         switch (qp_attr->qp_state) {
4269         case IB_QPS_INIT:
4270                 ret = cm_init_qp_init_attr(cm_id_priv, qp_attr, qp_attr_mask);
4271                 break;
4272         case IB_QPS_RTR:
4273                 ret = cm_init_qp_rtr_attr(cm_id_priv, qp_attr, qp_attr_mask);
4274                 break;
4275         case IB_QPS_RTS:
4276                 ret = cm_init_qp_rts_attr(cm_id_priv, qp_attr, qp_attr_mask);
4277                 break;
4278         default:
4279                 ret = -EINVAL;
4280                 break;
4281         }
4282         return ret;
4283 }
4284 EXPORT_SYMBOL(ib_cm_init_qp_attr);
4285
4286 static ssize_t cm_show_counter(struct ib_device *ibdev, u32 port_num,
4287                                struct ib_port_attribute *attr, char *buf)
4288 {
4289         struct cm_counter_attribute *cm_attr =
4290                 container_of(attr, struct cm_counter_attribute, attr);
4291         struct cm_device *cm_dev = ib_get_client_data(ibdev, &cm_client);
4292
4293         if (WARN_ON(!cm_dev))
4294                 return -EINVAL;
4295
4296         return sysfs_emit(
4297                 buf, "%ld\n",
4298                 atomic_long_read(
4299                         &cm_dev->port[port_num - 1]
4300                                  ->counters[cm_attr->group][cm_attr->index]));
4301 }
4302
4303 #define CM_COUNTER_ATTR(_name, _group, _index)                                 \
4304         {                                                                      \
4305                 .attr = __ATTR(_name, 0444, cm_show_counter, NULL),            \
4306                 .group = _group, .index = _index                               \
4307         }
4308
4309 #define CM_COUNTER_GROUP(_group, _name)                                        \
4310         static struct cm_counter_attribute cm_counter_attr_##_group[] = {      \
4311                 CM_COUNTER_ATTR(req, _group, CM_REQ_COUNTER),                  \
4312                 CM_COUNTER_ATTR(mra, _group, CM_MRA_COUNTER),                  \
4313                 CM_COUNTER_ATTR(rej, _group, CM_REJ_COUNTER),                  \
4314                 CM_COUNTER_ATTR(rep, _group, CM_REP_COUNTER),                  \
4315                 CM_COUNTER_ATTR(rtu, _group, CM_RTU_COUNTER),                  \
4316                 CM_COUNTER_ATTR(dreq, _group, CM_DREQ_COUNTER),                \
4317                 CM_COUNTER_ATTR(drep, _group, CM_DREP_COUNTER),                \
4318                 CM_COUNTER_ATTR(sidr_req, _group, CM_SIDR_REQ_COUNTER),        \
4319                 CM_COUNTER_ATTR(sidr_rep, _group, CM_SIDR_REP_COUNTER),        \
4320                 CM_COUNTER_ATTR(lap, _group, CM_LAP_COUNTER),                  \
4321                 CM_COUNTER_ATTR(apr, _group, CM_APR_COUNTER),                  \
4322         };                                                                     \
4323         static struct attribute *cm_counter_attrs_##_group[] = {               \
4324                 &cm_counter_attr_##_group[0].attr.attr,                        \
4325                 &cm_counter_attr_##_group[1].attr.attr,                        \
4326                 &cm_counter_attr_##_group[2].attr.attr,                        \
4327                 &cm_counter_attr_##_group[3].attr.attr,                        \
4328                 &cm_counter_attr_##_group[4].attr.attr,                        \
4329                 &cm_counter_attr_##_group[5].attr.attr,                        \
4330                 &cm_counter_attr_##_group[6].attr.attr,                        \
4331                 &cm_counter_attr_##_group[7].attr.attr,                        \
4332                 &cm_counter_attr_##_group[8].attr.attr,                        \
4333                 &cm_counter_attr_##_group[9].attr.attr,                        \
4334                 &cm_counter_attr_##_group[10].attr.attr,                       \
4335                 NULL,                                                          \
4336         };                                                                     \
4337         static const struct attribute_group cm_counter_group_##_group = {      \
4338                 .name = _name,                                                 \
4339                 .attrs = cm_counter_attrs_##_group,                            \
4340         };
4341
4342 CM_COUNTER_GROUP(CM_XMIT, "cm_tx_msgs")
4343 CM_COUNTER_GROUP(CM_XMIT_RETRIES, "cm_tx_retries")
4344 CM_COUNTER_GROUP(CM_RECV, "cm_rx_msgs")
4345 CM_COUNTER_GROUP(CM_RECV_DUPLICATES, "cm_rx_duplicates")
4346
4347 static const struct attribute_group *cm_counter_groups[] = {
4348         &cm_counter_group_CM_XMIT,
4349         &cm_counter_group_CM_XMIT_RETRIES,
4350         &cm_counter_group_CM_RECV,
4351         &cm_counter_group_CM_RECV_DUPLICATES,
4352         NULL,
4353 };
4354
4355 static int cm_add_one(struct ib_device *ib_device)
4356 {
4357         struct cm_device *cm_dev;
4358         struct cm_port *port;
4359         struct ib_mad_reg_req reg_req = {
4360                 .mgmt_class = IB_MGMT_CLASS_CM,
4361                 .mgmt_class_version = IB_CM_CLASS_VERSION,
4362         };
4363         struct ib_port_modify port_modify = {
4364                 .set_port_cap_mask = IB_PORT_CM_SUP
4365         };
4366         unsigned long flags;
4367         int ret;
4368         int count = 0;
4369         u32 i;
4370
4371         cm_dev = kzalloc(struct_size(cm_dev, port, ib_device->phys_port_cnt),
4372                          GFP_KERNEL);
4373         if (!cm_dev)
4374                 return -ENOMEM;
4375
4376         kref_init(&cm_dev->kref);
4377         spin_lock_init(&cm_dev->mad_agent_lock);
4378         cm_dev->ib_device = ib_device;
4379         cm_dev->ack_delay = ib_device->attrs.local_ca_ack_delay;
4380         cm_dev->going_down = 0;
4381
4382         ib_set_client_data(ib_device, &cm_client, cm_dev);
4383
4384         set_bit(IB_MGMT_METHOD_SEND, reg_req.method_mask);
4385         rdma_for_each_port (ib_device, i) {
4386                 if (!rdma_cap_ib_cm(ib_device, i))
4387                         continue;
4388
4389                 port = kzalloc(sizeof *port, GFP_KERNEL);
4390                 if (!port) {
4391                         ret = -ENOMEM;
4392                         goto error1;
4393                 }
4394
4395                 cm_dev->port[i-1] = port;
4396                 port->cm_dev = cm_dev;
4397                 port->port_num = i;
4398
4399                 ret = ib_port_register_client_groups(ib_device, i,
4400                                                      cm_counter_groups);
4401                 if (ret)
4402                         goto error1;
4403
4404                 port->mad_agent = ib_register_mad_agent(ib_device, i,
4405                                                         IB_QPT_GSI,
4406                                                         &reg_req,
4407                                                         0,
4408                                                         cm_send_handler,
4409                                                         cm_recv_handler,
4410                                                         port,
4411                                                         0);
4412                 if (IS_ERR(port->mad_agent)) {
4413                         ret = PTR_ERR(port->mad_agent);
4414                         goto error2;
4415                 }
4416
4417                 ret = ib_modify_port(ib_device, i, 0, &port_modify);
4418                 if (ret)
4419                         goto error3;
4420
4421                 count++;
4422         }
4423
4424         if (!count) {
4425                 ret = -EOPNOTSUPP;
4426                 goto free;
4427         }
4428
4429         write_lock_irqsave(&cm.device_lock, flags);
4430         list_add_tail(&cm_dev->list, &cm.device_list);
4431         write_unlock_irqrestore(&cm.device_lock, flags);
4432         return 0;
4433
4434 error3:
4435         ib_unregister_mad_agent(port->mad_agent);
4436 error2:
4437         ib_port_unregister_client_groups(ib_device, i, cm_counter_groups);
4438 error1:
4439         port_modify.set_port_cap_mask = 0;
4440         port_modify.clr_port_cap_mask = IB_PORT_CM_SUP;
4441         while (--i) {
4442                 if (!rdma_cap_ib_cm(ib_device, i))
4443                         continue;
4444
4445                 port = cm_dev->port[i-1];
4446                 ib_modify_port(ib_device, port->port_num, 0, &port_modify);
4447                 ib_unregister_mad_agent(port->mad_agent);
4448                 ib_port_unregister_client_groups(ib_device, i,
4449                                                  cm_counter_groups);
4450         }
4451 free:
4452         cm_device_put(cm_dev);
4453         return ret;
4454 }
4455
4456 static void cm_remove_one(struct ib_device *ib_device, void *client_data)
4457 {
4458         struct cm_device *cm_dev = client_data;
4459         struct cm_port *port;
4460         struct ib_port_modify port_modify = {
4461                 .clr_port_cap_mask = IB_PORT_CM_SUP
4462         };
4463         unsigned long flags;
4464         u32 i;
4465
4466         write_lock_irqsave(&cm.device_lock, flags);
4467         list_del(&cm_dev->list);
4468         write_unlock_irqrestore(&cm.device_lock, flags);
4469
4470         spin_lock_irq(&cm.lock);
4471         cm_dev->going_down = 1;
4472         spin_unlock_irq(&cm.lock);
4473
4474         rdma_for_each_port (ib_device, i) {
4475                 struct ib_mad_agent *mad_agent;
4476
4477                 if (!rdma_cap_ib_cm(ib_device, i))
4478                         continue;
4479
4480                 port = cm_dev->port[i-1];
4481                 mad_agent = port->mad_agent;
4482                 ib_modify_port(ib_device, port->port_num, 0, &port_modify);
4483                 /*
4484                  * We flush the queue here after the going_down set, this
4485                  * verify that no new works will be queued in the recv handler,
4486                  * after that we can call the unregister_mad_agent
4487                  */
4488                 flush_workqueue(cm.wq);
4489                 /*
4490                  * The above ensures no call paths from the work are running,
4491                  * the remaining paths all take the mad_agent_lock.
4492                  */
4493                 spin_lock(&cm_dev->mad_agent_lock);
4494                 port->mad_agent = NULL;
4495                 spin_unlock(&cm_dev->mad_agent_lock);
4496                 ib_unregister_mad_agent(mad_agent);
4497                 ib_port_unregister_client_groups(ib_device, i,
4498                                                  cm_counter_groups);
4499         }
4500
4501         cm_device_put(cm_dev);
4502 }
4503
4504 static int __init ib_cm_init(void)
4505 {
4506         int ret;
4507
4508         INIT_LIST_HEAD(&cm.device_list);
4509         rwlock_init(&cm.device_lock);
4510         spin_lock_init(&cm.lock);
4511         cm.listen_service_table = RB_ROOT;
4512         cm.listen_service_id = be64_to_cpu(IB_CM_ASSIGN_SERVICE_ID);
4513         cm.remote_id_table = RB_ROOT;
4514         cm.remote_qp_table = RB_ROOT;
4515         cm.remote_sidr_table = RB_ROOT;
4516         xa_init_flags(&cm.local_id_table, XA_FLAGS_ALLOC);
4517         get_random_bytes(&cm.random_id_operand, sizeof cm.random_id_operand);
4518         INIT_LIST_HEAD(&cm.timewait_list);
4519
4520         cm.wq = alloc_workqueue("ib_cm", 0, 1);
4521         if (!cm.wq) {
4522                 ret = -ENOMEM;
4523                 goto error2;
4524         }
4525
4526         ret = ib_register_client(&cm_client);
4527         if (ret)
4528                 goto error3;
4529
4530         return 0;
4531 error3:
4532         destroy_workqueue(cm.wq);
4533 error2:
4534         return ret;
4535 }
4536
4537 static void __exit ib_cm_cleanup(void)
4538 {
4539         struct cm_timewait_info *timewait_info, *tmp;
4540
4541         spin_lock_irq(&cm.lock);
4542         list_for_each_entry(timewait_info, &cm.timewait_list, list)
4543                 cancel_delayed_work(&timewait_info->work.work);
4544         spin_unlock_irq(&cm.lock);
4545
4546         ib_unregister_client(&cm_client);
4547         destroy_workqueue(cm.wq);
4548
4549         list_for_each_entry_safe(timewait_info, tmp, &cm.timewait_list, list) {
4550                 list_del(&timewait_info->list);
4551                 kfree(timewait_info);
4552         }
4553
4554         WARN_ON(!xa_empty(&cm.local_id_table));
4555 }
4556
4557 module_init(ib_cm_init);
4558 module_exit(ib_cm_cleanup);