drivers/infiniband/core/mad.c

   1 /*
   2  * Copyright (c) 2004-2007 Voltaire, Inc. All rights reserved.
   3  * Copyright (c) 2005 Intel Corporation.  All rights reserved.
   4  * Copyright (c) 2005 Mellanox Technologies Ltd.  All rights reserved.
   5  * Copyright (c) 2009 HNR Consulting. All rights reserved.
   6  * Copyright (c) 2014,2018 Intel Corporation.  All rights reserved.
   7  *
   8  * This software is available to you under a choice of one of two
   9  * licenses.  You may choose to be licensed under the terms of the GNU
  10  * General Public License (GPL) Version 2, available from the file
  11  * COPYING in the main directory of this source tree, or the
  12  * OpenIB.org BSD license below:
  13  *
  14  *     Redistribution and use in source and binary forms, with or
  15  *     without modification, are permitted provided that the following
  16  *     conditions are met:
  17  *
  18  *      - Redistributions of source code must retain the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer.
  21  *
  22  *      - Redistributions in binary form must reproduce the above
  23  *        copyright notice, this list of conditions and the following
  24  *        disclaimer in the documentation and/or other materials
  25  *        provided with the distribution.
  26  *
  27  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  28  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  29  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  30  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  31  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  32  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  33  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  34  * SOFTWARE.
  35  *
  36  */
  37
  38 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  39
  40 #include <linux/dma-mapping.h>
  41 #include <linux/slab.h>
  42 #include <linux/module.h>
  43 #include <linux/security.h>
  44 #include <linux/xarray.h>
  45 #include <rdma/ib_cache.h>
  46
  47 #include "mad_priv.h"
  48 #include "core_priv.h"
  49 #include "mad_rmpp.h"
  50 #include "smi.h"
  51 #include "opa_smi.h"
  52 #include "agent.h"
  53
  54 #define CREATE_TRACE_POINTS
  55 #include <trace/events/ib_mad.h>
  56
  57 #ifdef CONFIG_TRACEPOINTS
  58 static void create_mad_addr_info(struct ib_mad_send_wr_private *mad_send_wr,
  59                           struct ib_mad_qp_info *qp_info,
  60                           struct trace_event_raw_ib_mad_send_template *entry)
  61 {
  62         struct ib_ud_wr *wr = &mad_send_wr->send_wr;
  63         struct rdma_ah_attr attr = {};
  64
  65         rdma_query_ah(wr->ah, &attr);
  66
  67         /* These are common */
  68         entry->sl = attr.sl;
  69         entry->rqpn = wr->remote_qpn;
  70         entry->rqkey = wr->remote_qkey;
  71         entry->dlid = rdma_ah_get_dlid(&attr);
  72 }
  73 #endif
  74
  75 static int mad_sendq_size = IB_MAD_QP_SEND_SIZE;
  76 static int mad_recvq_size = IB_MAD_QP_RECV_SIZE;
  77
  78 module_param_named(send_queue_size, mad_sendq_size, int, 0444);
  79 MODULE_PARM_DESC(send_queue_size, "Size of send queue in number of work requests");
  80 module_param_named(recv_queue_size, mad_recvq_size, int, 0444);
  81 MODULE_PARM_DESC(recv_queue_size, "Size of receive queue in number of work requests");
  82
  83 static DEFINE_XARRAY_ALLOC1(ib_mad_clients);
  84 static u32 ib_mad_client_next;
  85 static struct list_head ib_mad_port_list;
  86
  87 /* Port list lock */
  88 static DEFINE_SPINLOCK(ib_mad_port_list_lock);
  89
  90 /* Forward declarations */
  91 static int method_in_use(struct ib_mad_mgmt_method_table **method,
  92                          struct ib_mad_reg_req *mad_reg_req);
  93 static void remove_mad_reg_req(struct ib_mad_agent_private *priv);
  94 static struct ib_mad_agent_private *find_mad_agent(
  95                                         struct ib_mad_port_private *port_priv,
  96                                         const struct ib_mad_hdr *mad);
  97 static int ib_mad_post_receive_mads(struct ib_mad_qp_info *qp_info,
  98                                     struct ib_mad_private *mad);
  99 static void cancel_mads(struct ib_mad_agent_private *mad_agent_priv);
 100 static void timeout_sends(struct work_struct *work);
 101 static void local_completions(struct work_struct *work);
 102 static int add_nonoui_reg_req(struct ib_mad_reg_req *mad_reg_req,
 103                               struct ib_mad_agent_private *agent_priv,
 104                               u8 mgmt_class);
 105 static int add_oui_reg_req(struct ib_mad_reg_req *mad_reg_req,
 106                            struct ib_mad_agent_private *agent_priv);
 107 static bool ib_mad_send_error(struct ib_mad_port_private *port_priv,
 108                               struct ib_wc *wc);
 109 static void ib_mad_send_done(struct ib_cq *cq, struct ib_wc *wc);
 110
 111 /*
 112  * Returns a ib_mad_port_private structure or NULL for a device/port
 113  * Assumes ib_mad_port_list_lock is being held
 114  */
 115 static inline struct ib_mad_port_private *
 116 __ib_get_mad_port(struct ib_device *device, u32 port_num)
 117 {
 118         struct ib_mad_port_private *entry;
 119
 120         list_for_each_entry(entry, &ib_mad_port_list, port_list) {
 121                 if (entry->device == device && entry->port_num == port_num)
 122                         return entry;
 123         }
 124         return NULL;
 125 }
 126
 127 /*
 128  * Wrapper function to return a ib_mad_port_private structure or NULL
 129  * for a device/port
 130  */
 131 static inline struct ib_mad_port_private *
 132 ib_get_mad_port(struct ib_device *device, u32 port_num)
 133 {
 134         struct ib_mad_port_private *entry;
 135         unsigned long flags;
 136
 137         spin_lock_irqsave(&ib_mad_port_list_lock, flags);
 138         entry = __ib_get_mad_port(device, port_num);
 139         spin_unlock_irqrestore(&ib_mad_port_list_lock, flags);
 140
 141         return entry;
 142 }
 143
 144 static inline u8 convert_mgmt_class(u8 mgmt_class)
 145 {
 146         /* Alias IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE to 0 */
 147         return mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE ?
 148                 0 : mgmt_class;
 149 }
 150
 151 static int get_spl_qp_index(enum ib_qp_type qp_type)
 152 {
 153         switch (qp_type) {
 154         case IB_QPT_SMI:
 155                 return 0;
 156         case IB_QPT_GSI:
 157                 return 1;
 158         default:
 159                 return -1;
 160         }
 161 }
 162
 163 static int vendor_class_index(u8 mgmt_class)
 164 {
 165         return mgmt_class - IB_MGMT_CLASS_VENDOR_RANGE2_START;
 166 }
 167
 168 static int is_vendor_class(u8 mgmt_class)
 169 {
 170         if ((mgmt_class < IB_MGMT_CLASS_VENDOR_RANGE2_START) ||
 171             (mgmt_class > IB_MGMT_CLASS_VENDOR_RANGE2_END))
 172                 return 0;
 173         return 1;
 174 }
 175
 176 static int is_vendor_oui(char *oui)
 177 {
 178         if (oui[0] || oui[1] || oui[2])
 179                 return 1;
 180         return 0;
 181 }
 182
 183 static int is_vendor_method_in_use(
 184                 struct ib_mad_mgmt_vendor_class *vendor_class,
 185                 struct ib_mad_reg_req *mad_reg_req)
 186 {
 187         struct ib_mad_mgmt_method_table *method;
 188         int i;
 189
 190         for (i = 0; i < MAX_MGMT_OUI; i++) {
 191                 if (!memcmp(vendor_class->oui[i], mad_reg_req->oui, 3)) {
 192                         method = vendor_class->method_table[i];
 193                         if (method) {
 194                                 if (method_in_use(&method, mad_reg_req))
 195                                         return 1;
 196                                 else
 197                                         break;
 198                         }
 199                 }
 200         }
 201         return 0;
 202 }
 203
 204 int ib_response_mad(const struct ib_mad_hdr *hdr)
 205 {
 206         return ((hdr->method & IB_MGMT_METHOD_RESP) ||
 207                 (hdr->method == IB_MGMT_METHOD_TRAP_REPRESS) ||
 208                 ((hdr->mgmt_class == IB_MGMT_CLASS_BM) &&
 209                  (hdr->attr_mod & IB_BM_ATTR_MOD_RESP)));
 210 }
 211 EXPORT_SYMBOL(ib_response_mad);
 212
 213 #define SOL_FC_MAX_DEFAULT_FRAC 4
 214 #define SOL_FC_MAX_SA_FRAC 32
 215
 216 static int get_sol_fc_max_outstanding(struct ib_mad_reg_req *mad_reg_req)
 217 {
 218         if (!mad_reg_req)
 219                 /* Send only agent */
 220                 return mad_recvq_size / SOL_FC_MAX_DEFAULT_FRAC;
 221
 222         switch (mad_reg_req->mgmt_class) {
 223         case IB_MGMT_CLASS_CM:
 224                 return mad_recvq_size / SOL_FC_MAX_DEFAULT_FRAC;
 225         case IB_MGMT_CLASS_SUBN_ADM:
 226                 return mad_recvq_size / SOL_FC_MAX_SA_FRAC;
 227         case IB_MGMT_CLASS_SUBN_LID_ROUTED:
 228         case IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE:
 229                 return min(mad_recvq_size, IB_MAD_QP_RECV_SIZE) /
 230                        SOL_FC_MAX_DEFAULT_FRAC;
 231         default:
 232                 return 0;
 233         }
 234 }
 235
 236 /*
 237  * ib_register_mad_agent - Register to send/receive MADs
 238  *
 239  * Context: Process context.
 240  */
 241 struct ib_mad_agent *ib_register_mad_agent(struct ib_device *device,
 242                                            u32 port_num,
 243                                            enum ib_qp_type qp_type,
 244                                            struct ib_mad_reg_req *mad_reg_req,
 245                                            u8 rmpp_version,
 246                                            ib_mad_send_handler send_handler,
 247                                            ib_mad_recv_handler recv_handler,
 248                                            void *context,
 249                                            u32 registration_flags)
 250 {
 251         struct ib_mad_port_private *port_priv;
 252         struct ib_mad_agent *ret = ERR_PTR(-EINVAL);
 253         struct ib_mad_agent_private *mad_agent_priv;
 254         struct ib_mad_reg_req *reg_req = NULL;
 255         struct ib_mad_mgmt_class_table *class;
 256         struct ib_mad_mgmt_vendor_class_table *vendor;
 257         struct ib_mad_mgmt_vendor_class *vendor_class;
 258         struct ib_mad_mgmt_method_table *method;
 259         int ret2, qpn;
 260         u8 mgmt_class, vclass;
 261
 262         if ((qp_type == IB_QPT_SMI && !rdma_cap_ib_smi(device, port_num)) ||
 263             (qp_type == IB_QPT_GSI && !rdma_cap_ib_cm(device, port_num)))
 264                 return ERR_PTR(-EPROTONOSUPPORT);
 265
 266         /* Validate parameters */
 267         qpn = get_spl_qp_index(qp_type);
 268         if (qpn == -1) {
 269                 dev_dbg_ratelimited(&device->dev, "%s: invalid QP Type %d\n",
 270                                     __func__, qp_type);
 271                 goto error1;
 272         }
 273
 274         if (rmpp_version && rmpp_version != IB_MGMT_RMPP_VERSION) {
 275                 dev_dbg_ratelimited(&device->dev,
 276                                     "%s: invalid RMPP Version %u\n",
 277                                     __func__, rmpp_version);
 278                 goto error1;
 279         }
 280
 281         /* Validate MAD registration request if supplied */
 282         if (mad_reg_req) {
 283                 if (mad_reg_req->mgmt_class_version >= MAX_MGMT_VERSION) {
 284                         dev_dbg_ratelimited(&device->dev,
 285                                             "%s: invalid Class Version %u\n",
 286                                             __func__,
 287                                             mad_reg_req->mgmt_class_version);
 288                         goto error1;
 289                 }
 290                 if (!recv_handler) {
 291                         dev_dbg_ratelimited(&device->dev,
 292                                             "%s: no recv_handler\n", __func__);
 293                         goto error1;
 294                 }
 295                 if (mad_reg_req->mgmt_class >= MAX_MGMT_CLASS) {
 296                         /*
 297                          * IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE is the only
 298                          * one in this range currently allowed
 299                          */
 300                         if (mad_reg_req->mgmt_class !=
 301                             IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) {
 302                                 dev_dbg_ratelimited(&device->dev,
 303                                         "%s: Invalid Mgmt Class 0x%x\n",
 304                                         __func__, mad_reg_req->mgmt_class);
 305                                 goto error1;
 306                         }
 307                 } else if (mad_reg_req->mgmt_class == 0) {
 308                         /*
 309                          * Class 0 is reserved in IBA and is used for
 310                          * aliasing of IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE
 311                          */
 312                         dev_dbg_ratelimited(&device->dev,
 313                                             "%s: Invalid Mgmt Class 0\n",
 314                                             __func__);
 315                         goto error1;
 316                 } else if (is_vendor_class(mad_reg_req->mgmt_class)) {
 317                         /*
 318                          * If class is in "new" vendor range,
 319                          * ensure supplied OUI is not zero
 320                          */
 321                         if (!is_vendor_oui(mad_reg_req->oui)) {
 322                                 dev_dbg_ratelimited(&device->dev,
 323                                         "%s: No OUI specified for class 0x%x\n",
 324                                         __func__,
 325                                         mad_reg_req->mgmt_class);
 326                                 goto error1;
 327                         }
 328                 }
 329                 /* Make sure class supplied is consistent with RMPP */
 330                 if (!ib_is_mad_class_rmpp(mad_reg_req->mgmt_class)) {
 331                         if (rmpp_version) {
 332                                 dev_dbg_ratelimited(&device->dev,
 333                                         "%s: RMPP version for non-RMPP class 0x%x\n",
 334                                         __func__, mad_reg_req->mgmt_class);
 335                                 goto error1;
 336                         }
 337                 }
 338
 339                 /* Make sure class supplied is consistent with QP type */
 340                 if (qp_type == IB_QPT_SMI) {
 341                         if ((mad_reg_req->mgmt_class !=
 342                                         IB_MGMT_CLASS_SUBN_LID_ROUTED) &&
 343                             (mad_reg_req->mgmt_class !=
 344                                         IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE)) {
 345                                 dev_dbg_ratelimited(&device->dev,
 346                                         "%s: Invalid SM QP type: class 0x%x\n",
 347                                         __func__, mad_reg_req->mgmt_class);
 348                                 goto error1;
 349                         }
 350                 } else {
 351                         if ((mad_reg_req->mgmt_class ==
 352                                         IB_MGMT_CLASS_SUBN_LID_ROUTED) ||
 353                             (mad_reg_req->mgmt_class ==
 354                                         IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE)) {
 355                                 dev_dbg_ratelimited(&device->dev,
 356                                         "%s: Invalid GS QP type: class 0x%x\n",
 357                                         __func__, mad_reg_req->mgmt_class);
 358                                 goto error1;
 359                         }
 360                 }
 361         } else {
 362                 /* No registration request supplied */
 363                 if (!send_handler)
 364                         goto error1;
 365                 if (registration_flags & IB_MAD_USER_RMPP)
 366                         goto error1;
 367         }
 368
 369         /* Validate device and port */
 370         port_priv = ib_get_mad_port(device, port_num);
 371         if (!port_priv) {
 372                 dev_dbg_ratelimited(&device->dev, "%s: Invalid port %u\n",
 373                                     __func__, port_num);
 374                 ret = ERR_PTR(-ENODEV);
 375                 goto error1;
 376         }
 377
 378         /* Verify the QP requested is supported. For example, Ethernet devices
 379          * will not have QP0.
 380          */
 381         if (!port_priv->qp_info[qpn].qp) {
 382                 dev_dbg_ratelimited(&device->dev, "%s: QP %d not supported\n",
 383                                     __func__, qpn);
 384                 ret = ERR_PTR(-EPROTONOSUPPORT);
 385                 goto error1;
 386         }
 387
 388         /* Allocate structures */
 389         mad_agent_priv = kzalloc(sizeof *mad_agent_priv, GFP_KERNEL);
 390         if (!mad_agent_priv) {
 391                 ret = ERR_PTR(-ENOMEM);
 392                 goto error1;
 393         }
 394
 395         if (mad_reg_req) {
 396                 reg_req = kmemdup(mad_reg_req, sizeof *reg_req, GFP_KERNEL);
 397                 if (!reg_req) {
 398                         ret = ERR_PTR(-ENOMEM);
 399                         goto error3;
 400                 }
 401         }
 402
 403         /* Now, fill in the various structures */
 404         mad_agent_priv->qp_info = &port_priv->qp_info[qpn];
 405         mad_agent_priv->reg_req = reg_req;
 406         mad_agent_priv->agent.rmpp_version = rmpp_version;
 407         mad_agent_priv->agent.device = device;
 408         mad_agent_priv->agent.recv_handler = recv_handler;
 409         mad_agent_priv->agent.send_handler = send_handler;
 410         mad_agent_priv->agent.context = context;
 411         mad_agent_priv->agent.qp = port_priv->qp_info[qpn].qp;
 412         mad_agent_priv->agent.port_num = port_num;
 413         mad_agent_priv->agent.flags = registration_flags;
 414         spin_lock_init(&mad_agent_priv->lock);
 415         INIT_LIST_HEAD(&mad_agent_priv->send_list);
 416         INIT_LIST_HEAD(&mad_agent_priv->wait_list);
 417         INIT_LIST_HEAD(&mad_agent_priv->rmpp_list);
 418         INIT_LIST_HEAD(&mad_agent_priv->backlog_list);
 419         INIT_DELAYED_WORK(&mad_agent_priv->timed_work, timeout_sends);
 420         INIT_LIST_HEAD(&mad_agent_priv->local_list);
 421         INIT_WORK(&mad_agent_priv->local_work, local_completions);
 422         refcount_set(&mad_agent_priv->refcount, 1);
 423         init_completion(&mad_agent_priv->comp);
 424         mad_agent_priv->sol_fc_send_count = 0;
 425         mad_agent_priv->sol_fc_wait_count = 0;
 426         mad_agent_priv->sol_fc_max =
 427                 recv_handler ? get_sol_fc_max_outstanding(mad_reg_req) : 0;
 428
 429         ret2 = ib_mad_agent_security_setup(&mad_agent_priv->agent, qp_type);
 430         if (ret2) {
 431                 ret = ERR_PTR(ret2);
 432                 goto error4;
 433         }
 434
 435         /*
 436          * The mlx4 driver uses the top byte to distinguish which virtual
 437          * function generated the MAD, so we must avoid using it.
 438          */
 439         ret2 = xa_alloc_cyclic(&ib_mad_clients, &mad_agent_priv->agent.hi_tid,
 440                         mad_agent_priv, XA_LIMIT(0, (1 << 24) - 1),
 441                         &ib_mad_client_next, GFP_KERNEL);
 442         if (ret2 < 0) {
 443                 ret = ERR_PTR(ret2);
 444                 goto error5;
 445         }
 446
 447         /*
 448          * Make sure MAD registration (if supplied)
 449          * is non overlapping with any existing ones
 450          */
 451         spin_lock_irq(&port_priv->reg_lock);
 452         if (mad_reg_req) {
 453                 mgmt_class = convert_mgmt_class(mad_reg_req->mgmt_class);
 454                 if (!is_vendor_class(mgmt_class)) {
 455                         class = port_priv->version[mad_reg_req->
 456                                                    mgmt_class_version].class;
 457                         if (class) {
 458                                 method = class->method_table[mgmt_class];
 459                                 if (method) {
 460                                         if (method_in_use(&method,
 461                                                            mad_reg_req))
 462                                                 goto error6;
 463                                 }
 464                         }
 465                         ret2 = add_nonoui_reg_req(mad_reg_req, mad_agent_priv,
 466                                                   mgmt_class);
 467                 } else {
 468                         /* "New" vendor class range */
 469                         vendor = port_priv->version[mad_reg_req->
 470                                                     mgmt_class_version].vendor;
 471                         if (vendor) {
 472                                 vclass = vendor_class_index(mgmt_class);
 473                                 vendor_class = vendor->vendor_class[vclass];
 474                                 if (vendor_class) {
 475                                         if (is_vendor_method_in_use(
 476                                                         vendor_class,
 477                                                         mad_reg_req))
 478                                                 goto error6;
 479                                 }
 480                         }
 481                         ret2 = add_oui_reg_req(mad_reg_req, mad_agent_priv);
 482                 }
 483                 if (ret2) {
 484                         ret = ERR_PTR(ret2);
 485                         goto error6;
 486                 }
 487         }
 488         spin_unlock_irq(&port_priv->reg_lock);
 489
 490         trace_ib_mad_create_agent(mad_agent_priv);
 491         return &mad_agent_priv->agent;
 492 error6:
 493         spin_unlock_irq(&port_priv->reg_lock);
 494         xa_erase(&ib_mad_clients, mad_agent_priv->agent.hi_tid);
 495 error5:
 496         ib_mad_agent_security_cleanup(&mad_agent_priv->agent);
 497 error4:
 498         kfree(reg_req);
 499 error3:
 500         kfree(mad_agent_priv);
 501 error1:
 502         return ret;
 503 }
 504 EXPORT_SYMBOL(ib_register_mad_agent);
 505
 506 static inline void deref_mad_agent(struct ib_mad_agent_private *mad_agent_priv)
 507 {
 508         if (refcount_dec_and_test(&mad_agent_priv->refcount))
 509                 complete(&mad_agent_priv->comp);
 510 }
 511
 512 static void unregister_mad_agent(struct ib_mad_agent_private *mad_agent_priv)
 513 {
 514         struct ib_mad_port_private *port_priv;
 515
 516         /* Note that we could still be handling received MADs */
 517         trace_ib_mad_unregister_agent(mad_agent_priv);
 518
 519         /*
 520          * Canceling all sends results in dropping received response
 521          * MADs, preventing us from queuing additional work
 522          */
 523         cancel_mads(mad_agent_priv);
 524         port_priv = mad_agent_priv->qp_info->port_priv;
 525         cancel_delayed_work(&mad_agent_priv->timed_work);
 526
 527         spin_lock_irq(&port_priv->reg_lock);
 528         remove_mad_reg_req(mad_agent_priv);
 529         spin_unlock_irq(&port_priv->reg_lock);
 530         xa_erase(&ib_mad_clients, mad_agent_priv->agent.hi_tid);
 531
 532         flush_workqueue(port_priv->wq);
 533
 534         deref_mad_agent(mad_agent_priv);
 535         wait_for_completion(&mad_agent_priv->comp);
 536         ib_cancel_rmpp_recvs(mad_agent_priv);
 537
 538         ib_mad_agent_security_cleanup(&mad_agent_priv->agent);
 539
 540         kfree(mad_agent_priv->reg_req);
 541         kfree_rcu(mad_agent_priv, rcu);
 542 }
 543
 544 /*
 545  * ib_unregister_mad_agent - Unregisters a client from using MAD services
 546  *
 547  * Context: Process context.
 548  */
 549 void ib_unregister_mad_agent(struct ib_mad_agent *mad_agent)
 550 {
 551         struct ib_mad_agent_private *mad_agent_priv;
 552
 553         mad_agent_priv = container_of(mad_agent,
 554                                       struct ib_mad_agent_private,
 555                                       agent);
 556         unregister_mad_agent(mad_agent_priv);
 557 }
 558 EXPORT_SYMBOL(ib_unregister_mad_agent);
 559
 560 static void dequeue_mad(struct ib_mad_list_head *mad_list)
 561 {
 562         struct ib_mad_queue *mad_queue;
 563         unsigned long flags;
 564
 565         mad_queue = mad_list->mad_queue;
 566         spin_lock_irqsave(&mad_queue->lock, flags);
 567         list_del(&mad_list->list);
 568         mad_queue->count--;
 569         spin_unlock_irqrestore(&mad_queue->lock, flags);
 570 }
 571
 572 static void build_smp_wc(struct ib_qp *qp, struct ib_cqe *cqe, u16 slid,
 573                 u16 pkey_index, u32 port_num, struct ib_wc *wc)
 574 {
 575         memset(wc, 0, sizeof *wc);
 576         wc->wr_cqe = cqe;
 577         wc->status = IB_WC_SUCCESS;
 578         wc->opcode = IB_WC_RECV;
 579         wc->pkey_index = pkey_index;
 580         wc->byte_len = sizeof(struct ib_mad) + sizeof(struct ib_grh);
 581         wc->src_qp = IB_QP0;
 582         wc->qp = qp;
 583         wc->slid = slid;
 584         wc->sl = 0;
 585         wc->dlid_path_bits = 0;
 586         wc->port_num = port_num;
 587 }
 588
 589 static size_t mad_priv_size(const struct ib_mad_private *mp)
 590 {
 591         return sizeof(struct ib_mad_private) + mp->mad_size;
 592 }
 593
 594 static struct ib_mad_private *alloc_mad_private(size_t mad_size, gfp_t flags)
 595 {
 596         size_t size = sizeof(struct ib_mad_private) + mad_size;
 597         struct ib_mad_private *ret = kzalloc(size, flags);
 598
 599         if (ret)
 600                 ret->mad_size = mad_size;
 601
 602         return ret;
 603 }
 604
 605 static size_t port_mad_size(const struct ib_mad_port_private *port_priv)
 606 {
 607         return rdma_max_mad_size(port_priv->device, port_priv->port_num);
 608 }
 609
 610 static size_t mad_priv_dma_size(const struct ib_mad_private *mp)
 611 {
 612         return sizeof(struct ib_grh) + mp->mad_size;
 613 }
 614
 615 /*
 616  * Return 0 if SMP is to be sent
 617  * Return 1 if SMP was consumed locally (whether or not solicited)
 618  * Return < 0 if error
 619  */
 620 static int handle_outgoing_dr_smp(struct ib_mad_agent_private *mad_agent_priv,
 621                                   struct ib_mad_send_wr_private *mad_send_wr)
 622 {
 623         int ret = 0;
 624         struct ib_smp *smp = mad_send_wr->send_buf.mad;
 625         struct opa_smp *opa_smp = (struct opa_smp *)smp;
 626         unsigned long flags;
 627         struct ib_mad_local_private *local;
 628         struct ib_mad_private *mad_priv;
 629         struct ib_mad_port_private *port_priv;
 630         struct ib_mad_agent_private *recv_mad_agent = NULL;
 631         struct ib_device *device = mad_agent_priv->agent.device;
 632         u32 port_num;
 633         struct ib_wc mad_wc;
 634         struct ib_ud_wr *send_wr = &mad_send_wr->send_wr;
 635         size_t mad_size = port_mad_size(mad_agent_priv->qp_info->port_priv);
 636         u16 out_mad_pkey_index = 0;
 637         u16 drslid;
 638         bool opa = rdma_cap_opa_mad(mad_agent_priv->qp_info->port_priv->device,
 639                                     mad_agent_priv->qp_info->port_priv->port_num);
 640
 641         if (rdma_cap_ib_switch(device) &&
 642             smp->mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE)
 643                 port_num = send_wr->port_num;
 644         else
 645                 port_num = mad_agent_priv->agent.port_num;
 646
 647         /*
 648          * Directed route handling starts if the initial LID routed part of
 649          * a request or the ending LID routed part of a response is empty.
 650          * If we are at the start of the LID routed part, don't update the
 651          * hop_ptr or hop_cnt.  See section 14.2.2, Vol 1 IB spec.
 652          */
 653         if (opa && smp->class_version == OPA_SM_CLASS_VERSION) {
 654                 u32 opa_drslid;
 655
 656                 trace_ib_mad_handle_out_opa_smi(opa_smp);
 657
 658                 if ((opa_get_smp_direction(opa_smp)
 659                      ? opa_smp->route.dr.dr_dlid : opa_smp->route.dr.dr_slid) ==
 660                      OPA_LID_PERMISSIVE &&
 661                      opa_smi_handle_dr_smp_send(opa_smp,
 662                                                 rdma_cap_ib_switch(device),
 663                                                 port_num) == IB_SMI_DISCARD) {
 664                         ret = -EINVAL;
 665                         dev_err(&device->dev, "OPA Invalid directed route\n");
 666                         goto out;
 667                 }
 668                 opa_drslid = be32_to_cpu(opa_smp->route.dr.dr_slid);
 669                 if (opa_drslid != be32_to_cpu(OPA_LID_PERMISSIVE) &&
 670                     opa_drslid & 0xffff0000) {
 671                         ret = -EINVAL;
 672                         dev_err(&device->dev, "OPA Invalid dr_slid 0x%x\n",
 673                                opa_drslid);
 674                         goto out;
 675                 }
 676                 drslid = (u16)(opa_drslid & 0x0000ffff);
 677
 678                 /* Check to post send on QP or process locally */
 679                 if (opa_smi_check_local_smp(opa_smp, device) == IB_SMI_DISCARD &&
 680                     opa_smi_check_local_returning_smp(opa_smp, device) == IB_SMI_DISCARD)
 681                         goto out;
 682         } else {
 683                 trace_ib_mad_handle_out_ib_smi(smp);
 684
 685                 if ((ib_get_smp_direction(smp) ? smp->dr_dlid : smp->dr_slid) ==
 686                      IB_LID_PERMISSIVE &&
 687                      smi_handle_dr_smp_send(smp, rdma_cap_ib_switch(device), port_num) ==
 688                      IB_SMI_DISCARD) {
 689                         ret = -EINVAL;
 690                         dev_err(&device->dev, "Invalid directed route\n");
 691                         goto out;
 692                 }
 693                 drslid = be16_to_cpu(smp->dr_slid);
 694
 695                 /* Check to post send on QP or process locally */
 696                 if (smi_check_local_smp(smp, device) == IB_SMI_DISCARD &&
 697                     smi_check_local_returning_smp(smp, device) == IB_SMI_DISCARD)
 698                         goto out;
 699         }
 700
 701         local = kmalloc(sizeof *local, GFP_ATOMIC);
 702         if (!local) {
 703                 ret = -ENOMEM;
 704                 goto out;
 705         }
 706         local->mad_priv = NULL;
 707         local->recv_mad_agent = NULL;
 708         mad_priv = alloc_mad_private(mad_size, GFP_ATOMIC);
 709         if (!mad_priv) {
 710                 ret = -ENOMEM;
 711                 kfree(local);
 712                 goto out;
 713         }
 714
 715         build_smp_wc(mad_agent_priv->agent.qp,
 716                      send_wr->wr.wr_cqe, drslid,
 717                      send_wr->pkey_index,
 718                      send_wr->port_num, &mad_wc);
 719
 720         if (opa && smp->base_version == OPA_MGMT_BASE_VERSION) {
 721                 mad_wc.byte_len = mad_send_wr->send_buf.hdr_len
 722                                         + mad_send_wr->send_buf.data_len
 723                                         + sizeof(struct ib_grh);
 724         }
 725
 726         /* No GRH for DR SMP */
 727         ret = device->ops.process_mad(device, 0, port_num, &mad_wc, NULL,
 728                                       (const struct ib_mad *)smp,
 729                                       (struct ib_mad *)mad_priv->mad, &mad_size,
 730                                       &out_mad_pkey_index);
 731         switch (ret) {
 732         case IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY:
 733                 if (ib_response_mad((const struct ib_mad_hdr *)mad_priv->mad) &&
 734                     mad_agent_priv->agent.recv_handler) {
 735                         local->mad_priv = mad_priv;
 736                         local->recv_mad_agent = mad_agent_priv;
 737                         /*
 738                          * Reference MAD agent until receive
 739                          * side of local completion handled
 740                          */
 741                         refcount_inc(&mad_agent_priv->refcount);
 742                 } else
 743                         kfree(mad_priv);
 744                 break;
 745         case IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_CONSUMED:
 746                 kfree(mad_priv);
 747                 break;
 748         case IB_MAD_RESULT_SUCCESS:
 749                 /* Treat like an incoming receive MAD */
 750                 port_priv = ib_get_mad_port(mad_agent_priv->agent.device,
 751                                             mad_agent_priv->agent.port_num);
 752                 if (port_priv) {
 753                         memcpy(mad_priv->mad, smp, mad_priv->mad_size);
 754                         recv_mad_agent = find_mad_agent(port_priv,
 755                                                         (const struct ib_mad_hdr *)mad_priv->mad);
 756                 }
 757                 if (!port_priv || !recv_mad_agent) {
 758                         /*
 759                          * No receiving agent so drop packet and
 760                          * generate send completion.
 761                          */
 762                         kfree(mad_priv);
 763                         break;
 764                 }
 765                 local->mad_priv = mad_priv;
 766                 local->recv_mad_agent = recv_mad_agent;
 767                 break;
 768         default:
 769                 kfree(mad_priv);
 770                 kfree(local);
 771                 ret = -EINVAL;
 772                 goto out;
 773         }
 774
 775         local->mad_send_wr = mad_send_wr;
 776         if (opa) {
 777                 local->mad_send_wr->send_wr.pkey_index = out_mad_pkey_index;
 778                 local->return_wc_byte_len = mad_size;
 779         }
 780         /* Reference MAD agent until send side of local completion handled */
 781         refcount_inc(&mad_agent_priv->refcount);
 782         /* Queue local completion to local list */
 783         spin_lock_irqsave(&mad_agent_priv->lock, flags);
 784         list_add_tail(&local->completion_list, &mad_agent_priv->local_list);
 785         spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
 786         queue_work(mad_agent_priv->qp_info->port_priv->wq,
 787                    &mad_agent_priv->local_work);
 788         ret = 1;
 789 out:
 790         return ret;
 791 }
 792
 793 static int get_pad_size(int hdr_len, int data_len, size_t mad_size)
 794 {
 795         int seg_size, pad;
 796
 797         seg_size = mad_size - hdr_len;
 798         if (data_len && seg_size) {
 799                 pad = seg_size - data_len % seg_size;
 800                 return pad == seg_size ? 0 : pad;
 801         } else
 802                 return seg_size;
 803 }
 804
 805 static void free_send_rmpp_list(struct ib_mad_send_wr_private *mad_send_wr)
 806 {
 807         struct ib_rmpp_segment *s, *t;
 808
 809         list_for_each_entry_safe(s, t, &mad_send_wr->rmpp_list, list) {
 810                 list_del(&s->list);
 811                 kfree(s);
 812         }
 813 }
 814
 815 static int alloc_send_rmpp_list(struct ib_mad_send_wr_private *send_wr,
 816                                 size_t mad_size, gfp_t gfp_mask)
 817 {
 818         struct ib_mad_send_buf *send_buf = &send_wr->send_buf;
 819         struct ib_rmpp_mad *rmpp_mad = send_buf->mad;
 820         struct ib_rmpp_segment *seg = NULL;
 821         int left, seg_size, pad;
 822
 823         send_buf->seg_size = mad_size - send_buf->hdr_len;
 824         send_buf->seg_rmpp_size = mad_size - IB_MGMT_RMPP_HDR;
 825         seg_size = send_buf->seg_size;
 826         pad = send_wr->pad;
 827
 828         /* Allocate data segments. */
 829         for (left = send_buf->data_len + pad; left > 0; left -= seg_size) {
 830                 seg = kmalloc(sizeof(*seg) + seg_size, gfp_mask);
 831                 if (!seg) {
 832                         free_send_rmpp_list(send_wr);
 833                         return -ENOMEM;
 834                 }
 835                 seg->num = ++send_buf->seg_count;
 836                 list_add_tail(&seg->list, &send_wr->rmpp_list);
 837         }
 838
 839         /* Zero any padding */
 840         if (pad)
 841                 memset(seg->data + seg_size - pad, 0, pad);
 842
 843         rmpp_mad->rmpp_hdr.rmpp_version = send_wr->mad_agent_priv->
 844                                           agent.rmpp_version;
 845         rmpp_mad->rmpp_hdr.rmpp_type = IB_MGMT_RMPP_TYPE_DATA;
 846         ib_set_rmpp_flags(&rmpp_mad->rmpp_hdr, IB_MGMT_RMPP_FLAG_ACTIVE);
 847
 848         send_wr->cur_seg = container_of(send_wr->rmpp_list.next,
 849                                         struct ib_rmpp_segment, list);
 850         send_wr->last_ack_seg = send_wr->cur_seg;
 851         return 0;
 852 }
 853
 854 int ib_mad_kernel_rmpp_agent(const struct ib_mad_agent *agent)
 855 {
 856         return agent->rmpp_version && !(agent->flags & IB_MAD_USER_RMPP);
 857 }
 858 EXPORT_SYMBOL(ib_mad_kernel_rmpp_agent);
 859
 860 struct ib_mad_send_buf *ib_create_send_mad(struct ib_mad_agent *mad_agent,
 861                                            u32 remote_qpn, u16 pkey_index,
 862                                            int rmpp_active, int hdr_len,
 863                                            int data_len, gfp_t gfp_mask,
 864                                            u8 base_version)
 865 {
 866         struct ib_mad_agent_private *mad_agent_priv;
 867         struct ib_mad_send_wr_private *mad_send_wr;
 868         int pad, message_size, ret, size;
 869         void *buf;
 870         size_t mad_size;
 871         bool opa;
 872
 873         mad_agent_priv = container_of(mad_agent, struct ib_mad_agent_private,
 874                                       agent);
 875
 876         opa = rdma_cap_opa_mad(mad_agent->device, mad_agent->port_num);
 877
 878         if (opa && base_version == OPA_MGMT_BASE_VERSION)
 879                 mad_size = sizeof(struct opa_mad);
 880         else
 881                 mad_size = sizeof(struct ib_mad);
 882
 883         pad = get_pad_size(hdr_len, data_len, mad_size);
 884         message_size = hdr_len + data_len + pad;
 885
 886         if (ib_mad_kernel_rmpp_agent(mad_agent)) {
 887                 if (!rmpp_active && message_size > mad_size)
 888                         return ERR_PTR(-EINVAL);
 889         } else
 890                 if (rmpp_active || message_size > mad_size)
 891                         return ERR_PTR(-EINVAL);
 892
 893         size = rmpp_active ? hdr_len : mad_size;
 894         buf = kzalloc(sizeof *mad_send_wr + size, gfp_mask);
 895         if (!buf)
 896                 return ERR_PTR(-ENOMEM);
 897
 898         mad_send_wr = buf + size;
 899         INIT_LIST_HEAD(&mad_send_wr->rmpp_list);
 900         mad_send_wr->send_buf.mad = buf;
 901         mad_send_wr->send_buf.hdr_len = hdr_len;
 902         mad_send_wr->send_buf.data_len = data_len;
 903         mad_send_wr->pad = pad;
 904
 905         mad_send_wr->mad_agent_priv = mad_agent_priv;
 906         mad_send_wr->sg_list[0].length = hdr_len;
 907         mad_send_wr->sg_list[0].lkey = mad_agent->qp->pd->local_dma_lkey;
 908
 909         /* OPA MADs don't have to be the full 2048 bytes */
 910         if (opa && base_version == OPA_MGMT_BASE_VERSION &&
 911             data_len < mad_size - hdr_len)
 912                 mad_send_wr->sg_list[1].length = data_len;
 913         else
 914                 mad_send_wr->sg_list[1].length = mad_size - hdr_len;
 915
 916         mad_send_wr->sg_list[1].lkey = mad_agent->qp->pd->local_dma_lkey;
 917
 918         mad_send_wr->mad_list.cqe.done = ib_mad_send_done;
 919
 920         mad_send_wr->send_wr.wr.wr_cqe = &mad_send_wr->mad_list.cqe;
 921         mad_send_wr->send_wr.wr.sg_list = mad_send_wr->sg_list;
 922         mad_send_wr->send_wr.wr.num_sge = 2;
 923         mad_send_wr->send_wr.wr.opcode = IB_WR_SEND;
 924         mad_send_wr->send_wr.wr.send_flags = IB_SEND_SIGNALED;
 925         mad_send_wr->send_wr.remote_qpn = remote_qpn;
 926         mad_send_wr->send_wr.remote_qkey = IB_QP_SET_QKEY;
 927         mad_send_wr->send_wr.pkey_index = pkey_index;
 928
 929         if (rmpp_active) {
 930                 ret = alloc_send_rmpp_list(mad_send_wr, mad_size, gfp_mask);
 931                 if (ret) {
 932                         kfree(buf);
 933                         return ERR_PTR(ret);
 934                 }
 935         }
 936
 937         mad_send_wr->send_buf.mad_agent = mad_agent;
 938         refcount_inc(&mad_agent_priv->refcount);
 939         return &mad_send_wr->send_buf;
 940 }
 941 EXPORT_SYMBOL(ib_create_send_mad);
 942
 943 int ib_get_mad_data_offset(u8 mgmt_class)
 944 {
 945         if (mgmt_class == IB_MGMT_CLASS_SUBN_ADM)
 946                 return IB_MGMT_SA_HDR;
 947         else if ((mgmt_class == IB_MGMT_CLASS_DEVICE_MGMT) ||
 948                  (mgmt_class == IB_MGMT_CLASS_DEVICE_ADM) ||
 949                  (mgmt_class == IB_MGMT_CLASS_BIS))
 950                 return IB_MGMT_DEVICE_HDR;
 951         else if ((mgmt_class >= IB_MGMT_CLASS_VENDOR_RANGE2_START) &&
 952                  (mgmt_class <= IB_MGMT_CLASS_VENDOR_RANGE2_END))
 953                 return IB_MGMT_VENDOR_HDR;
 954         else
 955                 return IB_MGMT_MAD_HDR;
 956 }
 957 EXPORT_SYMBOL(ib_get_mad_data_offset);
 958
 959 int ib_is_mad_class_rmpp(u8 mgmt_class)
 960 {
 961         if ((mgmt_class == IB_MGMT_CLASS_SUBN_ADM) ||
 962             (mgmt_class == IB_MGMT_CLASS_DEVICE_MGMT) ||
 963             (mgmt_class == IB_MGMT_CLASS_DEVICE_ADM) ||
 964             (mgmt_class == IB_MGMT_CLASS_BIS) ||
 965             ((mgmt_class >= IB_MGMT_CLASS_VENDOR_RANGE2_START) &&
 966              (mgmt_class <= IB_MGMT_CLASS_VENDOR_RANGE2_END)))
 967                 return 1;
 968         return 0;
 969 }
 970 EXPORT_SYMBOL(ib_is_mad_class_rmpp);
 971
 972 void *ib_get_rmpp_segment(struct ib_mad_send_buf *send_buf, int seg_num)
 973 {
 974         struct ib_mad_send_wr_private *mad_send_wr;
 975         struct list_head *list;
 976
 977         mad_send_wr = container_of(send_buf, struct ib_mad_send_wr_private,
 978                                    send_buf);
 979         list = &mad_send_wr->cur_seg->list;
 980
 981         if (mad_send_wr->cur_seg->num < seg_num) {
 982                 list_for_each_entry(mad_send_wr->cur_seg, list, list)
 983                         if (mad_send_wr->cur_seg->num == seg_num)
 984                                 break;
 985         } else if (mad_send_wr->cur_seg->num > seg_num) {
 986                 list_for_each_entry_reverse(mad_send_wr->cur_seg, list, list)
 987                         if (mad_send_wr->cur_seg->num == seg_num)
 988                                 break;
 989         }
 990         return mad_send_wr->cur_seg->data;
 991 }
 992 EXPORT_SYMBOL(ib_get_rmpp_segment);
 993
 994 static inline void *ib_get_payload(struct ib_mad_send_wr_private *mad_send_wr)
 995 {
 996         if (mad_send_wr->send_buf.seg_count)
 997                 return ib_get_rmpp_segment(&mad_send_wr->send_buf,
 998                                            mad_send_wr->seg_num);
 999         else
1000                 return mad_send_wr->send_buf.mad +
1001                        mad_send_wr->send_buf.hdr_len;
1002 }
1003
1004 void ib_free_send_mad(struct ib_mad_send_buf *send_buf)
1005 {
1006         struct ib_mad_agent_private *mad_agent_priv;
1007         struct ib_mad_send_wr_private *mad_send_wr;
1008
1009         mad_agent_priv = container_of(send_buf->mad_agent,
1010                                       struct ib_mad_agent_private, agent);
1011         mad_send_wr = container_of(send_buf, struct ib_mad_send_wr_private,
1012                                    send_buf);
1013
1014         free_send_rmpp_list(mad_send_wr);
1015         kfree(send_buf->mad);
1016         deref_mad_agent(mad_agent_priv);
1017 }
1018 EXPORT_SYMBOL(ib_free_send_mad);
1019
1020 int ib_send_mad(struct ib_mad_send_wr_private *mad_send_wr)
1021 {
1022         struct ib_mad_qp_info *qp_info;
1023         struct list_head *list;
1024         struct ib_mad_agent *mad_agent;
1025         struct ib_sge *sge;
1026         unsigned long flags;
1027         int ret;
1028
1029         /* Set WR ID to find mad_send_wr upon completion */
1030         qp_info = mad_send_wr->mad_agent_priv->qp_info;
1031         mad_send_wr->mad_list.mad_queue = &qp_info->send_queue;
1032         mad_send_wr->mad_list.cqe.done = ib_mad_send_done;
1033         mad_send_wr->send_wr.wr.wr_cqe = &mad_send_wr->mad_list.cqe;
1034
1035         mad_agent = mad_send_wr->send_buf.mad_agent;
1036         sge = mad_send_wr->sg_list;
1037         sge[0].addr = ib_dma_map_single(mad_agent->device,
1038                                         mad_send_wr->send_buf.mad,
1039                                         sge[0].length,
1040                                         DMA_TO_DEVICE);
1041         if (unlikely(ib_dma_mapping_error(mad_agent->device, sge[0].addr)))
1042                 return -ENOMEM;
1043
1044         mad_send_wr->header_mapping = sge[0].addr;
1045
1046         sge[1].addr = ib_dma_map_single(mad_agent->device,
1047                                         ib_get_payload(mad_send_wr),
1048                                         sge[1].length,
1049                                         DMA_TO_DEVICE);
1050         if (unlikely(ib_dma_mapping_error(mad_agent->device, sge[1].addr))) {
1051                 ib_dma_unmap_single(mad_agent->device,
1052                                     mad_send_wr->header_mapping,
1053                                     sge[0].length, DMA_TO_DEVICE);
1054                 return -ENOMEM;
1055         }
1056         mad_send_wr->payload_mapping = sge[1].addr;
1057
1058         spin_lock_irqsave(&qp_info->send_queue.lock, flags);
1059         if (qp_info->send_queue.count < qp_info->send_queue.max_active) {
1060                 trace_ib_mad_ib_send_mad(mad_send_wr, qp_info);
1061                 ret = ib_post_send(mad_agent->qp, &mad_send_wr->send_wr.wr,
1062                                    NULL);
1063                 list = &qp_info->send_queue.list;
1064         } else {
1065                 ret = 0;
1066                 list = &qp_info->overflow_list;
1067         }
1068
1069         if (!ret) {
1070                 qp_info->send_queue.count++;
1071                 list_add_tail(&mad_send_wr->mad_list.list, list);
1072         }
1073         spin_unlock_irqrestore(&qp_info->send_queue.lock, flags);
1074         if (ret) {
1075                 ib_dma_unmap_single(mad_agent->device,
1076                                     mad_send_wr->header_mapping,
1077                                     sge[0].length, DMA_TO_DEVICE);
1078                 ib_dma_unmap_single(mad_agent->device,
1079                                     mad_send_wr->payload_mapping,
1080                                     sge[1].length, DMA_TO_DEVICE);
1081         }
1082         return ret;
1083 }
1084
1085 static void handle_queued_state(struct ib_mad_send_wr_private *mad_send_wr,
1086                        struct ib_mad_agent_private *mad_agent_priv)
1087 {
1088         if (mad_send_wr->state == IB_MAD_STATE_WAIT_RESP) {
1089                 mad_agent_priv->sol_fc_wait_count--;
1090                 list_move_tail(&mad_send_wr->agent_list,
1091                                &mad_agent_priv->backlog_list);
1092         } else {
1093                 expect_mad_state(mad_send_wr, IB_MAD_STATE_INIT);
1094                 list_add_tail(&mad_send_wr->agent_list,
1095                               &mad_agent_priv->backlog_list);
1096         }
1097 }
1098
1099 static void handle_send_state(struct ib_mad_send_wr_private *mad_send_wr,
1100                        struct ib_mad_agent_private *mad_agent_priv)
1101 {
1102         if (mad_send_wr->state == IB_MAD_STATE_INIT) {
1103                 list_add_tail(&mad_send_wr->agent_list,
1104                               &mad_agent_priv->send_list);
1105         } else {
1106                 expect_mad_state2(mad_send_wr, IB_MAD_STATE_WAIT_RESP,
1107                                   IB_MAD_STATE_QUEUED);
1108                 list_move_tail(&mad_send_wr->agent_list,
1109                                &mad_agent_priv->send_list);
1110         }
1111
1112         if (mad_send_wr->is_solicited_fc) {
1113                 if (mad_send_wr->state == IB_MAD_STATE_WAIT_RESP)
1114                         mad_agent_priv->sol_fc_wait_count--;
1115                 mad_agent_priv->sol_fc_send_count++;
1116         }
1117 }
1118
1119 static void handle_wait_state(struct ib_mad_send_wr_private *mad_send_wr,
1120                        struct ib_mad_agent_private *mad_agent_priv)
1121 {
1122         struct ib_mad_send_wr_private *temp_mad_send_wr;
1123         struct list_head *list_item;
1124         unsigned long delay;
1125
1126         expect_mad_state3(mad_send_wr, IB_MAD_STATE_SEND_START,
1127                           IB_MAD_STATE_WAIT_RESP, IB_MAD_STATE_CANCELED);
1128         if (mad_send_wr->state == IB_MAD_STATE_SEND_START &&
1129             mad_send_wr->is_solicited_fc) {
1130                 mad_agent_priv->sol_fc_send_count--;
1131                 mad_agent_priv->sol_fc_wait_count++;
1132         }
1133
1134         list_del_init(&mad_send_wr->agent_list);
1135         delay = mad_send_wr->timeout;
1136         mad_send_wr->timeout += jiffies;
1137
1138         if (delay) {
1139                 list_for_each_prev(list_item,
1140                                    &mad_agent_priv->wait_list) {
1141                         temp_mad_send_wr = list_entry(
1142                                 list_item,
1143                                 struct ib_mad_send_wr_private,
1144                                 agent_list);
1145                         if (time_after(mad_send_wr->timeout,
1146                                        temp_mad_send_wr->timeout))
1147                                 break;
1148                 }
1149         } else {
1150                 list_item = &mad_agent_priv->wait_list;
1151         }
1152
1153         list_add(&mad_send_wr->agent_list, list_item);
1154 }
1155
1156 static void handle_early_resp_state(struct ib_mad_send_wr_private *mad_send_wr,
1157                             struct ib_mad_agent_private *mad_agent_priv)
1158 {
1159         expect_mad_state(mad_send_wr, IB_MAD_STATE_SEND_START);
1160         mad_agent_priv->sol_fc_send_count -= mad_send_wr->is_solicited_fc;
1161 }
1162
1163 static void handle_canceled_state(struct ib_mad_send_wr_private *mad_send_wr,
1164                          struct ib_mad_agent_private *mad_agent_priv)
1165 {
1166         not_expect_mad_state(mad_send_wr, IB_MAD_STATE_DONE);
1167         if (mad_send_wr->is_solicited_fc) {
1168                 if (mad_send_wr->state == IB_MAD_STATE_SEND_START)
1169                         mad_agent_priv->sol_fc_send_count--;
1170                 else if (mad_send_wr->state == IB_MAD_STATE_WAIT_RESP)
1171                         mad_agent_priv->sol_fc_wait_count--;
1172         }
1173 }
1174
1175 static void handle_done_state(struct ib_mad_send_wr_private *mad_send_wr,
1176                        struct ib_mad_agent_private *mad_agent_priv)
1177 {
1178         if (mad_send_wr->is_solicited_fc) {
1179                 if (mad_send_wr->state == IB_MAD_STATE_SEND_START)
1180                         mad_agent_priv->sol_fc_send_count--;
1181                 else if (mad_send_wr->state == IB_MAD_STATE_WAIT_RESP)
1182                         mad_agent_priv->sol_fc_wait_count--;
1183         }
1184
1185         list_del_init(&mad_send_wr->agent_list);
1186 }
1187
1188 void change_mad_state(struct ib_mad_send_wr_private *mad_send_wr,
1189                              enum ib_mad_state new_state)
1190 {
1191         struct ib_mad_agent_private *mad_agent_priv =
1192                 mad_send_wr->mad_agent_priv;
1193
1194         switch (new_state) {
1195         case IB_MAD_STATE_INIT:
1196                 break;
1197         case IB_MAD_STATE_QUEUED:
1198                 handle_queued_state(mad_send_wr, mad_agent_priv);
1199                 break;
1200         case IB_MAD_STATE_SEND_START:
1201                 handle_send_state(mad_send_wr, mad_agent_priv);
1202                 break;
1203         case IB_MAD_STATE_WAIT_RESP:
1204                 handle_wait_state(mad_send_wr, mad_agent_priv);
1205                 if (mad_send_wr->state == IB_MAD_STATE_CANCELED)
1206                         return;
1207                 break;
1208         case IB_MAD_STATE_EARLY_RESP:
1209                 handle_early_resp_state(mad_send_wr, mad_agent_priv);
1210                 break;
1211         case IB_MAD_STATE_CANCELED:
1212                 handle_canceled_state(mad_send_wr, mad_agent_priv);
1213                 break;
1214         case IB_MAD_STATE_DONE:
1215                 handle_done_state(mad_send_wr, mad_agent_priv);
1216                 break;
1217         }
1218
1219         mad_send_wr->state = new_state;
1220 }
1221
1222 static bool is_solicited_fc_mad(struct ib_mad_send_wr_private *mad_send_wr)
1223 {
1224         struct ib_rmpp_mad *rmpp_mad;
1225         u8 mgmt_class;
1226
1227         if (!mad_send_wr->timeout)
1228                 return 0;
1229
1230         rmpp_mad = mad_send_wr->send_buf.mad;
1231         if (mad_send_wr->mad_agent_priv->agent.rmpp_version &&
1232             (ib_get_rmpp_flags(&rmpp_mad->rmpp_hdr) & IB_MGMT_RMPP_FLAG_ACTIVE))
1233                 return 0;
1234
1235         mgmt_class =
1236                 ((struct ib_mad_hdr *)mad_send_wr->send_buf.mad)->mgmt_class;
1237         return mgmt_class == IB_MGMT_CLASS_CM ||
1238                mgmt_class == IB_MGMT_CLASS_SUBN_ADM ||
1239                mgmt_class == IB_MGMT_CLASS_SUBN_LID_ROUTED ||
1240                mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE;
1241 }
1242
1243 static bool mad_is_for_backlog(struct ib_mad_send_wr_private *mad_send_wr)
1244 {
1245         struct ib_mad_agent_private *mad_agent_priv =
1246                 mad_send_wr->mad_agent_priv;
1247
1248         if (!mad_send_wr->is_solicited_fc || !mad_agent_priv->sol_fc_max)
1249                 return false;
1250
1251         if (!list_empty(&mad_agent_priv->backlog_list))
1252                 return true;
1253
1254         return mad_agent_priv->sol_fc_send_count +
1255                        mad_agent_priv->sol_fc_wait_count >=
1256                mad_agent_priv->sol_fc_max;
1257 }
1258
1259 /*
1260  * ib_post_send_mad - Posts MAD(s) to the send queue of the QP associated
1261  *  with the registered client
1262  */
1263 int ib_post_send_mad(struct ib_mad_send_buf *send_buf,
1264                      struct ib_mad_send_buf **bad_send_buf)
1265 {
1266         struct ib_mad_agent_private *mad_agent_priv;
1267         struct ib_mad_send_buf *next_send_buf;
1268         struct ib_mad_send_wr_private *mad_send_wr;
1269         unsigned long flags;
1270         int ret = -EINVAL;
1271
1272         /* Walk list of send WRs and post each on send list */
1273         for (; send_buf; send_buf = next_send_buf) {
1274                 mad_send_wr = container_of(send_buf,
1275                                            struct ib_mad_send_wr_private,
1276                                            send_buf);
1277                 mad_agent_priv = mad_send_wr->mad_agent_priv;
1278
1279                 ret = ib_mad_enforce_security(mad_agent_priv,
1280                                               mad_send_wr->send_wr.pkey_index);
1281                 if (ret)
1282                         goto error;
1283
1284                 if (!send_buf->mad_agent->send_handler) {
1285                         ret = -EINVAL;
1286                         goto error;
1287                 }
1288
1289                 if (!ib_is_mad_class_rmpp(((struct ib_mad_hdr *) send_buf->mad)->mgmt_class)) {
1290                         if (mad_agent_priv->agent.rmpp_version) {
1291                                 ret = -EINVAL;
1292                                 goto error;
1293                         }
1294                 }
1295
1296                 /*
1297                  * Save pointer to next work request to post in case the
1298                  * current one completes, and the user modifies the work
1299                  * request associated with the completion
1300                  */
1301                 next_send_buf = send_buf->next;
1302                 mad_send_wr->send_wr.ah = send_buf->ah;
1303
1304                 if (((struct ib_mad_hdr *) send_buf->mad)->mgmt_class ==
1305                     IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) {
1306                         ret = handle_outgoing_dr_smp(mad_agent_priv,
1307                                                      mad_send_wr);
1308                         if (ret < 0)            /* error */
1309                                 goto error;
1310                         else if (ret == 1)      /* locally consumed */
1311                                 continue;
1312                 }
1313
1314                 mad_send_wr->tid = ((struct ib_mad_hdr *) send_buf->mad)->tid;
1315                 /* Timeout will be updated after send completes */
1316                 mad_send_wr->timeout = msecs_to_jiffies(send_buf->timeout_ms);
1317                 mad_send_wr->max_retries = send_buf->retries;
1318                 mad_send_wr->retries_left = send_buf->retries;
1319                 send_buf->retries = 0;
1320                 change_mad_state(mad_send_wr, IB_MAD_STATE_INIT);
1321
1322                 /* Reference MAD agent until send completes */
1323                 refcount_inc(&mad_agent_priv->refcount);
1324                 spin_lock_irqsave(&mad_agent_priv->lock, flags);
1325                 mad_send_wr->is_solicited_fc = is_solicited_fc_mad(mad_send_wr);
1326                 if (mad_is_for_backlog(mad_send_wr)) {
1327                         change_mad_state(mad_send_wr, IB_MAD_STATE_QUEUED);
1328                         spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
1329                         return 0;
1330                 }
1331
1332                 change_mad_state(mad_send_wr, IB_MAD_STATE_SEND_START);
1333                 spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
1334
1335                 if (ib_mad_kernel_rmpp_agent(&mad_agent_priv->agent)) {
1336                         ret = ib_send_rmpp_mad(mad_send_wr);
1337                         if (ret >= 0 && ret != IB_RMPP_RESULT_CONSUMED)
1338                                 ret = ib_send_mad(mad_send_wr);
1339                 } else
1340                         ret = ib_send_mad(mad_send_wr);
1341                 if (ret < 0) {
1342                         /* Fail send request */
1343                         spin_lock_irqsave(&mad_agent_priv->lock, flags);
1344                         change_mad_state(mad_send_wr, IB_MAD_STATE_DONE);
1345                         spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
1346                         deref_mad_agent(mad_agent_priv);
1347                         goto error;
1348                 }
1349         }
1350         return 0;
1351 error:
1352         if (bad_send_buf)
1353                 *bad_send_buf = send_buf;
1354         return ret;
1355 }
1356 EXPORT_SYMBOL(ib_post_send_mad);
1357
1358 /*
1359  * ib_free_recv_mad - Returns data buffers used to receive
1360  *  a MAD to the access layer
1361  */
1362 void ib_free_recv_mad(struct ib_mad_recv_wc *mad_recv_wc)
1363 {
1364         struct ib_mad_recv_buf *mad_recv_buf, *temp_recv_buf;
1365         struct ib_mad_private_header *mad_priv_hdr;
1366         struct ib_mad_private *priv;
1367         struct list_head free_list;
1368
1369         INIT_LIST_HEAD(&free_list);
1370         list_splice_init(&mad_recv_wc->rmpp_list, &free_list);
1371
1372         list_for_each_entry_safe(mad_recv_buf, temp_recv_buf,
1373                                         &free_list, list) {
1374                 mad_recv_wc = container_of(mad_recv_buf, struct ib_mad_recv_wc,
1375                                            recv_buf);
1376                 mad_priv_hdr = container_of(mad_recv_wc,
1377                                             struct ib_mad_private_header,
1378                                             recv_wc);
1379                 priv = container_of(mad_priv_hdr, struct ib_mad_private,
1380                                     header);
1381                 kfree(priv);
1382         }
1383 }
1384 EXPORT_SYMBOL(ib_free_recv_mad);
1385
1386 static int method_in_use(struct ib_mad_mgmt_method_table **method,
1387                          struct ib_mad_reg_req *mad_reg_req)
1388 {
1389         int i;
1390
1391         for_each_set_bit(i, mad_reg_req->method_mask, IB_MGMT_MAX_METHODS) {
1392                 if ((*method)->agent[i]) {
1393                         pr_err("Method %d already in use\n", i);
1394                         return -EINVAL;
1395                 }
1396         }
1397         return 0;
1398 }
1399
1400 static int allocate_method_table(struct ib_mad_mgmt_method_table **method)
1401 {
1402         /* Allocate management method table */
1403         *method = kzalloc(sizeof **method, GFP_ATOMIC);
1404         return (*method) ? 0 : (-ENOMEM);
1405 }
1406
1407 /*
1408  * Check to see if there are any methods still in use
1409  */
1410 static int check_method_table(struct ib_mad_mgmt_method_table *method)
1411 {
1412         int i;
1413
1414         for (i = 0; i < IB_MGMT_MAX_METHODS; i++)
1415                 if (method->agent[i])
1416                         return 1;
1417         return 0;
1418 }
1419
1420 /*
1421  * Check to see if there are any method tables for this class still in use
1422  */
1423 static int check_class_table(struct ib_mad_mgmt_class_table *class)
1424 {
1425         int i;
1426
1427         for (i = 0; i < MAX_MGMT_CLASS; i++)
1428                 if (class->method_table[i])
1429                         return 1;
1430         return 0;
1431 }
1432
1433 static int check_vendor_class(struct ib_mad_mgmt_vendor_class *vendor_class)
1434 {
1435         int i;
1436
1437         for (i = 0; i < MAX_MGMT_OUI; i++)
1438                 if (vendor_class->method_table[i])
1439                         return 1;
1440         return 0;
1441 }
1442
1443 static int find_vendor_oui(struct ib_mad_mgmt_vendor_class *vendor_class,
1444                            const char *oui)
1445 {
1446         int i;
1447
1448         for (i = 0; i < MAX_MGMT_OUI; i++)
1449                 /* Is there matching OUI for this vendor class ? */
1450                 if (!memcmp(vendor_class->oui[i], oui, 3))
1451                         return i;
1452
1453         return -1;
1454 }
1455
1456 static int check_vendor_table(struct ib_mad_mgmt_vendor_class_table *vendor)
1457 {
1458         int i;
1459
1460         for (i = 0; i < MAX_MGMT_VENDOR_RANGE2; i++)
1461                 if (vendor->vendor_class[i])
1462                         return 1;
1463
1464         return 0;
1465 }
1466
1467 static void remove_methods_mad_agent(struct ib_mad_mgmt_method_table *method,
1468                                      struct ib_mad_agent_private *agent)
1469 {
1470         int i;
1471
1472         /* Remove any methods for this mad agent */
1473         for (i = 0; i < IB_MGMT_MAX_METHODS; i++)
1474                 if (method->agent[i] == agent)
1475                         method->agent[i] = NULL;
1476 }
1477
1478 static int add_nonoui_reg_req(struct ib_mad_reg_req *mad_reg_req,
1479                               struct ib_mad_agent_private *agent_priv,
1480                               u8 mgmt_class)
1481 {
1482         struct ib_mad_port_private *port_priv;
1483         struct ib_mad_mgmt_class_table **class;
1484         struct ib_mad_mgmt_method_table **method;
1485         int i, ret;
1486
1487         port_priv = agent_priv->qp_info->port_priv;
1488         class = &port_priv->version[mad_reg_req->mgmt_class_version].class;
1489         if (!*class) {
1490                 /* Allocate management class table for "new" class version */
1491                 *class = kzalloc(sizeof **class, GFP_ATOMIC);
1492                 if (!*class) {
1493                         ret = -ENOMEM;
1494                         goto error1;
1495                 }
1496
1497                 /* Allocate method table for this management class */
1498                 method = &(*class)->method_table[mgmt_class];
1499                 if ((ret = allocate_method_table(method)))
1500                         goto error2;
1501         } else {
1502                 method = &(*class)->method_table[mgmt_class];
1503                 if (!*method) {
1504                         /* Allocate method table for this management class */
1505                         if ((ret = allocate_method_table(method)))
1506                                 goto error1;
1507                 }
1508         }
1509
1510         /* Now, make sure methods are not already in use */
1511         if (method_in_use(method, mad_reg_req))
1512                 goto error3;
1513
1514         /* Finally, add in methods being registered */
1515         for_each_set_bit(i, mad_reg_req->method_mask, IB_MGMT_MAX_METHODS)
1516                 (*method)->agent[i] = agent_priv;
1517
1518         return 0;
1519
1520 error3:
1521         /* Remove any methods for this mad agent */
1522         remove_methods_mad_agent(*method, agent_priv);
1523         /* Now, check to see if there are any methods in use */
1524         if (!check_method_table(*method)) {
1525                 /* If not, release management method table */
1526                 kfree(*method);
1527                 *method = NULL;
1528         }
1529         ret = -EINVAL;
1530         goto error1;
1531 error2:
1532         kfree(*class);
1533         *class = NULL;
1534 error1:
1535         return ret;
1536 }
1537
1538 static int add_oui_reg_req(struct ib_mad_reg_req *mad_reg_req,
1539                            struct ib_mad_agent_private *agent_priv)
1540 {
1541         struct ib_mad_port_private *port_priv;
1542         struct ib_mad_mgmt_vendor_class_table **vendor_table;
1543         struct ib_mad_mgmt_vendor_class_table *vendor = NULL;
1544         struct ib_mad_mgmt_vendor_class *vendor_class = NULL;
1545         struct ib_mad_mgmt_method_table **method;
1546         int i, ret = -ENOMEM;
1547         u8 vclass;
1548
1549         /* "New" vendor (with OUI) class */
1550         vclass = vendor_class_index(mad_reg_req->mgmt_class);
1551         port_priv = agent_priv->qp_info->port_priv;
1552         vendor_table = &port_priv->version[
1553                                 mad_reg_req->mgmt_class_version].vendor;
1554         if (!*vendor_table) {
1555                 /* Allocate mgmt vendor class table for "new" class version */
1556                 vendor = kzalloc(sizeof *vendor, GFP_ATOMIC);
1557                 if (!vendor)
1558                         goto error1;
1559
1560                 *vendor_table = vendor;
1561         }
1562         if (!(*vendor_table)->vendor_class[vclass]) {
1563                 /* Allocate table for this management vendor class */
1564                 vendor_class = kzalloc(sizeof *vendor_class, GFP_ATOMIC);
1565                 if (!vendor_class)
1566                         goto error2;
1567
1568                 (*vendor_table)->vendor_class[vclass] = vendor_class;
1569         }
1570         for (i = 0; i < MAX_MGMT_OUI; i++) {
1571                 /* Is there matching OUI for this vendor class ? */
1572                 if (!memcmp((*vendor_table)->vendor_class[vclass]->oui[i],
1573                             mad_reg_req->oui, 3)) {
1574                         method = &(*vendor_table)->vendor_class[
1575                                                 vclass]->method_table[i];
1576                         if (!*method)
1577                                 goto error3;
1578                         goto check_in_use;
1579                 }
1580         }
1581         for (i = 0; i < MAX_MGMT_OUI; i++) {
1582                 /* OUI slot available ? */
1583                 if (!is_vendor_oui((*vendor_table)->vendor_class[
1584                                 vclass]->oui[i])) {
1585                         method = &(*vendor_table)->vendor_class[
1586                                 vclass]->method_table[i];
1587                         /* Allocate method table for this OUI */
1588                         if (!*method) {
1589                                 ret = allocate_method_table(method);
1590                                 if (ret)
1591                                         goto error3;
1592                         }
1593                         memcpy((*vendor_table)->vendor_class[vclass]->oui[i],
1594                                mad_reg_req->oui, 3);
1595                         goto check_in_use;
1596                 }
1597         }
1598         dev_err(&agent_priv->agent.device->dev, "All OUI slots in use\n");
1599         goto error3;
1600
1601 check_in_use:
1602         /* Now, make sure methods are not already in use */
1603         if (method_in_use(method, mad_reg_req))
1604                 goto error4;
1605
1606         /* Finally, add in methods being registered */
1607         for_each_set_bit(i, mad_reg_req->method_mask, IB_MGMT_MAX_METHODS)
1608                 (*method)->agent[i] = agent_priv;
1609
1610         return 0;
1611
1612 error4:
1613         /* Remove any methods for this mad agent */
1614         remove_methods_mad_agent(*method, agent_priv);
1615         /* Now, check to see if there are any methods in use */
1616         if (!check_method_table(*method)) {
1617                 /* If not, release management method table */
1618                 kfree(*method);
1619                 *method = NULL;
1620         }
1621         ret = -EINVAL;
1622 error3:
1623         if (vendor_class) {
1624                 (*vendor_table)->vendor_class[vclass] = NULL;
1625                 kfree(vendor_class);
1626         }
1627 error2:
1628         if (vendor) {
1629                 *vendor_table = NULL;
1630                 kfree(vendor);
1631         }
1632 error1:
1633         return ret;
1634 }
1635
1636 static void remove_mad_reg_req(struct ib_mad_agent_private *agent_priv)
1637 {
1638         struct ib_mad_port_private *port_priv;
1639         struct ib_mad_mgmt_class_table *class;
1640         struct ib_mad_mgmt_method_table *method;
1641         struct ib_mad_mgmt_vendor_class_table *vendor;
1642         struct ib_mad_mgmt_vendor_class *vendor_class;
1643         int index;
1644         u8 mgmt_class;
1645
1646         /*
1647          * Was MAD registration request supplied
1648          * with original registration ?
1649          */
1650         if (!agent_priv->reg_req)
1651                 goto out;
1652
1653         port_priv = agent_priv->qp_info->port_priv;
1654         mgmt_class = convert_mgmt_class(agent_priv->reg_req->mgmt_class);
1655         class = port_priv->version[
1656                         agent_priv->reg_req->mgmt_class_version].class;
1657         if (!class)
1658                 goto vendor_check;
1659
1660         method = class->method_table[mgmt_class];
1661         if (method) {
1662                 /* Remove any methods for this mad agent */
1663                 remove_methods_mad_agent(method, agent_priv);
1664                 /* Now, check to see if there are any methods still in use */
1665                 if (!check_method_table(method)) {
1666                         /* If not, release management method table */
1667                         kfree(method);
1668                         class->method_table[mgmt_class] = NULL;
1669                         /* Any management classes left ? */
1670                         if (!check_class_table(class)) {
1671                                 /* If not, release management class table */
1672                                 kfree(class);
1673                                 port_priv->version[
1674                                         agent_priv->reg_req->
1675                                         mgmt_class_version].class = NULL;
1676                         }
1677                 }
1678         }
1679
1680 vendor_check:
1681         if (!is_vendor_class(mgmt_class))
1682                 goto out;
1683
1684         /* normalize mgmt_class to vendor range 2 */
1685         mgmt_class = vendor_class_index(agent_priv->reg_req->mgmt_class);
1686         vendor = port_priv->version[
1687                         agent_priv->reg_req->mgmt_class_version].vendor;
1688
1689         if (!vendor)
1690                 goto out;
1691
1692         vendor_class = vendor->vendor_class[mgmt_class];
1693         if (vendor_class) {
1694                 index = find_vendor_oui(vendor_class, agent_priv->reg_req->oui);
1695                 if (index < 0)
1696                         goto out;
1697                 method = vendor_class->method_table[index];
1698                 if (method) {
1699                         /* Remove any methods for this mad agent */
1700                         remove_methods_mad_agent(method, agent_priv);
1701                         /*
1702                          * Now, check to see if there are
1703                          * any methods still in use
1704                          */
1705                         if (!check_method_table(method)) {
1706                                 /* If not, release management method table */
1707                                 kfree(method);
1708                                 vendor_class->method_table[index] = NULL;
1709                                 memset(vendor_class->oui[index], 0, 3);
1710                                 /* Any OUIs left ? */
1711                                 if (!check_vendor_class(vendor_class)) {
1712                                         /* If not, release vendor class table */
1713                                         kfree(vendor_class);
1714                                         vendor->vendor_class[mgmt_class] = NULL;
1715                                         /* Any other vendor classes left ? */
1716                                         if (!check_vendor_table(vendor)) {
1717                                                 kfree(vendor);
1718                                                 port_priv->version[
1719                                                         agent_priv->reg_req->
1720                                                         mgmt_class_version].
1721                                                         vendor = NULL;
1722                                         }
1723                                 }
1724                         }
1725                 }
1726         }
1727
1728 out:
1729         return;
1730 }
1731
1732 static struct ib_mad_agent_private *
1733 find_mad_agent(struct ib_mad_port_private *port_priv,
1734                const struct ib_mad_hdr *mad_hdr)
1735 {
1736         struct ib_mad_agent_private *mad_agent = NULL;
1737         unsigned long flags;
1738
1739         if (ib_response_mad(mad_hdr)) {
1740                 u32 hi_tid;
1741
1742                 /*
1743                  * Routing is based on high 32 bits of transaction ID
1744                  * of MAD.
1745                  */
1746                 hi_tid = be64_to_cpu(mad_hdr->tid) >> 32;
1747                 rcu_read_lock();
1748                 mad_agent = xa_load(&ib_mad_clients, hi_tid);
1749                 if (mad_agent && !refcount_inc_not_zero(&mad_agent->refcount))
1750                         mad_agent = NULL;
1751                 rcu_read_unlock();
1752         } else {
1753                 struct ib_mad_mgmt_class_table *class;
1754                 struct ib_mad_mgmt_method_table *method;
1755                 struct ib_mad_mgmt_vendor_class_table *vendor;
1756                 struct ib_mad_mgmt_vendor_class *vendor_class;
1757                 const struct ib_vendor_mad *vendor_mad;
1758                 int index;
1759
1760                 spin_lock_irqsave(&port_priv->reg_lock, flags);
1761                 /*
1762                  * Routing is based on version, class, and method
1763                  * For "newer" vendor MADs, also based on OUI
1764                  */
1765                 if (mad_hdr->class_version >= MAX_MGMT_VERSION)
1766                         goto out;
1767                 if (!is_vendor_class(mad_hdr->mgmt_class)) {
1768                         class = port_priv->version[
1769                                         mad_hdr->class_version].class;
1770                         if (!class)
1771                                 goto out;
1772                         if (convert_mgmt_class(mad_hdr->mgmt_class) >=
1773                             ARRAY_SIZE(class->method_table))
1774                                 goto out;
1775                         method = class->method_table[convert_mgmt_class(
1776                                                         mad_hdr->mgmt_class)];
1777                         if (method)
1778                                 mad_agent = method->agent[mad_hdr->method &
1779                                                           ~IB_MGMT_METHOD_RESP];
1780                 } else {
1781                         vendor = port_priv->version[
1782                                         mad_hdr->class_version].vendor;
1783                         if (!vendor)
1784                                 goto out;
1785                         vendor_class = vendor->vendor_class[vendor_class_index(
1786                                                 mad_hdr->mgmt_class)];
1787                         if (!vendor_class)
1788                                 goto out;
1789                         /* Find matching OUI */
1790                         vendor_mad = (const struct ib_vendor_mad *)mad_hdr;
1791                         index = find_vendor_oui(vendor_class, vendor_mad->oui);
1792                         if (index == -1)
1793                                 goto out;
1794                         method = vendor_class->method_table[index];
1795                         if (method) {
1796                                 mad_agent = method->agent[mad_hdr->method &
1797                                                           ~IB_MGMT_METHOD_RESP];
1798                         }
1799                 }
1800                 if (mad_agent)
1801                         refcount_inc(&mad_agent->refcount);
1802 out:
1803                 spin_unlock_irqrestore(&port_priv->reg_lock, flags);
1804         }
1805
1806         if (mad_agent && !mad_agent->agent.recv_handler) {
1807                 dev_notice(&port_priv->device->dev,
1808                            "No receive handler for client %p on port %u\n",
1809                            &mad_agent->agent, port_priv->port_num);
1810                 deref_mad_agent(mad_agent);
1811                 mad_agent = NULL;
1812         }
1813
1814         return mad_agent;
1815 }
1816
1817 static int validate_mad(const struct ib_mad_hdr *mad_hdr,
1818                         const struct ib_mad_qp_info *qp_info,
1819                         bool opa)
1820 {
1821         int valid = 0;
1822         u32 qp_num = qp_info->qp->qp_num;
1823
1824         /* Make sure MAD base version is understood */
1825         if (mad_hdr->base_version != IB_MGMT_BASE_VERSION &&
1826             (!opa || mad_hdr->base_version != OPA_MGMT_BASE_VERSION)) {
1827                 pr_err("MAD received with unsupported base version %u %s\n",
1828                        mad_hdr->base_version, opa ? "(opa)" : "");
1829                 goto out;
1830         }
1831
1832         /* Filter SMI packets sent to other than QP0 */
1833         if ((mad_hdr->mgmt_class == IB_MGMT_CLASS_SUBN_LID_ROUTED) ||
1834             (mad_hdr->mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE)) {
1835                 if (qp_num == 0)
1836                         valid = 1;
1837         } else {
1838                 /* CM attributes other than ClassPortInfo only use Send method */
1839                 if ((mad_hdr->mgmt_class == IB_MGMT_CLASS_CM) &&
1840                     (mad_hdr->attr_id != IB_MGMT_CLASSPORTINFO_ATTR_ID) &&
1841                     (mad_hdr->method != IB_MGMT_METHOD_SEND))
1842                         goto out;
1843                 /* Filter GSI packets sent to QP0 */
1844                 if (qp_num != 0)
1845                         valid = 1;
1846         }
1847
1848 out:
1849         return valid;
1850 }
1851
1852 static int is_rmpp_data_mad(const struct ib_mad_agent_private *mad_agent_priv,
1853                             const struct ib_mad_hdr *mad_hdr)
1854 {
1855         struct ib_rmpp_mad *rmpp_mad;
1856
1857         rmpp_mad = (struct ib_rmpp_mad *)mad_hdr;
1858         return !mad_agent_priv->agent.rmpp_version ||
1859                 !ib_mad_kernel_rmpp_agent(&mad_agent_priv->agent) ||
1860                 !(ib_get_rmpp_flags(&rmpp_mad->rmpp_hdr) &
1861                                     IB_MGMT_RMPP_FLAG_ACTIVE) ||
1862                 (rmpp_mad->rmpp_hdr.rmpp_type == IB_MGMT_RMPP_TYPE_DATA);
1863 }
1864
1865 static inline int rcv_has_same_class(const struct ib_mad_send_wr_private *wr,
1866                                      const struct ib_mad_recv_wc *rwc)
1867 {
1868         return ((struct ib_mad_hdr *)(wr->send_buf.mad))->mgmt_class ==
1869                 rwc->recv_buf.mad->mad_hdr.mgmt_class;
1870 }
1871
1872 static inline int
1873 rcv_has_same_gid(const struct ib_mad_agent_private *mad_agent_priv,
1874                  const struct ib_mad_send_wr_private *wr,
1875                  const struct ib_mad_recv_wc *rwc)
1876 {
1877         struct rdma_ah_attr attr;
1878         u8 send_resp, rcv_resp;
1879         union ib_gid sgid;
1880         struct ib_device *device = mad_agent_priv->agent.device;
1881         u32 port_num = mad_agent_priv->agent.port_num;
1882         u8 lmc;
1883         bool has_grh;
1884
1885         send_resp = ib_response_mad((struct ib_mad_hdr *)wr->send_buf.mad);
1886         rcv_resp = ib_response_mad(&rwc->recv_buf.mad->mad_hdr);
1887
1888         if (send_resp == rcv_resp)
1889                 /* both requests, or both responses. GIDs different */
1890                 return 0;
1891
1892         if (rdma_query_ah(wr->send_buf.ah, &attr))
1893                 /* Assume not equal, to avoid false positives. */
1894                 return 0;
1895
1896         has_grh = !!(rdma_ah_get_ah_flags(&attr) & IB_AH_GRH);
1897         if (has_grh != !!(rwc->wc->wc_flags & IB_WC_GRH))
1898                 /* one has GID, other does not.  Assume different */
1899                 return 0;
1900
1901         if (!send_resp && rcv_resp) {
1902                 /* is request/response. */
1903                 if (!has_grh) {
1904                         if (ib_get_cached_lmc(device, port_num, &lmc))
1905                                 return 0;
1906                         return (!lmc || !((rdma_ah_get_path_bits(&attr) ^
1907                                            rwc->wc->dlid_path_bits) &
1908                                           ((1 << lmc) - 1)));
1909                 } else {
1910                         const struct ib_global_route *grh =
1911                                         rdma_ah_read_grh(&attr);
1912
1913                         if (rdma_query_gid(device, port_num,
1914                                            grh->sgid_index, &sgid))
1915                                 return 0;
1916                         return !memcmp(sgid.raw, rwc->recv_buf.grh->dgid.raw,
1917                                        16);
1918                 }
1919         }
1920
1921         if (!has_grh)
1922                 return rdma_ah_get_dlid(&attr) == rwc->wc->slid;
1923         else
1924                 return !memcmp(rdma_ah_read_grh(&attr)->dgid.raw,
1925                                rwc->recv_buf.grh->sgid.raw,
1926                                16);
1927 }
1928
1929 static inline int is_direct(u8 class)
1930 {
1931         return (class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE);
1932 }
1933
1934 struct ib_mad_send_wr_private*
1935 ib_find_send_mad(const struct ib_mad_agent_private *mad_agent_priv,
1936                  const struct ib_mad_recv_wc *wc)
1937 {
1938         struct ib_mad_send_wr_private *wr;
1939         const struct ib_mad_hdr *mad_hdr;
1940
1941         mad_hdr = &wc->recv_buf.mad->mad_hdr;
1942
1943         list_for_each_entry(wr, &mad_agent_priv->wait_list, agent_list) {
1944                 if ((wr->tid == mad_hdr->tid) &&
1945                     rcv_has_same_class(wr, wc) &&
1946                     /*
1947                      * Don't check GID for direct routed MADs.
1948                      * These might have permissive LIDs.
1949                      */
1950                     (is_direct(mad_hdr->mgmt_class) ||
1951                      rcv_has_same_gid(mad_agent_priv, wr, wc)))
1952                         return (wr->state != IB_MAD_STATE_CANCELED) ? wr : NULL;
1953         }
1954
1955         list_for_each_entry(wr, &mad_agent_priv->backlog_list, agent_list) {
1956                 if ((wr->tid == mad_hdr->tid) &&
1957                     rcv_has_same_class(wr, wc) &&
1958                     /*
1959                      * Don't check GID for direct routed MADs.
1960                      * These might have permissive LIDs.
1961                      */
1962                     (is_direct(mad_hdr->mgmt_class) ||
1963                      rcv_has_same_gid(mad_agent_priv, wr, wc)))
1964                         return (wr->state != IB_MAD_STATE_CANCELED) ? wr : NULL;
1965         }
1966
1967         /*
1968          * It's possible to receive the response before we've
1969          * been notified that the send has completed
1970          */
1971         list_for_each_entry(wr, &mad_agent_priv->send_list, agent_list) {
1972                 if (is_rmpp_data_mad(mad_agent_priv, wr->send_buf.mad) &&
1973                     wr->tid == mad_hdr->tid &&
1974                     wr->timeout &&
1975                     rcv_has_same_class(wr, wc) &&
1976                     /*
1977                      * Don't check GID for direct routed MADs.
1978                      * These might have permissive LIDs.
1979                      */
1980                     (is_direct(mad_hdr->mgmt_class) ||
1981                      rcv_has_same_gid(mad_agent_priv, wr, wc)))
1982                         /* Verify request has not been canceled */
1983                         return (wr->state != IB_MAD_STATE_CANCELED) ? wr : NULL;
1984         }
1985         return NULL;
1986 }
1987
1988 static void
1989 process_backlog_mads(struct ib_mad_agent_private *mad_agent_priv)
1990 {
1991         struct ib_mad_send_wr_private *mad_send_wr;
1992         struct ib_mad_send_wc mad_send_wc = {};
1993         unsigned long flags;
1994         int ret;
1995
1996         spin_lock_irqsave(&mad_agent_priv->lock, flags);
1997         while (!list_empty(&mad_agent_priv->backlog_list) &&
1998                (mad_agent_priv->sol_fc_send_count +
1999                         mad_agent_priv->sol_fc_wait_count <
2000                 mad_agent_priv->sol_fc_max)) {
2001                 mad_send_wr = list_entry(mad_agent_priv->backlog_list.next,
2002                                          struct ib_mad_send_wr_private,
2003                                          agent_list);
2004                 change_mad_state(mad_send_wr, IB_MAD_STATE_SEND_START);
2005                 spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
2006                 ret = ib_send_mad(mad_send_wr);
2007                 if (ret) {
2008                         spin_lock_irqsave(&mad_agent_priv->lock, flags);
2009                         deref_mad_agent(mad_agent_priv);
2010                         change_mad_state(mad_send_wr, IB_MAD_STATE_DONE);
2011                         spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
2012                         mad_send_wc.send_buf = &mad_send_wr->send_buf;
2013                         mad_send_wc.status = IB_WC_LOC_QP_OP_ERR;
2014                         mad_agent_priv->agent.send_handler(
2015                                 &mad_agent_priv->agent, &mad_send_wc);
2016                 }
2017
2018                 spin_lock_irqsave(&mad_agent_priv->lock, flags);
2019         }
2020
2021         spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
2022 }
2023
2024 void ib_mark_mad_done(struct ib_mad_send_wr_private *mad_send_wr)
2025 {
2026         mad_send_wr->timeout = 0;
2027         if (mad_send_wr->state == IB_MAD_STATE_WAIT_RESP ||
2028             mad_send_wr->state == IB_MAD_STATE_QUEUED)
2029                 change_mad_state(mad_send_wr, IB_MAD_STATE_DONE);
2030         else
2031                 change_mad_state(mad_send_wr, IB_MAD_STATE_EARLY_RESP);
2032 }
2033
2034 static void ib_mad_complete_recv(struct ib_mad_agent_private *mad_agent_priv,
2035                                  struct ib_mad_recv_wc *mad_recv_wc)
2036 {
2037         struct ib_mad_send_wr_private *mad_send_wr;
2038         struct ib_mad_send_wc mad_send_wc;
2039         unsigned long flags;
2040         bool is_mad_done;
2041         int ret;
2042
2043         INIT_LIST_HEAD(&mad_recv_wc->rmpp_list);
2044         ret = ib_mad_enforce_security(mad_agent_priv,
2045                                       mad_recv_wc->wc->pkey_index);
2046         if (ret) {
2047                 ib_free_recv_mad(mad_recv_wc);
2048                 deref_mad_agent(mad_agent_priv);
2049                 return;
2050         }
2051
2052         list_add(&mad_recv_wc->recv_buf.list, &mad_recv_wc->rmpp_list);
2053         if (ib_mad_kernel_rmpp_agent(&mad_agent_priv->agent)) {
2054                 mad_recv_wc = ib_process_rmpp_recv_wc(mad_agent_priv,
2055                                                       mad_recv_wc);
2056                 if (!mad_recv_wc) {
2057                         deref_mad_agent(mad_agent_priv);
2058                         return;
2059                 }
2060         }
2061
2062         /* Complete corresponding request */
2063         if (ib_response_mad(&mad_recv_wc->recv_buf.mad->mad_hdr)) {
2064                 spin_lock_irqsave(&mad_agent_priv->lock, flags);
2065                 mad_send_wr = ib_find_send_mad(mad_agent_priv, mad_recv_wc);
2066                 if (!mad_send_wr) {
2067                         spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
2068                         if (!ib_mad_kernel_rmpp_agent(&mad_agent_priv->agent)
2069                            && ib_is_mad_class_rmpp(mad_recv_wc->recv_buf.mad->mad_hdr.mgmt_class)
2070                            && (ib_get_rmpp_flags(&((struct ib_rmpp_mad *)mad_recv_wc->recv_buf.mad)->rmpp_hdr)
2071                                         & IB_MGMT_RMPP_FLAG_ACTIVE)) {
2072                                 /* user rmpp is in effect
2073                                  * and this is an active RMPP MAD
2074                                  */
2075                                 mad_agent_priv->agent.recv_handler(
2076                                                 &mad_agent_priv->agent, NULL,
2077                                                 mad_recv_wc);
2078                                 deref_mad_agent(mad_agent_priv);
2079                         } else {
2080                                 /* not user rmpp, revert to normal behavior and
2081                                  * drop the mad
2082                                  */
2083                                 ib_free_recv_mad(mad_recv_wc);
2084                                 deref_mad_agent(mad_agent_priv);
2085                                 return;
2086                         }
2087                 } else {
2088                         ib_mark_mad_done(mad_send_wr);
2089                         is_mad_done = (mad_send_wr->state == IB_MAD_STATE_DONE);
2090                         spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
2091
2092                         /* Defined behavior is to complete response before request */
2093                         mad_agent_priv->agent.recv_handler(
2094                                         &mad_agent_priv->agent,
2095                                         &mad_send_wr->send_buf,
2096                                         mad_recv_wc);
2097                         deref_mad_agent(mad_agent_priv);
2098
2099                         if (is_mad_done) {
2100                                 mad_send_wc.status = IB_WC_SUCCESS;
2101                                 mad_send_wc.vendor_err = 0;
2102                                 mad_send_wc.send_buf = &mad_send_wr->send_buf;
2103                                 ib_mad_complete_send_wr(mad_send_wr,
2104                                                         &mad_send_wc);
2105                         }
2106                 }
2107         } else {
2108                 mad_agent_priv->agent.recv_handler(&mad_agent_priv->agent, NULL,
2109                                                    mad_recv_wc);
2110                 deref_mad_agent(mad_agent_priv);
2111         }
2112 }
2113
2114 static enum smi_action handle_ib_smi(const struct ib_mad_port_private *port_priv,
2115                                      const struct ib_mad_qp_info *qp_info,
2116                                      const struct ib_wc *wc,
2117                                      u32 port_num,
2118                                      struct ib_mad_private *recv,
2119                                      struct ib_mad_private *response)
2120 {
2121         enum smi_forward_action retsmi;
2122         struct ib_smp *smp = (struct ib_smp *)recv->mad;
2123
2124         trace_ib_mad_handle_ib_smi(smp);
2125
2126         if (smi_handle_dr_smp_recv(smp,
2127                                    rdma_cap_ib_switch(port_priv->device),
2128                                    port_num,
2129                                    port_priv->device->phys_port_cnt) ==
2130                                    IB_SMI_DISCARD)
2131                 return IB_SMI_DISCARD;
2132
2133         retsmi = smi_check_forward_dr_smp(smp);
2134         if (retsmi == IB_SMI_LOCAL)
2135                 return IB_SMI_HANDLE;
2136
2137         if (retsmi == IB_SMI_SEND) { /* don't forward */
2138                 if (smi_handle_dr_smp_send(smp,
2139                                            rdma_cap_ib_switch(port_priv->device),
2140                                            port_num) == IB_SMI_DISCARD)
2141                         return IB_SMI_DISCARD;
2142
2143                 if (smi_check_local_smp(smp, port_priv->device) == IB_SMI_DISCARD)
2144                         return IB_SMI_DISCARD;
2145         } else if (rdma_cap_ib_switch(port_priv->device)) {
2146                 /* forward case for switches */
2147                 memcpy(response, recv, mad_priv_size(response));
2148                 response->header.recv_wc.wc = &response->header.wc;
2149                 response->header.recv_wc.recv_buf.mad = (struct ib_mad *)response->mad;
2150                 response->header.recv_wc.recv_buf.grh = &response->grh;
2151
2152                 agent_send_response((const struct ib_mad_hdr *)response->mad,
2153                                     &response->grh, wc,
2154                                     port_priv->device,
2155                                     smi_get_fwd_port(smp),
2156                                     qp_info->qp->qp_num,
2157                                     response->mad_size,
2158                                     false);
2159
2160                 return IB_SMI_DISCARD;
2161         }
2162         return IB_SMI_HANDLE;
2163 }
2164
2165 static bool generate_unmatched_resp(const struct ib_mad_private *recv,
2166                                     struct ib_mad_private *response,
2167                                     size_t *resp_len, bool opa)
2168 {
2169         const struct ib_mad_hdr *recv_hdr = (const struct ib_mad_hdr *)recv->mad;
2170         struct ib_mad_hdr *resp_hdr = (struct ib_mad_hdr *)response->mad;
2171
2172         if (recv_hdr->method == IB_MGMT_METHOD_GET ||
2173             recv_hdr->method == IB_MGMT_METHOD_SET) {
2174                 memcpy(response, recv, mad_priv_size(response));
2175                 response->header.recv_wc.wc = &response->header.wc;
2176                 response->header.recv_wc.recv_buf.mad = (struct ib_mad *)response->mad;
2177                 response->header.recv_wc.recv_buf.grh = &response->grh;
2178                 resp_hdr->method = IB_MGMT_METHOD_GET_RESP;
2179                 resp_hdr->status = cpu_to_be16(IB_MGMT_MAD_STATUS_UNSUPPORTED_METHOD_ATTRIB);
2180                 if (recv_hdr->mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE)
2181                         resp_hdr->status |= IB_SMP_DIRECTION;
2182
2183                 if (opa && recv_hdr->base_version == OPA_MGMT_BASE_VERSION) {
2184                         if (recv_hdr->mgmt_class ==
2185                             IB_MGMT_CLASS_SUBN_LID_ROUTED ||
2186                             recv_hdr->mgmt_class ==
2187                             IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE)
2188                                 *resp_len = opa_get_smp_header_size(
2189                                                         (struct opa_smp *)recv->mad);
2190                         else
2191                                 *resp_len = sizeof(struct ib_mad_hdr);
2192                 }
2193
2194                 return true;
2195         } else {
2196                 return false;
2197         }
2198 }
2199
2200 static enum smi_action
2201 handle_opa_smi(struct ib_mad_port_private *port_priv,
2202                struct ib_mad_qp_info *qp_info,
2203                struct ib_wc *wc,
2204                u32 port_num,
2205                struct ib_mad_private *recv,
2206                struct ib_mad_private *response)
2207 {
2208         enum smi_forward_action retsmi;
2209         struct opa_smp *smp = (struct opa_smp *)recv->mad;
2210
2211         trace_ib_mad_handle_opa_smi(smp);
2212
2213         if (opa_smi_handle_dr_smp_recv(smp,
2214                                    rdma_cap_ib_switch(port_priv->device),
2215                                    port_num,
2216                                    port_priv->device->phys_port_cnt) ==
2217                                    IB_SMI_DISCARD)
2218                 return IB_SMI_DISCARD;
2219
2220         retsmi = opa_smi_check_forward_dr_smp(smp);
2221         if (retsmi == IB_SMI_LOCAL)
2222                 return IB_SMI_HANDLE;
2223
2224         if (retsmi == IB_SMI_SEND) { /* don't forward */
2225                 if (opa_smi_handle_dr_smp_send(smp,
2226                                            rdma_cap_ib_switch(port_priv->device),
2227                                            port_num) == IB_SMI_DISCARD)
2228                         return IB_SMI_DISCARD;
2229
2230                 if (opa_smi_check_local_smp(smp, port_priv->device) ==
2231                     IB_SMI_DISCARD)
2232                         return IB_SMI_DISCARD;
2233
2234         } else if (rdma_cap_ib_switch(port_priv->device)) {
2235                 /* forward case for switches */
2236                 memcpy(response, recv, mad_priv_size(response));
2237                 response->header.recv_wc.wc = &response->header.wc;
2238                 response->header.recv_wc.recv_buf.opa_mad =
2239                                 (struct opa_mad *)response->mad;
2240                 response->header.recv_wc.recv_buf.grh = &response->grh;
2241
2242                 agent_send_response((const struct ib_mad_hdr *)response->mad,
2243                                     &response->grh, wc,
2244                                     port_priv->device,
2245                                     opa_smi_get_fwd_port(smp),
2246                                     qp_info->qp->qp_num,
2247                                     recv->header.wc.byte_len,
2248                                     true);
2249
2250                 return IB_SMI_DISCARD;
2251         }
2252
2253         return IB_SMI_HANDLE;
2254 }
2255
2256 static enum smi_action
2257 handle_smi(struct ib_mad_port_private *port_priv,
2258            struct ib_mad_qp_info *qp_info,
2259            struct ib_wc *wc,
2260            u32 port_num,
2261            struct ib_mad_private *recv,
2262            struct ib_mad_private *response,
2263            bool opa)
2264 {
2265         struct ib_mad_hdr *mad_hdr = (struct ib_mad_hdr *)recv->mad;
2266
2267         if (opa && mad_hdr->base_version == OPA_MGMT_BASE_VERSION &&
2268             mad_hdr->class_version == OPA_SM_CLASS_VERSION)
2269                 return handle_opa_smi(port_priv, qp_info, wc, port_num, recv,
2270                                       response);
2271
2272         return handle_ib_smi(port_priv, qp_info, wc, port_num, recv, response);
2273 }
2274
2275 static void ib_mad_recv_done(struct ib_cq *cq, struct ib_wc *wc)
2276 {
2277         struct ib_mad_port_private *port_priv = cq->cq_context;
2278         struct ib_mad_list_head *mad_list =
2279                 container_of(wc->wr_cqe, struct ib_mad_list_head, cqe);
2280         struct ib_mad_qp_info *qp_info;
2281         struct ib_mad_private_header *mad_priv_hdr;
2282         struct ib_mad_private *recv, *response = NULL;
2283         struct ib_mad_agent_private *mad_agent;
2284         u32 port_num;
2285         int ret = IB_MAD_RESULT_SUCCESS;
2286         size_t mad_size;
2287         u16 resp_mad_pkey_index = 0;
2288         bool opa;
2289
2290         if (list_empty_careful(&port_priv->port_list))
2291                 return;
2292
2293         if (wc->status != IB_WC_SUCCESS) {
2294                 /*
2295                  * Receive errors indicate that the QP has entered the error
2296                  * state - error handling/shutdown code will cleanup
2297                  */
2298                 return;
2299         }
2300
2301         qp_info = mad_list->mad_queue->qp_info;
2302         dequeue_mad(mad_list);
2303
2304         opa = rdma_cap_opa_mad(qp_info->port_priv->device,
2305                                qp_info->port_priv->port_num);
2306
2307         mad_priv_hdr = container_of(mad_list, struct ib_mad_private_header,
2308                                     mad_list);
2309         recv = container_of(mad_priv_hdr, struct ib_mad_private, header);
2310         ib_dma_unmap_single(port_priv->device,
2311                             recv->header.mapping,
2312                             mad_priv_dma_size(recv),
2313                             DMA_FROM_DEVICE);
2314
2315         /* Setup MAD receive work completion from "normal" work completion */
2316         recv->header.wc = *wc;
2317         recv->header.recv_wc.wc = &recv->header.wc;
2318
2319         if (opa && ((struct ib_mad_hdr *)(recv->mad))->base_version == OPA_MGMT_BASE_VERSION) {
2320                 recv->header.recv_wc.mad_len = wc->byte_len - sizeof(struct ib_grh);
2321                 recv->header.recv_wc.mad_seg_size = sizeof(struct opa_mad);
2322         } else {
2323                 recv->header.recv_wc.mad_len = sizeof(struct ib_mad);
2324                 recv->header.recv_wc.mad_seg_size = sizeof(struct ib_mad);
2325         }
2326
2327         recv->header.recv_wc.recv_buf.mad = (struct ib_mad *)recv->mad;
2328         recv->header.recv_wc.recv_buf.grh = &recv->grh;
2329
2330         /* Validate MAD */
2331         if (!validate_mad((const struct ib_mad_hdr *)recv->mad, qp_info, opa))
2332                 goto out;
2333
2334         trace_ib_mad_recv_done_handler(qp_info, wc,
2335                                        (struct ib_mad_hdr *)recv->mad);
2336
2337         mad_size = recv->mad_size;
2338         response = alloc_mad_private(mad_size, GFP_KERNEL);
2339         if (!response)
2340                 goto out;
2341
2342         if (rdma_cap_ib_switch(port_priv->device))
2343                 port_num = wc->port_num;
2344         else
2345                 port_num = port_priv->port_num;
2346
2347         if (((struct ib_mad_hdr *)recv->mad)->mgmt_class ==
2348             IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) {
2349                 if (handle_smi(port_priv, qp_info, wc, port_num, recv,
2350                                response, opa)
2351                     == IB_SMI_DISCARD)
2352                         goto out;
2353         }
2354
2355         /* Give driver "right of first refusal" on incoming MAD */
2356         if (port_priv->device->ops.process_mad) {
2357                 ret = port_priv->device->ops.process_mad(
2358                         port_priv->device, 0, port_priv->port_num, wc,
2359                         &recv->grh, (const struct ib_mad *)recv->mad,
2360                         (struct ib_mad *)response->mad, &mad_size,
2361                         &resp_mad_pkey_index);
2362
2363                 if (opa)
2364                         wc->pkey_index = resp_mad_pkey_index;
2365
2366                 if (ret & IB_MAD_RESULT_SUCCESS) {
2367                         if (ret & IB_MAD_RESULT_CONSUMED)
2368                                 goto out;
2369                         if (ret & IB_MAD_RESULT_REPLY) {
2370                                 agent_send_response((const struct ib_mad_hdr *)response->mad,
2371                                                     &recv->grh, wc,
2372                                                     port_priv->device,
2373                                                     port_num,
2374                                                     qp_info->qp->qp_num,
2375                                                     mad_size, opa);
2376                                 goto out;
2377                         }
2378                 }
2379         }
2380
2381         mad_agent = find_mad_agent(port_priv, (const struct ib_mad_hdr *)recv->mad);
2382         if (mad_agent) {
2383                 trace_ib_mad_recv_done_agent(mad_agent);
2384                 ib_mad_complete_recv(mad_agent, &recv->header.recv_wc);
2385                 /*
2386                  * recv is freed up in error cases in ib_mad_complete_recv
2387                  * or via recv_handler in ib_mad_complete_recv()
2388                  */
2389                 recv = NULL;
2390         } else if ((ret & IB_MAD_RESULT_SUCCESS) &&
2391                    generate_unmatched_resp(recv, response, &mad_size, opa)) {
2392                 agent_send_response((const struct ib_mad_hdr *)response->mad, &recv->grh, wc,
2393                                     port_priv->device, port_num,
2394                                     qp_info->qp->qp_num, mad_size, opa);
2395         }
2396
2397 out:
2398         /* Post another receive request for this QP */
2399         if (response) {
2400                 ib_mad_post_receive_mads(qp_info, response);
2401                 kfree(recv);
2402         } else
2403                 ib_mad_post_receive_mads(qp_info, recv);
2404 }
2405
2406 static void adjust_timeout(struct ib_mad_agent_private *mad_agent_priv)
2407 {
2408         struct ib_mad_send_wr_private *mad_send_wr;
2409         unsigned long delay;
2410
2411         if (list_empty(&mad_agent_priv->wait_list)) {
2412                 cancel_delayed_work(&mad_agent_priv->timed_work);
2413         } else {
2414                 mad_send_wr = list_entry(mad_agent_priv->wait_list.next,
2415                                          struct ib_mad_send_wr_private,
2416                                          agent_list);
2417
2418                 if (time_after(mad_agent_priv->timeout,
2419                                mad_send_wr->timeout)) {
2420                         mad_agent_priv->timeout = mad_send_wr->timeout;
2421                         delay = mad_send_wr->timeout - jiffies;
2422                         if ((long)delay <= 0)
2423                                 delay = 1;
2424                         mod_delayed_work(mad_agent_priv->qp_info->port_priv->wq,
2425                                          &mad_agent_priv->timed_work, delay);
2426                 }
2427         }
2428 }
2429
2430 static void wait_for_response(struct ib_mad_send_wr_private *mad_send_wr)
2431 {
2432         struct ib_mad_agent_private *mad_agent_priv;
2433         unsigned long delay;
2434
2435         mad_agent_priv = mad_send_wr->mad_agent_priv;
2436         delay = mad_send_wr->timeout;
2437         change_mad_state(mad_send_wr, IB_MAD_STATE_WAIT_RESP);
2438
2439         /* Reschedule a work item if we have a shorter timeout */
2440         if (mad_agent_priv->wait_list.next == &mad_send_wr->agent_list)
2441                 mod_delayed_work(mad_agent_priv->qp_info->port_priv->wq,
2442                                  &mad_agent_priv->timed_work, delay);
2443 }
2444
2445 void ib_reset_mad_timeout(struct ib_mad_send_wr_private *mad_send_wr,
2446                           unsigned long timeout_ms)
2447 {
2448         mad_send_wr->timeout = msecs_to_jiffies(timeout_ms);
2449         wait_for_response(mad_send_wr);
2450 }
2451
2452 /*
2453  * Process a send work completion
2454  */
2455 void ib_mad_complete_send_wr(struct ib_mad_send_wr_private *mad_send_wr,
2456                              struct ib_mad_send_wc *mad_send_wc)
2457 {
2458         struct ib_mad_agent_private     *mad_agent_priv;
2459         unsigned long                   flags;
2460         int                             ret;
2461
2462         mad_agent_priv = mad_send_wr->mad_agent_priv;
2463         spin_lock_irqsave(&mad_agent_priv->lock, flags);
2464         if (ib_mad_kernel_rmpp_agent(&mad_agent_priv->agent)) {
2465                 ret = ib_process_rmpp_send_wc(mad_send_wr, mad_send_wc);
2466                 if (ret == IB_RMPP_RESULT_CONSUMED)
2467                         goto done;
2468         } else
2469                 ret = IB_RMPP_RESULT_UNHANDLED;
2470
2471         if (mad_send_wr->state == IB_MAD_STATE_CANCELED)
2472                 mad_send_wc->status = IB_WC_WR_FLUSH_ERR;
2473         else if (mad_send_wr->state == IB_MAD_STATE_SEND_START &&
2474                  mad_send_wr->timeout) {
2475                 wait_for_response(mad_send_wr);
2476                 goto done;
2477         }
2478
2479         /* Remove send from MAD agent and notify client of completion */
2480         if (mad_send_wr->state != IB_MAD_STATE_DONE)
2481                 change_mad_state(mad_send_wr, IB_MAD_STATE_DONE);
2482         adjust_timeout(mad_agent_priv);
2483         spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
2484
2485         if (ret == IB_RMPP_RESULT_INTERNAL) {
2486                 ib_rmpp_send_handler(mad_send_wc);
2487         } else {
2488                 if (mad_send_wr->is_solicited_fc)
2489                         process_backlog_mads(mad_agent_priv);
2490                 mad_agent_priv->agent.send_handler(&mad_agent_priv->agent,
2491                                                    mad_send_wc);
2492         }
2493
2494         /* Release reference on agent taken when sending */
2495         deref_mad_agent(mad_agent_priv);
2496         return;
2497 done:
2498         spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
2499 }
2500
2501 static void ib_mad_send_done(struct ib_cq *cq, struct ib_wc *wc)
2502 {
2503         struct ib_mad_port_private *port_priv = cq->cq_context;
2504         struct ib_mad_list_head *mad_list =
2505                 container_of(wc->wr_cqe, struct ib_mad_list_head, cqe);
2506         struct ib_mad_send_wr_private   *mad_send_wr, *queued_send_wr;
2507         struct ib_mad_qp_info           *qp_info;
2508         struct ib_mad_queue             *send_queue;
2509         struct ib_mad_send_wc           mad_send_wc;
2510         unsigned long flags;
2511         int ret;
2512
2513         if (list_empty_careful(&port_priv->port_list))
2514                 return;
2515
2516         if (wc->status != IB_WC_SUCCESS) {
2517                 if (!ib_mad_send_error(port_priv, wc))
2518                         return;
2519         }
2520
2521         mad_send_wr = container_of(mad_list, struct ib_mad_send_wr_private,
2522                                    mad_list);
2523         send_queue = mad_list->mad_queue;
2524         qp_info = send_queue->qp_info;
2525
2526         trace_ib_mad_send_done_agent(mad_send_wr->mad_agent_priv);
2527         trace_ib_mad_send_done_handler(mad_send_wr, wc);
2528
2529 retry:
2530         ib_dma_unmap_single(mad_send_wr->send_buf.mad_agent->device,
2531                             mad_send_wr->header_mapping,
2532                             mad_send_wr->sg_list[0].length, DMA_TO_DEVICE);
2533         ib_dma_unmap_single(mad_send_wr->send_buf.mad_agent->device,
2534                             mad_send_wr->payload_mapping,
2535                             mad_send_wr->sg_list[1].length, DMA_TO_DEVICE);
2536         queued_send_wr = NULL;
2537         spin_lock_irqsave(&send_queue->lock, flags);
2538         list_del(&mad_list->list);
2539
2540         /* Move queued send to the send queue */
2541         if (send_queue->count-- > send_queue->max_active) {
2542                 mad_list = container_of(qp_info->overflow_list.next,
2543                                         struct ib_mad_list_head, list);
2544                 queued_send_wr = container_of(mad_list,
2545                                         struct ib_mad_send_wr_private,
2546                                         mad_list);
2547                 list_move_tail(&mad_list->list, &send_queue->list);
2548         }
2549         spin_unlock_irqrestore(&send_queue->lock, flags);
2550
2551         mad_send_wc.send_buf = &mad_send_wr->send_buf;
2552         mad_send_wc.status = wc->status;
2553         mad_send_wc.vendor_err = wc->vendor_err;
2554         ib_mad_complete_send_wr(mad_send_wr, &mad_send_wc);
2555
2556         if (queued_send_wr) {
2557                 trace_ib_mad_send_done_resend(queued_send_wr, qp_info);
2558                 ret = ib_post_send(qp_info->qp, &queued_send_wr->send_wr.wr,
2559                                    NULL);
2560                 if (ret) {
2561                         dev_err(&port_priv->device->dev,
2562                                 "ib_post_send failed: %d\n", ret);
2563                         mad_send_wr = queued_send_wr;
2564                         wc->status = IB_WC_LOC_QP_OP_ERR;
2565                         goto retry;
2566                 }
2567         }
2568 }
2569
2570 static void mark_sends_for_retry(struct ib_mad_qp_info *qp_info)
2571 {
2572         struct ib_mad_send_wr_private *mad_send_wr;
2573         struct ib_mad_list_head *mad_list;
2574         unsigned long flags;
2575
2576         spin_lock_irqsave(&qp_info->send_queue.lock, flags);
2577         list_for_each_entry(mad_list, &qp_info->send_queue.list, list) {
2578                 mad_send_wr = container_of(mad_list,
2579                                            struct ib_mad_send_wr_private,
2580                                            mad_list);
2581                 mad_send_wr->retry = 1;
2582         }
2583         spin_unlock_irqrestore(&qp_info->send_queue.lock, flags);
2584 }
2585
2586 static bool ib_mad_send_error(struct ib_mad_port_private *port_priv,
2587                 struct ib_wc *wc)
2588 {
2589         struct ib_mad_list_head *mad_list =
2590                 container_of(wc->wr_cqe, struct ib_mad_list_head, cqe);
2591         struct ib_mad_qp_info *qp_info = mad_list->mad_queue->qp_info;
2592         struct ib_mad_send_wr_private *mad_send_wr;
2593         int ret;
2594
2595         /*
2596          * Send errors will transition the QP to SQE - move
2597          * QP to RTS and repost flushed work requests
2598          */
2599         mad_send_wr = container_of(mad_list, struct ib_mad_send_wr_private,
2600                                    mad_list);
2601         if (wc->status == IB_WC_WR_FLUSH_ERR) {
2602                 if (mad_send_wr->retry) {
2603                         /* Repost send */
2604                         mad_send_wr->retry = 0;
2605                         trace_ib_mad_error_handler(mad_send_wr, qp_info);
2606                         ret = ib_post_send(qp_info->qp, &mad_send_wr->send_wr.wr,
2607                                            NULL);
2608                         if (!ret)
2609                                 return false;
2610                 }
2611         } else {
2612                 struct ib_qp_attr *attr;
2613
2614                 /* Transition QP to RTS and fail offending send */
2615                 attr = kmalloc(sizeof *attr, GFP_KERNEL);
2616                 if (attr) {
2617                         attr->qp_state = IB_QPS_RTS;
2618                         attr->cur_qp_state = IB_QPS_SQE;
2619                         ret = ib_modify_qp(qp_info->qp, attr,
2620                                            IB_QP_STATE | IB_QP_CUR_STATE);
2621                         kfree(attr);
2622                         if (ret)
2623                                 dev_err(&port_priv->device->dev,
2624                                         "%s - ib_modify_qp to RTS: %d\n",
2625                                         __func__, ret);
2626                         else
2627                                 mark_sends_for_retry(qp_info);
2628                 }
2629         }
2630
2631         return true;
2632 }
2633
2634 static void clear_mad_error_list(struct list_head *list,
2635                                  enum ib_wc_status wc_status,
2636                                  struct ib_mad_agent_private *mad_agent_priv)
2637 {
2638         struct ib_mad_send_wr_private *mad_send_wr, *n;
2639         struct ib_mad_send_wc mad_send_wc;
2640
2641         mad_send_wc.status = wc_status;
2642         mad_send_wc.vendor_err = 0;
2643
2644         list_for_each_entry_safe(mad_send_wr, n, list, agent_list) {
2645                 mad_send_wc.send_buf = &mad_send_wr->send_buf;
2646                 mad_agent_priv->agent.send_handler(&mad_agent_priv->agent,
2647                                                    &mad_send_wc);
2648                 deref_mad_agent(mad_agent_priv);
2649         }
2650 }
2651
2652 static void cancel_mads(struct ib_mad_agent_private *mad_agent_priv)
2653 {
2654         unsigned long flags;
2655         struct ib_mad_send_wr_private *mad_send_wr, *temp_mad_send_wr;
2656         struct list_head cancel_list;
2657
2658         INIT_LIST_HEAD(&cancel_list);
2659
2660         spin_lock_irqsave(&mad_agent_priv->lock, flags);
2661         list_for_each_entry_safe(mad_send_wr, temp_mad_send_wr,
2662                                  &mad_agent_priv->send_list, agent_list)
2663                 change_mad_state(mad_send_wr, IB_MAD_STATE_CANCELED);
2664
2665         /* Empty wait & backlog list to prevent receives from finding request */
2666         list_for_each_entry_safe(mad_send_wr, temp_mad_send_wr,
2667                                  &mad_agent_priv->wait_list, agent_list) {
2668                 change_mad_state(mad_send_wr, IB_MAD_STATE_DONE);
2669                 list_add_tail(&mad_send_wr->agent_list, &cancel_list);
2670         }
2671
2672         list_for_each_entry_safe(mad_send_wr, temp_mad_send_wr,
2673                                  &mad_agent_priv->backlog_list, agent_list) {
2674                 change_mad_state(mad_send_wr, IB_MAD_STATE_DONE);
2675                 list_add_tail(&mad_send_wr->agent_list, &cancel_list);
2676         }
2677
2678         spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
2679         /* Report all cancelled requests */
2680         clear_mad_error_list(&cancel_list, IB_WC_WR_FLUSH_ERR, mad_agent_priv);
2681 }
2682
2683 static struct ib_mad_send_wr_private*
2684 find_send_wr(struct ib_mad_agent_private *mad_agent_priv,
2685              struct ib_mad_send_buf *send_buf)
2686 {
2687         struct ib_mad_send_wr_private *mad_send_wr;
2688
2689         list_for_each_entry(mad_send_wr, &mad_agent_priv->wait_list,
2690                             agent_list) {
2691                 if (&mad_send_wr->send_buf == send_buf)
2692                         return mad_send_wr;
2693         }
2694
2695         list_for_each_entry(mad_send_wr, &mad_agent_priv->send_list,
2696                             agent_list) {
2697                 if (is_rmpp_data_mad(mad_agent_priv,
2698                                      mad_send_wr->send_buf.mad) &&
2699                     &mad_send_wr->send_buf == send_buf)
2700                         return mad_send_wr;
2701         }
2702
2703         list_for_each_entry(mad_send_wr, &mad_agent_priv->backlog_list,
2704                             agent_list) {
2705                 if (&mad_send_wr->send_buf == send_buf)
2706                         return mad_send_wr;
2707         }
2708
2709         return NULL;
2710 }
2711
2712 int ib_modify_mad(struct ib_mad_send_buf *send_buf, u32 timeout_ms)
2713 {
2714         struct ib_mad_agent_private *mad_agent_priv;
2715         struct ib_mad_send_wr_private *mad_send_wr;
2716         unsigned long flags;
2717         int active;
2718
2719         if (!send_buf)
2720                 return -EINVAL;
2721
2722         mad_agent_priv = container_of(send_buf->mad_agent,
2723                                       struct ib_mad_agent_private, agent);
2724         spin_lock_irqsave(&mad_agent_priv->lock, flags);
2725         mad_send_wr = find_send_wr(mad_agent_priv, send_buf);
2726         if (!mad_send_wr || mad_send_wr->state == IB_MAD_STATE_CANCELED) {
2727                 spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
2728                 return -EINVAL;
2729         }
2730
2731         active = ((mad_send_wr->state == IB_MAD_STATE_SEND_START) ||
2732                   (mad_send_wr->state == IB_MAD_STATE_EARLY_RESP) ||
2733                   (mad_send_wr->state == IB_MAD_STATE_QUEUED && timeout_ms));
2734         if (!timeout_ms)
2735                 change_mad_state(mad_send_wr, IB_MAD_STATE_CANCELED);
2736
2737         mad_send_wr->send_buf.timeout_ms = timeout_ms;
2738         if (active)
2739                 mad_send_wr->timeout = msecs_to_jiffies(timeout_ms);
2740         else
2741                 ib_reset_mad_timeout(mad_send_wr, timeout_ms);
2742
2743         spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
2744         return 0;
2745 }
2746 EXPORT_SYMBOL(ib_modify_mad);
2747
2748 static void local_completions(struct work_struct *work)
2749 {
2750         struct ib_mad_agent_private *mad_agent_priv;
2751         struct ib_mad_local_private *local;
2752         struct ib_mad_agent_private *recv_mad_agent;
2753         unsigned long flags;
2754         int free_mad;
2755         struct ib_wc wc;
2756         struct ib_mad_send_wc mad_send_wc;
2757         bool opa;
2758
2759         mad_agent_priv =
2760                 container_of(work, struct ib_mad_agent_private, local_work);
2761
2762         opa = rdma_cap_opa_mad(mad_agent_priv->qp_info->port_priv->device,
2763                                mad_agent_priv->qp_info->port_priv->port_num);
2764
2765         spin_lock_irqsave(&mad_agent_priv->lock, flags);
2766         while (!list_empty(&mad_agent_priv->local_list)) {
2767                 local = list_entry(mad_agent_priv->local_list.next,
2768                                    struct ib_mad_local_private,
2769                                    completion_list);
2770                 list_del(&local->completion_list);
2771                 spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
2772                 free_mad = 0;
2773                 if (local->mad_priv) {
2774                         u8 base_version;
2775                         recv_mad_agent = local->recv_mad_agent;
2776                         if (!recv_mad_agent) {
2777                                 dev_err(&mad_agent_priv->agent.device->dev,
2778                                         "No receive MAD agent for local completion\n");
2779                                 free_mad = 1;
2780                                 goto local_send_completion;
2781                         }
2782
2783                         /*
2784                          * Defined behavior is to complete response
2785                          * before request
2786                          */
2787                         build_smp_wc(recv_mad_agent->agent.qp,
2788                                      local->mad_send_wr->send_wr.wr.wr_cqe,
2789                                      be16_to_cpu(IB_LID_PERMISSIVE),
2790                                      local->mad_send_wr->send_wr.pkey_index,
2791                                      recv_mad_agent->agent.port_num, &wc);
2792
2793                         local->mad_priv->header.recv_wc.wc = &wc;
2794
2795                         base_version = ((struct ib_mad_hdr *)(local->mad_priv->mad))->base_version;
2796                         if (opa && base_version == OPA_MGMT_BASE_VERSION) {
2797                                 local->mad_priv->header.recv_wc.mad_len = local->return_wc_byte_len;
2798                                 local->mad_priv->header.recv_wc.mad_seg_size = sizeof(struct opa_mad);
2799                         } else {
2800                                 local->mad_priv->header.recv_wc.mad_len = sizeof(struct ib_mad);
2801                                 local->mad_priv->header.recv_wc.mad_seg_size = sizeof(struct ib_mad);
2802                         }
2803
2804                         INIT_LIST_HEAD(&local->mad_priv->header.recv_wc.rmpp_list);
2805                         list_add(&local->mad_priv->header.recv_wc.recv_buf.list,
2806                                  &local->mad_priv->header.recv_wc.rmpp_list);
2807                         local->mad_priv->header.recv_wc.recv_buf.grh = NULL;
2808                         local->mad_priv->header.recv_wc.recv_buf.mad =
2809                                                 (struct ib_mad *)local->mad_priv->mad;
2810                         recv_mad_agent->agent.recv_handler(
2811                                                 &recv_mad_agent->agent,
2812                                                 &local->mad_send_wr->send_buf,
2813                                                 &local->mad_priv->header.recv_wc);
2814                         spin_lock_irqsave(&recv_mad_agent->lock, flags);
2815                         deref_mad_agent(recv_mad_agent);
2816                         spin_unlock_irqrestore(&recv_mad_agent->lock, flags);
2817                 }
2818
2819 local_send_completion:
2820                 /* Complete send */
2821                 mad_send_wc.status = IB_WC_SUCCESS;
2822                 mad_send_wc.vendor_err = 0;
2823                 mad_send_wc.send_buf = &local->mad_send_wr->send_buf;
2824                 mad_agent_priv->agent.send_handler(&mad_agent_priv->agent,
2825                                                    &mad_send_wc);
2826
2827                 spin_lock_irqsave(&mad_agent_priv->lock, flags);
2828                 deref_mad_agent(mad_agent_priv);
2829                 if (free_mad)
2830                         kfree(local->mad_priv);
2831                 kfree(local);
2832         }
2833         spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
2834 }
2835
2836 static int retry_send(struct ib_mad_send_wr_private *mad_send_wr)
2837 {
2838         int ret;
2839
2840         if (!mad_send_wr->retries_left)
2841                 return -ETIMEDOUT;
2842
2843         mad_send_wr->retries_left--;
2844         mad_send_wr->send_buf.retries++;
2845
2846         mad_send_wr->timeout = msecs_to_jiffies(mad_send_wr->send_buf.timeout_ms);
2847         if (mad_send_wr->is_solicited_fc &&
2848             !list_empty(&mad_send_wr->mad_agent_priv->backlog_list)) {
2849                 change_mad_state(mad_send_wr, IB_MAD_STATE_QUEUED);
2850                 return 0;
2851         }
2852
2853         if (ib_mad_kernel_rmpp_agent(&mad_send_wr->mad_agent_priv->agent)) {
2854                 ret = ib_retry_rmpp(mad_send_wr);
2855                 switch (ret) {
2856                 case IB_RMPP_RESULT_UNHANDLED:
2857                         ret = ib_send_mad(mad_send_wr);
2858                         break;
2859                 case IB_RMPP_RESULT_CONSUMED:
2860                         ret = 0;
2861                         break;
2862                 default:
2863                         ret = -ECOMM;
2864                         break;
2865                 }
2866         } else
2867                 ret = ib_send_mad(mad_send_wr);
2868
2869         if (!ret)
2870                 change_mad_state(mad_send_wr, IB_MAD_STATE_SEND_START);
2871
2872         return ret;
2873 }
2874
2875 static void timeout_sends(struct work_struct *work)
2876 {
2877         struct ib_mad_send_wr_private *mad_send_wr;
2878         struct ib_mad_agent_private *mad_agent_priv;
2879         struct list_head timeout_list;
2880         struct list_head cancel_list;
2881         struct list_head *list_item;
2882         unsigned long flags, delay;
2883
2884         mad_agent_priv = container_of(work, struct ib_mad_agent_private,
2885                                       timed_work.work);
2886         INIT_LIST_HEAD(&timeout_list);
2887         INIT_LIST_HEAD(&cancel_list);
2888
2889         spin_lock_irqsave(&mad_agent_priv->lock, flags);
2890         while (!list_empty(&mad_agent_priv->wait_list)) {
2891                 mad_send_wr = list_entry(mad_agent_priv->wait_list.next,
2892                                          struct ib_mad_send_wr_private,
2893                                          agent_list);
2894
2895                 if (time_after(mad_send_wr->timeout, jiffies)) {
2896                         delay = mad_send_wr->timeout - jiffies;
2897                         if ((long)delay <= 0)
2898                                 delay = 1;
2899                         queue_delayed_work(mad_agent_priv->qp_info->
2900                                            port_priv->wq,
2901                                            &mad_agent_priv->timed_work, delay);
2902                         break;
2903                 }
2904
2905                 if (mad_send_wr->state == IB_MAD_STATE_CANCELED)
2906                         list_item = &cancel_list;
2907                 else if (retry_send(mad_send_wr))
2908                         list_item = &timeout_list;
2909                 else
2910                         continue;
2911
2912                 change_mad_state(mad_send_wr, IB_MAD_STATE_DONE);
2913                 list_add_tail(&mad_send_wr->agent_list, list_item);
2914         }
2915
2916         spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
2917         process_backlog_mads(mad_agent_priv);
2918         clear_mad_error_list(&timeout_list, IB_WC_RESP_TIMEOUT_ERR,
2919                              mad_agent_priv);
2920         clear_mad_error_list(&cancel_list, IB_WC_WR_FLUSH_ERR, mad_agent_priv);
2921 }
2922
2923 /*
2924  * Allocate receive MADs and post receive WRs for them
2925  */
2926 static int ib_mad_post_receive_mads(struct ib_mad_qp_info *qp_info,
2927                                     struct ib_mad_private *mad)
2928 {
2929         unsigned long flags;
2930         struct ib_mad_private *mad_priv;
2931         struct ib_sge sg_list;
2932         struct ib_recv_wr recv_wr;
2933         struct ib_mad_queue *recv_queue = &qp_info->recv_queue;
2934         int ret = 0;
2935
2936         /* Initialize common scatter list fields */
2937         sg_list.lkey = qp_info->port_priv->pd->local_dma_lkey;
2938
2939         /* Initialize common receive WR fields */
2940         recv_wr.next = NULL;
2941         recv_wr.sg_list = &sg_list;
2942         recv_wr.num_sge = 1;
2943
2944         while (true) {
2945                 /* Allocate and map receive buffer */
2946                 if (mad) {
2947                         mad_priv = mad;
2948                         mad = NULL;
2949                 } else {
2950                         mad_priv = alloc_mad_private(port_mad_size(qp_info->port_priv),
2951                                                      GFP_ATOMIC);
2952                         if (!mad_priv)
2953                                 return -ENOMEM;
2954                 }
2955                 sg_list.length = mad_priv_dma_size(mad_priv);
2956                 sg_list.addr = ib_dma_map_single(qp_info->port_priv->device,
2957                                                  &mad_priv->grh,
2958                                                  mad_priv_dma_size(mad_priv),
2959                                                  DMA_FROM_DEVICE);
2960                 if (unlikely(ib_dma_mapping_error(qp_info->port_priv->device,
2961                                                   sg_list.addr))) {
2962                         ret = -ENOMEM;
2963                         goto free_mad_priv;
2964                 }
2965                 mad_priv->header.mapping = sg_list.addr;
2966                 mad_priv->header.mad_list.mad_queue = recv_queue;
2967                 mad_priv->header.mad_list.cqe.done = ib_mad_recv_done;
2968                 recv_wr.wr_cqe = &mad_priv->header.mad_list.cqe;
2969                 spin_lock_irqsave(&recv_queue->lock, flags);
2970                 if (recv_queue->count >= recv_queue->max_active) {
2971                         /* Fully populated the receive queue */
2972                         spin_unlock_irqrestore(&recv_queue->lock, flags);
2973                         break;
2974                 }
2975                 recv_queue->count++;
2976                 list_add_tail(&mad_priv->header.mad_list.list,
2977                               &recv_queue->list);
2978                 spin_unlock_irqrestore(&recv_queue->lock, flags);
2979
2980                 ret = ib_post_recv(qp_info->qp, &recv_wr, NULL);
2981                 if (ret) {
2982                         spin_lock_irqsave(&recv_queue->lock, flags);
2983                         list_del(&mad_priv->header.mad_list.list);
2984                         recv_queue->count--;
2985                         spin_unlock_irqrestore(&recv_queue->lock, flags);
2986                         dev_err(&qp_info->port_priv->device->dev,
2987                                 "ib_post_recv failed: %d\n", ret);
2988                         break;
2989                 }
2990         }
2991
2992         ib_dma_unmap_single(qp_info->port_priv->device,
2993                             mad_priv->header.mapping,
2994                             mad_priv_dma_size(mad_priv), DMA_FROM_DEVICE);
2995 free_mad_priv:
2996         kfree(mad_priv);
2997         return ret;
2998 }
2999
3000 /*
3001  * Return all the posted receive MADs
3002  */
3003 static void cleanup_recv_queue(struct ib_mad_qp_info *qp_info)
3004 {
3005         struct ib_mad_private_header *mad_priv_hdr;
3006         struct ib_mad_private *recv;
3007         struct ib_mad_list_head *mad_list;
3008
3009         if (!qp_info->qp)
3010                 return;
3011
3012         while (!list_empty(&qp_info->recv_queue.list)) {
3013
3014                 mad_list = list_entry(qp_info->recv_queue.list.next,
3015                                       struct ib_mad_list_head, list);
3016                 mad_priv_hdr = container_of(mad_list,
3017                                             struct ib_mad_private_header,
3018                                             mad_list);
3019                 recv = container_of(mad_priv_hdr, struct ib_mad_private,
3020                                     header);
3021
3022                 /* Remove from posted receive MAD list */
3023                 list_del(&mad_list->list);
3024
3025                 ib_dma_unmap_single(qp_info->port_priv->device,
3026                                     recv->header.mapping,
3027                                     mad_priv_dma_size(recv),
3028                                     DMA_FROM_DEVICE);
3029                 kfree(recv);
3030         }
3031
3032         qp_info->recv_queue.count = 0;
3033 }
3034
3035 /*
3036  * Start the port
3037  */
3038 static int ib_mad_port_start(struct ib_mad_port_private *port_priv)
3039 {
3040         int ret, i;
3041         struct ib_qp_attr *attr;
3042         struct ib_qp *qp;
3043         u16 pkey_index;
3044
3045         attr = kmalloc(sizeof *attr, GFP_KERNEL);
3046         if (!attr)
3047                 return -ENOMEM;
3048
3049         ret = ib_find_pkey(port_priv->device, port_priv->port_num,
3050                            IB_DEFAULT_PKEY_FULL, &pkey_index);
3051         if (ret)
3052                 pkey_index = 0;
3053
3054         for (i = 0; i < IB_MAD_QPS_CORE; i++) {
3055                 qp = port_priv->qp_info[i].qp;
3056                 if (!qp)
3057                         continue;
3058
3059                 /*
3060                  * PKey index for QP1 is irrelevant but
3061                  * one is needed for the Reset to Init transition
3062                  */
3063                 attr->qp_state = IB_QPS_INIT;
3064                 attr->pkey_index = pkey_index;
3065                 attr->qkey = (qp->qp_num == 0) ? 0 : IB_QP1_QKEY;
3066                 ret = ib_modify_qp(qp, attr, IB_QP_STATE |
3067                                              IB_QP_PKEY_INDEX | IB_QP_QKEY);
3068                 if (ret) {
3069                         dev_err(&port_priv->device->dev,
3070                                 "Couldn't change QP%d state to INIT: %d\n",
3071                                 i, ret);
3072                         goto out;
3073                 }
3074
3075                 attr->qp_state = IB_QPS_RTR;
3076                 ret = ib_modify_qp(qp, attr, IB_QP_STATE);
3077                 if (ret) {
3078                         dev_err(&port_priv->device->dev,
3079                                 "Couldn't change QP%d state to RTR: %d\n",
3080                                 i, ret);
3081                         goto out;
3082                 }
3083
3084                 attr->qp_state = IB_QPS_RTS;
3085                 attr->sq_psn = IB_MAD_SEND_Q_PSN;
3086                 ret = ib_modify_qp(qp, attr, IB_QP_STATE | IB_QP_SQ_PSN);
3087                 if (ret) {
3088                         dev_err(&port_priv->device->dev,
3089                                 "Couldn't change QP%d state to RTS: %d\n",
3090                                 i, ret);
3091                         goto out;
3092                 }
3093         }
3094
3095         ret = ib_req_notify_cq(port_priv->cq, IB_CQ_NEXT_COMP);
3096         if (ret) {
3097                 dev_err(&port_priv->device->dev,
3098                         "Failed to request completion notification: %d\n",
3099                         ret);
3100                 goto out;
3101         }
3102
3103         for (i = 0; i < IB_MAD_QPS_CORE; i++) {
3104                 if (!port_priv->qp_info[i].qp)
3105                         continue;
3106
3107                 ret = ib_mad_post_receive_mads(&port_priv->qp_info[i], NULL);
3108                 if (ret) {
3109                         dev_err(&port_priv->device->dev,
3110                                 "Couldn't post receive WRs\n");
3111                         goto out;
3112                 }
3113         }
3114 out:
3115         kfree(attr);
3116         return ret;
3117 }
3118
3119 static void qp_event_handler(struct ib_event *event, void *qp_context)
3120 {
3121         struct ib_mad_qp_info   *qp_info = qp_context;
3122
3123         /* It's worse than that! He's dead, Jim! */
3124         dev_err(&qp_info->port_priv->device->dev,
3125                 "Fatal error (%d) on MAD QP (%u)\n",
3126                 event->event, qp_info->qp->qp_num);
3127 }
3128
3129 static void init_mad_queue(struct ib_mad_qp_info *qp_info,
3130                            struct ib_mad_queue *mad_queue)
3131 {
3132         mad_queue->qp_info = qp_info;
3133         mad_queue->count = 0;
3134         spin_lock_init(&mad_queue->lock);
3135         INIT_LIST_HEAD(&mad_queue->list);
3136 }
3137
3138 static void init_mad_qp(struct ib_mad_port_private *port_priv,
3139                         struct ib_mad_qp_info *qp_info)
3140 {
3141         qp_info->port_priv = port_priv;
3142         init_mad_queue(qp_info, &qp_info->send_queue);
3143         init_mad_queue(qp_info, &qp_info->recv_queue);
3144         INIT_LIST_HEAD(&qp_info->overflow_list);
3145 }
3146
3147 static int create_mad_qp(struct ib_mad_qp_info *qp_info,
3148                          enum ib_qp_type qp_type)
3149 {
3150         struct ib_qp_init_attr  qp_init_attr;
3151         int ret;
3152
3153         memset(&qp_init_attr, 0, sizeof qp_init_attr);
3154         qp_init_attr.send_cq = qp_info->port_priv->cq;
3155         qp_init_attr.recv_cq = qp_info->port_priv->cq;
3156         qp_init_attr.sq_sig_type = IB_SIGNAL_ALL_WR;
3157         qp_init_attr.cap.max_send_wr = mad_sendq_size;
3158         qp_init_attr.cap.max_recv_wr = mad_recvq_size;
3159         qp_init_attr.cap.max_send_sge = IB_MAD_SEND_REQ_MAX_SG;
3160         qp_init_attr.cap.max_recv_sge = IB_MAD_RECV_REQ_MAX_SG;
3161         qp_init_attr.qp_type = qp_type;
3162         qp_init_attr.port_num = qp_info->port_priv->port_num;
3163         qp_init_attr.qp_context = qp_info;
3164         qp_init_attr.event_handler = qp_event_handler;
3165         qp_info->qp = ib_create_qp(qp_info->port_priv->pd, &qp_init_attr);
3166         if (IS_ERR(qp_info->qp)) {
3167                 dev_err(&qp_info->port_priv->device->dev,
3168                         "Couldn't create ib_mad QP%d\n",
3169                         get_spl_qp_index(qp_type));
3170                 ret = PTR_ERR(qp_info->qp);
3171                 goto error;
3172         }
3173         /* Use minimum queue sizes unless the CQ is resized */
3174         qp_info->send_queue.max_active = mad_sendq_size;
3175         qp_info->recv_queue.max_active = mad_recvq_size;
3176         return 0;
3177
3178 error:
3179         return ret;
3180 }
3181
3182 static void destroy_mad_qp(struct ib_mad_qp_info *qp_info)
3183 {
3184         if (!qp_info->qp)
3185                 return;
3186
3187         ib_destroy_qp(qp_info->qp);
3188 }
3189
3190 /*
3191  * Open the port
3192  * Create the QP, PD, MR, and CQ if needed
3193  */
3194 static int ib_mad_port_open(struct ib_device *device,
3195                             u32 port_num)
3196 {
3197         int ret, cq_size;
3198         struct ib_mad_port_private *port_priv;
3199         unsigned long flags;
3200         int has_smi;
3201
3202         if (WARN_ON(rdma_max_mad_size(device, port_num) < IB_MGMT_MAD_SIZE))
3203                 return -EFAULT;
3204
3205         if (WARN_ON(rdma_cap_opa_mad(device, port_num) &&
3206                     rdma_max_mad_size(device, port_num) < OPA_MGMT_MAD_SIZE))
3207                 return -EFAULT;
3208
3209         /* Create new device info */
3210         port_priv = kzalloc(sizeof *port_priv, GFP_KERNEL);
3211         if (!port_priv)
3212                 return -ENOMEM;
3213
3214         port_priv->device = device;
3215         port_priv->port_num = port_num;
3216         spin_lock_init(&port_priv->reg_lock);
3217         init_mad_qp(port_priv, &port_priv->qp_info[0]);
3218         init_mad_qp(port_priv, &port_priv->qp_info[1]);
3219
3220         cq_size = mad_sendq_size + mad_recvq_size;
3221         has_smi = rdma_cap_ib_smi(device, port_num);
3222         if (has_smi)
3223                 cq_size *= 2;
3224
3225         port_priv->pd = ib_alloc_pd(device, 0);
3226         if (IS_ERR(port_priv->pd)) {
3227                 dev_err(&device->dev, "Couldn't create ib_mad PD\n");
3228                 ret = PTR_ERR(port_priv->pd);
3229                 goto error3;
3230         }
3231
3232         port_priv->cq = ib_alloc_cq(port_priv->device, port_priv, cq_size, 0,
3233                         IB_POLL_UNBOUND_WORKQUEUE);
3234         if (IS_ERR(port_priv->cq)) {
3235                 dev_err(&device->dev, "Couldn't create ib_mad CQ\n");
3236                 ret = PTR_ERR(port_priv->cq);
3237                 goto error4;
3238         }
3239
3240         if (has_smi) {
3241                 ret = create_mad_qp(&port_priv->qp_info[0], IB_QPT_SMI);
3242                 if (ret)
3243                         goto error6;
3244         }
3245
3246         if (rdma_cap_ib_cm(device, port_num)) {
3247                 ret = create_mad_qp(&port_priv->qp_info[1], IB_QPT_GSI);
3248                 if (ret)
3249                         goto error7;
3250         }
3251
3252         port_priv->wq = alloc_ordered_workqueue("ib_mad%u", WQ_MEM_RECLAIM,
3253                                                 port_num);
3254         if (!port_priv->wq) {
3255                 ret = -ENOMEM;
3256                 goto error8;
3257         }
3258
3259         spin_lock_irqsave(&ib_mad_port_list_lock, flags);
3260         list_add_tail(&port_priv->port_list, &ib_mad_port_list);
3261         spin_unlock_irqrestore(&ib_mad_port_list_lock, flags);
3262
3263         ret = ib_mad_port_start(port_priv);
3264         if (ret) {
3265                 dev_err(&device->dev, "Couldn't start port\n");
3266                 goto error9;
3267         }
3268
3269         return 0;
3270
3271 error9:
3272         spin_lock_irqsave(&ib_mad_port_list_lock, flags);
3273         list_del_init(&port_priv->port_list);
3274         spin_unlock_irqrestore(&ib_mad_port_list_lock, flags);
3275
3276         destroy_workqueue(port_priv->wq);
3277 error8:
3278         destroy_mad_qp(&port_priv->qp_info[1]);
3279 error7:
3280         destroy_mad_qp(&port_priv->qp_info[0]);
3281 error6:
3282         ib_free_cq(port_priv->cq);
3283         cleanup_recv_queue(&port_priv->qp_info[1]);
3284         cleanup_recv_queue(&port_priv->qp_info[0]);
3285 error4:
3286         ib_dealloc_pd(port_priv->pd);
3287 error3:
3288         kfree(port_priv);
3289
3290         return ret;
3291 }
3292
3293 /*
3294  * Close the port
3295  * If there are no classes using the port, free the port
3296  * resources (CQ, MR, PD, QP) and remove the port's info structure
3297  */
3298 static int ib_mad_port_close(struct ib_device *device, u32 port_num)
3299 {
3300         struct ib_mad_port_private *port_priv;
3301         unsigned long flags;
3302
3303         spin_lock_irqsave(&ib_mad_port_list_lock, flags);
3304         port_priv = __ib_get_mad_port(device, port_num);
3305         if (port_priv == NULL) {
3306                 spin_unlock_irqrestore(&ib_mad_port_list_lock, flags);
3307                 dev_err(&device->dev, "Port %u not found\n", port_num);
3308                 return -ENODEV;
3309         }
3310         list_del_init(&port_priv->port_list);
3311         spin_unlock_irqrestore(&ib_mad_port_list_lock, flags);
3312
3313         destroy_workqueue(port_priv->wq);
3314         destroy_mad_qp(&port_priv->qp_info[1]);
3315         destroy_mad_qp(&port_priv->qp_info[0]);
3316         ib_free_cq(port_priv->cq);
3317         ib_dealloc_pd(port_priv->pd);
3318         cleanup_recv_queue(&port_priv->qp_info[1]);
3319         cleanup_recv_queue(&port_priv->qp_info[0]);
3320         /* XXX: Handle deallocation of MAD registration tables */
3321
3322         kfree(port_priv);
3323
3324         return 0;
3325 }
3326
3327 static int ib_mad_init_device(struct ib_device *device)
3328 {
3329         int start, i;
3330         unsigned int count = 0;
3331         int ret;
3332
3333         start = rdma_start_port(device);
3334
3335         for (i = start; i <= rdma_end_port(device); i++) {
3336                 if (!rdma_cap_ib_mad(device, i))
3337                         continue;
3338
3339                 ret = ib_mad_port_open(device, i);
3340                 if (ret) {
3341                         dev_err(&device->dev, "Couldn't open port %d\n", i);
3342                         goto error;
3343                 }
3344                 ret = ib_agent_port_open(device, i);
3345                 if (ret) {
3346                         dev_err(&device->dev,
3347                                 "Couldn't open port %d for agents\n", i);
3348                         goto error_agent;
3349                 }
3350                 count++;
3351         }
3352         if (!count)
3353                 return -EOPNOTSUPP;
3354
3355         return 0;
3356
3357 error_agent:
3358         if (ib_mad_port_close(device, i))
3359                 dev_err(&device->dev, "Couldn't close port %d\n", i);
3360
3361 error:
3362         while (--i >= start) {
3363                 if (!rdma_cap_ib_mad(device, i))
3364                         continue;
3365
3366                 if (ib_agent_port_close(device, i))
3367                         dev_err(&device->dev,
3368                                 "Couldn't close port %d for agents\n", i);
3369                 if (ib_mad_port_close(device, i))
3370                         dev_err(&device->dev, "Couldn't close port %d\n", i);
3371         }
3372         return ret;
3373 }
3374
3375 static void ib_mad_remove_device(struct ib_device *device, void *client_data)
3376 {
3377         unsigned int i;
3378
3379         rdma_for_each_port (device, i) {
3380                 if (!rdma_cap_ib_mad(device, i))
3381                         continue;
3382
3383                 if (ib_agent_port_close(device, i))
3384                         dev_err(&device->dev,
3385                                 "Couldn't close port %u for agents\n", i);
3386                 if (ib_mad_port_close(device, i))
3387                         dev_err(&device->dev, "Couldn't close port %u\n", i);
3388         }
3389 }
3390
3391 static struct ib_client mad_client = {
3392         .name   = "mad",
3393         .add = ib_mad_init_device,
3394         .remove = ib_mad_remove_device
3395 };
3396
3397 int ib_mad_init(void)
3398 {
3399         mad_recvq_size = min(mad_recvq_size, IB_MAD_QP_MAX_SIZE);
3400         mad_recvq_size = max(mad_recvq_size, IB_MAD_QP_MIN_SIZE);
3401
3402         mad_sendq_size = min(mad_sendq_size, IB_MAD_QP_MAX_SIZE);
3403         mad_sendq_size = max(mad_sendq_size, IB_MAD_QP_MIN_SIZE);
3404
3405         INIT_LIST_HEAD(&ib_mad_port_list);
3406
3407         if (ib_register_client(&mad_client)) {
3408                 pr_err("Couldn't register ib_mad client\n");
3409                 return -EINVAL;
3410         }
3411
3412         return 0;
3413 }
3414
3415 void ib_mad_cleanup(void)
3416 {
3417         ib_unregister_client(&mad_client);
3418 }