projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'mm-hotfixes-stable-2025-09-17-21-10' of git://git.kernel.org/pub/scm/linux...
[linux-block.git] / drivers / infiniband / core / mad.c
CommitLineData
1da177e4 1/*
de493d47 2 * Copyright (c) 2004-2007 Voltaire, Inc. All rights reserved.
fa619a77
HR		 3 * Copyright (c) 2005 Intel Corporation. All rights reserved.
4 * Copyright (c) 2005 Mellanox Technologies Ltd. All rights reserved.
b76aabc3 5 * Copyright (c) 2009 HNR Consulting. All rights reserved.
4d60cad5 6 * Copyright (c) 2014,2018 Intel Corporation. All rights reserved.
1da177e4
LT		 7 *
8 * This software is available to you under a choice of one of two
9 * licenses. You may choose to be licensed under the terms of the GNU
10 * General Public License (GPL) Version 2, available from the file
11 * COPYING in the main directory of this source tree, or the
12 * OpenIB.org BSD license below:
13 *
14 * Redistribution and use in source and binary forms, with or
15 * without modification, are permitted provided that the following
16 * conditions are met:
17 *
18 * - Redistributions of source code must retain the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer.
21 *
22 * - Redistributions in binary form must reproduce the above
23 * copyright notice, this list of conditions and the following
24 * disclaimer in the documentation and/or other materials
25 * provided with the distribution.
26 *
27 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
28 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
29 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
30 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
31 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
32 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
33 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34 * SOFTWARE.
35 *
1da177e4 36 */
7ef5d4b0
IW		 37
38#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
39
1da177e4 40#include <linux/dma-mapping.h>
5a0e3ad6 41#include <linux/slab.h>
e4dd23d7 42#include <linux/module.h>
47a2b338 43#include <linux/security.h>
949a2370 44#include <linux/xarray.h>
9874e746 45#include <rdma/ib_cache.h>
1da177e4
LT		 46
47#include "mad_priv.h"
47a2b338 48#include "core_priv.h"
fa619a77 49#include "mad_rmpp.h"
1da177e4 50#include "smi.h"
8e4349d1 51#include "opa_smi.h"
1da177e4 52#include "agent.h"
1da177e4 53
4d60cad5
IW		 54#define CREATE_TRACE_POINTS
55#include <trace/events/ib_mad.h>
56
57#ifdef CONFIG_TRACEPOINTS
58static void create_mad_addr_info(struct ib_mad_send_wr_private *mad_send_wr,
59 struct ib_mad_qp_info *qp_info,
60 struct trace_event_raw_ib_mad_send_template *entry)
61{
4d60cad5
IW		 62 struct ib_ud_wr *wr = &mad_send_wr->send_wr;
63 struct rdma_ah_attr attr = {};
64
65 rdma_query_ah(wr->ah, &attr);
66
67 /* These are common */
68 entry->sl = attr.sl;
4d60cad5
IW		 69 entry->rqpn = wr->remote_qpn;
70 entry->rqkey = wr->remote_qkey;
71 entry->dlid = rdma_ah_get_dlid(&attr);
72}
73#endif
74
16933955
RD		 75static int mad_sendq_size = IB_MAD_QP_SEND_SIZE;
76static int mad_recvq_size = IB_MAD_QP_RECV_SIZE;
b76aabc3
HR		 77
78module_param_named(send_queue_size, mad_sendq_size, int, 0444);
79MODULE_PARM_DESC(send_queue_size, "Size of send queue in number of work requests");
80module_param_named(recv_queue_size, mad_recvq_size, int, 0444);
81MODULE_PARM_DESC(recv_queue_size, "Size of receive queue in number of work requests");
82
949a2370
MW		 83static DEFINE_XARRAY_ALLOC1(ib_mad_clients);
84static u32 ib_mad_client_next;
1da177e4 85static struct list_head ib_mad_port_list;
1da177e4
LT		 86
87/* Port list lock */
6276e08a 88static DEFINE_SPINLOCK(ib_mad_port_list_lock);
1da177e4
LT		 89
90/* Forward declarations */
91static int method_in_use(struct ib_mad_mgmt_method_table **method,
92 struct ib_mad_reg_req *mad_reg_req);
93static void remove_mad_reg_req(struct ib_mad_agent_private *priv);
94static struct ib_mad_agent_private *find_mad_agent(
95 struct ib_mad_port_private *port_priv,
d94bd266 96 const struct ib_mad_hdr *mad);
1da177e4
LT		 97static int ib_mad_post_receive_mads(struct ib_mad_qp_info *qp_info,
98 struct ib_mad_private *mad);
99static void cancel_mads(struct ib_mad_agent_private *mad_agent_priv);
c4028958
DH		 100static void timeout_sends(struct work_struct *work);
101static void local_completions(struct work_struct *work);
1da177e4
LT		 102static int add_nonoui_reg_req(struct ib_mad_reg_req *mad_reg_req,
103 struct ib_mad_agent_private *agent_priv,
104 u8 mgmt_class);
105static int add_oui_reg_req(struct ib_mad_reg_req *mad_reg_req,
106 struct ib_mad_agent_private *agent_priv);
d53e11fd
CH		 107static bool ib_mad_send_error(struct ib_mad_port_private *port_priv,
108 struct ib_wc *wc);
109static void ib_mad_send_done(struct ib_cq *cq, struct ib_wc *wc);
1da177e4
LT		 110
111/*
112 * Returns a ib_mad_port_private structure or NULL for a device/port
113 * Assumes ib_mad_port_list_lock is being held
114 */
115static inline struct ib_mad_port_private *
1fb7f897 116__ib_get_mad_port(struct ib_device *device, u32 port_num)
1da177e4
LT		 117{
118 struct ib_mad_port_private *entry;
119
120 list_for_each_entry(entry, &ib_mad_port_list, port_list) {
121 if (entry->device == device && entry->port_num == port_num)
122 return entry;
123 }
124 return NULL;
125}
126
127/*
128 * Wrapper function to return a ib_mad_port_private structure or NULL
129 * for a device/port
130 */
131static inline struct ib_mad_port_private *
1fb7f897 132ib_get_mad_port(struct ib_device *device, u32 port_num)
1da177e4
LT		 133{
134 struct ib_mad_port_private *entry;
135 unsigned long flags;
136
137 spin_lock_irqsave(&ib_mad_port_list_lock, flags);
138 entry = __ib_get_mad_port(device, port_num);
139 spin_unlock_irqrestore(&ib_mad_port_list_lock, flags);
140
141 return entry;
142}
143
144static inline u8 convert_mgmt_class(u8 mgmt_class)
145{
146 /* Alias IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE to 0 */
147 return mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE ?
148 0 : mgmt_class;
149}
150
151static int get_spl_qp_index(enum ib_qp_type qp_type)
152{
b6eb7011 153 switch (qp_type) {
1da177e4
LT		 154 case IB_QPT_SMI:
155 return 0;
156 case IB_QPT_GSI:
157 return 1;
158 default:
159 return -1;
160 }
161}
162
163static int vendor_class_index(u8 mgmt_class)
164{
165 return mgmt_class - IB_MGMT_CLASS_VENDOR_RANGE2_START;
166}
167
168static int is_vendor_class(u8 mgmt_class)
169{
170 if ((mgmt_class < IB_MGMT_CLASS_VENDOR_RANGE2_START) ||
171 (mgmt_class > IB_MGMT_CLASS_VENDOR_RANGE2_END))
172 return 0;
173 return 1;
174}
175
176static int is_vendor_oui(char *oui)
177{
178 if (oui[0] || oui[1] || oui[2])
179 return 1;
180 return 0;
181}
182
183static int is_vendor_method_in_use(
184 struct ib_mad_mgmt_vendor_class *vendor_class,
185 struct ib_mad_reg_req *mad_reg_req)
186{
187 struct ib_mad_mgmt_method_table *method;
188 int i;
189
190 for (i = 0; i < MAX_MGMT_OUI; i++) {
191 if (!memcmp(vendor_class->oui[i], mad_reg_req->oui, 3)) {
192 method = vendor_class->method_table[i];
193 if (method) {
194 if (method_in_use(&method, mad_reg_req))
195 return 1;
196 else
197 break;
198 }
199 }
200 }
201 return 0;
202}
203
96909308 204int ib_response_mad(const struct ib_mad_hdr *hdr)
2527e681 205{
96909308
IW		 206 return ((hdr->method & IB_MGMT_METHOD_RESP) ||
207 (hdr->method == IB_MGMT_METHOD_TRAP_REPRESS) ||
208 ((hdr->mgmt_class == IB_MGMT_CLASS_BM) &&
209 (hdr->attr_mod & IB_BM_ATTR_MOD_RESP)));
2527e681
SH		 210}
211EXPORT_SYMBOL(ib_response_mad);
212
314cb74c
OHT		 213#define SOL_FC_MAX_DEFAULT_FRAC 4
214#define SOL_FC_MAX_SA_FRAC 32
215
216static int get_sol_fc_max_outstanding(struct ib_mad_reg_req *mad_reg_req)
217{
218 if (!mad_reg_req)
219 /* Send only agent */
220 return mad_recvq_size / SOL_FC_MAX_DEFAULT_FRAC;
221
222 switch (mad_reg_req->mgmt_class) {
223 case IB_MGMT_CLASS_CM:
224 return mad_recvq_size / SOL_FC_MAX_DEFAULT_FRAC;
225 case IB_MGMT_CLASS_SUBN_ADM:
226 return mad_recvq_size / SOL_FC_MAX_SA_FRAC;
227 case IB_MGMT_CLASS_SUBN_LID_ROUTED:
228 case IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE:
229 return min(mad_recvq_size, IB_MAD_QP_RECV_SIZE) /
230 SOL_FC_MAX_DEFAULT_FRAC;
231 default:
232 return 0;
233 }
234}
235
1da177e4
LT		 236/*
237 * ib_register_mad_agent - Register to send/receive MADs
0c271c43
MW		 238 *
239 * Context: Process context.
1da177e4
LT		 240 */
241struct ib_mad_agent *ib_register_mad_agent(struct ib_device *device,
1fb7f897 242 u32 port_num,
1da177e4
LT		 243 enum ib_qp_type qp_type,
244 struct ib_mad_reg_req *mad_reg_req,
245 u8 rmpp_version,
246 ib_mad_send_handler send_handler,
247 ib_mad_recv_handler recv_handler,
0f29b46d
IW		 248 void *context,
249 u32 registration_flags)
1da177e4
LT		 250{
251 struct ib_mad_port_private *port_priv;
252 struct ib_mad_agent *ret = ERR_PTR(-EINVAL);
253 struct ib_mad_agent_private *mad_agent_priv;
254 struct ib_mad_reg_req *reg_req = NULL;
255 struct ib_mad_mgmt_class_table *class;
256 struct ib_mad_mgmt_vendor_class_table *vendor;
257 struct ib_mad_mgmt_vendor_class *vendor_class;
258 struct ib_mad_mgmt_method_table *method;
259 int ret2, qpn;
1da177e4
LT		 260 u8 mgmt_class, vclass;
261
798bba01
PP		 262 if ((qp_type == IB_QPT_SMI && !rdma_cap_ib_smi(device, port_num)) ||
263 (qp_type == IB_QPT_GSI && !rdma_cap_ib_cm(device, port_num)))
264 return ERR_PTR(-EPROTONOSUPPORT);
265
1da177e4
LT		 266 /* Validate parameters */
267 qpn = get_spl_qp_index(qp_type);
9ad13a42 268 if (qpn == -1) {
f9d08f1e
PP		 269 dev_dbg_ratelimited(&device->dev, "%s: invalid QP Type %d\n",
270 __func__, qp_type);
1da177e4 271 goto error1;
9ad13a42 272 }
1da177e4 273
9ad13a42 274 if (rmpp_version && rmpp_version != IB_MGMT_RMPP_VERSION) {
f9d08f1e
PP		 275 dev_dbg_ratelimited(&device->dev,
276 "%s: invalid RMPP Version %u\n",
277 __func__, rmpp_version);
fa619a77 278 goto error1;
9ad13a42 279 }
1da177e4
LT		 280
281 /* Validate MAD registration request if supplied */
282 if (mad_reg_req) {
9ad13a42 283 if (mad_reg_req->mgmt_class_version >= MAX_MGMT_VERSION) {
f9d08f1e
PP		 284 dev_dbg_ratelimited(&device->dev,
285 "%s: invalid Class Version %u\n",
286 __func__,
287 mad_reg_req->mgmt_class_version);
1da177e4 288 goto error1;
9ad13a42
IW		 289 }
290 if (!recv_handler) {
f9d08f1e
PP		 291 dev_dbg_ratelimited(&device->dev,
292 "%s: no recv_handler\n", __func__);
1da177e4 293 goto error1;
9ad13a42 294 }
1da177e4
LT		 295 if (mad_reg_req->mgmt_class >= MAX_MGMT_CLASS) {
296 /*
297 * IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE is the only
298 * one in this range currently allowed
299 */
300 if (mad_reg_req->mgmt_class !=
9ad13a42 301 IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) {
f9d08f1e
PP		 302 dev_dbg_ratelimited(&device->dev,
303 "%s: Invalid Mgmt Class 0x%x\n",
304 __func__, mad_reg_req->mgmt_class);
1da177e4 305 goto error1;
9ad13a42 306 }
1da177e4
LT		 307 } else if (mad_reg_req->mgmt_class == 0) {
308 /*
309 * Class 0 is reserved in IBA and is used for
310 * aliasing of IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE
311 */
f9d08f1e
PP		 312 dev_dbg_ratelimited(&device->dev,
313 "%s: Invalid Mgmt Class 0\n",
314 __func__);
1da177e4
LT		 315 goto error1;
316 } else if (is_vendor_class(mad_reg_req->mgmt_class)) {
317 /*
318 * If class is in "new" vendor range,
319 * ensure supplied OUI is not zero
320 */
9ad13a42 321 if (!is_vendor_oui(mad_reg_req->oui)) {
f9d08f1e
PP		 322 dev_dbg_ratelimited(&device->dev,
323 "%s: No OUI specified for class 0x%x\n",
324 __func__,
325 mad_reg_req->mgmt_class);
1da177e4 326 goto error1;
9ad13a42 327 }
1da177e4 328 }
618a3c03 329 /* Make sure class supplied is consistent with RMPP */
64cb9c6a 330 if (!ib_is_mad_class_rmpp(mad_reg_req->mgmt_class)) {
9ad13a42 331 if (rmpp_version) {
f9d08f1e
PP		 332 dev_dbg_ratelimited(&device->dev,
333 "%s: RMPP version for non-RMPP class 0x%x\n",
334 __func__, mad_reg_req->mgmt_class);
618a3c03 335 goto error1;
9ad13a42 336 }
618a3c03 337 }
1471cb6c 338
1da177e4
LT		 339 /* Make sure class supplied is consistent with QP type */
340 if (qp_type == IB_QPT_SMI) {
341 if ((mad_reg_req->mgmt_class !=
342 IB_MGMT_CLASS_SUBN_LID_ROUTED) &&
343 (mad_reg_req->mgmt_class !=
9ad13a42 344 IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE)) {
f9d08f1e
PP		 345 dev_dbg_ratelimited(&device->dev,
346 "%s: Invalid SM QP type: class 0x%x\n",
347 __func__, mad_reg_req->mgmt_class);
1da177e4 348 goto error1;
9ad13a42 349 }
1da177e4
LT		 350 } else {
351 if ((mad_reg_req->mgmt_class ==
352 IB_MGMT_CLASS_SUBN_LID_ROUTED) ||
353 (mad_reg_req->mgmt_class ==
9ad13a42 354 IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE)) {
f9d08f1e
PP		 355 dev_dbg_ratelimited(&device->dev,
356 "%s: Invalid GS QP type: class 0x%x\n",
357 __func__, mad_reg_req->mgmt_class);
1da177e4 358 goto error1;
9ad13a42 359 }
1da177e4
LT		 360 }
361 } else {
362 /* No registration request supplied */
363 if (!send_handler)
364 goto error1;
1471cb6c
IW		 365 if (registration_flags & IB_MAD_USER_RMPP)
366 goto error1;
1da177e4
LT		 367 }
368
369 /* Validate device and port */
370 port_priv = ib_get_mad_port(device, port_num);
371 if (!port_priv) {
3cea7b4a 372 dev_dbg_ratelimited(&device->dev, "%s: Invalid port %u\n",
f9d08f1e 373 __func__, port_num);
1da177e4
LT		 374 ret = ERR_PTR(-ENODEV);
375 goto error1;
376 }
377
f9d08f1e
PP		 378 /* Verify the QP requested is supported. For example, Ethernet devices
379 * will not have QP0.
380 */
c8367c4c 381 if (!port_priv->qp_info[qpn].qp) {
f9d08f1e
PP		 382 dev_dbg_ratelimited(&device->dev, "%s: QP %d not supported\n",
383 __func__, qpn);
c8367c4c
IW		 384 ret = ERR_PTR(-EPROTONOSUPPORT);
385 goto error1;
386 }
387
1da177e4 388 /* Allocate structures */
de6eb66b 389 mad_agent_priv = kzalloc(sizeof *mad_agent_priv, GFP_KERNEL);
1da177e4
LT		 390 if (!mad_agent_priv) {
391 ret = ERR_PTR(-ENOMEM);
392 goto error1;
393 }
b82cab6b 394
1da177e4 395 if (mad_reg_req) {
9893e742 396 reg_req = kmemdup(mad_reg_req, sizeof *reg_req, GFP_KERNEL);
1da177e4
LT		 397 if (!reg_req) {
398 ret = ERR_PTR(-ENOMEM);
b82cab6b 399 goto error3;
1da177e4 400 }
1da177e4
LT		 401 }
402
403 /* Now, fill in the various structures */
1da177e4
LT		 404 mad_agent_priv->qp_info = &port_priv->qp_info[qpn];
405 mad_agent_priv->reg_req = reg_req;
fa619a77 406 mad_agent_priv->agent.rmpp_version = rmpp_version;
1da177e4
LT		 407 mad_agent_priv->agent.device = device;
408 mad_agent_priv->agent.recv_handler = recv_handler;
409 mad_agent_priv->agent.send_handler = send_handler;
410 mad_agent_priv->agent.context = context;
411 mad_agent_priv->agent.qp = port_priv->qp_info[qpn].qp;
412 mad_agent_priv->agent.port_num = port_num;
0f29b46d 413 mad_agent_priv->agent.flags = registration_flags;
d9620a4c
RC		 414 spin_lock_init(&mad_agent_priv->lock);
415 INIT_LIST_HEAD(&mad_agent_priv->send_list);
416 INIT_LIST_HEAD(&mad_agent_priv->wait_list);
d9620a4c 417 INIT_LIST_HEAD(&mad_agent_priv->rmpp_list);
314cb74c 418 INIT_LIST_HEAD(&mad_agent_priv->backlog_list);
d9620a4c
RC		 419 INIT_DELAYED_WORK(&mad_agent_priv->timed_work, timeout_sends);
420 INIT_LIST_HEAD(&mad_agent_priv->local_list);
421 INIT_WORK(&mad_agent_priv->local_work, local_completions);
e41c4253 422 refcount_set(&mad_agent_priv->refcount, 1);
d9620a4c 423 init_completion(&mad_agent_priv->comp);
314cb74c
OHT		 424 mad_agent_priv->sol_fc_send_count = 0;
425 mad_agent_priv->sol_fc_wait_count = 0;
426 mad_agent_priv->sol_fc_max =
8ab05a54
VD		 427 recv_handler ? get_sol_fc_max_outstanding(mad_reg_req) : 0;
428
47a2b338
DJ		 429 ret2 = ib_mad_agent_security_setup(&mad_agent_priv->agent, qp_type);
430 if (ret2) {
431 ret = ERR_PTR(ret2);
432 goto error4;
433 }
434
949a2370
MW		 435 /*
436 * The mlx4 driver uses the top byte to distinguish which virtual
437 * function generated the MAD, so we must avoid using it.
438 */
439 ret2 = xa_alloc_cyclic(&ib_mad_clients, &mad_agent_priv->agent.hi_tid,
440 mad_agent_priv, XA_LIMIT(0, (1 << 24) - 1),
441 &ib_mad_client_next, GFP_KERNEL);
9a41e38a 442 if (ret2 < 0) {
443 ret = ERR_PTR(ret2);
444 goto error5;
445 }
1da177e4
LT		 446
447 /*
448 * Make sure MAD registration (if supplied)
449 * is non overlapping with any existing ones
450 */
9a41e38a 451 spin_lock_irq(&port_priv->reg_lock);
1da177e4
LT		 452 if (mad_reg_req) {
453 mgmt_class = convert_mgmt_class(mad_reg_req->mgmt_class);
454 if (!is_vendor_class(mgmt_class)) {
455 class = port_priv->version[mad_reg_req->
456 mgmt_class_version].class;
457 if (class) {
458 method = class->method_table[mgmt_class];
459 if (method) {
460 if (method_in_use(&method,
461 mad_reg_req))
9a41e38a 462 goto error6;
1da177e4
LT		 463 }
464 }
465 ret2 = add_nonoui_reg_req(mad_reg_req, mad_agent_priv,
466 mgmt_class);
467 } else {
468 /* "New" vendor class range */
469 vendor = port_priv->version[mad_reg_req->
470 mgmt_class_version].vendor;
471 if (vendor) {
472 vclass = vendor_class_index(mgmt_class);
473 vendor_class = vendor->vendor_class[vclass];
474 if (vendor_class) {
475 if (is_vendor_method_in_use(
476 vendor_class,
477 mad_reg_req))
9a41e38a 478 goto error6;
1da177e4
LT		 479 }
480 }
481 ret2 = add_oui_reg_req(mad_reg_req, mad_agent_priv);
482 }
483 if (ret2) {
484 ret = ERR_PTR(ret2);
9a41e38a 485 goto error6;
1da177e4
LT		 486 }
487 }
0c271c43 488 spin_unlock_irq(&port_priv->reg_lock);
1da177e4 489
0e65bae2 490 trace_ib_mad_create_agent(mad_agent_priv);
1da177e4 491 return &mad_agent_priv->agent;
9a41e38a 492error6:
0c271c43 493 spin_unlock_irq(&port_priv->reg_lock);
949a2370 494 xa_erase(&ib_mad_clients, mad_agent_priv->agent.hi_tid);
9a41e38a 495error5:
47a2b338
DJ		 496 ib_mad_agent_security_cleanup(&mad_agent_priv->agent);
497error4:
1da177e4 498 kfree(reg_req);
b82cab6b 499error3:
2012a116 500 kfree(mad_agent_priv);
1da177e4
LT		 501error1:
502 return ret;
503}
504EXPORT_SYMBOL(ib_register_mad_agent);
505
1b52fa98
SH		 506static inline void deref_mad_agent(struct ib_mad_agent_private *mad_agent_priv)
507{
e41c4253 508 if (refcount_dec_and_test(&mad_agent_priv->refcount))
1b52fa98
SH		 509 complete(&mad_agent_priv->comp);
510}
511
1da177e4
LT		 512static void unregister_mad_agent(struct ib_mad_agent_private *mad_agent_priv)
513{
514 struct ib_mad_port_private *port_priv;
1da177e4
LT		 515
516 /* Note that we could still be handling received MADs */
0e65bae2 517 trace_ib_mad_unregister_agent(mad_agent_priv);
1da177e4
LT		 518
519 /*
520 * Canceling all sends results in dropping received response
521 * MADs, preventing us from queuing additional work
522 */
523 cancel_mads(mad_agent_priv);
1da177e4 524 port_priv = mad_agent_priv->qp_info->port_priv;
1da177e4 525 cancel_delayed_work(&mad_agent_priv->timed_work);
1da177e4 526
0c271c43 527 spin_lock_irq(&port_priv->reg_lock);
1da177e4 528 remove_mad_reg_req(mad_agent_priv);
0c271c43 529 spin_unlock_irq(&port_priv->reg_lock);
949a2370 530 xa_erase(&ib_mad_clients, mad_agent_priv->agent.hi_tid);
1da177e4 531
b82cab6b 532 flush_workqueue(port_priv->wq);
1da177e4 533
1b52fa98
SH		 534 deref_mad_agent(mad_agent_priv);
535 wait_for_completion(&mad_agent_priv->comp);
116a1b9f 536 ib_cancel_rmpp_recvs(mad_agent_priv);
1da177e4 537
47a2b338
DJ		 538 ib_mad_agent_security_cleanup(&mad_agent_priv->agent);
539
6044ec88 540 kfree(mad_agent_priv->reg_req);
9a41e38a 541 kfree_rcu(mad_agent_priv, rcu);
1da177e4
LT		 542}
543
1da177e4
LT		 544/*
545 * ib_unregister_mad_agent - Unregisters a client from using MAD services
0c271c43
MW		 546 *
547 * Context: Process context.
1da177e4 548 */
8d2216be 549void ib_unregister_mad_agent(struct ib_mad_agent *mad_agent)
1da177e4
LT		 550{
551 struct ib_mad_agent_private *mad_agent_priv;
04c349a9
MG		 552
553 mad_agent_priv = container_of(mad_agent,
554 struct ib_mad_agent_private,
555 agent);
556 unregister_mad_agent(mad_agent_priv);
1da177e4
LT		 557}
558EXPORT_SYMBOL(ib_unregister_mad_agent);
559
560static void dequeue_mad(struct ib_mad_list_head *mad_list)
561{
562 struct ib_mad_queue *mad_queue;
563 unsigned long flags;
564
1da177e4
LT		 565 mad_queue = mad_list->mad_queue;
566 spin_lock_irqsave(&mad_queue->lock, flags);
567 list_del(&mad_list->list);
568 mad_queue->count--;
569 spin_unlock_irqrestore(&mad_queue->lock, flags);
570}
571
d53e11fd 572static void build_smp_wc(struct ib_qp *qp, struct ib_cqe *cqe, u16 slid,
1fb7f897 573 u16 pkey_index, u32 port_num, struct ib_wc *wc)
1da177e4
LT		 574{
575 memset(wc, 0, sizeof *wc);
d53e11fd 576 wc->wr_cqe = cqe;
1da177e4
LT		 577 wc->status = IB_WC_SUCCESS;
578 wc->opcode = IB_WC_RECV;
579 wc->pkey_index = pkey_index;
580 wc->byte_len = sizeof(struct ib_mad) + sizeof(struct ib_grh);
581 wc->src_qp = IB_QP0;
062dbb69 582 wc->qp = qp;
1da177e4
LT		 583 wc->slid = slid;
584 wc->sl = 0;
585 wc->dlid_path_bits = 0;
586 wc->port_num = port_num;
587}
588
c9082e51
IW		 589static size_t mad_priv_size(const struct ib_mad_private *mp)
590{
591 return sizeof(struct ib_mad_private) + mp->mad_size;
592}
593
594static struct ib_mad_private *alloc_mad_private(size_t mad_size, gfp_t flags)
595{
596 size_t size = sizeof(struct ib_mad_private) + mad_size;
597 struct ib_mad_private *ret = kzalloc(size, flags);
598
599 if (ret)
600 ret->mad_size = mad_size;
601
602 return ret;
603}
604
605static size_t port_mad_size(const struct ib_mad_port_private *port_priv)
606{
607 return rdma_max_mad_size(port_priv->device, port_priv->port_num);
608}
609
610static size_t mad_priv_dma_size(const struct ib_mad_private *mp)
611{
612 return sizeof(struct ib_grh) + mp->mad_size;
613}
614
1da177e4
LT		 615/*
616 * Return 0 if SMP is to be sent
617 * Return 1 if SMP was consumed locally (whether or not solicited)
618 * Return < 0 if error
619 */
620static int handle_outgoing_dr_smp(struct ib_mad_agent_private *mad_agent_priv,
34816ad9 621 struct ib_mad_send_wr_private *mad_send_wr)
1da177e4 622{
de493d47 623 int ret = 0;
34816ad9 624 struct ib_smp *smp = mad_send_wr->send_buf.mad;
8e4349d1 625 struct opa_smp *opa_smp = (struct opa_smp *)smp;
1da177e4
LT		 626 unsigned long flags;
627 struct ib_mad_local_private *local;
628 struct ib_mad_private *mad_priv;
629 struct ib_mad_port_private *port_priv;
630 struct ib_mad_agent_private *recv_mad_agent = NULL;
631 struct ib_device *device = mad_agent_priv->agent.device;
1fb7f897 632 u32 port_num;
1da177e4 633 struct ib_wc mad_wc;
e622f2f4 634 struct ib_ud_wr *send_wr = &mad_send_wr->send_wr;
c9082e51 635 size_t mad_size = port_mad_size(mad_agent_priv->qp_info->port_priv);
4cd7c947 636 u16 out_mad_pkey_index = 0;
8e4349d1
IW		 637 u16 drslid;
638 bool opa = rdma_cap_opa_mad(mad_agent_priv->qp_info->port_priv->device,
639 mad_agent_priv->qp_info->port_priv->port_num);
1da177e4 640
4139032b 641 if (rdma_cap_ib_switch(device) &&
1bae4dbf 642 smp->mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE)
e622f2f4 643 port_num = send_wr->port_num;
1bae4dbf
HR		 644 else
645 port_num = mad_agent_priv->agent.port_num;
646
8cf3f04f
RC		 647 /*
648 * Directed route handling starts if the initial LID routed part of
649 * a request or the ending LID routed part of a response is empty.
650 * If we are at the start of the LID routed part, don't update the
651 * hop_ptr or hop_cnt. See section 14.2.2, Vol 1 IB spec.
652 */
9fa240bb 653 if (opa && smp->class_version == OPA_SM_CLASS_VERSION) {
8e4349d1
IW		 654 u32 opa_drslid;
655
2ccfbb70
IW		 656 trace_ib_mad_handle_out_opa_smi(opa_smp);
657
8e4349d1
IW		 658 if ((opa_get_smp_direction(opa_smp)
659 ? opa_smp->route.dr.dr_dlid : opa_smp->route.dr.dr_slid) ==
660 OPA_LID_PERMISSIVE &&
4139032b
HR		 661 opa_smi_handle_dr_smp_send(opa_smp,
662 rdma_cap_ib_switch(device),
8e4349d1
IW		 663 port_num) == IB_SMI_DISCARD) {
664 ret = -EINVAL;
665 dev_err(&device->dev, "OPA Invalid directed route\n");
666 goto out;
667 }
668 opa_drslid = be32_to_cpu(opa_smp->route.dr.dr_slid);
cd4cd565 669 if (opa_drslid != be32_to_cpu(OPA_LID_PERMISSIVE) &&
8e4349d1
IW		 670 opa_drslid & 0xffff0000) {
671 ret = -EINVAL;
672 dev_err(&device->dev, "OPA Invalid dr_slid 0x%x\n",
673 opa_drslid);
674 goto out;
675 }
676 drslid = (u16)(opa_drslid & 0x0000ffff);
de493d47 677
8e4349d1
IW		 678 /* Check to post send on QP or process locally */
679 if (opa_smi_check_local_smp(opa_smp, device) == IB_SMI_DISCARD &&
680 opa_smi_check_local_returning_smp(opa_smp, device) == IB_SMI_DISCARD)
681 goto out;
682 } else {
2ccfbb70
IW		 683 trace_ib_mad_handle_out_ib_smi(smp);
684
8e4349d1
IW		 685 if ((ib_get_smp_direction(smp) ? smp->dr_dlid : smp->dr_slid) ==
686 IB_LID_PERMISSIVE &&
4139032b 687 smi_handle_dr_smp_send(smp, rdma_cap_ib_switch(device), port_num) ==
8e4349d1
IW		 688 IB_SMI_DISCARD) {
689 ret = -EINVAL;
690 dev_err(&device->dev, "Invalid directed route\n");
691 goto out;
692 }
693 drslid = be16_to_cpu(smp->dr_slid);
694
695 /* Check to post send on QP or process locally */
696 if (smi_check_local_smp(smp, device) == IB_SMI_DISCARD &&
697 smi_check_local_returning_smp(smp, device) == IB_SMI_DISCARD)
698 goto out;
699 }
1da177e4
LT		 700
701 local = kmalloc(sizeof *local, GFP_ATOMIC);
702 if (!local) {
703 ret = -ENOMEM;
1da177e4
LT		 704 goto out;
705 }
706 local->mad_priv = NULL;
707 local->recv_mad_agent = NULL;
c9082e51 708 mad_priv = alloc_mad_private(mad_size, GFP_ATOMIC);
1da177e4
LT		 709 if (!mad_priv) {
710 ret = -ENOMEM;
1da177e4
LT		 711 kfree(local);
712 goto out;
713 }
714
062dbb69 715 build_smp_wc(mad_agent_priv->agent.qp,
d53e11fd 716 send_wr->wr.wr_cqe, drslid,
e622f2f4
CH		 717 send_wr->pkey_index,
718 send_wr->port_num, &mad_wc);
1da177e4 719
8e4349d1
IW		 720 if (opa && smp->base_version == OPA_MGMT_BASE_VERSION) {
721 mad_wc.byte_len = mad_send_wr->send_buf.hdr_len
722 + mad_send_wr->send_buf.data_len
723 + sizeof(struct ib_grh);
724 }
725
1da177e4 726 /* No GRH for DR SMP */
3023a1e9 727 ret = device->ops.process_mad(device, 0, port_num, &mad_wc, NULL,
e26e7b88
LR		 728 (const struct ib_mad *)smp,
729 (struct ib_mad *)mad_priv->mad, &mad_size,
730 &out_mad_pkey_index);
b6eb7011 731 switch (ret) {
1da177e4 732 case IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY:
c9082e51 733 if (ib_response_mad((const struct ib_mad_hdr *)mad_priv->mad) &&
1da177e4
LT		 734 mad_agent_priv->agent.recv_handler) {
735 local->mad_priv = mad_priv;
736 local->recv_mad_agent = mad_agent_priv;
737 /*
738 * Reference MAD agent until receive
739 * side of local completion handled
740 */
e41c4253 741 refcount_inc(&mad_agent_priv->refcount);
1da177e4 742 } else
c9082e51 743 kfree(mad_priv);
1da177e4
LT		 744 break;
745 case IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_CONSUMED:
c9082e51 746 kfree(mad_priv);
4780c195 747 break;
1da177e4
LT		 748 case IB_MAD_RESULT_SUCCESS:
749 /* Treat like an incoming receive MAD */
1da177e4
LT		 750 port_priv = ib_get_mad_port(mad_agent_priv->agent.device,
751 mad_agent_priv->agent.port_num);
752 if (port_priv) {
c9082e51 753 memcpy(mad_priv->mad, smp, mad_priv->mad_size);
1da177e4 754 recv_mad_agent = find_mad_agent(port_priv,
c9082e51 755 (const struct ib_mad_hdr *)mad_priv->mad);
1da177e4
LT		 756 }
757 if (!port_priv || !recv_mad_agent) {
4780c195
RC		 758 /*
759 * No receiving agent so drop packet and
760 * generate send completion.
761 */
c9082e51 762 kfree(mad_priv);
4780c195 763 break;
1da177e4
LT		 764 }
765 local->mad_priv = mad_priv;
766 local->recv_mad_agent = recv_mad_agent;
767 break;
768 default:
c9082e51 769 kfree(mad_priv);
1da177e4
LT		 770 kfree(local);
771 ret = -EINVAL;
772 goto out;
773 }
774
34816ad9 775 local->mad_send_wr = mad_send_wr;
8e4349d1 776 if (opa) {
e622f2f4 777 local->mad_send_wr->send_wr.pkey_index = out_mad_pkey_index;
8e4349d1
IW		 778 local->return_wc_byte_len = mad_size;
779 }
1da177e4 780 /* Reference MAD agent until send side of local completion handled */
e41c4253 781 refcount_inc(&mad_agent_priv->refcount);
1da177e4
LT		 782 /* Queue local completion to local list */
783 spin_lock_irqsave(&mad_agent_priv->lock, flags);
784 list_add_tail(&local->completion_list, &mad_agent_priv->local_list);
785 spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
786 queue_work(mad_agent_priv->qp_info->port_priv->wq,
b82cab6b 787 &mad_agent_priv->local_work);
1da177e4
LT		 788 ret = 1;
789out:
790 return ret;
791}
792
548ead17 793static int get_pad_size(int hdr_len, int data_len, size_t mad_size)
824c8ae7
HR		 794{
795 int seg_size, pad;
796
548ead17 797 seg_size = mad_size - hdr_len;
824c8ae7
HR		 798 if (data_len && seg_size) {
799 pad = seg_size - data_len % seg_size;
f36e1793 800 return pad == seg_size ? 0 : pad;
824c8ae7 801 } else
f36e1793
JM		 802 return seg_size;
803}
804
805static void free_send_rmpp_list(struct ib_mad_send_wr_private *mad_send_wr)
806{
807 struct ib_rmpp_segment *s, *t;
808
809 list_for_each_entry_safe(s, t, &mad_send_wr->rmpp_list, list) {
810 list_del(&s->list);
811 kfree(s);
812 }
813}
814
815static int alloc_send_rmpp_list(struct ib_mad_send_wr_private *send_wr,
548ead17 816 size_t mad_size, gfp_t gfp_mask)
f36e1793
JM		 817{
818 struct ib_mad_send_buf *send_buf = &send_wr->send_buf;
819 struct ib_rmpp_mad *rmpp_mad = send_buf->mad;
820 struct ib_rmpp_segment *seg = NULL;
821 int left, seg_size, pad;
822
548ead17
IW		 823 send_buf->seg_size = mad_size - send_buf->hdr_len;
824 send_buf->seg_rmpp_size = mad_size - IB_MGMT_RMPP_HDR;
f36e1793
JM		 825 seg_size = send_buf->seg_size;
826 pad = send_wr->pad;
827
828 /* Allocate data segments. */
829 for (left = send_buf->data_len + pad; left > 0; left -= seg_size) {
f681967a 830 seg = kmalloc(sizeof(*seg) + seg_size, gfp_mask);
f36e1793 831 if (!seg) {
f36e1793
JM		 832 free_send_rmpp_list(send_wr);
833 return -ENOMEM;
834 }
835 seg->num = ++send_buf->seg_count;
836 list_add_tail(&seg->list, &send_wr->rmpp_list);
837 }
838
839 /* Zero any padding */
840 if (pad)
841 memset(seg->data + seg_size - pad, 0, pad);
842
843 rmpp_mad->rmpp_hdr.rmpp_version = send_wr->mad_agent_priv->
844 agent.rmpp_version;
845 rmpp_mad->rmpp_hdr.rmpp_type = IB_MGMT_RMPP_TYPE_DATA;
846 ib_set_rmpp_flags(&rmpp_mad->rmpp_hdr, IB_MGMT_RMPP_FLAG_ACTIVE);
847
848 send_wr->cur_seg = container_of(send_wr->rmpp_list.next,
849 struct ib_rmpp_segment, list);
850 send_wr->last_ack_seg = send_wr->cur_seg;
851 return 0;
824c8ae7
HR		 852}
853
f766c58f 854int ib_mad_kernel_rmpp_agent(const struct ib_mad_agent *agent)
1471cb6c
IW		 855{
856 return agent->rmpp_version && !(agent->flags & IB_MAD_USER_RMPP);
857}
858EXPORT_SYMBOL(ib_mad_kernel_rmpp_agent);
859
f681967a
WL		 860struct ib_mad_send_buf *ib_create_send_mad(struct ib_mad_agent *mad_agent,
861 u32 remote_qpn, u16 pkey_index,
862 int rmpp_active, int hdr_len,
863 int data_len, gfp_t gfp_mask,
864 u8 base_version)
824c8ae7
HR		 865{
866 struct ib_mad_agent_private *mad_agent_priv;
34816ad9 867 struct ib_mad_send_wr_private *mad_send_wr;
f36e1793 868 int pad, message_size, ret, size;
824c8ae7 869 void *buf;
548ead17
IW		 870 size_t mad_size;
871 bool opa;
824c8ae7 872
34816ad9
SH		 873 mad_agent_priv = container_of(mad_agent, struct ib_mad_agent_private,
874 agent);
548ead17
IW		 875
876 opa = rdma_cap_opa_mad(mad_agent->device, mad_agent->port_num);
877
878 if (opa && base_version == OPA_MGMT_BASE_VERSION)
879 mad_size = sizeof(struct opa_mad);
880 else
881 mad_size = sizeof(struct ib_mad);
882
883 pad = get_pad_size(hdr_len, data_len, mad_size);
f36e1793 884 message_size = hdr_len + data_len + pad;
824c8ae7 885
1471cb6c 886 if (ib_mad_kernel_rmpp_agent(mad_agent)) {
548ead17 887 if (!rmpp_active && message_size > mad_size)
1471cb6c
IW		 888 return ERR_PTR(-EINVAL);
889 } else
548ead17 890 if (rmpp_active || message_size > mad_size)
1471cb6c 891 return ERR_PTR(-EINVAL);
fa619a77 892
548ead17 893 size = rmpp_active ? hdr_len : mad_size;
f36e1793 894 buf = kzalloc(sizeof *mad_send_wr + size, gfp_mask);
824c8ae7
HR		 895 if (!buf)
896 return ERR_PTR(-ENOMEM);
34816ad9 897
f36e1793
JM		 898 mad_send_wr = buf + size;
899 INIT_LIST_HEAD(&mad_send_wr->rmpp_list);
34816ad9 900 mad_send_wr->send_buf.mad = buf;
f36e1793
JM		 901 mad_send_wr->send_buf.hdr_len = hdr_len;
902 mad_send_wr->send_buf.data_len = data_len;
903 mad_send_wr->pad = pad;
34816ad9
SH		 904
905 mad_send_wr->mad_agent_priv = mad_agent_priv;
f36e1793 906 mad_send_wr->sg_list[0].length = hdr_len;
4be90bc6 907 mad_send_wr->sg_list[0].lkey = mad_agent->qp->pd->local_dma_lkey;
548ead17
IW		 908
909 /* OPA MADs don't have to be the full 2048 bytes */
910 if (opa && base_version == OPA_MGMT_BASE_VERSION &&
911 data_len < mad_size - hdr_len)
912 mad_send_wr->sg_list[1].length = data_len;
913 else
914 mad_send_wr->sg_list[1].length = mad_size - hdr_len;
915
4be90bc6 916 mad_send_wr->sg_list[1].lkey = mad_agent->qp->pd->local_dma_lkey;
34816ad9 917
d53e11fd
CH		 918 mad_send_wr->mad_list.cqe.done = ib_mad_send_done;
919
920 mad_send_wr->send_wr.wr.wr_cqe = &mad_send_wr->mad_list.cqe;
e622f2f4
CH		 921 mad_send_wr->send_wr.wr.sg_list = mad_send_wr->sg_list;
922 mad_send_wr->send_wr.wr.num_sge = 2;
923 mad_send_wr->send_wr.wr.opcode = IB_WR_SEND;
924 mad_send_wr->send_wr.wr.send_flags = IB_SEND_SIGNALED;
925 mad_send_wr->send_wr.remote_qpn = remote_qpn;
926 mad_send_wr->send_wr.remote_qkey = IB_QP_SET_QKEY;
927 mad_send_wr->send_wr.pkey_index = pkey_index;
fa619a77
HR		 928
929 if (rmpp_active) {
548ead17 930 ret = alloc_send_rmpp_list(mad_send_wr, mad_size, gfp_mask);
f36e1793
JM		 931 if (ret) {
932 kfree(buf);
933 return ERR_PTR(ret);
934 }
fa619a77
HR		 935 }
936
34816ad9 937 mad_send_wr->send_buf.mad_agent = mad_agent;
e41c4253 938 refcount_inc(&mad_agent_priv->refcount);
34816ad9 939 return &mad_send_wr->send_buf;
824c8ae7
HR		 940}
941EXPORT_SYMBOL(ib_create_send_mad);
942
618a3c03
HR		 943int ib_get_mad_data_offset(u8 mgmt_class)
944{
945 if (mgmt_class == IB_MGMT_CLASS_SUBN_ADM)
946 return IB_MGMT_SA_HDR;
947 else if ((mgmt_class == IB_MGMT_CLASS_DEVICE_MGMT) ||
948 (mgmt_class == IB_MGMT_CLASS_DEVICE_ADM) ||
949 (mgmt_class == IB_MGMT_CLASS_BIS))
950 return IB_MGMT_DEVICE_HDR;
951 else if ((mgmt_class >= IB_MGMT_CLASS_VENDOR_RANGE2_START) &&
952 (mgmt_class <= IB_MGMT_CLASS_VENDOR_RANGE2_END))
953 return IB_MGMT_VENDOR_HDR;
954 else
955 return IB_MGMT_MAD_HDR;
956}
957EXPORT_SYMBOL(ib_get_mad_data_offset);
958
959int ib_is_mad_class_rmpp(u8 mgmt_class)
960{
961 if ((mgmt_class == IB_MGMT_CLASS_SUBN_ADM) ||
962 (mgmt_class == IB_MGMT_CLASS_DEVICE_MGMT) ||
963 (mgmt_class == IB_MGMT_CLASS_DEVICE_ADM) ||
964 (mgmt_class == IB_MGMT_CLASS_BIS) ||
965 ((mgmt_class >= IB_MGMT_CLASS_VENDOR_RANGE2_START) &&
966 (mgmt_class <= IB_MGMT_CLASS_VENDOR_RANGE2_END)))
967 return 1;
968 return 0;
969}
970EXPORT_SYMBOL(ib_is_mad_class_rmpp);
971
f36e1793
JM		 972void *ib_get_rmpp_segment(struct ib_mad_send_buf *send_buf, int seg_num)
973{
974 struct ib_mad_send_wr_private *mad_send_wr;
975 struct list_head *list;
976
977 mad_send_wr = container_of(send_buf, struct ib_mad_send_wr_private,
978 send_buf);
979 list = &mad_send_wr->cur_seg->list;
980
981 if (mad_send_wr->cur_seg->num < seg_num) {
982 list_for_each_entry(mad_send_wr->cur_seg, list, list)
983 if (mad_send_wr->cur_seg->num == seg_num)
984 break;
985 } else if (mad_send_wr->cur_seg->num > seg_num) {
986 list_for_each_entry_reverse(mad_send_wr->cur_seg, list, list)
987 if (mad_send_wr->cur_seg->num == seg_num)
988 break;
989 }
990 return mad_send_wr->cur_seg->data;
991}
992EXPORT_SYMBOL(ib_get_rmpp_segment);
993
994static inline void *ib_get_payload(struct ib_mad_send_wr_private *mad_send_wr)
995{
996 if (mad_send_wr->send_buf.seg_count)
997 return ib_get_rmpp_segment(&mad_send_wr->send_buf,
998 mad_send_wr->seg_num);
999 else
1000 return mad_send_wr->send_buf.mad +
1001 mad_send_wr->send_buf.hdr_len;
1002}
1003
824c8ae7
HR		 1004void ib_free_send_mad(struct ib_mad_send_buf *send_buf)
1005{
1006 struct ib_mad_agent_private *mad_agent_priv;
f36e1793 1007 struct ib_mad_send_wr_private *mad_send_wr;
824c8ae7
HR		 1008
1009 mad_agent_priv = container_of(send_buf->mad_agent,
1010 struct ib_mad_agent_private, agent);
f36e1793
JM		 1011 mad_send_wr = container_of(send_buf, struct ib_mad_send_wr_private,
1012 send_buf);
824c8ae7 1013
f36e1793
JM		 1014 free_send_rmpp_list(mad_send_wr);
1015 kfree(send_buf->mad);
1b52fa98 1016 deref_mad_agent(mad_agent_priv);
824c8ae7
HR		 1017}
1018EXPORT_SYMBOL(ib_free_send_mad);
1019
fa619a77 1020int ib_send_mad(struct ib_mad_send_wr_private *mad_send_wr)
1da177e4
LT		 1021{
1022 struct ib_mad_qp_info *qp_info;
cabe3cbc 1023 struct list_head *list;
34816ad9
SH		 1024 struct ib_mad_agent *mad_agent;
1025 struct ib_sge *sge;
1da177e4
LT		 1026 unsigned long flags;
1027 int ret;
1028
f8197a4e 1029 /* Set WR ID to find mad_send_wr upon completion */
d760ce8f 1030 qp_info = mad_send_wr->mad_agent_priv->qp_info;
1da177e4 1031 mad_send_wr->mad_list.mad_queue = &qp_info->send_queue;
d53e11fd
CH		 1032 mad_send_wr->mad_list.cqe.done = ib_mad_send_done;
1033 mad_send_wr->send_wr.wr.wr_cqe = &mad_send_wr->mad_list.cqe;
1da177e4 1034
34816ad9
SH		 1035 mad_agent = mad_send_wr->send_buf.mad_agent;
1036 sge = mad_send_wr->sg_list;
1527106f
RC		 1037 sge[0].addr = ib_dma_map_single(mad_agent->device,
1038 mad_send_wr->send_buf.mad,
1039 sge[0].length,
1040 DMA_TO_DEVICE);
2c34e68f
YB		 1041 if (unlikely(ib_dma_mapping_error(mad_agent->device, sge[0].addr)))
1042 return -ENOMEM;
1043
1527106f
RC		 1044 mad_send_wr->header_mapping = sge[0].addr;
1045
1046 sge[1].addr = ib_dma_map_single(mad_agent->device,
1047 ib_get_payload(mad_send_wr),
1048 sge[1].length,
1049 DMA_TO_DEVICE);
2c34e68f
YB		 1050 if (unlikely(ib_dma_mapping_error(mad_agent->device, sge[1].addr))) {
1051 ib_dma_unmap_single(mad_agent->device,
1052 mad_send_wr->header_mapping,
1053 sge[0].length, DMA_TO_DEVICE);
1054 return -ENOMEM;
1055 }
1527106f 1056 mad_send_wr->payload_mapping = sge[1].addr;
34816ad9 1057
1da177e4 1058 spin_lock_irqsave(&qp_info->send_queue.lock, flags);
cabe3cbc 1059 if (qp_info->send_queue.count < qp_info->send_queue.max_active) {
4d60cad5 1060 trace_ib_mad_ib_send_mad(mad_send_wr, qp_info);
e622f2f4 1061 ret = ib_post_send(mad_agent->qp, &mad_send_wr->send_wr.wr,
1fec77bf 1062 NULL);
cabe3cbc 1063 list = &qp_info->send_queue.list;
1da177e4 1064 } else {
1da177e4 1065 ret = 0;
cabe3cbc 1066 list = &qp_info->overflow_list;
1da177e4 1067 }
cabe3cbc
HR		 1068
1069 if (!ret) {
1070 qp_info->send_queue.count++;
1071 list_add_tail(&mad_send_wr->mad_list.list, list);
1072 }
1073 spin_unlock_irqrestore(&qp_info->send_queue.lock, flags);
f36e1793 1074 if (ret) {
1527106f
RC		 1075 ib_dma_unmap_single(mad_agent->device,
1076 mad_send_wr->header_mapping,
1077 sge[0].length, DMA_TO_DEVICE);
1078 ib_dma_unmap_single(mad_agent->device,
1079 mad_send_wr->payload_mapping,
1080 sge[1].length, DMA_TO_DEVICE);
f36e1793 1081 }
1da177e4
LT		 1082 return ret;
1083}
1084
314cb74c
OHT		 1085static void handle_queued_state(struct ib_mad_send_wr_private *mad_send_wr,
1086 struct ib_mad_agent_private *mad_agent_priv)
1087{
1088 if (mad_send_wr->state == IB_MAD_STATE_WAIT_RESP) {
1089 mad_agent_priv->sol_fc_wait_count--;
1090 list_move_tail(&mad_send_wr->agent_list,
1091 &mad_agent_priv->backlog_list);
1092 } else {
1093 expect_mad_state(mad_send_wr, IB_MAD_STATE_INIT);
1094 list_add_tail(&mad_send_wr->agent_list,
1095 &mad_agent_priv->backlog_list);
1096 }
1097}
1098
1cf0d899
OHT		 1099static void handle_send_state(struct ib_mad_send_wr_private *mad_send_wr,
1100 struct ib_mad_agent_private *mad_agent_priv)
1101{
1102 if (mad_send_wr->state == IB_MAD_STATE_INIT) {
1103 list_add_tail(&mad_send_wr->agent_list,
1104 &mad_agent_priv->send_list);
1105 } else {
314cb74c
OHT		 1106 expect_mad_state2(mad_send_wr, IB_MAD_STATE_WAIT_RESP,
1107 IB_MAD_STATE_QUEUED);
1cf0d899
OHT		 1108 list_move_tail(&mad_send_wr->agent_list,
1109 &mad_agent_priv->send_list);
1110 }
314cb74c
OHT		 1111
1112 if (mad_send_wr->is_solicited_fc) {
1113 if (mad_send_wr->state == IB_MAD_STATE_WAIT_RESP)
1114 mad_agent_priv->sol_fc_wait_count--;
1115 mad_agent_priv->sol_fc_send_count++;
1116 }
1cf0d899
OHT		 1117}
1118
1119static void handle_wait_state(struct ib_mad_send_wr_private *mad_send_wr,
1120 struct ib_mad_agent_private *mad_agent_priv)
1121{
1122 struct ib_mad_send_wr_private *temp_mad_send_wr;
1123 struct list_head *list_item;
1124 unsigned long delay;
1125
1126 expect_mad_state3(mad_send_wr, IB_MAD_STATE_SEND_START,
1127 IB_MAD_STATE_WAIT_RESP, IB_MAD_STATE_CANCELED);
314cb74c
OHT		 1128 if (mad_send_wr->state == IB_MAD_STATE_SEND_START &&
1129 mad_send_wr->is_solicited_fc) {
1130 mad_agent_priv->sol_fc_send_count--;
1131 mad_agent_priv->sol_fc_wait_count++;
1132 }
1cf0d899 1133
314cb74c 1134 list_del_init(&mad_send_wr->agent_list);
1cf0d899
OHT		 1135 delay = mad_send_wr->timeout;
1136 mad_send_wr->timeout += jiffies;
1137
1138 if (delay) {
1139 list_for_each_prev(list_item,
1140 &mad_agent_priv->wait_list) {
1141 temp_mad_send_wr = list_entry(
1142 list_item,
1143 struct ib_mad_send_wr_private,
1144 agent_list);
1145 if (time_after(mad_send_wr->timeout,
1146 temp_mad_send_wr->timeout))
1147 break;
1148 }
1149 } else {
1150 list_item = &mad_agent_priv->wait_list;
1151 }
1152
1153 list_add(&mad_send_wr->agent_list, list_item);
1154}
1155
1156static void handle_early_resp_state(struct ib_mad_send_wr_private *mad_send_wr,
1157 struct ib_mad_agent_private *mad_agent_priv)
1158{
1159 expect_mad_state(mad_send_wr, IB_MAD_STATE_SEND_START);
314cb74c 1160 mad_agent_priv->sol_fc_send_count -= mad_send_wr->is_solicited_fc;
1cf0d899
OHT		 1161}
1162
1163static void handle_canceled_state(struct ib_mad_send_wr_private *mad_send_wr,
1164 struct ib_mad_agent_private *mad_agent_priv)
1165{
1166 not_expect_mad_state(mad_send_wr, IB_MAD_STATE_DONE);
314cb74c
OHT		 1167 if (mad_send_wr->is_solicited_fc) {
1168 if (mad_send_wr->state == IB_MAD_STATE_SEND_START)
1169 mad_agent_priv->sol_fc_send_count--;
1170 else if (mad_send_wr->state == IB_MAD_STATE_WAIT_RESP)
1171 mad_agent_priv->sol_fc_wait_count--;
1172 }
1cf0d899
OHT		 1173}
1174
1175static void handle_done_state(struct ib_mad_send_wr_private *mad_send_wr,
1176 struct ib_mad_agent_private *mad_agent_priv)
1177{
314cb74c
OHT		 1178 if (mad_send_wr->is_solicited_fc) {
1179 if (mad_send_wr->state == IB_MAD_STATE_SEND_START)
1180 mad_agent_priv->sol_fc_send_count--;
1181 else if (mad_send_wr->state == IB_MAD_STATE_WAIT_RESP)
1182 mad_agent_priv->sol_fc_wait_count--;
1183 }
1184
1cf0d899
OHT		 1185 list_del_init(&mad_send_wr->agent_list);
1186}
1187
1188void change_mad_state(struct ib_mad_send_wr_private *mad_send_wr,
1189 enum ib_mad_state new_state)
1190{
1191 struct ib_mad_agent_private *mad_agent_priv =
1192 mad_send_wr->mad_agent_priv;
1193
1194 switch (new_state) {
1195 case IB_MAD_STATE_INIT:
1196 break;
314cb74c
OHT		 1197 case IB_MAD_STATE_QUEUED:
1198 handle_queued_state(mad_send_wr, mad_agent_priv);
1199 break;
1cf0d899
OHT		 1200 case IB_MAD_STATE_SEND_START:
1201 handle_send_state(mad_send_wr, mad_agent_priv);
1202 break;
1203 case IB_MAD_STATE_WAIT_RESP:
1204 handle_wait_state(mad_send_wr, mad_agent_priv);
1205 if (mad_send_wr->state == IB_MAD_STATE_CANCELED)
1206 return;
1207 break;
1208 case IB_MAD_STATE_EARLY_RESP:
1209 handle_early_resp_state(mad_send_wr, mad_agent_priv);
1210 break;
1211 case IB_MAD_STATE_CANCELED:
1212 handle_canceled_state(mad_send_wr, mad_agent_priv);
1213 break;
1214 case IB_MAD_STATE_DONE:
1215 handle_done_state(mad_send_wr, mad_agent_priv);
1216 break;
1217 }
1218
1219 mad_send_wr->state = new_state;
1220}
1221
314cb74c
OHT		 1222static bool is_solicited_fc_mad(struct ib_mad_send_wr_private *mad_send_wr)
1223{
1224 struct ib_rmpp_mad *rmpp_mad;
1225 u8 mgmt_class;
1226
1227 if (!mad_send_wr->timeout)
1228 return 0;
1229
1230 rmpp_mad = mad_send_wr->send_buf.mad;
1231 if (mad_send_wr->mad_agent_priv->agent.rmpp_version &&
1232 (ib_get_rmpp_flags(&rmpp_mad->rmpp_hdr) & IB_MGMT_RMPP_FLAG_ACTIVE))
1233 return 0;
1234
1235 mgmt_class =
1236 ((struct ib_mad_hdr *)mad_send_wr->send_buf.mad)->mgmt_class;
1237 return mgmt_class == IB_MGMT_CLASS_CM ||
1238 mgmt_class == IB_MGMT_CLASS_SUBN_ADM ||
1239 mgmt_class == IB_MGMT_CLASS_SUBN_LID_ROUTED ||
1240 mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE;
1241}
1242
1243static bool mad_is_for_backlog(struct ib_mad_send_wr_private *mad_send_wr)
1244{
1245 struct ib_mad_agent_private *mad_agent_priv =
1246 mad_send_wr->mad_agent_priv;
1247
1248 if (!mad_send_wr->is_solicited_fc || !mad_agent_priv->sol_fc_max)
1249 return false;
1250
1251 if (!list_empty(&mad_agent_priv->backlog_list))
1252 return true;
1253
1254 return mad_agent_priv->sol_fc_send_count +
1255 mad_agent_priv->sol_fc_wait_count >=
1256 mad_agent_priv->sol_fc_max;
1257}
1258
1da177e4
LT		 1259/*
1260 * ib_post_send_mad - Posts MAD(s) to the send queue of the QP associated
1261 * with the registered client
1262 */
34816ad9
SH		 1263int ib_post_send_mad(struct ib_mad_send_buf *send_buf,
1264 struct ib_mad_send_buf **bad_send_buf)
1da177e4 1265{
1da177e4 1266 struct ib_mad_agent_private *mad_agent_priv;
34816ad9
SH		 1267 struct ib_mad_send_buf *next_send_buf;
1268 struct ib_mad_send_wr_private *mad_send_wr;
1269 unsigned long flags;
1270 int ret = -EINVAL;
1da177e4
LT		 1271
1272 /* Walk list of send WRs and post each on send list */
34816ad9 1273 for (; send_buf; send_buf = next_send_buf) {
34816ad9
SH		 1274 mad_send_wr = container_of(send_buf,
1275 struct ib_mad_send_wr_private,
1276 send_buf);
1277 mad_agent_priv = mad_send_wr->mad_agent_priv;
1da177e4 1278
47a2b338
DJ		 1279 ret = ib_mad_enforce_security(mad_agent_priv,
1280 mad_send_wr->send_wr.pkey_index);
1281 if (ret)
1282 goto error;
1283
8ab05a54 1284 if (!send_buf->mad_agent->send_handler) {
34816ad9
SH		 1285 ret = -EINVAL;
1286 goto error;
1da177e4
LT		 1287 }
1288
618a3c03
HR		 1289 if (!ib_is_mad_class_rmpp(((struct ib_mad_hdr *) send_buf->mad)->mgmt_class)) {
1290 if (mad_agent_priv->agent.rmpp_version) {
1291 ret = -EINVAL;
1292 goto error;
1293 }
1294 }
1295
1da177e4
LT		 1296 /*
1297 * Save pointer to next work request to post in case the
1298 * current one completes, and the user modifies the work
1299 * request associated with the completion
1300 */
34816ad9 1301 next_send_buf = send_buf->next;
e622f2f4 1302 mad_send_wr->send_wr.ah = send_buf->ah;
1da177e4 1303
34816ad9
SH		 1304 if (((struct ib_mad_hdr *) send_buf->mad)->mgmt_class ==
1305 IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) {
1306 ret = handle_outgoing_dr_smp(mad_agent_priv,
1307 mad_send_wr);
1da177e4 1308 if (ret < 0) /* error */
34816ad9 1309 goto error;
1da177e4 1310 else if (ret == 1) /* locally consumed */
34816ad9 1311 continue;
1da177e4
LT		 1312 }
1313
34816ad9 1314 mad_send_wr->tid = ((struct ib_mad_hdr *) send_buf->mad)->tid;
1da177e4 1315 /* Timeout will be updated after send completes */
34816ad9 1316 mad_send_wr->timeout = msecs_to_jiffies(send_buf->timeout_ms);
4fc8cd49
SH		 1317 mad_send_wr->max_retries = send_buf->retries;
1318 mad_send_wr->retries_left = send_buf->retries;
1319 send_buf->retries = 0;
1cf0d899 1320 change_mad_state(mad_send_wr, IB_MAD_STATE_INIT);
1da177e4
LT		 1321
1322 /* Reference MAD agent until send completes */
e41c4253 1323 refcount_inc(&mad_agent_priv->refcount);
1da177e4 1324 spin_lock_irqsave(&mad_agent_priv->lock, flags);
314cb74c
OHT		 1325 mad_send_wr->is_solicited_fc = is_solicited_fc_mad(mad_send_wr);
1326 if (mad_is_for_backlog(mad_send_wr)) {
1327 change_mad_state(mad_send_wr, IB_MAD_STATE_QUEUED);
1328 spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
1329 return 0;
1330 }
1331
1cf0d899 1332 change_mad_state(mad_send_wr, IB_MAD_STATE_SEND_START);
1da177e4
LT		 1333 spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
1334
1471cb6c 1335 if (ib_mad_kernel_rmpp_agent(&mad_agent_priv->agent)) {
fa619a77
HR		 1336 ret = ib_send_rmpp_mad(mad_send_wr);
1337 if (ret >= 0 && ret != IB_RMPP_RESULT_CONSUMED)
1338 ret = ib_send_mad(mad_send_wr);
1339 } else
1340 ret = ib_send_mad(mad_send_wr);
1341 if (ret < 0) {
1da177e4
LT		 1342 /* Fail send request */
1343 spin_lock_irqsave(&mad_agent_priv->lock, flags);
1cf0d899 1344 change_mad_state(mad_send_wr, IB_MAD_STATE_DONE);
1da177e4 1345 spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
b9af0e2d 1346 deref_mad_agent(mad_agent_priv);
34816ad9 1347 goto error;
1da177e4 1348 }
1da177e4
LT		 1349 }
1350 return 0;
34816ad9
SH		 1351error:
1352 if (bad_send_buf)
1353 *bad_send_buf = send_buf;
1da177e4
LT		 1354 return ret;
1355}
1356EXPORT_SYMBOL(ib_post_send_mad);
1357
1358/*
1359 * ib_free_recv_mad - Returns data buffers used to receive
1360 * a MAD to the access layer
1361 */
1362void ib_free_recv_mad(struct ib_mad_recv_wc *mad_recv_wc)
1363{
fa619a77 1364 struct ib_mad_recv_buf *mad_recv_buf, *temp_recv_buf;
1da177e4
LT		 1365 struct ib_mad_private_header *mad_priv_hdr;
1366 struct ib_mad_private *priv;
fa619a77 1367 struct list_head free_list;
1da177e4 1368
fa619a77
HR		 1369 INIT_LIST_HEAD(&free_list);
1370 list_splice_init(&mad_recv_wc->rmpp_list, &free_list);
1da177e4 1371
fa619a77
HR		 1372 list_for_each_entry_safe(mad_recv_buf, temp_recv_buf,
1373 &free_list, list) {
1374 mad_recv_wc = container_of(mad_recv_buf, struct ib_mad_recv_wc,
1375 recv_buf);
1da177e4
LT		 1376 mad_priv_hdr = container_of(mad_recv_wc,
1377 struct ib_mad_private_header,
1378 recv_wc);
1379 priv = container_of(mad_priv_hdr, struct ib_mad_private,
1380 header);
c9082e51 1381 kfree(priv);
1da177e4 1382 }
1da177e4
LT		 1383}
1384EXPORT_SYMBOL(ib_free_recv_mad);
1385
1da177e4
LT		 1386static int method_in_use(struct ib_mad_mgmt_method_table **method,
1387 struct ib_mad_reg_req *mad_reg_req)
1388{
1389 int i;
1390
19b629f5 1391 for_each_set_bit(i, mad_reg_req->method_mask, IB_MGMT_MAX_METHODS) {
1da177e4 1392 if ((*method)->agent[i]) {
7ef5d4b0 1393 pr_err("Method %d already in use\n", i);
1da177e4
LT		 1394 return -EINVAL;
1395 }
1396 }
1397 return 0;
1398}
1399
1400static int allocate_method_table(struct ib_mad_mgmt_method_table **method)
1401{
1402 /* Allocate management method table */
de6eb66b 1403 *method = kzalloc(sizeof **method, GFP_ATOMIC);
27162432 1404 return (*method) ? 0 : (-ENOMEM);
1da177e4
LT		 1405}
1406
1407/*
1408 * Check to see if there are any methods still in use
1409 */
1410static int check_method_table(struct ib_mad_mgmt_method_table *method)
1411{
1412 int i;
1413
1414 for (i = 0; i < IB_MGMT_MAX_METHODS; i++)
1415 if (method->agent[i])
1416 return 1;
1417 return 0;
1418}
1419
1420/*
1421 * Check to see if there are any method tables for this class still in use
1422 */
1423static int check_class_table(struct ib_mad_mgmt_class_table *class)
1424{
1425 int i;
1426
1427 for (i = 0; i < MAX_MGMT_CLASS; i++)
1428 if (class->method_table[i])
1429 return 1;
1430 return 0;
1431}
1432
1433static int check_vendor_class(struct ib_mad_mgmt_vendor_class *vendor_class)
1434{
1435 int i;
1436
1437 for (i = 0; i < MAX_MGMT_OUI; i++)
1438 if (vendor_class->method_table[i])
1439 return 1;
1440 return 0;
1441}
1442
1443static int find_vendor_oui(struct ib_mad_mgmt_vendor_class *vendor_class,
d94bd266 1444 const char *oui)
1da177e4
LT		 1445{
1446 int i;
1447
1448 for (i = 0; i < MAX_MGMT_OUI; i++)
3cd96564
RD		 1449 /* Is there matching OUI for this vendor class ? */
1450 if (!memcmp(vendor_class->oui[i], oui, 3))
1da177e4
LT		 1451 return i;
1452
1453 return -1;
1454}
1455
1456static int check_vendor_table(struct ib_mad_mgmt_vendor_class_table *vendor)
1457{
1458 int i;
1459
1460 for (i = 0; i < MAX_MGMT_VENDOR_RANGE2; i++)
1461 if (vendor->vendor_class[i])
1462 return 1;
1463
1464 return 0;
1465}
1466
1467static void remove_methods_mad_agent(struct ib_mad_mgmt_method_table *method,
1468 struct ib_mad_agent_private *agent)
1469{
1470 int i;
1471
1472 /* Remove any methods for this mad agent */
b6eb7011
WL		 1473 for (i = 0; i < IB_MGMT_MAX_METHODS; i++)
1474 if (method->agent[i] == agent)
1da177e4 1475 method->agent[i] = NULL;
1da177e4
LT		 1476}
1477
1478static int add_nonoui_reg_req(struct ib_mad_reg_req *mad_reg_req,
1479 struct ib_mad_agent_private *agent_priv,
1480 u8 mgmt_class)
1481{
1482 struct ib_mad_port_private *port_priv;
1483 struct ib_mad_mgmt_class_table **class;
1484 struct ib_mad_mgmt_method_table **method;
1485 int i, ret;
1486
1487 port_priv = agent_priv->qp_info->port_priv;
1488 class = &port_priv->version[mad_reg_req->mgmt_class_version].class;
1489 if (!*class) {
1490 /* Allocate management class table for "new" class version */
de6eb66b 1491 *class = kzalloc(sizeof **class, GFP_ATOMIC);
1da177e4 1492 if (!*class) {
1da177e4
LT		 1493 ret = -ENOMEM;
1494 goto error1;
1495 }
de6eb66b 1496
1da177e4
LT		 1497 /* Allocate method table for this management class */
1498 method = &(*class)->method_table[mgmt_class];
1499 if ((ret = allocate_method_table(method)))
1500 goto error2;
1501 } else {
1502 method = &(*class)->method_table[mgmt_class];
1503 if (!*method) {
1504 /* Allocate method table for this management class */
1505 if ((ret = allocate_method_table(method)))
1506 goto error1;
1507 }
1508 }
1509
1510 /* Now, make sure methods are not already in use */
1511 if (method_in_use(method, mad_reg_req))
1512 goto error3;
1513
1514 /* Finally, add in methods being registered */
19b629f5 1515 for_each_set_bit(i, mad_reg_req->method_mask, IB_MGMT_MAX_METHODS)
1da177e4 1516 (*method)->agent[i] = agent_priv;
19b629f5 1517
1da177e4
LT		 1518 return 0;
1519
1520error3:
1521 /* Remove any methods for this mad agent */
1522 remove_methods_mad_agent(*method, agent_priv);
1523 /* Now, check to see if there are any methods in use */
1524 if (!check_method_table(*method)) {
1525 /* If not, release management method table */
1526 kfree(*method);
1527 *method = NULL;
1528 }
1529 ret = -EINVAL;
1530 goto error1;
1531error2:
1532 kfree(*class);
1533 *class = NULL;
1534error1:
1535 return ret;
1536}
1537
1538static int add_oui_reg_req(struct ib_mad_reg_req *mad_reg_req,
1539 struct ib_mad_agent_private *agent_priv)
1540{
1541 struct ib_mad_port_private *port_priv;
1542 struct ib_mad_mgmt_vendor_class_table **vendor_table;
1543 struct ib_mad_mgmt_vendor_class_table *vendor = NULL;
1544 struct ib_mad_mgmt_vendor_class *vendor_class = NULL;
1545 struct ib_mad_mgmt_method_table **method;
1546 int i, ret = -ENOMEM;
1547 u8 vclass;
1548
1549 /* "New" vendor (with OUI) class */
1550 vclass = vendor_class_index(mad_reg_req->mgmt_class);
1551 port_priv = agent_priv->qp_info->port_priv;
1552 vendor_table = &port_priv->version[
1553 mad_reg_req->mgmt_class_version].vendor;
1554 if (!*vendor_table) {
1555 /* Allocate mgmt vendor class table for "new" class version */
de6eb66b 1556 vendor = kzalloc(sizeof *vendor, GFP_ATOMIC);
27162432 1557 if (!vendor)
1da177e4 1558 goto error1;
de6eb66b 1559
1da177e4
LT		 1560 *vendor_table = vendor;
1561 }
1562 if (!(*vendor_table)->vendor_class[vclass]) {
1563 /* Allocate table for this management vendor class */
de6eb66b 1564 vendor_class = kzalloc(sizeof *vendor_class, GFP_ATOMIC);
27162432 1565 if (!vendor_class)
1da177e4 1566 goto error2;
de6eb66b 1567
1da177e4
LT		 1568 (*vendor_table)->vendor_class[vclass] = vendor_class;
1569 }
1570 for (i = 0; i < MAX_MGMT_OUI; i++) {
1571 /* Is there matching OUI for this vendor class ? */
1572 if (!memcmp((*vendor_table)->vendor_class[vclass]->oui[i],
1573 mad_reg_req->oui, 3)) {
1574 method = &(*vendor_table)->vendor_class[
1575 vclass]->method_table[i];
2468b82d
LR		 1576 if (!*method)
1577 goto error3;
1da177e4
LT		 1578 goto check_in_use;
1579 }
1580 }
1581 for (i = 0; i < MAX_MGMT_OUI; i++) {
1582 /* OUI slot available ? */
1583 if (!is_vendor_oui((*vendor_table)->vendor_class[
1584 vclass]->oui[i])) {
1585 method = &(*vendor_table)->vendor_class[
1586 vclass]->method_table[i];
1da177e4 1587 /* Allocate method table for this OUI */
2468b82d
LR		 1588 if (!*method) {
1589 ret = allocate_method_table(method);
1590 if (ret)
1591 goto error3;
1592 }
1da177e4
LT		 1593 memcpy((*vendor_table)->vendor_class[vclass]->oui[i],
1594 mad_reg_req->oui, 3);
1595 goto check_in_use;
1596 }
1597 }
7ef5d4b0 1598 dev_err(&agent_priv->agent.device->dev, "All OUI slots in use\n");
1da177e4
LT		 1599 goto error3;
1600
1601check_in_use:
1602 /* Now, make sure methods are not already in use */
1603 if (method_in_use(method, mad_reg_req))
1604 goto error4;
1605
1606 /* Finally, add in methods being registered */
19b629f5 1607 for_each_set_bit(i, mad_reg_req->method_mask, IB_MGMT_MAX_METHODS)
1da177e4 1608 (*method)->agent[i] = agent_priv;
19b629f5 1609
1da177e4
LT		 1610 return 0;
1611
1612error4:
1613 /* Remove any methods for this mad agent */
1614 remove_methods_mad_agent(*method, agent_priv);
1615 /* Now, check to see if there are any methods in use */
1616 if (!check_method_table(*method)) {
1617 /* If not, release management method table */
1618 kfree(*method);
1619 *method = NULL;
1620 }
1621 ret = -EINVAL;
1622error3:
1623 if (vendor_class) {
1624 (*vendor_table)->vendor_class[vclass] = NULL;
1625 kfree(vendor_class);
1626 }
1627error2:
1628 if (vendor) {
1629 *vendor_table = NULL;
1630 kfree(vendor);
1631 }
1632error1:
1633 return ret;
1634}
1635
1636static void remove_mad_reg_req(struct ib_mad_agent_private *agent_priv)
1637{
1638 struct ib_mad_port_private *port_priv;
1639 struct ib_mad_mgmt_class_table *class;
1640 struct ib_mad_mgmt_method_table *method;
1641 struct ib_mad_mgmt_vendor_class_table *vendor;
1642 struct ib_mad_mgmt_vendor_class *vendor_class;
1643 int index;
1644 u8 mgmt_class;
1645
1646 /*
1647 * Was MAD registration request supplied
1648 * with original registration ?
1649 */
b6eb7011 1650 if (!agent_priv->reg_req)
1da177e4 1651 goto out;
1da177e4
LT		 1652
1653 port_priv = agent_priv->qp_info->port_priv;
1654 mgmt_class = convert_mgmt_class(agent_priv->reg_req->mgmt_class);
1655 class = port_priv->version[
1656 agent_priv->reg_req->mgmt_class_version].class;
1657 if (!class)
1658 goto vendor_check;
1659
1660 method = class->method_table[mgmt_class];
1661 if (method) {
1662 /* Remove any methods for this mad agent */
1663 remove_methods_mad_agent(method, agent_priv);
1664 /* Now, check to see if there are any methods still in use */
1665 if (!check_method_table(method)) {
1666 /* If not, release management method table */
2190d10d
BVA		 1667 kfree(method);
1668 class->method_table[mgmt_class] = NULL;
1669 /* Any management classes left ? */
1da177e4
LT		 1670 if (!check_class_table(class)) {
1671 /* If not, release management class table */
1672 kfree(class);
1673 port_priv->version[
1674 agent_priv->reg_req->
1675 mgmt_class_version].class = NULL;
1676 }
1677 }
1678 }
1679
1680vendor_check:
1681 if (!is_vendor_class(mgmt_class))
1682 goto out;
1683
1684 /* normalize mgmt_class to vendor range 2 */
1685 mgmt_class = vendor_class_index(agent_priv->reg_req->mgmt_class);
1686 vendor = port_priv->version[
1687 agent_priv->reg_req->mgmt_class_version].vendor;
1688
1689 if (!vendor)
1690 goto out;
1691
1692 vendor_class = vendor->vendor_class[mgmt_class];
1693 if (vendor_class) {
1694 index = find_vendor_oui(vendor_class, agent_priv->reg_req->oui);
1695 if (index < 0)
1696 goto out;
1697 method = vendor_class->method_table[index];
1698 if (method) {
1699 /* Remove any methods for this mad agent */
1700 remove_methods_mad_agent(method, agent_priv);
1701 /*
1702 * Now, check to see if there are
1703 * any methods still in use
1704 */
1705 if (!check_method_table(method)) {
1706 /* If not, release management method table */
1707 kfree(method);
1708 vendor_class->method_table[index] = NULL;
1709 memset(vendor_class->oui[index], 0, 3);
1710 /* Any OUIs left ? */
1711 if (!check_vendor_class(vendor_class)) {
1712 /* If not, release vendor class table */
1713 kfree(vendor_class);
1714 vendor->vendor_class[mgmt_class] = NULL;
1715 /* Any other vendor classes left ? */
1716 if (!check_vendor_table(vendor)) {
1717 kfree(vendor);
1718 port_priv->version[
1719 agent_priv->reg_req->
1720 mgmt_class_version].
1721 vendor = NULL;
1722 }
1723 }
1724 }
1725 }
1726 }
1727
1728out:
1729 return;
1730}
1731
1da177e4
LT		 1732static struct ib_mad_agent_private *
1733find_mad_agent(struct ib_mad_port_private *port_priv,
d94bd266 1734 const struct ib_mad_hdr *mad_hdr)
1da177e4
LT		 1735{
1736 struct ib_mad_agent_private *mad_agent = NULL;
1737 unsigned long flags;
1738
d94bd266 1739 if (ib_response_mad(mad_hdr)) {
1da177e4 1740 u32 hi_tid;
1da177e4
LT		 1741
1742 /*
1743 * Routing is based on high 32 bits of transaction ID
1744 * of MAD.
1745 */
d94bd266 1746 hi_tid = be64_to_cpu(mad_hdr->tid) >> 32;
9a41e38a 1747 rcu_read_lock();
949a2370 1748 mad_agent = xa_load(&ib_mad_clients, hi_tid);
e41c4253 1749 if (mad_agent && !refcount_inc_not_zero(&mad_agent->refcount))
9a41e38a 1750 mad_agent = NULL;
1751 rcu_read_unlock();
1da177e4
LT		 1752 } else {
1753 struct ib_mad_mgmt_class_table *class;
1754 struct ib_mad_mgmt_method_table *method;
1755 struct ib_mad_mgmt_vendor_class_table *vendor;
1756 struct ib_mad_mgmt_vendor_class *vendor_class;
d94bd266 1757 const struct ib_vendor_mad *vendor_mad;
1da177e4
LT		 1758 int index;
1759
9a41e38a 1760 spin_lock_irqsave(&port_priv->reg_lock, flags);
1da177e4
LT		 1761 /*
1762 * Routing is based on version, class, and method
1763 * For "newer" vendor MADs, also based on OUI
1764 */
d94bd266 1765 if (mad_hdr->class_version >= MAX_MGMT_VERSION)
1da177e4 1766 goto out;
d94bd266 1767 if (!is_vendor_class(mad_hdr->mgmt_class)) {
1da177e4 1768 class = port_priv->version[
d94bd266 1769 mad_hdr->class_version].class;
1da177e4
LT		 1770 if (!class)
1771 goto out;
d94bd266 1772 if (convert_mgmt_class(mad_hdr->mgmt_class) >=
2fe2f378 1773 ARRAY_SIZE(class->method_table))
b7ab0b19 1774 goto out;
1da177e4 1775 method = class->method_table[convert_mgmt_class(
d94bd266 1776 mad_hdr->mgmt_class)];
1da177e4 1777 if (method)
d94bd266 1778 mad_agent = method->agent[mad_hdr->method &
1da177e4
LT		 1779 ~IB_MGMT_METHOD_RESP];
1780 } else {
1781 vendor = port_priv->version[
d94bd266 1782 mad_hdr->class_version].vendor;
1da177e4
LT		 1783 if (!vendor)
1784 goto out;
1785 vendor_class = vendor->vendor_class[vendor_class_index(
d94bd266 1786 mad_hdr->mgmt_class)];
1da177e4
LT		 1787 if (!vendor_class)
1788 goto out;
1789 /* Find matching OUI */
d94bd266 1790 vendor_mad = (const struct ib_vendor_mad *)mad_hdr;
1da177e4
LT		 1791 index = find_vendor_oui(vendor_class, vendor_mad->oui);
1792 if (index == -1)
1793 goto out;
1794 method = vendor_class->method_table[index];
1795 if (method) {
d94bd266 1796 mad_agent = method->agent[mad_hdr->method &
1da177e4
LT		 1797 ~IB_MGMT_METHOD_RESP];
1798 }
1799 }
9a41e38a 1800 if (mad_agent)
e41c4253 1801 refcount_inc(&mad_agent->refcount);
9a41e38a 1802out:
1803 spin_unlock_irqrestore(&port_priv->reg_lock, flags);
1da177e4
LT		 1804 }
1805
9a41e38a 1806 if (mad_agent && !mad_agent->agent.recv_handler) {
1807 dev_notice(&port_priv->device->dev,
1fb7f897 1808 "No receive handler for client %p on port %u\n",
9a41e38a 1809 &mad_agent->agent, port_priv->port_num);
1810 deref_mad_agent(mad_agent);
1811 mad_agent = NULL;
1da177e4 1812 }
1da177e4
LT		 1813
1814 return mad_agent;
1815}
1816
8e4349d1
IW		 1817static int validate_mad(const struct ib_mad_hdr *mad_hdr,
1818 const struct ib_mad_qp_info *qp_info,
1819 bool opa)
1da177e4
LT		 1820{
1821 int valid = 0;
8e4349d1 1822 u32 qp_num = qp_info->qp->qp_num;
1da177e4
LT		 1823
1824 /* Make sure MAD base version is understood */
8e4349d1
IW		 1825 if (mad_hdr->base_version != IB_MGMT_BASE_VERSION &&
1826 (!opa || mad_hdr->base_version != OPA_MGMT_BASE_VERSION)) {
3cea7b4a 1827 pr_err("MAD received with unsupported base version %u %s\n",
8e4349d1 1828 mad_hdr->base_version, opa ? "(opa)" : "");
1da177e4
LT		 1829 goto out;
1830 }
1831
1832 /* Filter SMI packets sent to other than QP0 */
77f60833
IW		 1833 if ((mad_hdr->mgmt_class == IB_MGMT_CLASS_SUBN_LID_ROUTED) ||
1834 (mad_hdr->mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE)) {
1da177e4
LT		 1835 if (qp_num == 0)
1836 valid = 1;
1837 } else {
53370886
HR		 1838 /* CM attributes other than ClassPortInfo only use Send method */
1839 if ((mad_hdr->mgmt_class == IB_MGMT_CLASS_CM) &&
1840 (mad_hdr->attr_id != IB_MGMT_CLASSPORTINFO_ATTR_ID) &&
1841 (mad_hdr->method != IB_MGMT_METHOD_SEND))
1842 goto out;
1da177e4
LT		 1843 /* Filter GSI packets sent to QP0 */
1844 if (qp_num != 0)
1845 valid = 1;
1846 }
1847
1848out:
1849 return valid;
1850}
1851
f766c58f
IW		 1852static int is_rmpp_data_mad(const struct ib_mad_agent_private *mad_agent_priv,
1853 const struct ib_mad_hdr *mad_hdr)
fa619a77
HR		 1854{
1855 struct ib_rmpp_mad *rmpp_mad;
1856
1857 rmpp_mad = (struct ib_rmpp_mad *)mad_hdr;
1858 return !mad_agent_priv->agent.rmpp_version ||
1471cb6c 1859 !ib_mad_kernel_rmpp_agent(&mad_agent_priv->agent) ||
fa619a77
HR		 1860 !(ib_get_rmpp_flags(&rmpp_mad->rmpp_hdr) &
1861 IB_MGMT_RMPP_FLAG_ACTIVE) ||
1862 (rmpp_mad->rmpp_hdr.rmpp_type == IB_MGMT_RMPP_TYPE_DATA);
1863}
1864
8bf4b30c
IW		 1865static inline int rcv_has_same_class(const struct ib_mad_send_wr_private *wr,
1866 const struct ib_mad_recv_wc *rwc)
fa9656bb 1867{
8bf4b30c 1868 return ((struct ib_mad_hdr *)(wr->send_buf.mad))->mgmt_class ==
fa9656bb
JM		 1869 rwc->recv_buf.mad->mad_hdr.mgmt_class;
1870}
1871
f681967a
WL		 1872static inline int
1873rcv_has_same_gid(const struct ib_mad_agent_private *mad_agent_priv,
1874 const struct ib_mad_send_wr_private *wr,
1875 const struct ib_mad_recv_wc *rwc)
fa9656bb 1876{
90898850 1877 struct rdma_ah_attr attr;
fa9656bb 1878 u8 send_resp, rcv_resp;
9874e746
JM		 1879 union ib_gid sgid;
1880 struct ib_device *device = mad_agent_priv->agent.device;
1fb7f897 1881 u32 port_num = mad_agent_priv->agent.port_num;
9874e746 1882 u8 lmc;
d8966fcd 1883 bool has_grh;
fa9656bb 1884
96909308
IW		 1885 send_resp = ib_response_mad((struct ib_mad_hdr *)wr->send_buf.mad);
1886 rcv_resp = ib_response_mad(&rwc->recv_buf.mad->mad_hdr);
fa9656bb 1887
fa9656bb
JM		 1888 if (send_resp == rcv_resp)
1889 /* both requests, or both responses. GIDs different */
1890 return 0;
1891
bfbfd661 1892 if (rdma_query_ah(wr->send_buf.ah, &attr))
fa9656bb
JM		 1893 /* Assume not equal, to avoid false positives. */
1894 return 0;
1895
d8966fcd
DC		 1896 has_grh = !!(rdma_ah_get_ah_flags(&attr) & IB_AH_GRH);
1897 if (has_grh != !!(rwc->wc->wc_flags & IB_WC_GRH))
fa9656bb
JM		 1898 /* one has GID, other does not. Assume different */
1899 return 0;
9874e746
JM		 1900
1901 if (!send_resp && rcv_resp) {
1902 /* is request/response. */
d8966fcd 1903 if (!has_grh) {
9874e746
JM		 1904 if (ib_get_cached_lmc(device, port_num, &lmc))
1905 return 0;
d8966fcd 1906 return (!lmc || !((rdma_ah_get_path_bits(&attr) ^
9874e746
JM		 1907 rwc->wc->dlid_path_bits) &
1908 ((1 << lmc) - 1)));
1909 } else {
d8966fcd
DC		 1910 const struct ib_global_route *grh =
1911 rdma_ah_read_grh(&attr);
1912
1dfce294
PP		 1913 if (rdma_query_gid(device, port_num,
1914 grh->sgid_index, &sgid))
9874e746
JM		 1915 return 0;
1916 return !memcmp(sgid.raw, rwc->recv_buf.grh->dgid.raw,
1917 16);
1918 }
1919 }
1920
d8966fcd
DC		 1921 if (!has_grh)
1922 return rdma_ah_get_dlid(&attr) == rwc->wc->slid;
9874e746 1923 else
d8966fcd
DC		 1924 return !memcmp(rdma_ah_read_grh(&attr)->dgid.raw,
1925 rwc->recv_buf.grh->sgid.raw,
9874e746
JM		 1926 16);
1927}
1928
1929static inline int is_direct(u8 class)
1930{
1931 return (class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE);
fa9656bb 1932}
9874e746 1933
fa619a77 1934struct ib_mad_send_wr_private*
f766c58f
IW		 1935ib_find_send_mad(const struct ib_mad_agent_private *mad_agent_priv,
1936 const struct ib_mad_recv_wc *wc)
1da177e4 1937{
9874e746 1938 struct ib_mad_send_wr_private *wr;
83a1d228 1939 const struct ib_mad_hdr *mad_hdr;
fa9656bb 1940
83a1d228 1941 mad_hdr = &wc->recv_buf.mad->mad_hdr;
9874e746
JM		 1942
1943 list_for_each_entry(wr, &mad_agent_priv->wait_list, agent_list) {
83a1d228 1944 if ((wr->tid == mad_hdr->tid) &&
9874e746
JM		 1945 rcv_has_same_class(wr, wc) &&
1946 /*
1947 * Don't check GID for direct routed MADs.
1948 * These might have permissive LIDs.
1949 */
83a1d228 1950 (is_direct(mad_hdr->mgmt_class) ||
9874e746 1951 rcv_has_same_gid(mad_agent_priv, wr, wc)))
1cf0d899 1952 return (wr->state != IB_MAD_STATE_CANCELED) ? wr : NULL;
1da177e4
LT		 1953 }
1954
314cb74c
OHT		 1955 list_for_each_entry(wr, &mad_agent_priv->backlog_list, agent_list) {
1956 if ((wr->tid == mad_hdr->tid) &&
1957 rcv_has_same_class(wr, wc) &&
1958 /*
1959 * Don't check GID for direct routed MADs.
1960 * These might have permissive LIDs.
1961 */
1962 (is_direct(mad_hdr->mgmt_class) ||
1963 rcv_has_same_gid(mad_agent_priv, wr, wc)))
1964 return (wr->state != IB_MAD_STATE_CANCELED) ? wr : NULL;
1965 }
1966
1da177e4
LT		 1967 /*
1968 * It's possible to receive the response before we've
1969 * been notified that the send has completed
1970 */
9874e746 1971 list_for_each_entry(wr, &mad_agent_priv->send_list, agent_list) {
c597eee5 1972 if (is_rmpp_data_mad(mad_agent_priv, wr->send_buf.mad) &&
83a1d228 1973 wr->tid == mad_hdr->tid &&
9874e746
JM		 1974 wr->timeout &&
1975 rcv_has_same_class(wr, wc) &&
1976 /*
1977 * Don't check GID for direct routed MADs.
1978 * These might have permissive LIDs.
1979 */
83a1d228 1980 (is_direct(mad_hdr->mgmt_class) ||
9874e746 1981 rcv_has_same_gid(mad_agent_priv, wr, wc)))
1da177e4 1982 /* Verify request has not been canceled */
1cf0d899 1983 return (wr->state != IB_MAD_STATE_CANCELED) ? wr : NULL;
1da177e4
LT		 1984 }
1985 return NULL;
1986}
1987
314cb74c
OHT		 1988static void
1989process_backlog_mads(struct ib_mad_agent_private *mad_agent_priv)
1990{
1991 struct ib_mad_send_wr_private *mad_send_wr;
1992 struct ib_mad_send_wc mad_send_wc = {};
1993 unsigned long flags;
1994 int ret;
1995
1996 spin_lock_irqsave(&mad_agent_priv->lock, flags);
1997 while (!list_empty(&mad_agent_priv->backlog_list) &&
1998 (mad_agent_priv->sol_fc_send_count +
1999 mad_agent_priv->sol_fc_wait_count <
2000 mad_agent_priv->sol_fc_max)) {
2001 mad_send_wr = list_entry(mad_agent_priv->backlog_list.next,
2002 struct ib_mad_send_wr_private,
2003 agent_list);
2004 change_mad_state(mad_send_wr, IB_MAD_STATE_SEND_START);
2005 spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
2006 ret = ib_send_mad(mad_send_wr);
2007 if (ret) {
2008 spin_lock_irqsave(&mad_agent_priv->lock, flags);
2009 deref_mad_agent(mad_agent_priv);
2010 change_mad_state(mad_send_wr, IB_MAD_STATE_DONE);
2011 spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
2012 mad_send_wc.send_buf = &mad_send_wr->send_buf;
2013 mad_send_wc.status = IB_WC_LOC_QP_OP_ERR;
2014 mad_agent_priv->agent.send_handler(
2015 &mad_agent_priv->agent, &mad_send_wc);
2016 }
2017
2018 spin_lock_irqsave(&mad_agent_priv->lock, flags);
2019 }
2020
2021 spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
2022}
2023
fa619a77 2024void ib_mark_mad_done(struct ib_mad_send_wr_private *mad_send_wr)
6a0c435e
HR		 2025{
2026 mad_send_wr->timeout = 0;
314cb74c
OHT		 2027 if (mad_send_wr->state == IB_MAD_STATE_WAIT_RESP ||
2028 mad_send_wr->state == IB_MAD_STATE_QUEUED)
1cf0d899
OHT		 2029 change_mad_state(mad_send_wr, IB_MAD_STATE_DONE);
2030 else
2031 change_mad_state(mad_send_wr, IB_MAD_STATE_EARLY_RESP);
6a0c435e
HR		 2032}
2033
1da177e4 2034static void ib_mad_complete_recv(struct ib_mad_agent_private *mad_agent_priv,
4a0754fa 2035 struct ib_mad_recv_wc *mad_recv_wc)
1da177e4
LT		 2036{
2037 struct ib_mad_send_wr_private *mad_send_wr;
2038 struct ib_mad_send_wc mad_send_wc;
2039 unsigned long flags;
1cf0d899 2040 bool is_mad_done;
47a2b338
DJ		 2041 int ret;
2042
89548bca 2043 INIT_LIST_HEAD(&mad_recv_wc->rmpp_list);
47a2b338
DJ		 2044 ret = ib_mad_enforce_security(mad_agent_priv,
2045 mad_recv_wc->wc->pkey_index);
2046 if (ret) {
2047 ib_free_recv_mad(mad_recv_wc);
2048 deref_mad_agent(mad_agent_priv);
89548bca 2049 return;
47a2b338 2050 }
1da177e4 2051
fa619a77 2052 list_add(&mad_recv_wc->recv_buf.list, &mad_recv_wc->rmpp_list);
1471cb6c 2053 if (ib_mad_kernel_rmpp_agent(&mad_agent_priv->agent)) {
fa619a77
HR		 2054 mad_recv_wc = ib_process_rmpp_recv_wc(mad_agent_priv,
2055 mad_recv_wc);
2056 if (!mad_recv_wc) {
1b52fa98 2057 deref_mad_agent(mad_agent_priv);
fa619a77
HR		 2058 return;
2059 }
2060 }
2061
1da177e4 2062 /* Complete corresponding request */
96909308 2063 if (ib_response_mad(&mad_recv_wc->recv_buf.mad->mad_hdr)) {
1da177e4 2064 spin_lock_irqsave(&mad_agent_priv->lock, flags);
fa9656bb 2065 mad_send_wr = ib_find_send_mad(mad_agent_priv, mad_recv_wc);
1da177e4
LT		 2066 if (!mad_send_wr) {
2067 spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
1471cb6c
IW		 2068 if (!ib_mad_kernel_rmpp_agent(&mad_agent_priv->agent)
2069 && ib_is_mad_class_rmpp(mad_recv_wc->recv_buf.mad->mad_hdr.mgmt_class)
2070 && (ib_get_rmpp_flags(&((struct ib_rmpp_mad *)mad_recv_wc->recv_buf.mad)->rmpp_hdr)
2071 & IB_MGMT_RMPP_FLAG_ACTIVE)) {
2072 /* user rmpp is in effect
2073 * and this is an active RMPP MAD
2074 */
ca281265
CH		 2075 mad_agent_priv->agent.recv_handler(
2076 &mad_agent_priv->agent, NULL,
2077 mad_recv_wc);
b9af0e2d 2078 deref_mad_agent(mad_agent_priv);
1471cb6c
IW		 2079 } else {
2080 /* not user rmpp, revert to normal behavior and
26caea5f
WL		 2081 * drop the mad
2082 */
1471cb6c
IW		 2083 ib_free_recv_mad(mad_recv_wc);
2084 deref_mad_agent(mad_agent_priv);
2085 return;
2086 }
2087 } else {
2088 ib_mark_mad_done(mad_send_wr);
1cf0d899 2089 is_mad_done = (mad_send_wr->state == IB_MAD_STATE_DONE);
1471cb6c 2090 spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
1da177e4 2091
1471cb6c 2092 /* Defined behavior is to complete response before request */
ca281265
CH		 2093 mad_agent_priv->agent.recv_handler(
2094 &mad_agent_priv->agent,
2095 &mad_send_wr->send_buf,
2096 mad_recv_wc);
b9af0e2d 2097 deref_mad_agent(mad_agent_priv);
1da177e4 2098
1cf0d899
OHT		 2099 if (is_mad_done) {
2100 mad_send_wc.status = IB_WC_SUCCESS;
2101 mad_send_wc.vendor_err = 0;
2102 mad_send_wc.send_buf = &mad_send_wr->send_buf;
2103 ib_mad_complete_send_wr(mad_send_wr,
2104 &mad_send_wc);
2105 }
1471cb6c 2106 }
1da177e4 2107 } else {
ca281265 2108 mad_agent_priv->agent.recv_handler(&mad_agent_priv->agent, NULL,
4a0754fa 2109 mad_recv_wc);
1b52fa98 2110 deref_mad_agent(mad_agent_priv);
1da177e4
LT		 2111 }
2112}
2113
e11ae8aa
IW		 2114static enum smi_action handle_ib_smi(const struct ib_mad_port_private *port_priv,
2115 const struct ib_mad_qp_info *qp_info,
2116 const struct ib_wc *wc,
1fb7f897 2117 u32 port_num,
e11ae8aa
IW		 2118 struct ib_mad_private *recv,
2119 struct ib_mad_private *response)
2120{
2121 enum smi_forward_action retsmi;
c9082e51 2122 struct ib_smp *smp = (struct ib_smp *)recv->mad;
e11ae8aa 2123
2ccfbb70
IW		 2124 trace_ib_mad_handle_ib_smi(smp);
2125
c9082e51 2126 if (smi_handle_dr_smp_recv(smp,
4139032b 2127 rdma_cap_ib_switch(port_priv->device),
e11ae8aa
IW		 2128 port_num,
2129 port_priv->device->phys_port_cnt) ==
2130 IB_SMI_DISCARD)
2131 return IB_SMI_DISCARD;
2132
c9082e51 2133 retsmi = smi_check_forward_dr_smp(smp);
e11ae8aa
IW		 2134 if (retsmi == IB_SMI_LOCAL)
2135 return IB_SMI_HANDLE;
2136
2137 if (retsmi == IB_SMI_SEND) { /* don't forward */
c9082e51 2138 if (smi_handle_dr_smp_send(smp,
4139032b 2139 rdma_cap_ib_switch(port_priv->device),
e11ae8aa
IW		 2140 port_num) == IB_SMI_DISCARD)
2141 return IB_SMI_DISCARD;
2142
c9082e51 2143 if (smi_check_local_smp(smp, port_priv->device) == IB_SMI_DISCARD)
e11ae8aa 2144 return IB_SMI_DISCARD;
4139032b 2145 } else if (rdma_cap_ib_switch(port_priv->device)) {
e11ae8aa 2146 /* forward case for switches */
c9082e51 2147 memcpy(response, recv, mad_priv_size(response));
e11ae8aa 2148 response->header.recv_wc.wc = &response->header.wc;
c9082e51 2149 response->header.recv_wc.recv_buf.mad = (struct ib_mad *)response->mad;
e11ae8aa
IW		 2150 response->header.recv_wc.recv_buf.grh = &response->grh;
2151
c9082e51 2152 agent_send_response((const struct ib_mad_hdr *)response->mad,
e11ae8aa
IW		 2153 &response->grh, wc,
2154 port_priv->device,
c9082e51
IW		 2155 smi_get_fwd_port(smp),
2156 qp_info->qp->qp_num,
8e4349d1
IW		 2157 response->mad_size,
2158 false);
e11ae8aa
IW		 2159
2160 return IB_SMI_DISCARD;
2161 }
2162 return IB_SMI_HANDLE;
2163}
2164
c9082e51 2165static bool generate_unmatched_resp(const struct ib_mad_private *recv,
8e4349d1
IW		 2166 struct ib_mad_private *response,
2167 size_t *resp_len, bool opa)
0b307043 2168{
c9082e51
IW		 2169 const struct ib_mad_hdr *recv_hdr = (const struct ib_mad_hdr *)recv->mad;
2170 struct ib_mad_hdr *resp_hdr = (struct ib_mad_hdr *)response->mad;
2171
2172 if (recv_hdr->method == IB_MGMT_METHOD_GET ||
2173 recv_hdr->method == IB_MGMT_METHOD_SET) {
2174 memcpy(response, recv, mad_priv_size(response));
0b307043 2175 response->header.recv_wc.wc = &response->header.wc;
c9082e51 2176 response->header.recv_wc.recv_buf.mad = (struct ib_mad *)response->mad;
0b307043 2177 response->header.recv_wc.recv_buf.grh = &response->grh;
c9082e51
IW		 2178 resp_hdr->method = IB_MGMT_METHOD_GET_RESP;
2179 resp_hdr->status = cpu_to_be16(IB_MGMT_MAD_STATUS_UNSUPPORTED_METHOD_ATTRIB);
2180 if (recv_hdr->mgmt_class == IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE)
2181 resp_hdr->status |= IB_SMP_DIRECTION;
0b307043 2182
8e4349d1
IW		 2183 if (opa && recv_hdr->base_version == OPA_MGMT_BASE_VERSION) {
2184 if (recv_hdr->mgmt_class ==
2185 IB_MGMT_CLASS_SUBN_LID_ROUTED ||
2186 recv_hdr->mgmt_class ==
2187 IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE)
2188 *resp_len = opa_get_smp_header_size(
2189 (struct opa_smp *)recv->mad);
2190 else
2191 *resp_len = sizeof(struct ib_mad_hdr);
2192 }
2193
0b307043
ST		 2194 return true;
2195 } else {
2196 return false;
2197 }
2198}
8e4349d1
IW		 2199
2200static enum smi_action
2201handle_opa_smi(struct ib_mad_port_private *port_priv,
2202 struct ib_mad_qp_info *qp_info,
2203 struct ib_wc *wc,
1fb7f897 2204 u32 port_num,
8e4349d1
IW		 2205 struct ib_mad_private *recv,
2206 struct ib_mad_private *response)
2207{
2208 enum smi_forward_action retsmi;
2209 struct opa_smp *smp = (struct opa_smp *)recv->mad;
2210
2ccfbb70
IW		 2211 trace_ib_mad_handle_opa_smi(smp);
2212
8e4349d1 2213 if (opa_smi_handle_dr_smp_recv(smp,
4139032b 2214 rdma_cap_ib_switch(port_priv->device),
8e4349d1
IW		 2215 port_num,
2216 port_priv->device->phys_port_cnt) ==
2217 IB_SMI_DISCARD)
2218 return IB_SMI_DISCARD;
2219
2220 retsmi = opa_smi_check_forward_dr_smp(smp);
2221 if (retsmi == IB_SMI_LOCAL)
2222 return IB_SMI_HANDLE;
2223
2224 if (retsmi == IB_SMI_SEND) { /* don't forward */
2225 if (opa_smi_handle_dr_smp_send(smp,
4139032b 2226 rdma_cap_ib_switch(port_priv->device),
8e4349d1
IW		 2227 port_num) == IB_SMI_DISCARD)
2228 return IB_SMI_DISCARD;
2229
2230 if (opa_smi_check_local_smp(smp, port_priv->device) ==
2231 IB_SMI_DISCARD)
2232 return IB_SMI_DISCARD;
2233
4139032b 2234 } else if (rdma_cap_ib_switch(port_priv->device)) {
8e4349d1
IW		 2235 /* forward case for switches */
2236 memcpy(response, recv, mad_priv_size(response));
2237 response->header.recv_wc.wc = &response->header.wc;
2238 response->header.recv_wc.recv_buf.opa_mad =
2239 (struct opa_mad *)response->mad;
2240 response->header.recv_wc.recv_buf.grh = &response->grh;
2241
2242 agent_send_response((const struct ib_mad_hdr *)response->mad,
2243 &response->grh, wc,
2244 port_priv->device,
2245 opa_smi_get_fwd_port(smp),
2246 qp_info->qp->qp_num,
2247 recv->header.wc.byte_len,
2248 true);
2249
2250 return IB_SMI_DISCARD;
2251 }
2252
2253 return IB_SMI_HANDLE;
2254}
2255
2256static enum smi_action
2257handle_smi(struct ib_mad_port_private *port_priv,
2258 struct ib_mad_qp_info *qp_info,
2259 struct ib_wc *wc,
1fb7f897 2260 u32 port_num,
8e4349d1
IW		 2261 struct ib_mad_private *recv,
2262 struct ib_mad_private *response,
2263 bool opa)
2264{
2265 struct ib_mad_hdr *mad_hdr = (struct ib_mad_hdr *)recv->mad;
2266
2267 if (opa && mad_hdr->base_version == OPA_MGMT_BASE_VERSION &&
9fa240bb 2268 mad_hdr->class_version == OPA_SM_CLASS_VERSION)
8e4349d1
IW		 2269 return handle_opa_smi(port_priv, qp_info, wc, port_num, recv,
2270 response);
2271
2272 return handle_ib_smi(port_priv, qp_info, wc, port_num, recv, response);
2273}
2274
d53e11fd 2275static void ib_mad_recv_done(struct ib_cq *cq, struct ib_wc *wc)
1da177e4 2276{
d53e11fd
CH		 2277 struct ib_mad_port_private *port_priv = cq->cq_context;
2278 struct ib_mad_list_head *mad_list =
2279 container_of(wc->wr_cqe, struct ib_mad_list_head, cqe);
1da177e4
LT		 2280 struct ib_mad_qp_info *qp_info;
2281 struct ib_mad_private_header *mad_priv_hdr;
445d6807 2282 struct ib_mad_private *recv, *response = NULL;
1da177e4 2283 struct ib_mad_agent_private *mad_agent;
1fb7f897 2284 u32 port_num;
a9e74323 2285 int ret = IB_MAD_RESULT_SUCCESS;
4cd7c947
IW		 2286 size_t mad_size;
2287 u16 resp_mad_pkey_index = 0;
8e4349d1 2288 bool opa;
1da177e4 2289
d53e11fd
CH		 2290 if (list_empty_careful(&port_priv->port_list))
2291 return;
2292
2293 if (wc->status != IB_WC_SUCCESS) {
2294 /*
2295 * Receive errors indicate that the QP has entered the error
2296 * state - error handling/shutdown code will cleanup
2297 */
2298 return;
2299 }
2300
1da177e4
LT		 2301 qp_info = mad_list->mad_queue->qp_info;
2302 dequeue_mad(mad_list);
2303
8e4349d1
IW		 2304 opa = rdma_cap_opa_mad(qp_info->port_priv->device,
2305 qp_info->port_priv->port_num);
2306
1da177e4
LT		 2307 mad_priv_hdr = container_of(mad_list, struct ib_mad_private_header,
2308 mad_list);
2309 recv = container_of(mad_priv_hdr, struct ib_mad_private, header);
1527106f
RC		 2310 ib_dma_unmap_single(port_priv->device,
2311 recv->header.mapping,
c9082e51 2312 mad_priv_dma_size(recv),
1527106f 2313 DMA_FROM_DEVICE);
1da177e4
LT		 2314
2315 /* Setup MAD receive work completion from "normal" work completion */
24239aff
SH		 2316 recv->header.wc = *wc;
2317 recv->header.recv_wc.wc = &recv->header.wc;
8e4349d1
IW		 2318
2319 if (opa && ((struct ib_mad_hdr *)(recv->mad))->base_version == OPA_MGMT_BASE_VERSION) {
2320 recv->header.recv_wc.mad_len = wc->byte_len - sizeof(struct ib_grh);
2321 recv->header.recv_wc.mad_seg_size = sizeof(struct opa_mad);
2322 } else {
2323 recv->header.recv_wc.mad_len = sizeof(struct ib_mad);
2324 recv->header.recv_wc.mad_seg_size = sizeof(struct ib_mad);
2325 }
2326
c9082e51 2327 recv->header.recv_wc.recv_buf.mad = (struct ib_mad *)recv->mad;
1da177e4
LT		 2328 recv->header.recv_wc.recv_buf.grh = &recv->grh;
2329
1da177e4 2330 /* Validate MAD */
8e4349d1 2331 if (!validate_mad((const struct ib_mad_hdr *)recv->mad, qp_info, opa))
1da177e4
LT		 2332 goto out;
2333
821bf1de
IW		 2334 trace_ib_mad_recv_done_handler(qp_info, wc,
2335 (struct ib_mad_hdr *)recv->mad);
2336
4cd7c947
IW		 2337 mad_size = recv->mad_size;
2338 response = alloc_mad_private(mad_size, GFP_KERNEL);
27162432 2339 if (!response)
445d6807 2340 goto out;
445d6807 2341
4139032b 2342 if (rdma_cap_ib_switch(port_priv->device))
1bae4dbf
HR		 2343 port_num = wc->port_num;
2344 else
2345 port_num = port_priv->port_num;
2346
c9082e51 2347 if (((struct ib_mad_hdr *)recv->mad)->mgmt_class ==
1da177e4 2348 IB_MGMT_CLASS_SUBN_DIRECTED_ROUTE) {
8e4349d1
IW		 2349 if (handle_smi(port_priv, qp_info, wc, port_num, recv,
2350 response, opa)
e11ae8aa 2351 == IB_SMI_DISCARD)
1da177e4 2352 goto out;
1da177e4
LT		 2353 }
2354
1da177e4 2355 /* Give driver "right of first refusal" on incoming MAD */
3023a1e9
KH		 2356 if (port_priv->device->ops.process_mad) {
2357 ret = port_priv->device->ops.process_mad(
2358 port_priv->device, 0, port_priv->port_num, wc,
e26e7b88
LR		 2359 &recv->grh, (const struct ib_mad *)recv->mad,
2360 (struct ib_mad *)response->mad, &mad_size,
2361 &resp_mad_pkey_index);
8e4349d1
IW		 2362
2363 if (opa)
2364 wc->pkey_index = resp_mad_pkey_index;
2365
1da177e4
LT		 2366 if (ret & IB_MAD_RESULT_SUCCESS) {
2367 if (ret & IB_MAD_RESULT_CONSUMED)
2368 goto out;
2369 if (ret & IB_MAD_RESULT_REPLY) {
c9082e51 2370 agent_send_response((const struct ib_mad_hdr *)response->mad,
34816ad9
SH		 2371 &recv->grh, wc,
2372 port_priv->device,
1bae4dbf 2373 port_num,
c9082e51 2374 qp_info->qp->qp_num,
8e4349d1 2375 mad_size, opa);
1da177e4
LT		 2376 goto out;
2377 }
2378 }
2379 }
2380
c9082e51 2381 mad_agent = find_mad_agent(port_priv, (const struct ib_mad_hdr *)recv->mad);
1da177e4 2382 if (mad_agent) {
0e65bae2 2383 trace_ib_mad_recv_done_agent(mad_agent);
4a0754fa 2384 ib_mad_complete_recv(mad_agent, &recv->header.recv_wc);
1da177e4
LT		 2385 /*
2386 * recv is freed up in error cases in ib_mad_complete_recv
2387 * or via recv_handler in ib_mad_complete_recv()
2388 */
2389 recv = NULL;
a9e74323 2390 } else if ((ret & IB_MAD_RESULT_SUCCESS) &&
8e4349d1 2391 generate_unmatched_resp(recv, response, &mad_size, opa)) {
c9082e51
IW		 2392 agent_send_response((const struct ib_mad_hdr *)response->mad, &recv->grh, wc,
2393 port_priv->device, port_num,
8e4349d1 2394 qp_info->qp->qp_num, mad_size, opa);
1da177e4
LT		 2395 }
2396
2397out:
2398 /* Post another receive request for this QP */
2399 if (response) {
2400 ib_mad_post_receive_mads(qp_info, response);
c9082e51 2401 kfree(recv);
1da177e4
LT		 2402 } else
2403 ib_mad_post_receive_mads(qp_info, recv);
2404}
2405
2406static void adjust_timeout(struct ib_mad_agent_private *mad_agent_priv)
2407{
2408 struct ib_mad_send_wr_private *mad_send_wr;
2409 unsigned long delay;
2410
2411 if (list_empty(&mad_agent_priv->wait_list)) {
136b5721 2412 cancel_delayed_work(&mad_agent_priv->timed_work);
1da177e4
LT		 2413 } else {
2414 mad_send_wr = list_entry(mad_agent_priv->wait_list.next,
2415 struct ib_mad_send_wr_private,
2416 agent_list);
2417
2418 if (time_after(mad_agent_priv->timeout,
2419 mad_send_wr->timeout)) {
2420 mad_agent_priv->timeout = mad_send_wr->timeout;
1da177e4
LT		 2421 delay = mad_send_wr->timeout - jiffies;
2422 if ((long)delay <= 0)
2423 delay = 1;
e7c2f967
TH		 2424 mod_delayed_work(mad_agent_priv->qp_info->port_priv->wq,
2425 &mad_agent_priv->timed_work, delay);
1da177e4
LT		 2426 }
2427 }
2428}
2429
d760ce8f 2430static void wait_for_response(struct ib_mad_send_wr_private *mad_send_wr)
1da177e4 2431{
d760ce8f 2432 struct ib_mad_agent_private *mad_agent_priv;
1da177e4
LT		 2433 unsigned long delay;
2434
d760ce8f 2435 mad_agent_priv = mad_send_wr->mad_agent_priv;
1da177e4 2436 delay = mad_send_wr->timeout;
1cf0d899 2437 change_mad_state(mad_send_wr, IB_MAD_STATE_WAIT_RESP);
1da177e4
LT		 2438
2439 /* Reschedule a work item if we have a shorter timeout */
e7c2f967
TH		 2440 if (mad_agent_priv->wait_list.next == &mad_send_wr->agent_list)
2441 mod_delayed_work(mad_agent_priv->qp_info->port_priv->wq,
2442 &mad_agent_priv->timed_work, delay);
1da177e4
LT		 2443}
2444
03b61ad2 2445void ib_reset_mad_timeout(struct ib_mad_send_wr_private *mad_send_wr,
dbace111 2446 unsigned long timeout_ms)
03b61ad2
HR		 2447{
2448 mad_send_wr->timeout = msecs_to_jiffies(timeout_ms);
2449 wait_for_response(mad_send_wr);
2450}
2451
1da177e4
LT		 2452/*
2453 * Process a send work completion
2454 */
fa619a77
HR		 2455void ib_mad_complete_send_wr(struct ib_mad_send_wr_private *mad_send_wr,
2456 struct ib_mad_send_wc *mad_send_wc)
1da177e4
LT		 2457{
2458 struct ib_mad_agent_private *mad_agent_priv;
2459 unsigned long flags;
fa619a77 2460 int ret;
1da177e4 2461
d760ce8f 2462 mad_agent_priv = mad_send_wr->mad_agent_priv;
1da177e4 2463 spin_lock_irqsave(&mad_agent_priv->lock, flags);
1471cb6c 2464 if (ib_mad_kernel_rmpp_agent(&mad_agent_priv->agent)) {
fa619a77
HR		 2465 ret = ib_process_rmpp_send_wc(mad_send_wr, mad_send_wc);
2466 if (ret == IB_RMPP_RESULT_CONSUMED)
2467 goto done;
2468 } else
2469 ret = IB_RMPP_RESULT_UNHANDLED;
2470
1cf0d899
OHT		 2471 if (mad_send_wr->state == IB_MAD_STATE_CANCELED)
2472 mad_send_wc->status = IB_WC_WR_FLUSH_ERR;
2473 else if (mad_send_wr->state == IB_MAD_STATE_SEND_START &&
2474 mad_send_wr->timeout) {
2475 wait_for_response(mad_send_wr);
fa619a77 2476 goto done;
1da177e4
LT		 2477 }
2478
2479 /* Remove send from MAD agent and notify client of completion */
1cf0d899
OHT		 2480 if (mad_send_wr->state != IB_MAD_STATE_DONE)
2481 change_mad_state(mad_send_wr, IB_MAD_STATE_DONE);
1da177e4
LT		 2482 adjust_timeout(mad_agent_priv);
2483 spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
2484
314cb74c 2485 if (ret == IB_RMPP_RESULT_INTERNAL) {
34816ad9 2486 ib_rmpp_send_handler(mad_send_wc);
314cb74c
OHT		 2487 } else {
2488 if (mad_send_wr->is_solicited_fc)
2489 process_backlog_mads(mad_agent_priv);
fa619a77
HR		 2490 mad_agent_priv->agent.send_handler(&mad_agent_priv->agent,
2491 mad_send_wc);
314cb74c 2492 }
1da177e4
LT		 2493
2494 /* Release reference on agent taken when sending */
1b52fa98 2495 deref_mad_agent(mad_agent_priv);
fa619a77
HR		 2496 return;
2497done:
2498 spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
1da177e4
LT		 2499}
2500
d53e11fd 2501static void ib_mad_send_done(struct ib_cq *cq, struct ib_wc *wc)
1da177e4 2502{
d53e11fd
CH		 2503 struct ib_mad_port_private *port_priv = cq->cq_context;
2504 struct ib_mad_list_head *mad_list =
2505 container_of(wc->wr_cqe, struct ib_mad_list_head, cqe);
1da177e4 2506 struct ib_mad_send_wr_private *mad_send_wr, *queued_send_wr;
1da177e4
LT		 2507 struct ib_mad_qp_info *qp_info;
2508 struct ib_mad_queue *send_queue;
34816ad9 2509 struct ib_mad_send_wc mad_send_wc;
1da177e4
LT		 2510 unsigned long flags;
2511 int ret;
2512
d53e11fd
CH		 2513 if (list_empty_careful(&port_priv->port_list))
2514 return;
2515
2516 if (wc->status != IB_WC_SUCCESS) {
2517 if (!ib_mad_send_error(port_priv, wc))
2518 return;
2519 }
2520
1da177e4
LT		 2521 mad_send_wr = container_of(mad_list, struct ib_mad_send_wr_private,
2522 mad_list);
2523 send_queue = mad_list->mad_queue;
2524 qp_info = send_queue->qp_info;
2525
0e65bae2 2526 trace_ib_mad_send_done_agent(mad_send_wr->mad_agent_priv);
4d60cad5
IW		 2527 trace_ib_mad_send_done_handler(mad_send_wr, wc);
2528
1da177e4 2529retry:
1527106f
RC		 2530 ib_dma_unmap_single(mad_send_wr->send_buf.mad_agent->device,
2531 mad_send_wr->header_mapping,
2532 mad_send_wr->sg_list[0].length, DMA_TO_DEVICE);
2533 ib_dma_unmap_single(mad_send_wr->send_buf.mad_agent->device,
2534 mad_send_wr->payload_mapping,
2535 mad_send_wr->sg_list[1].length, DMA_TO_DEVICE);
1da177e4
LT		 2536 queued_send_wr = NULL;
2537 spin_lock_irqsave(&send_queue->lock, flags);
2538 list_del(&mad_list->list);
2539
2540 /* Move queued send to the send queue */
2541 if (send_queue->count-- > send_queue->max_active) {
2542 mad_list = container_of(qp_info->overflow_list.next,
2543 struct ib_mad_list_head, list);
2544 queued_send_wr = container_of(mad_list,
2545 struct ib_mad_send_wr_private,
2546 mad_list);
179e0917 2547 list_move_tail(&mad_list->list, &send_queue->list);
1da177e4
LT		 2548 }
2549 spin_unlock_irqrestore(&send_queue->lock, flags);
2550
34816ad9
SH		 2551 mad_send_wc.send_buf = &mad_send_wr->send_buf;
2552 mad_send_wc.status = wc->status;
2553 mad_send_wc.vendor_err = wc->vendor_err;
34816ad9 2554 ib_mad_complete_send_wr(mad_send_wr, &mad_send_wc);
1da177e4
LT		 2555
2556 if (queued_send_wr) {
4d60cad5 2557 trace_ib_mad_send_done_resend(queued_send_wr, qp_info);
e622f2f4 2558 ret = ib_post_send(qp_info->qp, &queued_send_wr->send_wr.wr,
1fec77bf 2559 NULL);
1da177e4 2560 if (ret) {
7ef5d4b0
IW		 2561 dev_err(&port_priv->device->dev,
2562 "ib_post_send failed: %d\n", ret);
1da177e4
LT		 2563 mad_send_wr = queued_send_wr;
2564 wc->status = IB_WC_LOC_QP_OP_ERR;
2565 goto retry;
2566 }
2567 }
2568}
2569
2570static void mark_sends_for_retry(struct ib_mad_qp_info *qp_info)
2571{
2572 struct ib_mad_send_wr_private *mad_send_wr;
2573 struct ib_mad_list_head *mad_list;
2574 unsigned long flags;
2575
2576 spin_lock_irqsave(&qp_info->send_queue.lock, flags);
2577 list_for_each_entry(mad_list, &qp_info->send_queue.list, list) {
2578 mad_send_wr = container_of(mad_list,
2579 struct ib_mad_send_wr_private,
2580 mad_list);
2581 mad_send_wr->retry = 1;
2582 }
2583 spin_unlock_irqrestore(&qp_info->send_queue.lock, flags);
2584}
2585
d53e11fd
CH		 2586static bool ib_mad_send_error(struct ib_mad_port_private *port_priv,
2587 struct ib_wc *wc)
1da177e4 2588{
d53e11fd
CH		 2589 struct ib_mad_list_head *mad_list =
2590 container_of(wc->wr_cqe, struct ib_mad_list_head, cqe);
2591 struct ib_mad_qp_info *qp_info = mad_list->mad_queue->qp_info;
1da177e4
LT		 2592 struct ib_mad_send_wr_private *mad_send_wr;
2593 int ret;
2594
1da177e4
LT		 2595 /*
2596 * Send errors will transition the QP to SQE - move
2597 * QP to RTS and repost flushed work requests
2598 */
2599 mad_send_wr = container_of(mad_list, struct ib_mad_send_wr_private,
2600 mad_list);
2601 if (wc->status == IB_WC_WR_FLUSH_ERR) {
2602 if (mad_send_wr->retry) {
2603 /* Repost send */
1da177e4 2604 mad_send_wr->retry = 0;
4d60cad5 2605 trace_ib_mad_error_handler(mad_send_wr, qp_info);
e622f2f4 2606 ret = ib_post_send(qp_info->qp, &mad_send_wr->send_wr.wr,
1fec77bf 2607 NULL);
d53e11fd
CH		 2608 if (!ret)
2609 return false;
2610 }
1da177e4
LT		 2611 } else {
2612 struct ib_qp_attr *attr;
2613
2614 /* Transition QP to RTS and fail offending send */
2615 attr = kmalloc(sizeof *attr, GFP_KERNEL);
2616 if (attr) {
2617 attr->qp_state = IB_QPS_RTS;
2618 attr->cur_qp_state = IB_QPS_SQE;
2619 ret = ib_modify_qp(qp_info->qp, attr,
2620 IB_QP_STATE | IB_QP_CUR_STATE);
2621 kfree(attr);
2622 if (ret)
7ef5d4b0 2623 dev_err(&port_priv->device->dev,
d53e11fd
CH		 2624 "%s - ib_modify_qp to RTS: %d\n",
2625 __func__, ret);
1da177e4
LT		 2626 else
2627 mark_sends_for_retry(qp_info);
2628 }
1da177e4 2629 }
1da177e4 2630
d53e11fd 2631 return true;
1da177e4
LT		 2632}
2633
1cf0d899
OHT		 2634static void clear_mad_error_list(struct list_head *list,
2635 enum ib_wc_status wc_status,
2636 struct ib_mad_agent_private *mad_agent_priv)
2637{
2638 struct ib_mad_send_wr_private *mad_send_wr, *n;
2639 struct ib_mad_send_wc mad_send_wc;
2640
2641 mad_send_wc.status = wc_status;
2642 mad_send_wc.vendor_err = 0;
2643
2644 list_for_each_entry_safe(mad_send_wr, n, list, agent_list) {
2645 mad_send_wc.send_buf = &mad_send_wr->send_buf;
2646 mad_agent_priv->agent.send_handler(&mad_agent_priv->agent,
2647 &mad_send_wc);
2648 deref_mad_agent(mad_agent_priv);
2649 }
2650}
2651
1da177e4
LT		 2652static void cancel_mads(struct ib_mad_agent_private *mad_agent_priv)
2653{
2654 unsigned long flags;
2655 struct ib_mad_send_wr_private *mad_send_wr, *temp_mad_send_wr;
1da177e4
LT		 2656 struct list_head cancel_list;
2657
2658 INIT_LIST_HEAD(&cancel_list);
2659
2660 spin_lock_irqsave(&mad_agent_priv->lock, flags);
2661 list_for_each_entry_safe(mad_send_wr, temp_mad_send_wr,
1cf0d899
OHT		 2662 &mad_agent_priv->send_list, agent_list)
2663 change_mad_state(mad_send_wr, IB_MAD_STATE_CANCELED);
1da177e4 2664
314cb74c 2665 /* Empty wait & backlog list to prevent receives from finding request */
1cf0d899
OHT		 2666 list_for_each_entry_safe(mad_send_wr, temp_mad_send_wr,
2667 &mad_agent_priv->wait_list, agent_list) {
2668 change_mad_state(mad_send_wr, IB_MAD_STATE_DONE);
2669 list_add_tail(&mad_send_wr->agent_list, &cancel_list);
2670 }
1da177e4 2671
314cb74c
OHT		 2672 list_for_each_entry_safe(mad_send_wr, temp_mad_send_wr,
2673 &mad_agent_priv->backlog_list, agent_list) {
2674 change_mad_state(mad_send_wr, IB_MAD_STATE_DONE);
2675 list_add_tail(&mad_send_wr->agent_list, &cancel_list);
2676 }
2677
2678 spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
1da177e4 2679 /* Report all cancelled requests */
1cf0d899 2680 clear_mad_error_list(&cancel_list, IB_WC_WR_FLUSH_ERR, mad_agent_priv);
1da177e4
LT		 2681}
2682
2683static struct ib_mad_send_wr_private*
34816ad9
SH		 2684find_send_wr(struct ib_mad_agent_private *mad_agent_priv,
2685 struct ib_mad_send_buf *send_buf)
1da177e4
LT		 2686{
2687 struct ib_mad_send_wr_private *mad_send_wr;
2688
2689 list_for_each_entry(mad_send_wr, &mad_agent_priv->wait_list,
2690 agent_list) {
34816ad9 2691 if (&mad_send_wr->send_buf == send_buf)
1da177e4
LT		 2692 return mad_send_wr;
2693 }
2694
2695 list_for_each_entry(mad_send_wr, &mad_agent_priv->send_list,
2696 agent_list) {
c597eee5
IW		 2697 if (is_rmpp_data_mad(mad_agent_priv,
2698 mad_send_wr->send_buf.mad) &&
34816ad9 2699 &mad_send_wr->send_buf == send_buf)
1da177e4
LT		 2700 return mad_send_wr;
2701 }
314cb74c
OHT		 2702
2703 list_for_each_entry(mad_send_wr, &mad_agent_priv->backlog_list,
2704 agent_list) {
2705 if (&mad_send_wr->send_buf == send_buf)
2706 return mad_send_wr;
2707 }
2708
1da177e4
LT		 2709 return NULL;
2710}
2711
70076a41 2712int ib_modify_mad(struct ib_mad_send_buf *send_buf, u32 timeout_ms)
1da177e4
LT		 2713{
2714 struct ib_mad_agent_private *mad_agent_priv;
2715 struct ib_mad_send_wr_private *mad_send_wr;
2716 unsigned long flags;
cabe3cbc 2717 int active;
1da177e4 2718
70076a41
MZ		 2719 if (!send_buf)
2720 return -EINVAL;
2721
2722 mad_agent_priv = container_of(send_buf->mad_agent,
2723 struct ib_mad_agent_private, agent);
1da177e4 2724 spin_lock_irqsave(&mad_agent_priv->lock, flags);
34816ad9 2725 mad_send_wr = find_send_wr(mad_agent_priv, send_buf);
1cf0d899 2726 if (!mad_send_wr || mad_send_wr->state == IB_MAD_STATE_CANCELED) {
1da177e4 2727 spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
03b61ad2 2728 return -EINVAL;
1da177e4
LT		 2729 }
2730
314cb74c
OHT		 2731 active = ((mad_send_wr->state == IB_MAD_STATE_SEND_START) ||
2732 (mad_send_wr->state == IB_MAD_STATE_EARLY_RESP) ||
2733 (mad_send_wr->state == IB_MAD_STATE_QUEUED && timeout_ms));
1cf0d899
OHT		 2734 if (!timeout_ms)
2735 change_mad_state(mad_send_wr, IB_MAD_STATE_CANCELED);
1da177e4 2736
34816ad9 2737 mad_send_wr->send_buf.timeout_ms = timeout_ms;
cabe3cbc 2738 if (active)
03b61ad2
HR		 2739 mad_send_wr->timeout = msecs_to_jiffies(timeout_ms);
2740 else
2741 ib_reset_mad_timeout(mad_send_wr, timeout_ms);
2742
1da177e4 2743 spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
03b61ad2
HR		 2744 return 0;
2745}
2746EXPORT_SYMBOL(ib_modify_mad);
1da177e4 2747
c4028958 2748static void local_completions(struct work_struct *work)
1da177e4
LT		 2749{
2750 struct ib_mad_agent_private *mad_agent_priv;
2751 struct ib_mad_local_private *local;
2752 struct ib_mad_agent_private *recv_mad_agent;
2753 unsigned long flags;
1d9bc6d6 2754 int free_mad;
1da177e4
LT		 2755 struct ib_wc wc;
2756 struct ib_mad_send_wc mad_send_wc;
8e4349d1 2757 bool opa;
1da177e4 2758
c4028958
DH		 2759 mad_agent_priv =
2760 container_of(work, struct ib_mad_agent_private, local_work);
1da177e4 2761
8e4349d1
IW		 2762 opa = rdma_cap_opa_mad(mad_agent_priv->qp_info->port_priv->device,
2763 mad_agent_priv->qp_info->port_priv->port_num);
2764
1da177e4
LT		 2765 spin_lock_irqsave(&mad_agent_priv->lock, flags);
2766 while (!list_empty(&mad_agent_priv->local_list)) {
2767 local = list_entry(mad_agent_priv->local_list.next,
2768 struct ib_mad_local_private,
2769 completion_list);
37289efe 2770 list_del(&local->completion_list);
1da177e4 2771 spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
1d9bc6d6 2772 free_mad = 0;
1da177e4 2773 if (local->mad_priv) {
8e4349d1 2774 u8 base_version;
1da177e4
LT		 2775 recv_mad_agent = local->recv_mad_agent;
2776 if (!recv_mad_agent) {
7ef5d4b0
IW		 2777 dev_err(&mad_agent_priv->agent.device->dev,
2778 "No receive MAD agent for local completion\n");
1d9bc6d6 2779 free_mad = 1;
1da177e4
LT		 2780 goto local_send_completion;
2781 }
2782
2783 /*
2784 * Defined behavior is to complete response
2785 * before request
2786 */
062dbb69 2787 build_smp_wc(recv_mad_agent->agent.qp,
d53e11fd 2788 local->mad_send_wr->send_wr.wr.wr_cqe,
97f52eb4 2789 be16_to_cpu(IB_LID_PERMISSIVE),
e622f2f4 2790 local->mad_send_wr->send_wr.pkey_index,
8e4349d1 2791 recv_mad_agent->agent.port_num, &wc);
1da177e4
LT		 2792
2793 local->mad_priv->header.recv_wc.wc = &wc;
8e4349d1
IW		 2794
2795 base_version = ((struct ib_mad_hdr *)(local->mad_priv->mad))->base_version;
2796 if (opa && base_version == OPA_MGMT_BASE_VERSION) {
2797 local->mad_priv->header.recv_wc.mad_len = local->return_wc_byte_len;
2798 local->mad_priv->header.recv_wc.mad_seg_size = sizeof(struct opa_mad);
2799 } else {
2800 local->mad_priv->header.recv_wc.mad_len = sizeof(struct ib_mad);
2801 local->mad_priv->header.recv_wc.mad_seg_size = sizeof(struct ib_mad);
2802 }
2803
fa619a77
HR		 2804 INIT_LIST_HEAD(&local->mad_priv->header.recv_wc.rmpp_list);
2805 list_add(&local->mad_priv->header.recv_wc.recv_buf.list,
2806 &local->mad_priv->header.recv_wc.rmpp_list);
1da177e4
LT		 2807 local->mad_priv->header.recv_wc.recv_buf.grh = NULL;
2808 local->mad_priv->header.recv_wc.recv_buf.mad =
c9082e51 2809 (struct ib_mad *)local->mad_priv->mad;
1da177e4
LT		 2810 recv_mad_agent->agent.recv_handler(
2811 &recv_mad_agent->agent,
ca281265 2812 &local->mad_send_wr->send_buf,
1da177e4
LT		 2813 &local->mad_priv->header.recv_wc);
2814 spin_lock_irqsave(&recv_mad_agent->lock, flags);
b9af0e2d 2815 deref_mad_agent(recv_mad_agent);
1da177e4
LT		 2816 spin_unlock_irqrestore(&recv_mad_agent->lock, flags);
2817 }
2818
2819local_send_completion:
2820 /* Complete send */
2821 mad_send_wc.status = IB_WC_SUCCESS;
2822 mad_send_wc.vendor_err = 0;
34816ad9 2823 mad_send_wc.send_buf = &local->mad_send_wr->send_buf;
1da177e4
LT		 2824 mad_agent_priv->agent.send_handler(&mad_agent_priv->agent,
2825 &mad_send_wc);
2826
2827 spin_lock_irqsave(&mad_agent_priv->lock, flags);
b9af0e2d 2828 deref_mad_agent(mad_agent_priv);
1d9bc6d6 2829 if (free_mad)
c9082e51 2830 kfree(local->mad_priv);
1da177e4
LT		 2831 kfree(local);
2832 }
2833 spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
2834}
2835
f75b7a52
HR		 2836static int retry_send(struct ib_mad_send_wr_private *mad_send_wr)
2837{
2838 int ret;
2839
4fc8cd49 2840 if (!mad_send_wr->retries_left)
f75b7a52
HR		 2841 return -ETIMEDOUT;
2842
4fc8cd49
SH		 2843 mad_send_wr->retries_left--;
2844 mad_send_wr->send_buf.retries++;
2845
34816ad9 2846 mad_send_wr->timeout = msecs_to_jiffies(mad_send_wr->send_buf.timeout_ms);
314cb74c
OHT		 2847 if (mad_send_wr->is_solicited_fc &&
2848 !list_empty(&mad_send_wr->mad_agent_priv->backlog_list)) {
2849 change_mad_state(mad_send_wr, IB_MAD_STATE_QUEUED);
2850 return 0;
2851 }
f75b7a52 2852
1471cb6c 2853 if (ib_mad_kernel_rmpp_agent(&mad_send_wr->mad_agent_priv->agent)) {
fa619a77
HR		 2854 ret = ib_retry_rmpp(mad_send_wr);
2855 switch (ret) {
2856 case IB_RMPP_RESULT_UNHANDLED:
2857 ret = ib_send_mad(mad_send_wr);
2858 break;
2859 case IB_RMPP_RESULT_CONSUMED:
2860 ret = 0;
2861 break;
2862 default:
2863 ret = -ECOMM;
2864 break;
2865 }
2866 } else
2867 ret = ib_send_mad(mad_send_wr);
f75b7a52 2868
1cf0d899
OHT		 2869 if (!ret)
2870 change_mad_state(mad_send_wr, IB_MAD_STATE_SEND_START);
2871
f75b7a52
HR		 2872 return ret;
2873}
2874
c4028958 2875static void timeout_sends(struct work_struct *work)
1da177e4 2876{
1cf0d899 2877 struct ib_mad_send_wr_private *mad_send_wr;
1da177e4 2878 struct ib_mad_agent_private *mad_agent_priv;
1cf0d899
OHT		 2879 struct list_head timeout_list;
2880 struct list_head cancel_list;
2881 struct list_head *list_item;
1da177e4
LT		 2882 unsigned long flags, delay;
2883
c4028958
DH		 2884 mad_agent_priv = container_of(work, struct ib_mad_agent_private,
2885 timed_work.work);
1cf0d899
OHT		 2886 INIT_LIST_HEAD(&timeout_list);
2887 INIT_LIST_HEAD(&cancel_list);
1da177e4
LT		 2888
2889 spin_lock_irqsave(&mad_agent_priv->lock, flags);
2890 while (!list_empty(&mad_agent_priv->wait_list)) {
2891 mad_send_wr = list_entry(mad_agent_priv->wait_list.next,
2892 struct ib_mad_send_wr_private,
2893 agent_list);
2894
2895 if (time_after(mad_send_wr->timeout, jiffies)) {
2896 delay = mad_send_wr->timeout - jiffies;
2897 if ((long)delay <= 0)
2898 delay = 1;
2899 queue_delayed_work(mad_agent_priv->qp_info->
2900 port_priv->wq,
2901 &mad_agent_priv->timed_work, delay);
2902 break;
2903 }
2904
1cf0d899
OHT		 2905 if (mad_send_wr->state == IB_MAD_STATE_CANCELED)
2906 list_item = &cancel_list;
2907 else if (retry_send(mad_send_wr))
2908 list_item = &timeout_list;
2909 else
f75b7a52
HR		 2910 continue;
2911
1cf0d899
OHT		 2912 change_mad_state(mad_send_wr, IB_MAD_STATE_DONE);
2913 list_add_tail(&mad_send_wr->agent_list, list_item);
2a777679 2914 }
1da177e4 2915
1cf0d899 2916 spin_unlock_irqrestore(&mad_agent_priv->lock, flags);
314cb74c 2917 process_backlog_mads(mad_agent_priv);
1cf0d899
OHT		 2918 clear_mad_error_list(&timeout_list, IB_WC_RESP_TIMEOUT_ERR,
2919 mad_agent_priv);
2920 clear_mad_error_list(&cancel_list, IB_WC_WR_FLUSH_ERR, mad_agent_priv);
1da177e4
LT		 2921}
2922
1da177e4
LT		 2923/*
2924 * Allocate receive MADs and post receive WRs for them
2925 */
2926static int ib_mad_post_receive_mads(struct ib_mad_qp_info *qp_info,
2927 struct ib_mad_private *mad)
2928{
2929 unsigned long flags;
1da177e4
LT		 2930 struct ib_mad_private *mad_priv;
2931 struct ib_sge sg_list;
1fec77bf 2932 struct ib_recv_wr recv_wr;
1da177e4 2933 struct ib_mad_queue *recv_queue = &qp_info->recv_queue;
37826f0a 2934 int ret = 0;
1da177e4
LT		 2935
2936 /* Initialize common scatter list fields */
4be90bc6 2937 sg_list.lkey = qp_info->port_priv->pd->local_dma_lkey;
1da177e4
LT		 2938
2939 /* Initialize common receive WR fields */
2940 recv_wr.next = NULL;
2941 recv_wr.sg_list = &sg_list;
2942 recv_wr.num_sge = 1;
2943
37826f0a 2944 while (true) {
1da177e4
LT		 2945 /* Allocate and map receive buffer */
2946 if (mad) {
2947 mad_priv = mad;
2948 mad = NULL;
2949 } else {
c9082e51
IW		 2950 mad_priv = alloc_mad_private(port_mad_size(qp_info->port_priv),
2951 GFP_ATOMIC);
37826f0a
MS		 2952 if (!mad_priv)
2953 return -ENOMEM;
1da177e4 2954 }
c9082e51 2955 sg_list.length = mad_priv_dma_size(mad_priv);
1527106f
RC		 2956 sg_list.addr = ib_dma_map_single(qp_info->port_priv->device,
2957 &mad_priv->grh,
c9082e51 2958 mad_priv_dma_size(mad_priv),
1527106f 2959 DMA_FROM_DEVICE);
2c34e68f
YB		 2960 if (unlikely(ib_dma_mapping_error(qp_info->port_priv->device,
2961 sg_list.addr))) {
2962 ret = -ENOMEM;
37826f0a 2963 goto free_mad_priv;
2c34e68f 2964 }
1527106f 2965 mad_priv->header.mapping = sg_list.addr;
1da177e4 2966 mad_priv->header.mad_list.mad_queue = recv_queue;
d53e11fd
CH		 2967 mad_priv->header.mad_list.cqe.done = ib_mad_recv_done;
2968 recv_wr.wr_cqe = &mad_priv->header.mad_list.cqe;
1da177e4 2969 spin_lock_irqsave(&recv_queue->lock, flags);
37826f0a
MS		 2970 if (recv_queue->count >= recv_queue->max_active) {
2971 /* Fully populated the receive queue */
2972 spin_unlock_irqrestore(&recv_queue->lock, flags);
2973 break;
2974 }
2975 recv_queue->count++;
2976 list_add_tail(&mad_priv->header.mad_list.list,
2977 &recv_queue->list);
1da177e4 2978 spin_unlock_irqrestore(&recv_queue->lock, flags);
37826f0a 2979
1fec77bf 2980 ret = ib_post_recv(qp_info->qp, &recv_wr, NULL);
1da177e4
LT		 2981 if (ret) {
2982 spin_lock_irqsave(&recv_queue->lock, flags);
2983 list_del(&mad_priv->header.mad_list.list);
2984 recv_queue->count--;
2985 spin_unlock_irqrestore(&recv_queue->lock, flags);
7ef5d4b0
IW		 2986 dev_err(&qp_info->port_priv->device->dev,
2987 "ib_post_recv failed: %d\n", ret);
1da177e4
LT		 2988 break;
2989 }
37826f0a 2990 }
1da177e4 2991
37826f0a
MS		 2992 ib_dma_unmap_single(qp_info->port_priv->device,
2993 mad_priv->header.mapping,
2994 mad_priv_dma_size(mad_priv), DMA_FROM_DEVICE);
2995free_mad_priv:
2996 kfree(mad_priv);
1da177e4
LT		 2997 return ret;
2998}
2999
3000/*
3001 * Return all the posted receive MADs
3002 */
3003static void cleanup_recv_queue(struct ib_mad_qp_info *qp_info)
3004{
3005 struct ib_mad_private_header *mad_priv_hdr;
3006 struct ib_mad_private *recv;
3007 struct ib_mad_list_head *mad_list;
3008
fac70d51
EC		 3009 if (!qp_info->qp)
3010 return;
3011
1da177e4
LT		 3012 while (!list_empty(&qp_info->recv_queue.list)) {
3013
3014 mad_list = list_entry(qp_info->recv_queue.list.next,
3015 struct ib_mad_list_head, list);
3016 mad_priv_hdr = container_of(mad_list,
3017 struct ib_mad_private_header,
3018 mad_list);
3019 recv = container_of(mad_priv_hdr, struct ib_mad_private,
3020 header);
3021
3022 /* Remove from posted receive MAD list */
3023 list_del(&mad_list->list);
3024
1527106f
RC		 3025 ib_dma_unmap_single(qp_info->port_priv->device,
3026 recv->header.mapping,
c9082e51 3027 mad_priv_dma_size(recv),
1527106f 3028 DMA_FROM_DEVICE);
c9082e51 3029 kfree(recv);
1da177e4
LT		 3030 }
3031
3032 qp_info->recv_queue.count = 0;
3033}
3034
3035/*
3036 * Start the port
3037 */
3038static int ib_mad_port_start(struct ib_mad_port_private *port_priv)
3039{
3040 int ret, i;
3041 struct ib_qp_attr *attr;
3042 struct ib_qp *qp;
ef5ed416 3043 u16 pkey_index;
1da177e4
LT		 3044
3045 attr = kmalloc(sizeof *attr, GFP_KERNEL);
27162432 3046 if (!attr)
1da177e4 3047 return -ENOMEM;
1da177e4 3048
ef5ed416
JM		 3049 ret = ib_find_pkey(port_priv->device, port_priv->port_num,
3050 IB_DEFAULT_PKEY_FULL, &pkey_index);
3051 if (ret)
3052 pkey_index = 0;
3053
1da177e4
LT		 3054 for (i = 0; i < IB_MAD_QPS_CORE; i++) {
3055 qp = port_priv->qp_info[i].qp;
fac70d51
EC		 3056 if (!qp)
3057 continue;
3058
1da177e4
LT		 3059 /*
3060 * PKey index for QP1 is irrelevant but
3061 * one is needed for the Reset to Init transition
3062 */
3063 attr->qp_state = IB_QPS_INIT;
ef5ed416 3064 attr->pkey_index = pkey_index;
1da177e4
LT		 3065 attr->qkey = (qp->qp_num == 0) ? 0 : IB_QP1_QKEY;
3066 ret = ib_modify_qp(qp, attr, IB_QP_STATE |
3067 IB_QP_PKEY_INDEX | IB_QP_QKEY);
3068 if (ret) {
7ef5d4b0
IW		 3069 dev_err(&port_priv->device->dev,
3070 "Couldn't change QP%d state to INIT: %d\n",
3071 i, ret);
1da177e4
LT		 3072 goto out;
3073 }
3074
3075 attr->qp_state = IB_QPS_RTR;
3076 ret = ib_modify_qp(qp, attr, IB_QP_STATE);
3077 if (ret) {
7ef5d4b0
IW		 3078 dev_err(&port_priv->device->dev,
3079 "Couldn't change QP%d state to RTR: %d\n",
3080 i, ret);
1da177e4
LT		 3081 goto out;
3082 }
3083
3084 attr->qp_state = IB_QPS_RTS;
3085 attr->sq_psn = IB_MAD_SEND_Q_PSN;
3086 ret = ib_modify_qp(qp, attr, IB_QP_STATE | IB_QP_SQ_PSN);
3087 if (ret) {
7ef5d4b0
IW		 3088 dev_err(&port_priv->device->dev,
3089 "Couldn't change QP%d state to RTS: %d\n",
3090 i, ret);
1da177e4
LT		 3091 goto out;
3092 }
3093 }
3094
3095 ret = ib_req_notify_cq(port_priv->cq, IB_CQ_NEXT_COMP);
3096 if (ret) {
7ef5d4b0
IW		 3097 dev_err(&port_priv->device->dev,
3098 "Failed to request completion notification: %d\n",
3099 ret);
1da177e4
LT		 3100 goto out;
3101 }
3102
3103 for (i = 0; i < IB_MAD_QPS_CORE; i++) {
fac70d51
EC		 3104 if (!port_priv->qp_info[i].qp)
3105 continue;
3106
1da177e4
LT		 3107 ret = ib_mad_post_receive_mads(&port_priv->qp_info[i], NULL);
3108 if (ret) {
7ef5d4b0
IW		 3109 dev_err(&port_priv->device->dev,
3110 "Couldn't post receive WRs\n");
1da177e4
LT		 3111 goto out;
3112 }
3113 }
3114out:
3115 kfree(attr);
3116 return ret;
3117}
3118
3119static void qp_event_handler(struct ib_event *event, void *qp_context)
3120{
3121 struct ib_mad_qp_info *qp_info = qp_context;
3122
3123 /* It's worse than that! He's dead, Jim! */
7ef5d4b0 3124 dev_err(&qp_info->port_priv->device->dev,
3cea7b4a 3125 "Fatal error (%d) on MAD QP (%u)\n",
1da177e4
LT		 3126 event->event, qp_info->qp->qp_num);
3127}
3128
3129static void init_mad_queue(struct ib_mad_qp_info *qp_info,
3130 struct ib_mad_queue *mad_queue)
3131{
3132 mad_queue->qp_info = qp_info;
3133 mad_queue->count = 0;
3134 spin_lock_init(&mad_queue->lock);
3135 INIT_LIST_HEAD(&mad_queue->list);
3136}
3137
3138static void init_mad_qp(struct ib_mad_port_private *port_priv,
3139 struct ib_mad_qp_info *qp_info)
3140{
3141 qp_info->port_priv = port_priv;
3142 init_mad_queue(qp_info, &qp_info->send_queue);
3143 init_mad_queue(qp_info, &qp_info->recv_queue);
3144 INIT_LIST_HEAD(&qp_info->overflow_list);
1da177e4
LT		 3145}
3146
3147static int create_mad_qp(struct ib_mad_qp_info *qp_info,
3148 enum ib_qp_type qp_type)
3149{
3150 struct ib_qp_init_attr qp_init_attr;
3151 int ret;
3152
3153 memset(&qp_init_attr, 0, sizeof qp_init_attr);
3154 qp_init_attr.send_cq = qp_info->port_priv->cq;
3155 qp_init_attr.recv_cq = qp_info->port_priv->cq;
3156 qp_init_attr.sq_sig_type = IB_SIGNAL_ALL_WR;
b76aabc3
HR		 3157 qp_init_attr.cap.max_send_wr = mad_sendq_size;
3158 qp_init_attr.cap.max_recv_wr = mad_recvq_size;
1da177e4
LT		 3159 qp_init_attr.cap.max_send_sge = IB_MAD_SEND_REQ_MAX_SG;
3160 qp_init_attr.cap.max_recv_sge = IB_MAD_RECV_REQ_MAX_SG;
3161 qp_init_attr.qp_type = qp_type;
3162 qp_init_attr.port_num = qp_info->port_priv->port_num;
3163 qp_init_attr.qp_context = qp_info;
3164 qp_init_attr.event_handler = qp_event_handler;
3165 qp_info->qp = ib_create_qp(qp_info->port_priv->pd, &qp_init_attr);
3166 if (IS_ERR(qp_info->qp)) {
7ef5d4b0
IW		 3167 dev_err(&qp_info->port_priv->device->dev,
3168 "Couldn't create ib_mad QP%d\n",
3169 get_spl_qp_index(qp_type));
1da177e4
LT		 3170 ret = PTR_ERR(qp_info->qp);
3171 goto error;
3172 }
3173 /* Use minimum queue sizes unless the CQ is resized */
b76aabc3
HR		 3174 qp_info->send_queue.max_active = mad_sendq_size;
3175 qp_info->recv_queue.max_active = mad_recvq_size;
1da177e4
LT		 3176 return 0;
3177
3178error:
3179 return ret;
3180}
3181
3182static void destroy_mad_qp(struct ib_mad_qp_info *qp_info)
3183{
fac70d51
EC		 3184 if (!qp_info->qp)
3185 return;
3186
1da177e4 3187 ib_destroy_qp(qp_info->qp);
1da177e4
LT		 3188}
3189
3190/*
3191 * Open the port
3192 * Create the QP, PD, MR, and CQ if needed
3193 */
3194static int ib_mad_port_open(struct ib_device *device,
1fb7f897 3195 u32 port_num)
1da177e4
LT		 3196{
3197 int ret, cq_size;
3198 struct ib_mad_port_private *port_priv;
3199 unsigned long flags;
fac70d51 3200 int has_smi;
1da177e4 3201
337877a4
IW		 3202 if (WARN_ON(rdma_max_mad_size(device, port_num) < IB_MGMT_MAD_SIZE))
3203 return -EFAULT;
3204
548ead17
IW		 3205 if (WARN_ON(rdma_cap_opa_mad(device, port_num) &&
3206 rdma_max_mad_size(device, port_num) < OPA_MGMT_MAD_SIZE))
3207 return -EFAULT;
3208
1da177e4 3209 /* Create new device info */
de6eb66b 3210 port_priv = kzalloc(sizeof *port_priv, GFP_KERNEL);
27162432 3211 if (!port_priv)
1da177e4 3212 return -ENOMEM;
de6eb66b 3213
1da177e4
LT		 3214 port_priv->device = device;
3215 port_priv->port_num = port_num;
3216 spin_lock_init(&port_priv->reg_lock);
1da177e4
LT		 3217 init_mad_qp(port_priv, &port_priv->qp_info[0]);
3218 init_mad_qp(port_priv, &port_priv->qp_info[1]);
3219
fac70d51 3220 cq_size = mad_sendq_size + mad_recvq_size;
29541e3a 3221 has_smi = rdma_cap_ib_smi(device, port_num);
fac70d51
EC		 3222 if (has_smi)
3223 cq_size *= 2;
3224
770b7d96
JM		 3225 port_priv->pd = ib_alloc_pd(device, 0);
3226 if (IS_ERR(port_priv->pd)) {
3227 dev_err(&device->dev, "Couldn't create ib_mad PD\n");
3228 ret = PTR_ERR(port_priv->pd);
3229 goto error3;
3230 }
3231
d53e11fd 3232 port_priv->cq = ib_alloc_cq(port_priv->device, port_priv, cq_size, 0,
f794809a 3233 IB_POLL_UNBOUND_WORKQUEUE);
1da177e4 3234 if (IS_ERR(port_priv->cq)) {
7ef5d4b0 3235 dev_err(&device->dev, "Couldn't create ib_mad CQ\n");
1da177e4 3236 ret = PTR_ERR(port_priv->cq);
1da177e4
LT		 3237 goto error4;
3238 }
3239
fac70d51
EC		 3240 if (has_smi) {
3241 ret = create_mad_qp(&port_priv->qp_info[0], IB_QPT_SMI);
3242 if (ret)
3243 goto error6;
3244 }
a9e0faca
MZ		 3245
3246 if (rdma_cap_ib_cm(device, port_num)) {
3247 ret = create_mad_qp(&port_priv->qp_info[1], IB_QPT_GSI);
3248 if (ret)
3249 goto error7;
3250 }
1da177e4 3251
7229d7b6
JR		 3252 port_priv->wq = alloc_ordered_workqueue("ib_mad%u", WQ_MEM_RECLAIM,
3253 port_num);
1da177e4
LT		 3254 if (!port_priv->wq) {
3255 ret = -ENOMEM;
3256 goto error8;
3257 }
1da177e4 3258
dc05980d
MT		 3259 spin_lock_irqsave(&ib_mad_port_list_lock, flags);
3260 list_add_tail(&port_priv->port_list, &ib_mad_port_list);
3261 spin_unlock_irqrestore(&ib_mad_port_list_lock, flags);
3262
1da177e4
LT		 3263 ret = ib_mad_port_start(port_priv);
3264 if (ret) {
7ef5d4b0 3265 dev_err(&device->dev, "Couldn't start port\n");
1da177e4
LT		 3266 goto error9;
3267 }
3268
1da177e4
LT		 3269 return 0;
3270
3271error9:
dc05980d
MT		 3272 spin_lock_irqsave(&ib_mad_port_list_lock, flags);
3273 list_del_init(&port_priv->port_list);
3274 spin_unlock_irqrestore(&ib_mad_port_list_lock, flags);
3275
1da177e4
LT		 3276 destroy_workqueue(port_priv->wq);
3277error8:
3278 destroy_mad_qp(&port_priv->qp_info[1]);
3279error7:
3280 destroy_mad_qp(&port_priv->qp_info[0]);
3281error6:
d53e11fd 3282 ib_free_cq(port_priv->cq);
1da177e4
LT		 3283 cleanup_recv_queue(&port_priv->qp_info[1]);
3284 cleanup_recv_queue(&port_priv->qp_info[0]);
770b7d96
JM		 3285error4:
3286 ib_dealloc_pd(port_priv->pd);
1da177e4
LT		 3287error3:
3288 kfree(port_priv);
3289
3290 return ret;
3291}
3292
3293/*
3294 * Close the port
3295 * If there are no classes using the port, free the port
3296 * resources (CQ, MR, PD, QP) and remove the port's info structure
3297 */
1fb7f897 3298static int ib_mad_port_close(struct ib_device *device, u32 port_num)
1da177e4
LT		 3299{
3300 struct ib_mad_port_private *port_priv;
3301 unsigned long flags;
3302
3303 spin_lock_irqsave(&ib_mad_port_list_lock, flags);
3304 port_priv = __ib_get_mad_port(device, port_num);
3305 if (port_priv == NULL) {
3306 spin_unlock_irqrestore(&ib_mad_port_list_lock, flags);
1fb7f897 3307 dev_err(&device->dev, "Port %u not found\n", port_num);
1da177e4
LT		 3308 return -ENODEV;
3309 }
dc05980d 3310 list_del_init(&port_priv->port_list);
1da177e4
LT		 3311 spin_unlock_irqrestore(&ib_mad_port_list_lock, flags);
3312
1da177e4
LT		 3313 destroy_workqueue(port_priv->wq);
3314 destroy_mad_qp(&port_priv->qp_info[1]);
3315 destroy_mad_qp(&port_priv->qp_info[0]);
d53e11fd 3316 ib_free_cq(port_priv->cq);
770b7d96 3317 ib_dealloc_pd(port_priv->pd);
1da177e4
LT		 3318 cleanup_recv_queue(&port_priv->qp_info[1]);
3319 cleanup_recv_queue(&port_priv->qp_info[0]);
3320 /* XXX: Handle deallocation of MAD registration tables */
3321
3322 kfree(port_priv);
3323
3324 return 0;
3325}
3326
11a0ae4c 3327static int ib_mad_init_device(struct ib_device *device)
1da177e4 3328{
4139032b 3329 int start, i;
11a0ae4c
JG		 3330 unsigned int count = 0;
3331 int ret;
1da177e4 3332
4139032b 3333 start = rdma_start_port(device);
4ab6fb7e 3334
4139032b 3335 for (i = start; i <= rdma_end_port(device); i++) {
c757dea8 3336 if (!rdma_cap_ib_mad(device, i))
827f2a8b
MW		 3337 continue;
3338
11a0ae4c
JG		 3339 ret = ib_mad_port_open(device, i);
3340 if (ret) {
7ef5d4b0 3341 dev_err(&device->dev, "Couldn't open port %d\n", i);
4ab6fb7e 3342 goto error;
1da177e4 3343 }
11a0ae4c
JG		 3344 ret = ib_agent_port_open(device, i);
3345 if (ret) {
7ef5d4b0
IW		 3346 dev_err(&device->dev,
3347 "Couldn't open port %d for agents\n", i);
4ab6fb7e 3348 goto error_agent;
1da177e4 3349 }
11a0ae4c 3350 count++;
1da177e4 3351 }
11a0ae4c
JG		 3352 if (!count)
3353 return -EOPNOTSUPP;
3354
3355 return 0;
1da177e4 3356
4ab6fb7e
RD		 3357error_agent:
3358 if (ib_mad_port_close(device, i))
7ef5d4b0 3359 dev_err(&device->dev, "Couldn't close port %d\n", i);
4ab6fb7e
RD		 3360
3361error:
827f2a8b 3362 while (--i >= start) {
c757dea8 3363 if (!rdma_cap_ib_mad(device, i))
827f2a8b 3364 continue;
4ab6fb7e 3365
4ab6fb7e 3366 if (ib_agent_port_close(device, i))
7ef5d4b0
IW		 3367 dev_err(&device->dev,
3368 "Couldn't close port %d for agents\n", i);
4ab6fb7e 3369 if (ib_mad_port_close(device, i))
7ef5d4b0 3370 dev_err(&device->dev, "Couldn't close port %d\n", i);
1da177e4 3371 }
11a0ae4c 3372 return ret;
1da177e4
LT		 3373}
3374
7c1eb45a 3375static void ib_mad_remove_device(struct ib_device *device, void *client_data)
1da177e4 3376{
ea1075ed 3377 unsigned int i;
827f2a8b 3378
ea1075ed 3379 rdma_for_each_port (device, i) {
c757dea8 3380 if (!rdma_cap_ib_mad(device, i))
827f2a8b
MW		 3381 continue;
3382
3383 if (ib_agent_port_close(device, i))
7ef5d4b0 3384 dev_err(&device->dev,
3cea7b4a 3385 "Couldn't close port %u for agents\n", i);
827f2a8b 3386 if (ib_mad_port_close(device, i))
3cea7b4a 3387 dev_err(&device->dev, "Couldn't close port %u\n", i);
1da177e4
LT		 3388 }
3389}
3390
3391static struct ib_client mad_client = {
3392 .name = "mad",
3393 .add = ib_mad_init_device,
3394 .remove = ib_mad_remove_device
3395};
3396
4c2cb422 3397int ib_mad_init(void)
1da177e4 3398{
b76aabc3
HR		 3399 mad_recvq_size = min(mad_recvq_size, IB_MAD_QP_MAX_SIZE);
3400 mad_recvq_size = max(mad_recvq_size, IB_MAD_QP_MIN_SIZE);
3401
3402 mad_sendq_size = min(mad_sendq_size, IB_MAD_QP_MAX_SIZE);
3403 mad_sendq_size = max(mad_sendq_size, IB_MAD_QP_MIN_SIZE);
3404
1da177e4
LT		 3405 INIT_LIST_HEAD(&ib_mad_port_list);
3406
3407 if (ib_register_client(&mad_client)) {
7ef5d4b0 3408 pr_err("Couldn't register ib_mad client\n");
c9082e51 3409 return -EINVAL;
1da177e4
LT		 3410 }
3411
3412 return 0;
1da177e4
LT		 3413}
3414
4c2cb422 3415void ib_mad_cleanup(void)
1da177e4
LT		 3416{
3417 ib_unregister_client(&mad_client);
1da177e4 3418}
Linux 6.x block layer and io_uring tree(s)