projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
RDMA/cma: Allow user to specify AF_IB when binding
[linux-2.6-block.git] / drivers / infiniband / core / cma.c
CommitLineData
e51060f0
SH		 1/*
2 * Copyright (c) 2005 Voltaire Inc. All rights reserved.
3 * Copyright (c) 2002-2005, Network Appliance, Inc. All rights reserved.
4 * Copyright (c) 1999-2005, Mellanox Technologies, Inc. All rights reserved.
5 * Copyright (c) 2005-2006 Intel Corporation. All rights reserved.
6 *
a9474917
SH		 7 * This software is available to you under a choice of one of two
8 * licenses. You may choose to be licensed under the terms of the GNU
9 * General Public License (GPL) Version 2, available from the file
10 * COPYING in the main directory of this source tree, or the
11 * OpenIB.org BSD license below:
e51060f0 12 *
a9474917
SH		 13 * Redistribution and use in source and binary forms, with or
14 * without modification, are permitted provided that the following
15 * conditions are met:
e51060f0 16 *
a9474917
SH		 17 * - Redistributions of source code must retain the above
18 * copyright notice, this list of conditions and the following
19 * disclaimer.
e51060f0 20 *
a9474917
SH		 21 * - Redistributions in binary form must reproduce the above
22 * copyright notice, this list of conditions and the following
23 * disclaimer in the documentation and/or other materials
24 * provided with the distribution.
e51060f0 25 *
a9474917
SH		 26 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
27 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
28 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
29 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
30 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
31 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
32 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33 * SOFTWARE.
e51060f0
SH		 34 */
35
36#include <linux/completion.h>
37#include <linux/in.h>
38#include <linux/in6.h>
39#include <linux/mutex.h>
40#include <linux/random.h>
41#include <linux/idr.h>
07ebafba 42#include <linux/inetdevice.h>
5a0e3ad6 43#include <linux/slab.h>
e4dd23d7 44#include <linux/module.h>
366cddb4 45#include <net/route.h>
e51060f0
SH		 46
47#include <net/tcp.h>
1f5175ad 48#include <net/ipv6.h>
e51060f0
SH		 49
50#include <rdma/rdma_cm.h>
51#include <rdma/rdma_cm_ib.h>
753f618a 52#include <rdma/rdma_netlink.h>
2e2d190c 53#include <rdma/ib.h>
e51060f0
SH		 54#include <rdma/ib_cache.h>
55#include <rdma/ib_cm.h>
56#include <rdma/ib_sa.h>
07ebafba 57#include <rdma/iw_cm.h>
e51060f0
SH		 58
59MODULE_AUTHOR("Sean Hefty");
60MODULE_DESCRIPTION("Generic RDMA CM Agent");
61MODULE_LICENSE("Dual BSD/GPL");
62
63#define CMA_CM_RESPONSE_TIMEOUT 20
d5bb7599 64#define CMA_MAX_CM_RETRIES 15
dcb3f974 65#define CMA_CM_MRA_SETTING (IB_CM_MRA_FLAG_DELAY | 24)
3c86aa70 66#define CMA_IBOE_PACKET_LIFETIME 18
e51060f0
SH		 67
68static void cma_add_one(struct ib_device *device);
69static void cma_remove_one(struct ib_device *device);
70
71static struct ib_client cma_client = {
72 .name = "cma",
73 .add = cma_add_one,
74 .remove = cma_remove_one
75};
76
c1a0b23b 77static struct ib_sa_client sa_client;
7a118df3 78static struct rdma_addr_client addr_client;
e51060f0
SH		 79static LIST_HEAD(dev_list);
80static LIST_HEAD(listen_any_list);
81static DEFINE_MUTEX(lock);
82static struct workqueue_struct *cma_wq;
83static DEFINE_IDR(sdp_ps);
84static DEFINE_IDR(tcp_ps);
628e5f6d 85static DEFINE_IDR(udp_ps);
c8f6a362 86static DEFINE_IDR(ipoib_ps);
2d2e9415 87static DEFINE_IDR(ib_ps);
e51060f0
SH		 88
89struct cma_device {
90 struct list_head list;
91 struct ib_device *device;
e51060f0
SH		 92 struct completion comp;
93 atomic_t refcount;
94 struct list_head id_list;
95};
96
e51060f0
SH		 97struct rdma_bind_list {
98 struct idr *ps;
99 struct hlist_head owners;
100 unsigned short port;
101};
102
68602120
SH		 103enum {
104 CMA_OPTION_AFONLY,
105};
106
e51060f0
SH		 107/*
108 * Device removal can occur at anytime, so we need extra handling to
109 * serialize notifying the user of device removal with other callbacks.
110 * We do this by disabling removal notification while a callback is in process,
111 * and reporting it after the callback completes.
112 */
113struct rdma_id_private {
114 struct rdma_cm_id id;
115
116 struct rdma_bind_list *bind_list;
117 struct hlist_node node;
d02d1f53
SH		 118 struct list_head list; /* listen_any_list or cma_device.list */
119 struct list_head listen_list; /* per device listens */
e51060f0 120 struct cma_device *cma_dev;
c8f6a362 121 struct list_head mc_list;
e51060f0 122
d02d1f53 123 int internal_id;
550e5ca7 124 enum rdma_cm_state state;
e51060f0 125 spinlock_t lock;
c5483388
SH		 126 struct mutex qp_mutex;
127
e51060f0
SH		 128 struct completion comp;
129 atomic_t refcount;
de910bd9 130 struct mutex handler_mutex;
e51060f0
SH		 131
132 int backlog;
133 int timeout_ms;
134 struct ib_sa_query *query;
135 int query_id;
136 union {
137 struct ib_cm_id *ib;
07ebafba 138 struct iw_cm_id *iw;
e51060f0
SH		 139 } cm_id;
140
141 u32 seq_num;
c8f6a362 142 u32 qkey;
e51060f0 143 u32 qp_num;
83e9502d 144 pid_t owner;
68602120 145 u32 options;
e51060f0 146 u8 srq;
a81c994d 147 u8 tos;
a9bb7912 148 u8 reuseaddr;
5b0ec991 149 u8 afonly;
e51060f0
SH		 150};
151
c8f6a362
SH		 152struct cma_multicast {
153 struct rdma_id_private *id_priv;
154 union {
155 struct ib_sa_multicast *ib;
156 } multicast;
157 struct list_head list;
158 void *context;
3f446754 159 struct sockaddr_storage addr;
3c86aa70 160 struct kref mcref;
c8f6a362
SH		 161};
162
e51060f0
SH		 163struct cma_work {
164 struct work_struct work;
165 struct rdma_id_private *id;
550e5ca7
NM		 166 enum rdma_cm_state old_state;
167 enum rdma_cm_state new_state;
e51060f0
SH		 168 struct rdma_cm_event event;
169};
170
dd5bdff8
OG		 171struct cma_ndev_work {
172 struct work_struct work;
173 struct rdma_id_private *id;
174 struct rdma_cm_event event;
175};
176
3c86aa70
EC		 177struct iboe_mcast_work {
178 struct work_struct work;
179 struct rdma_id_private *id;
180 struct cma_multicast *mc;
181};
182
e51060f0
SH		 183union cma_ip_addr {
184 struct in6_addr ip6;
185 struct {
1b90c137
AV		 186 __be32 pad[3];
187 __be32 addr;
e51060f0
SH		 188 } ip4;
189};
190
191struct cma_hdr {
192 u8 cma_version;
193 u8 ip_version; /* IP version: 7:4 */
1b90c137 194 __be16 port;
e51060f0
SH		 195 union cma_ip_addr src_addr;
196 union cma_ip_addr dst_addr;
197};
198
199struct sdp_hh {
200 u8 bsdh[16];
201 u8 sdp_version; /* Major version: 7:4 */
202 u8 ip_version; /* IP version: 7:4 */
203 u8 sdp_specific1[10];
1b90c137
AV		 204 __be16 port;
205 __be16 sdp_specific2;
e51060f0
SH		 206 union cma_ip_addr src_addr;
207 union cma_ip_addr dst_addr;
208};
209
210struct sdp_hah {
211 u8 bsdh[16];
212 u8 sdp_version;
213};
214
215#define CMA_VERSION 0x00
216#define SDP_MAJ_VERSION 0x2
217
550e5ca7 218static int cma_comp(struct rdma_id_private *id_priv, enum rdma_cm_state comp)
e51060f0
SH		 219{
220 unsigned long flags;
221 int ret;
222
223 spin_lock_irqsave(&id_priv->lock, flags);
224 ret = (id_priv->state == comp);
225 spin_unlock_irqrestore(&id_priv->lock, flags);
226 return ret;
227}
228
229static int cma_comp_exch(struct rdma_id_private *id_priv,
550e5ca7 230 enum rdma_cm_state comp, enum rdma_cm_state exch)
e51060f0
SH		 231{
232 unsigned long flags;
233 int ret;
234
235 spin_lock_irqsave(&id_priv->lock, flags);
236 if ((ret = (id_priv->state == comp)))
237 id_priv->state = exch;
238 spin_unlock_irqrestore(&id_priv->lock, flags);
239 return ret;
240}
241
550e5ca7
NM		 242static enum rdma_cm_state cma_exch(struct rdma_id_private *id_priv,
243 enum rdma_cm_state exch)
e51060f0
SH		 244{
245 unsigned long flags;
550e5ca7 246 enum rdma_cm_state old;
e51060f0
SH		 247
248 spin_lock_irqsave(&id_priv->lock, flags);
249 old = id_priv->state;
250 id_priv->state = exch;
251 spin_unlock_irqrestore(&id_priv->lock, flags);
252 return old;
253}
254
255static inline u8 cma_get_ip_ver(struct cma_hdr *hdr)
256{
257 return hdr->ip_version >> 4;
258}
259
260static inline void cma_set_ip_ver(struct cma_hdr *hdr, u8 ip_ver)
261{
262 hdr->ip_version = (ip_ver << 4) | (hdr->ip_version & 0xF);
263}
264
265static inline u8 sdp_get_majv(u8 sdp_version)
266{
267 return sdp_version >> 4;
268}
269
270static inline u8 sdp_get_ip_ver(struct sdp_hh *hh)
271{
272 return hh->ip_version >> 4;
273}
274
275static inline void sdp_set_ip_ver(struct sdp_hh *hh, u8 ip_ver)
276{
277 hh->ip_version = (ip_ver << 4) | (hh->ip_version & 0xF);
278}
279
280static void cma_attach_to_dev(struct rdma_id_private *id_priv,
281 struct cma_device *cma_dev)
282{
283 atomic_inc(&cma_dev->refcount);
284 id_priv->cma_dev = cma_dev;
285 id_priv->id.device = cma_dev->device;
3c86aa70
EC		 286 id_priv->id.route.addr.dev_addr.transport =
287 rdma_node_get_transport(cma_dev->device->node_type);
e51060f0
SH		 288 list_add_tail(&id_priv->list, &cma_dev->id_list);
289}
290
291static inline void cma_deref_dev(struct cma_device *cma_dev)
292{
293 if (atomic_dec_and_test(&cma_dev->refcount))
294 complete(&cma_dev->comp);
295}
296
3c86aa70
EC		 297static inline void release_mc(struct kref *kref)
298{
299 struct cma_multicast *mc = container_of(kref, struct cma_multicast, mcref);
300
301 kfree(mc->multicast.ib);
302 kfree(mc);
303}
304
a396d43a 305static void cma_release_dev(struct rdma_id_private *id_priv)
e51060f0 306{
a396d43a 307 mutex_lock(&lock);
e51060f0
SH		 308 list_del(&id_priv->list);
309 cma_deref_dev(id_priv->cma_dev);
310 id_priv->cma_dev = NULL;
a396d43a 311 mutex_unlock(&lock);
e51060f0
SH		 312}
313
d2ca39f2 314static int cma_set_qkey(struct rdma_id_private *id_priv)
c8f6a362
SH		 315{
316 struct ib_sa_mcmember_rec rec;
317 int ret = 0;
318
d2ca39f2
YE		 319 if (id_priv->qkey)
320 return 0;
321
322 switch (id_priv->id.ps) {
c8f6a362 323 case RDMA_PS_UDP:
d2ca39f2 324 id_priv->qkey = RDMA_UDP_QKEY;
c8f6a362
SH		 325 break;
326 case RDMA_PS_IPOIB:
d2ca39f2
YE		 327 ib_addr_get_mgid(&id_priv->id.route.addr.dev_addr, &rec.mgid);
328 ret = ib_sa_get_mcmember_rec(id_priv->id.device,
329 id_priv->id.port_num, &rec.mgid,
330 &rec);
331 if (!ret)
332 id_priv->qkey = be32_to_cpu(rec.qkey);
c8f6a362
SH		 333 break;
334 default:
335 break;
336 }
337 return ret;
338}
339
3c86aa70
EC		 340static int find_gid_port(struct ib_device *device, union ib_gid *gid, u8 port_num)
341{
342 int i;
343 int err;
344 struct ib_port_attr props;
345 union ib_gid tmp;
346
347 err = ib_query_port(device, port_num, &props);
348 if (err)
63f05be2 349 return err;
3c86aa70
EC		 350
351 for (i = 0; i < props.gid_tbl_len; ++i) {
352 err = ib_query_gid(device, port_num, i, &tmp);
353 if (err)
63f05be2 354 return err;
3c86aa70
EC		 355 if (!memcmp(&tmp, gid, sizeof tmp))
356 return 0;
357 }
358
63f05be2 359 return -EADDRNOTAVAIL;
3c86aa70
EC		 360}
361
680f920a
SH		 362static void cma_translate_ib(struct sockaddr_ib *sib, struct rdma_dev_addr *dev_addr)
363{
364 dev_addr->dev_type = ARPHRD_INFINIBAND;
365 rdma_addr_set_sgid(dev_addr, (union ib_gid *) &sib->sib_addr);
366 ib_addr_set_pkey(dev_addr, ntohs(sib->sib_pkey));
367}
368
369static int cma_translate_addr(struct sockaddr *addr, struct rdma_dev_addr *dev_addr)
370{
371 int ret;
372
373 if (addr->sa_family != AF_IB) {
374 ret = rdma_translate_ip(addr, dev_addr);
375 } else {
376 cma_translate_ib((struct sockaddr_ib *) addr, dev_addr);
377 ret = 0;
378 }
379
380 return ret;
381}
382
07ebafba 383static int cma_acquire_dev(struct rdma_id_private *id_priv)
e51060f0 384{
c8f6a362 385 struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
e51060f0 386 struct cma_device *cma_dev;
3c86aa70 387 union ib_gid gid, iboe_gid;
e51060f0 388 int ret = -ENODEV;
3c86aa70
EC		 389 u8 port;
390 enum rdma_link_layer dev_ll = dev_addr->dev_type == ARPHRD_INFINIBAND ?
391 IB_LINK_LAYER_INFINIBAND : IB_LINK_LAYER_ETHERNET;
e51060f0 392
2efdd6a0
MS		 393 if (dev_ll != IB_LINK_LAYER_INFINIBAND &&
394 id_priv->id.ps == RDMA_PS_IPOIB)
395 return -EINVAL;
396
a396d43a 397 mutex_lock(&lock);
3c86aa70
EC		 398 iboe_addr_get_sgid(dev_addr, &iboe_gid);
399 memcpy(&gid, dev_addr->src_dev_addr +
400 rdma_addr_gid_offset(dev_addr), sizeof gid);
e51060f0 401 list_for_each_entry(cma_dev, &dev_list, list) {
3c86aa70
EC		 402 for (port = 1; port <= cma_dev->device->phys_port_cnt; ++port) {
403 if (rdma_port_get_link_layer(cma_dev->device, port) == dev_ll) {
404 if (rdma_node_get_transport(cma_dev->device->node_type) == RDMA_TRANSPORT_IB &&
405 rdma_port_get_link_layer(cma_dev->device, port) == IB_LINK_LAYER_ETHERNET)
406 ret = find_gid_port(cma_dev->device, &iboe_gid, port);
407 else
408 ret = find_gid_port(cma_dev->device, &gid, port);
409
410 if (!ret) {
411 id_priv->id.port_num = port;
412 goto out;
63f05be2 413 }
3c86aa70 414 }
e51060f0
SH		 415 }
416 }
3c86aa70
EC		 417
418out:
419 if (!ret)
420 cma_attach_to_dev(id_priv, cma_dev);
421
a396d43a 422 mutex_unlock(&lock);
e51060f0
SH		 423 return ret;
424}
425
e51060f0
SH		 426static void cma_deref_id(struct rdma_id_private *id_priv)
427{
428 if (atomic_dec_and_test(&id_priv->refcount))
429 complete(&id_priv->comp);
430}
431
de910bd9 432static int cma_disable_callback(struct rdma_id_private *id_priv,
550e5ca7 433 enum rdma_cm_state state)
8aa08602 434{
de910bd9
OG		 435 mutex_lock(&id_priv->handler_mutex);
436 if (id_priv->state != state) {
437 mutex_unlock(&id_priv->handler_mutex);
438 return -EINVAL;
439 }
440 return 0;
e51060f0
SH		 441}
442
443struct rdma_cm_id *rdma_create_id(rdma_cm_event_handler event_handler,
b26f9b99
SH		 444 void *context, enum rdma_port_space ps,
445 enum ib_qp_type qp_type)
e51060f0
SH		 446{
447 struct rdma_id_private *id_priv;
448
449 id_priv = kzalloc(sizeof *id_priv, GFP_KERNEL);
450 if (!id_priv)
451 return ERR_PTR(-ENOMEM);
452
83e9502d 453 id_priv->owner = task_pid_nr(current);
550e5ca7 454 id_priv->state = RDMA_CM_IDLE;
e51060f0
SH		 455 id_priv->id.context = context;
456 id_priv->id.event_handler = event_handler;
457 id_priv->id.ps = ps;
b26f9b99 458 id_priv->id.qp_type = qp_type;
e51060f0 459 spin_lock_init(&id_priv->lock);
c5483388 460 mutex_init(&id_priv->qp_mutex);
e51060f0
SH		 461 init_completion(&id_priv->comp);
462 atomic_set(&id_priv->refcount, 1);
de910bd9 463 mutex_init(&id_priv->handler_mutex);
e51060f0 464 INIT_LIST_HEAD(&id_priv->listen_list);
c8f6a362 465 INIT_LIST_HEAD(&id_priv->mc_list);
e51060f0
SH		 466 get_random_bytes(&id_priv->seq_num, sizeof id_priv->seq_num);
467
468 return &id_priv->id;
469}
470EXPORT_SYMBOL(rdma_create_id);
471
c8f6a362 472static int cma_init_ud_qp(struct rdma_id_private *id_priv, struct ib_qp *qp)
e51060f0
SH		 473{
474 struct ib_qp_attr qp_attr;
c8f6a362 475 int qp_attr_mask, ret;
e51060f0 476
c8f6a362
SH		 477 qp_attr.qp_state = IB_QPS_INIT;
478 ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
e51060f0
SH		 479 if (ret)
480 return ret;
481
c8f6a362
SH		 482 ret = ib_modify_qp(qp, &qp_attr, qp_attr_mask);
483 if (ret)
484 return ret;
485
486 qp_attr.qp_state = IB_QPS_RTR;
487 ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE);
488 if (ret)
489 return ret;
490
491 qp_attr.qp_state = IB_QPS_RTS;
492 qp_attr.sq_psn = 0;
493 ret = ib_modify_qp(qp, &qp_attr, IB_QP_STATE | IB_QP_SQ_PSN);
494
495 return ret;
e51060f0
SH		 496}
497
c8f6a362 498static int cma_init_conn_qp(struct rdma_id_private *id_priv, struct ib_qp *qp)
07ebafba
TT		 499{
500 struct ib_qp_attr qp_attr;
c8f6a362 501 int qp_attr_mask, ret;
07ebafba
TT		 502
503 qp_attr.qp_state = IB_QPS_INIT;
c8f6a362
SH		 504 ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
505 if (ret)
506 return ret;
07ebafba 507
c8f6a362 508 return ib_modify_qp(qp, &qp_attr, qp_attr_mask);
07ebafba
TT		 509}
510
e51060f0
SH		 511int rdma_create_qp(struct rdma_cm_id *id, struct ib_pd *pd,
512 struct ib_qp_init_attr *qp_init_attr)
513{
514 struct rdma_id_private *id_priv;
515 struct ib_qp *qp;
516 int ret;
517
518 id_priv = container_of(id, struct rdma_id_private, id);
519 if (id->device != pd->device)
520 return -EINVAL;
521
522 qp = ib_create_qp(pd, qp_init_attr);
523 if (IS_ERR(qp))
524 return PTR_ERR(qp);
525
b26f9b99 526 if (id->qp_type == IB_QPT_UD)
c8f6a362
SH		 527 ret = cma_init_ud_qp(id_priv, qp);
528 else
529 ret = cma_init_conn_qp(id_priv, qp);
e51060f0
SH		 530 if (ret)
531 goto err;
532
533 id->qp = qp;
534 id_priv->qp_num = qp->qp_num;
e51060f0
SH		 535 id_priv->srq = (qp->srq != NULL);
536 return 0;
537err:
538 ib_destroy_qp(qp);
539 return ret;
540}
541EXPORT_SYMBOL(rdma_create_qp);
542
543void rdma_destroy_qp(struct rdma_cm_id *id)
544{
c5483388
SH		 545 struct rdma_id_private *id_priv;
546
547 id_priv = container_of(id, struct rdma_id_private, id);
548 mutex_lock(&id_priv->qp_mutex);
549 ib_destroy_qp(id_priv->id.qp);
550 id_priv->id.qp = NULL;
551 mutex_unlock(&id_priv->qp_mutex);
e51060f0
SH		 552}
553EXPORT_SYMBOL(rdma_destroy_qp);
554
5851bb89
SH		 555static int cma_modify_qp_rtr(struct rdma_id_private *id_priv,
556 struct rdma_conn_param *conn_param)
e51060f0
SH		 557{
558 struct ib_qp_attr qp_attr;
559 int qp_attr_mask, ret;
560
c5483388
SH		 561 mutex_lock(&id_priv->qp_mutex);
562 if (!id_priv->id.qp) {
563 ret = 0;
564 goto out;
565 }
e51060f0
SH		 566
567 /* Need to update QP attributes from default values. */
568 qp_attr.qp_state = IB_QPS_INIT;
c5483388 569 ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
e51060f0 570 if (ret)
c5483388 571 goto out;
e51060f0 572
c5483388 573 ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
e51060f0 574 if (ret)
c5483388 575 goto out;
e51060f0
SH		 576
577 qp_attr.qp_state = IB_QPS_RTR;
c5483388 578 ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
e51060f0 579 if (ret)
c5483388 580 goto out;
e51060f0 581
5851bb89
SH		 582 if (conn_param)
583 qp_attr.max_dest_rd_atomic = conn_param->responder_resources;
c5483388
SH		 584 ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
585out:
586 mutex_unlock(&id_priv->qp_mutex);
587 return ret;
e51060f0
SH		 588}
589
5851bb89
SH		 590static int cma_modify_qp_rts(struct rdma_id_private *id_priv,
591 struct rdma_conn_param *conn_param)
e51060f0
SH		 592{
593 struct ib_qp_attr qp_attr;
594 int qp_attr_mask, ret;
595
c5483388
SH		 596 mutex_lock(&id_priv->qp_mutex);
597 if (!id_priv->id.qp) {
598 ret = 0;
599 goto out;
600 }
e51060f0
SH		 601
602 qp_attr.qp_state = IB_QPS_RTS;
c5483388 603 ret = rdma_init_qp_attr(&id_priv->id, &qp_attr, &qp_attr_mask);
e51060f0 604 if (ret)
c5483388 605 goto out;
e51060f0 606
5851bb89
SH		 607 if (conn_param)
608 qp_attr.max_rd_atomic = conn_param->initiator_depth;
c5483388
SH		 609 ret = ib_modify_qp(id_priv->id.qp, &qp_attr, qp_attr_mask);
610out:
611 mutex_unlock(&id_priv->qp_mutex);
612 return ret;
e51060f0
SH		 613}
614
c5483388 615static int cma_modify_qp_err(struct rdma_id_private *id_priv)
e51060f0
SH		 616{
617 struct ib_qp_attr qp_attr;
c5483388 618 int ret;
e51060f0 619
c5483388
SH		 620 mutex_lock(&id_priv->qp_mutex);
621 if (!id_priv->id.qp) {
622 ret = 0;
623 goto out;
624 }
e51060f0
SH		 625
626 qp_attr.qp_state = IB_QPS_ERR;
c5483388
SH		 627 ret = ib_modify_qp(id_priv->id.qp, &qp_attr, IB_QP_STATE);
628out:
629 mutex_unlock(&id_priv->qp_mutex);
630 return ret;
e51060f0
SH		 631}
632
c8f6a362
SH		 633static int cma_ib_init_qp_attr(struct rdma_id_private *id_priv,
634 struct ib_qp_attr *qp_attr, int *qp_attr_mask)
635{
636 struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
637 int ret;
3c86aa70
EC		 638 u16 pkey;
639
640 if (rdma_port_get_link_layer(id_priv->id.device, id_priv->id.port_num) ==
641 IB_LINK_LAYER_INFINIBAND)
642 pkey = ib_addr_get_pkey(dev_addr);
643 else
644 pkey = 0xffff;
c8f6a362
SH		 645
646 ret = ib_find_cached_pkey(id_priv->id.device, id_priv->id.port_num,
3c86aa70 647 pkey, &qp_attr->pkey_index);
c8f6a362
SH		 648 if (ret)
649 return ret;
650
651 qp_attr->port_num = id_priv->id.port_num;
652 *qp_attr_mask = IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT;
653
b26f9b99 654 if (id_priv->id.qp_type == IB_QPT_UD) {
d2ca39f2
YE		 655 ret = cma_set_qkey(id_priv);
656 if (ret)
657 return ret;
658
c8f6a362
SH		 659 qp_attr->qkey = id_priv->qkey;
660 *qp_attr_mask |= IB_QP_QKEY;
661 } else {
662 qp_attr->qp_access_flags = 0;
663 *qp_attr_mask |= IB_QP_ACCESS_FLAGS;
664 }
665 return 0;
666}
667
e51060f0
SH		 668int rdma_init_qp_attr(struct rdma_cm_id *id, struct ib_qp_attr *qp_attr,
669 int *qp_attr_mask)
670{
671 struct rdma_id_private *id_priv;
c8f6a362 672 int ret = 0;
e51060f0
SH		 673
674 id_priv = container_of(id, struct rdma_id_private, id);
07ebafba
TT		 675 switch (rdma_node_get_transport(id_priv->id.device->node_type)) {
676 case RDMA_TRANSPORT_IB:
b26f9b99 677 if (!id_priv->cm_id.ib || (id_priv->id.qp_type == IB_QPT_UD))
c8f6a362
SH		 678 ret = cma_ib_init_qp_attr(id_priv, qp_attr, qp_attr_mask);
679 else
680 ret = ib_cm_init_qp_attr(id_priv->cm_id.ib, qp_attr,
681 qp_attr_mask);
e51060f0
SH		 682 if (qp_attr->qp_state == IB_QPS_RTR)
683 qp_attr->rq_psn = id_priv->seq_num;
684 break;
07ebafba 685 case RDMA_TRANSPORT_IWARP:
c8f6a362 686 if (!id_priv->cm_id.iw) {
8f076531 687 qp_attr->qp_access_flags = 0;
c8f6a362
SH		 688 *qp_attr_mask = IB_QP_STATE | IB_QP_ACCESS_FLAGS;
689 } else
690 ret = iw_cm_init_qp_attr(id_priv->cm_id.iw, qp_attr,
691 qp_attr_mask);
07ebafba 692 break;
e51060f0
SH		 693 default:
694 ret = -ENOSYS;
695 break;
696 }
697
698 return ret;
699}
700EXPORT_SYMBOL(rdma_init_qp_attr);
701
702static inline int cma_zero_addr(struct sockaddr *addr)
703{
2e2d190c
SH		 704 switch (addr->sa_family) {
705 case AF_INET:
706 return ipv4_is_zeronet(((struct sockaddr_in *)addr)->sin_addr.s_addr);
707 case AF_INET6:
708 return ipv6_addr_any(&((struct sockaddr_in6 *) addr)->sin6_addr);
709 case AF_IB:
710 return ib_addr_any(&((struct sockaddr_ib *) addr)->sib_addr);
711 default:
712 return 0;
e51060f0
SH		 713 }
714}
715
716static inline int cma_loopback_addr(struct sockaddr *addr)
717{
2e2d190c
SH		 718 switch (addr->sa_family) {
719 case AF_INET:
720 return ipv4_is_loopback(((struct sockaddr_in *) addr)->sin_addr.s_addr);
721 case AF_INET6:
722 return ipv6_addr_loopback(&((struct sockaddr_in6 *) addr)->sin6_addr);
723 case AF_IB:
724 return ib_addr_loopback(&((struct sockaddr_ib *) addr)->sib_addr);
725 default:
726 return 0;
727 }
e51060f0
SH		 728}
729
730static inline int cma_any_addr(struct sockaddr *addr)
731{
732 return cma_zero_addr(addr) || cma_loopback_addr(addr);
733}
734
43b752da
HS		 735static int cma_addr_cmp(struct sockaddr *src, struct sockaddr *dst)
736{
737 if (src->sa_family != dst->sa_family)
738 return -1;
739
740 switch (src->sa_family) {
741 case AF_INET:
742 return ((struct sockaddr_in *) src)->sin_addr.s_addr !=
743 ((struct sockaddr_in *) dst)->sin_addr.s_addr;
2e2d190c 744 case AF_INET6:
43b752da
HS		 745 return ipv6_addr_cmp(&((struct sockaddr_in6 *) src)->sin6_addr,
746 &((struct sockaddr_in6 *) dst)->sin6_addr);
2e2d190c
SH		 747 default:
748 return ib_addr_cmp(&((struct sockaddr_ib *) src)->sib_addr,
749 &((struct sockaddr_ib *) dst)->sib_addr);
43b752da
HS		 750 }
751}
752
58afdcb7 753static __be16 cma_port(struct sockaddr *addr)
628e5f6d 754{
58afdcb7
SH		 755 struct sockaddr_ib *sib;
756
757 switch (addr->sa_family) {
758 case AF_INET:
628e5f6d 759 return ((struct sockaddr_in *) addr)->sin_port;
58afdcb7 760 case AF_INET6:
628e5f6d 761 return ((struct sockaddr_in6 *) addr)->sin6_port;
58afdcb7
SH		 762 case AF_IB:
763 sib = (struct sockaddr_ib *) addr;
764 return htons((u16) (be64_to_cpu(sib->sib_sid) &
765 be64_to_cpu(sib->sib_sid_mask)));
766 default:
767 return 0;
768 }
628e5f6d
SH		 769}
770
e51060f0
SH		 771static inline int cma_any_port(struct sockaddr *addr)
772{
628e5f6d 773 return !cma_port(addr);
e51060f0
SH		 774}
775
776static int cma_get_net_info(void *hdr, enum rdma_port_space ps,
1b90c137 777 u8 *ip_ver, __be16 *port,
e51060f0
SH		 778 union cma_ip_addr **src, union cma_ip_addr **dst)
779{
780 switch (ps) {
781 case RDMA_PS_SDP:
782 if (sdp_get_majv(((struct sdp_hh *) hdr)->sdp_version) !=
783 SDP_MAJ_VERSION)
784 return -EINVAL;
785
786 *ip_ver = sdp_get_ip_ver(hdr);
787 *port = ((struct sdp_hh *) hdr)->port;
788 *src = &((struct sdp_hh *) hdr)->src_addr;
789 *dst = &((struct sdp_hh *) hdr)->dst_addr;
790 break;
791 default:
792 if (((struct cma_hdr *) hdr)->cma_version != CMA_VERSION)
793 return -EINVAL;
794
795 *ip_ver = cma_get_ip_ver(hdr);
796 *port = ((struct cma_hdr *) hdr)->port;
797 *src = &((struct cma_hdr *) hdr)->src_addr;
798 *dst = &((struct cma_hdr *) hdr)->dst_addr;
799 break;
800 }
801
802 if (*ip_ver != 4 && *ip_ver != 6)
803 return -EINVAL;
804 return 0;
805}
806
807static void cma_save_net_info(struct rdma_addr *addr,
808 struct rdma_addr *listen_addr,
1b90c137 809 u8 ip_ver, __be16 port,
e51060f0
SH		 810 union cma_ip_addr *src, union cma_ip_addr *dst)
811{
812 struct sockaddr_in *listen4, *ip4;
813 struct sockaddr_in6 *listen6, *ip6;
814
815 switch (ip_ver) {
816 case 4:
817 listen4 = (struct sockaddr_in *) &listen_addr->src_addr;
818 ip4 = (struct sockaddr_in *) &addr->src_addr;
819 ip4->sin_family = listen4->sin_family;
820 ip4->sin_addr.s_addr = dst->ip4.addr;
821 ip4->sin_port = listen4->sin_port;
822
823 ip4 = (struct sockaddr_in *) &addr->dst_addr;
824 ip4->sin_family = listen4->sin_family;
825 ip4->sin_addr.s_addr = src->ip4.addr;
826 ip4->sin_port = port;
827 break;
828 case 6:
829 listen6 = (struct sockaddr_in6 *) &listen_addr->src_addr;
830 ip6 = (struct sockaddr_in6 *) &addr->src_addr;
831 ip6->sin6_family = listen6->sin6_family;
832 ip6->sin6_addr = dst->ip6;
833 ip6->sin6_port = listen6->sin6_port;
834
835 ip6 = (struct sockaddr_in6 *) &addr->dst_addr;
836 ip6->sin6_family = listen6->sin6_family;
837 ip6->sin6_addr = src->ip6;
838 ip6->sin6_port = port;
839 break;
840 default:
841 break;
842 }
843}
844
845static inline int cma_user_data_offset(enum rdma_port_space ps)
846{
847 switch (ps) {
848 case RDMA_PS_SDP:
849 return 0;
850 default:
851 return sizeof(struct cma_hdr);
852 }
853}
854
e51060f0
SH		 855static void cma_cancel_route(struct rdma_id_private *id_priv)
856{
3c86aa70
EC		 857 switch (rdma_port_get_link_layer(id_priv->id.device, id_priv->id.port_num)) {
858 case IB_LINK_LAYER_INFINIBAND:
e51060f0
SH		 859 if (id_priv->query)
860 ib_sa_cancel_query(id_priv->query_id, id_priv->query);
861 break;
862 default:
863 break;
864 }
865}
866
e51060f0
SH		 867static void cma_cancel_listens(struct rdma_id_private *id_priv)
868{
869 struct rdma_id_private *dev_id_priv;
870
d02d1f53
SH		 871 /*
872 * Remove from listen_any_list to prevent added devices from spawning
873 * additional listen requests.
874 */
e51060f0
SH		 875 mutex_lock(&lock);
876 list_del(&id_priv->list);
877
878 while (!list_empty(&id_priv->listen_list)) {
879 dev_id_priv = list_entry(id_priv->listen_list.next,
880 struct rdma_id_private, listen_list);
d02d1f53
SH		 881 /* sync with device removal to avoid duplicate destruction */
882 list_del_init(&dev_id_priv->list);
883 list_del(&dev_id_priv->listen_list);
884 mutex_unlock(&lock);
885
886 rdma_destroy_id(&dev_id_priv->id);
887 mutex_lock(&lock);
e51060f0
SH		 888 }
889 mutex_unlock(&lock);
890}
891
892static void cma_cancel_operation(struct rdma_id_private *id_priv,
550e5ca7 893 enum rdma_cm_state state)
e51060f0
SH		 894{
895 switch (state) {
550e5ca7 896 case RDMA_CM_ADDR_QUERY:
e51060f0
SH		 897 rdma_addr_cancel(&id_priv->id.route.addr.dev_addr);
898 break;
550e5ca7 899 case RDMA_CM_ROUTE_QUERY:
e51060f0
SH		 900 cma_cancel_route(id_priv);
901 break;
550e5ca7 902 case RDMA_CM_LISTEN:
3f446754
RD		 903 if (cma_any_addr((struct sockaddr *) &id_priv->id.route.addr.src_addr)
904 && !id_priv->cma_dev)
e51060f0
SH		 905 cma_cancel_listens(id_priv);
906 break;
907 default:
908 break;
909 }
910}
911
912static void cma_release_port(struct rdma_id_private *id_priv)
913{
914 struct rdma_bind_list *bind_list = id_priv->bind_list;
915
916 if (!bind_list)
917 return;
918
919 mutex_lock(&lock);
920 hlist_del(&id_priv->node);
921 if (hlist_empty(&bind_list->owners)) {
922 idr_remove(bind_list->ps, bind_list->port);
923 kfree(bind_list);
924 }
925 mutex_unlock(&lock);
926}
927
c8f6a362
SH		 928static void cma_leave_mc_groups(struct rdma_id_private *id_priv)
929{
930 struct cma_multicast *mc;
931
932 while (!list_empty(&id_priv->mc_list)) {
933 mc = container_of(id_priv->mc_list.next,
934 struct cma_multicast, list);
935 list_del(&mc->list);
3c86aa70
EC		 936 switch (rdma_port_get_link_layer(id_priv->cma_dev->device, id_priv->id.port_num)) {
937 case IB_LINK_LAYER_INFINIBAND:
938 ib_sa_free_multicast(mc->multicast.ib);
939 kfree(mc);
940 break;
941 case IB_LINK_LAYER_ETHERNET:
942 kref_put(&mc->mcref, release_mc);
943 break;
944 default:
945 break;
946 }
c8f6a362
SH		 947 }
948}
949
e51060f0
SH		 950void rdma_destroy_id(struct rdma_cm_id *id)
951{
952 struct rdma_id_private *id_priv;
550e5ca7 953 enum rdma_cm_state state;
e51060f0
SH		 954
955 id_priv = container_of(id, struct rdma_id_private, id);
550e5ca7 956 state = cma_exch(id_priv, RDMA_CM_DESTROYING);
e51060f0
SH		 957 cma_cancel_operation(id_priv, state);
958
a396d43a
SH		 959 /*
960 * Wait for any active callback to finish. New callbacks will find
961 * the id_priv state set to destroying and abort.
962 */
963 mutex_lock(&id_priv->handler_mutex);
964 mutex_unlock(&id_priv->handler_mutex);
965
e51060f0 966 if (id_priv->cma_dev) {
3c86aa70 967 switch (rdma_node_get_transport(id_priv->id.device->node_type)) {
07ebafba 968 case RDMA_TRANSPORT_IB:
0c9361fc 969 if (id_priv->cm_id.ib)
e51060f0
SH		 970 ib_destroy_cm_id(id_priv->cm_id.ib);
971 break;
07ebafba 972 case RDMA_TRANSPORT_IWARP:
0c9361fc 973 if (id_priv->cm_id.iw)
07ebafba
TT		 974 iw_destroy_cm_id(id_priv->cm_id.iw);
975 break;
e51060f0
SH		 976 default:
977 break;
978 }
c8f6a362 979 cma_leave_mc_groups(id_priv);
a396d43a 980 cma_release_dev(id_priv);
e51060f0
SH		 981 }
982
983 cma_release_port(id_priv);
984 cma_deref_id(id_priv);
985 wait_for_completion(&id_priv->comp);
986
d02d1f53
SH		 987 if (id_priv->internal_id)
988 cma_deref_id(id_priv->id.context);
989
e51060f0
SH		 990 kfree(id_priv->id.route.path_rec);
991 kfree(id_priv);
992}
993EXPORT_SYMBOL(rdma_destroy_id);
994
995static int cma_rep_recv(struct rdma_id_private *id_priv)
996{
997 int ret;
998
5851bb89 999 ret = cma_modify_qp_rtr(id_priv, NULL);
e51060f0
SH		 1000 if (ret)
1001 goto reject;
1002
5851bb89 1003 ret = cma_modify_qp_rts(id_priv, NULL);
e51060f0
SH		 1004 if (ret)
1005 goto reject;
1006
1007 ret = ib_send_cm_rtu(id_priv->cm_id.ib, NULL, 0);
1008 if (ret)
1009 goto reject;
1010
1011 return 0;
1012reject:
c5483388 1013 cma_modify_qp_err(id_priv);
e51060f0
SH		 1014 ib_send_cm_rej(id_priv->cm_id.ib, IB_CM_REJ_CONSUMER_DEFINED,
1015 NULL, 0, NULL, 0);
1016 return ret;
1017}
1018
1019static int cma_verify_rep(struct rdma_id_private *id_priv, void *data)
1020{
1021 if (id_priv->id.ps == RDMA_PS_SDP &&
1022 sdp_get_majv(((struct sdp_hah *) data)->sdp_version) !=
1023 SDP_MAJ_VERSION)
1024 return -EINVAL;
1025
1026 return 0;
1027}
1028
a1b1b61f
SH		 1029static void cma_set_rep_event_data(struct rdma_cm_event *event,
1030 struct ib_cm_rep_event_param *rep_data,
1031 void *private_data)
1032{
1033 event->param.conn.private_data = private_data;
1034 event->param.conn.private_data_len = IB_CM_REP_PRIVATE_DATA_SIZE;
1035 event->param.conn.responder_resources = rep_data->responder_resources;
1036 event->param.conn.initiator_depth = rep_data->initiator_depth;
1037 event->param.conn.flow_control = rep_data->flow_control;
1038 event->param.conn.rnr_retry_count = rep_data->rnr_retry_count;
1039 event->param.conn.srq = rep_data->srq;
1040 event->param.conn.qp_num = rep_data->remote_qpn;
1041}
1042
e51060f0
SH		 1043static int cma_ib_handler(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event)
1044{
1045 struct rdma_id_private *id_priv = cm_id->context;
a1b1b61f
SH		 1046 struct rdma_cm_event event;
1047 int ret = 0;
e51060f0 1048
38ca83a5 1049 if ((ib_event->event != IB_CM_TIMEWAIT_EXIT &&
550e5ca7 1050 cma_disable_callback(id_priv, RDMA_CM_CONNECT)) ||
38ca83a5 1051 (ib_event->event == IB_CM_TIMEWAIT_EXIT &&
550e5ca7 1052 cma_disable_callback(id_priv, RDMA_CM_DISCONNECT)))
8aa08602 1053 return 0;
e51060f0 1054
a1b1b61f 1055 memset(&event, 0, sizeof event);
e51060f0
SH		 1056 switch (ib_event->event) {
1057 case IB_CM_REQ_ERROR:
1058 case IB_CM_REP_ERROR:
a1b1b61f
SH		 1059 event.event = RDMA_CM_EVENT_UNREACHABLE;
1060 event.status = -ETIMEDOUT;
e51060f0
SH		 1061 break;
1062 case IB_CM_REP_RECEIVED:
a1b1b61f
SH		 1063 event.status = cma_verify_rep(id_priv, ib_event->private_data);
1064 if (event.status)
1065 event.event = RDMA_CM_EVENT_CONNECT_ERROR;
e51060f0 1066 else if (id_priv->id.qp && id_priv->id.ps != RDMA_PS_SDP) {
a1b1b61f
SH		 1067 event.status = cma_rep_recv(id_priv);
1068 event.event = event.status ? RDMA_CM_EVENT_CONNECT_ERROR :
1069 RDMA_CM_EVENT_ESTABLISHED;
e51060f0 1070 } else
a1b1b61f
SH		 1071 event.event = RDMA_CM_EVENT_CONNECT_RESPONSE;
1072 cma_set_rep_event_data(&event, &ib_event->param.rep_rcvd,
1073 ib_event->private_data);
e51060f0
SH		 1074 break;
1075 case IB_CM_RTU_RECEIVED:
0fe313b0
SH		 1076 case IB_CM_USER_ESTABLISHED:
1077 event.event = RDMA_CM_EVENT_ESTABLISHED;
e51060f0
SH		 1078 break;
1079 case IB_CM_DREQ_ERROR:
a1b1b61f 1080 event.status = -ETIMEDOUT; /* fall through */
e51060f0
SH		 1081 case IB_CM_DREQ_RECEIVED:
1082 case IB_CM_DREP_RECEIVED:
550e5ca7
NM		 1083 if (!cma_comp_exch(id_priv, RDMA_CM_CONNECT,
1084 RDMA_CM_DISCONNECT))
e51060f0 1085 goto out;
a1b1b61f 1086 event.event = RDMA_CM_EVENT_DISCONNECTED;
e51060f0
SH		 1087 break;
1088 case IB_CM_TIMEWAIT_EXIT:
38ca83a5
AV		 1089 event.event = RDMA_CM_EVENT_TIMEWAIT_EXIT;
1090 break;
e51060f0
SH		 1091 case IB_CM_MRA_RECEIVED:
1092 /* ignore event */
1093 goto out;
1094 case IB_CM_REJ_RECEIVED:
c5483388 1095 cma_modify_qp_err(id_priv);
a1b1b61f
SH		 1096 event.status = ib_event->param.rej_rcvd.reason;
1097 event.event = RDMA_CM_EVENT_REJECTED;
1098 event.param.conn.private_data = ib_event->private_data;
1099 event.param.conn.private_data_len = IB_CM_REJ_PRIVATE_DATA_SIZE;
e51060f0
SH		 1100 break;
1101 default:
468f2239 1102 printk(KERN_ERR "RDMA CMA: unexpected IB CM event: %d\n",
e51060f0
SH		 1103 ib_event->event);
1104 goto out;
1105 }
1106
a1b1b61f 1107 ret = id_priv->id.event_handler(&id_priv->id, &event);
e51060f0
SH		 1108 if (ret) {
1109 /* Destroy the CM ID by returning a non-zero value. */
1110 id_priv->cm_id.ib = NULL;
550e5ca7 1111 cma_exch(id_priv, RDMA_CM_DESTROYING);
de910bd9 1112 mutex_unlock(&id_priv->handler_mutex);
e51060f0
SH		 1113 rdma_destroy_id(&id_priv->id);
1114 return ret;
1115 }
1116out:
de910bd9 1117 mutex_unlock(&id_priv->handler_mutex);
e51060f0
SH		 1118 return ret;
1119}
1120
628e5f6d
SH		 1121static struct rdma_id_private *cma_new_conn_id(struct rdma_cm_id *listen_id,
1122 struct ib_cm_event *ib_event)
e51060f0
SH		 1123{
1124 struct rdma_id_private *id_priv;
1125 struct rdma_cm_id *id;
1126 struct rdma_route *rt;
1127 union cma_ip_addr *src, *dst;
1b90c137 1128 __be16 port;
e51060f0 1129 u8 ip_ver;
64c5e613 1130 int ret;
e51060f0 1131
3f168d2b
KK		 1132 if (cma_get_net_info(ib_event->private_data, listen_id->ps,
1133 &ip_ver, &port, &src, &dst))
0c9361fc 1134 return NULL;
3f168d2b 1135
e51060f0 1136 id = rdma_create_id(listen_id->event_handler, listen_id->context,
b26f9b99 1137 listen_id->ps, ib_event->param.req_rcvd.qp_type);
e51060f0 1138 if (IS_ERR(id))
0c9361fc 1139 return NULL;
3f168d2b
KK		 1140
1141 cma_save_net_info(&id->route.addr, &listen_id->route.addr,
1142 ip_ver, port, src, dst);
e51060f0
SH		 1143
1144 rt = &id->route;
1145 rt->num_paths = ib_event->param.req_rcvd.alternate_path ? 2 : 1;
3f168d2b
KK		 1146 rt->path_rec = kmalloc(sizeof *rt->path_rec * rt->num_paths,
1147 GFP_KERNEL);
e51060f0 1148 if (!rt->path_rec)
0c9361fc 1149 goto err;
e51060f0 1150
e51060f0
SH		 1151 rt->path_rec[0] = *ib_event->param.req_rcvd.primary_path;
1152 if (rt->num_paths == 2)
1153 rt->path_rec[1] = *ib_event->param.req_rcvd.alternate_path;
1154
6f8372b6
SH		 1155 if (cma_any_addr((struct sockaddr *) &rt->addr.src_addr)) {
1156 rt->addr.dev_addr.dev_type = ARPHRD_INFINIBAND;
1157 rdma_addr_set_sgid(&rt->addr.dev_addr, &rt->path_rec[0].sgid);
46ea5061 1158 ib_addr_set_pkey(&rt->addr.dev_addr, be16_to_cpu(rt->path_rec[0].pkey));
6f8372b6 1159 } else {
680f920a
SH		 1160 ret = cma_translate_addr((struct sockaddr *) &rt->addr.src_addr,
1161 &rt->addr.dev_addr);
6f8372b6 1162 if (ret)
0c9361fc 1163 goto err;
6f8372b6
SH		 1164 }
1165 rdma_addr_set_dgid(&rt->addr.dev_addr, &rt->path_rec[0].dgid);
e51060f0
SH		 1166
1167 id_priv = container_of(id, struct rdma_id_private, id);
550e5ca7 1168 id_priv->state = RDMA_CM_CONNECT;
e51060f0 1169 return id_priv;
3f168d2b 1170
3f168d2b 1171err:
0c9361fc 1172 rdma_destroy_id(id);
e51060f0
SH		 1173 return NULL;
1174}
1175
628e5f6d
SH		 1176static struct rdma_id_private *cma_new_udp_id(struct rdma_cm_id *listen_id,
1177 struct ib_cm_event *ib_event)
1178{
1179 struct rdma_id_private *id_priv;
1180 struct rdma_cm_id *id;
1181 union cma_ip_addr *src, *dst;
1b90c137 1182 __be16 port;
628e5f6d
SH		 1183 u8 ip_ver;
1184 int ret;
1185
1186 id = rdma_create_id(listen_id->event_handler, listen_id->context,
b26f9b99 1187 listen_id->ps, IB_QPT_UD);
628e5f6d
SH		 1188 if (IS_ERR(id))
1189 return NULL;
1190
1191
1192 if (cma_get_net_info(ib_event->private_data, listen_id->ps,
1193 &ip_ver, &port, &src, &dst))
1194 goto err;
1195
1196 cma_save_net_info(&id->route.addr, &listen_id->route.addr,
1197 ip_ver, port, src, dst);
1198
6f8372b6 1199 if (!cma_any_addr((struct sockaddr *) &id->route.addr.src_addr)) {
680f920a
SH		 1200 ret = cma_translate_addr((struct sockaddr *) &id->route.addr.src_addr,
1201 &id->route.addr.dev_addr);
6f8372b6
SH		 1202 if (ret)
1203 goto err;
1204 }
628e5f6d
SH		 1205
1206 id_priv = container_of(id, struct rdma_id_private, id);
550e5ca7 1207 id_priv->state = RDMA_CM_CONNECT;
628e5f6d
SH		 1208 return id_priv;
1209err:
1210 rdma_destroy_id(id);
1211 return NULL;
1212}
1213
a1b1b61f
SH		 1214static void cma_set_req_event_data(struct rdma_cm_event *event,
1215 struct ib_cm_req_event_param *req_data,
1216 void *private_data, int offset)
1217{
1218 event->param.conn.private_data = private_data + offset;
1219 event->param.conn.private_data_len = IB_CM_REQ_PRIVATE_DATA_SIZE - offset;
1220 event->param.conn.responder_resources = req_data->responder_resources;
1221 event->param.conn.initiator_depth = req_data->initiator_depth;
1222 event->param.conn.flow_control = req_data->flow_control;
1223 event->param.conn.retry_count = req_data->retry_count;
1224 event->param.conn.rnr_retry_count = req_data->rnr_retry_count;
1225 event->param.conn.srq = req_data->srq;
1226 event->param.conn.qp_num = req_data->remote_qpn;
1227}
1228
9595480c
HS		 1229static int cma_check_req_qp_type(struct rdma_cm_id *id, struct ib_cm_event *ib_event)
1230{
4dd81e89 1231 return (((ib_event->event == IB_CM_REQ_RECEIVED) &&
9595480c
HS		 1232 (ib_event->param.req_rcvd.qp_type == id->qp_type)) ||
1233 ((ib_event->event == IB_CM_SIDR_REQ_RECEIVED) &&
1234 (id->qp_type == IB_QPT_UD)) ||
1235 (!id->qp_type));
1236}
1237
e51060f0
SH		 1238static int cma_req_handler(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event)
1239{
1240 struct rdma_id_private *listen_id, *conn_id;
a1b1b61f 1241 struct rdma_cm_event event;
e51060f0
SH		 1242 int offset, ret;
1243
1244 listen_id = cm_id->context;
9595480c
HS		 1245 if (!cma_check_req_qp_type(&listen_id->id, ib_event))
1246 return -EINVAL;
1247
550e5ca7 1248 if (cma_disable_callback(listen_id, RDMA_CM_LISTEN))
8aa08602 1249 return -ECONNABORTED;
e51060f0 1250
628e5f6d
SH		 1251 memset(&event, 0, sizeof event);
1252 offset = cma_user_data_offset(listen_id->id.ps);
1253 event.event = RDMA_CM_EVENT_CONNECT_REQUEST;
9595480c 1254 if (ib_event->event == IB_CM_SIDR_REQ_RECEIVED) {
628e5f6d
SH		 1255 conn_id = cma_new_udp_id(&listen_id->id, ib_event);
1256 event.param.ud.private_data = ib_event->private_data + offset;
1257 event.param.ud.private_data_len =
1258 IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE - offset;
1259 } else {
1260 conn_id = cma_new_conn_id(&listen_id->id, ib_event);
1261 cma_set_req_event_data(&event, &ib_event->param.req_rcvd,
1262 ib_event->private_data, offset);
1263 }
e51060f0
SH		 1264 if (!conn_id) {
1265 ret = -ENOMEM;
b6cec8aa 1266 goto err1;
e51060f0
SH		 1267 }
1268
de910bd9 1269 mutex_lock_nested(&conn_id->handler_mutex, SINGLE_DEPTH_NESTING);
07ebafba 1270 ret = cma_acquire_dev(conn_id);
a1a733f6 1271 if (ret)
b6cec8aa 1272 goto err2;
e51060f0
SH		 1273
1274 conn_id->cm_id.ib = cm_id;
1275 cm_id->context = conn_id;
1276 cm_id->cm_handler = cma_ib_handler;
1277
25ae21a1
SH		 1278 /*
1279 * Protect against the user destroying conn_id from another thread
1280 * until we're done accessing it.
1281 */
1282 atomic_inc(&conn_id->refcount);
a1b1b61f 1283 ret = conn_id->id.event_handler(&conn_id->id, &event);
b6cec8aa
SH		 1284 if (ret)
1285 goto err3;
1286
1287 /*
1288 * Acquire mutex to prevent user executing rdma_destroy_id()
1289 * while we're accessing the cm_id.
1290 */
1291 mutex_lock(&lock);
1292 if (cma_comp(conn_id, RDMA_CM_CONNECT) && (conn_id->id.qp_type != IB_QPT_UD))
1293 ib_send_cm_mra(cm_id, CMA_CM_MRA_SETTING, NULL, 0);
1294 mutex_unlock(&lock);
1295 mutex_unlock(&conn_id->handler_mutex);
1296 mutex_unlock(&listen_id->handler_mutex);
25ae21a1 1297 cma_deref_id(conn_id);
b6cec8aa 1298 return 0;
a1a733f6 1299
b6cec8aa
SH		 1300err3:
1301 cma_deref_id(conn_id);
a1a733f6
KK		 1302 /* Destroy the CM ID by returning a non-zero value. */
1303 conn_id->cm_id.ib = NULL;
b6cec8aa 1304err2:
550e5ca7 1305 cma_exch(conn_id, RDMA_CM_DESTROYING);
de910bd9 1306 mutex_unlock(&conn_id->handler_mutex);
b6cec8aa 1307err1:
de910bd9 1308 mutex_unlock(&listen_id->handler_mutex);
b6cec8aa
SH		 1309 if (conn_id)
1310 rdma_destroy_id(&conn_id->id);
e51060f0
SH		 1311 return ret;
1312}
1313
1314static __be64 cma_get_service_id(enum rdma_port_space ps, struct sockaddr *addr)
1315{
628e5f6d 1316 return cpu_to_be64(((u64)ps << 16) + be16_to_cpu(cma_port(addr)));
e51060f0
SH		 1317}
1318
1319static void cma_set_compare_data(enum rdma_port_space ps, struct sockaddr *addr,
1320 struct ib_cm_compare_data *compare)
1321{
1322 struct cma_hdr *cma_data, *cma_mask;
1323 struct sdp_hh *sdp_data, *sdp_mask;
1b90c137 1324 __be32 ip4_addr;
e51060f0
SH		 1325 struct in6_addr ip6_addr;
1326
1327 memset(compare, 0, sizeof *compare);
1328 cma_data = (void *) compare->data;
1329 cma_mask = (void *) compare->mask;
1330 sdp_data = (void *) compare->data;
1331 sdp_mask = (void *) compare->mask;
1332
1333 switch (addr->sa_family) {
1334 case AF_INET:
1335 ip4_addr = ((struct sockaddr_in *) addr)->sin_addr.s_addr;
1336 if (ps == RDMA_PS_SDP) {
1337 sdp_set_ip_ver(sdp_data, 4);
1338 sdp_set_ip_ver(sdp_mask, 0xF);
1339 sdp_data->dst_addr.ip4.addr = ip4_addr;
1b90c137 1340 sdp_mask->dst_addr.ip4.addr = htonl(~0);
e51060f0
SH		 1341 } else {
1342 cma_set_ip_ver(cma_data, 4);
1343 cma_set_ip_ver(cma_mask, 0xF);
406b6a25
SH		 1344 if (!cma_any_addr(addr)) {
1345 cma_data->dst_addr.ip4.addr = ip4_addr;
1346 cma_mask->dst_addr.ip4.addr = htonl(~0);
1347 }
e51060f0
SH		 1348 }
1349 break;
1350 case AF_INET6:
1351 ip6_addr = ((struct sockaddr_in6 *) addr)->sin6_addr;
1352 if (ps == RDMA_PS_SDP) {
1353 sdp_set_ip_ver(sdp_data, 6);
1354 sdp_set_ip_ver(sdp_mask, 0xF);
1355 sdp_data->dst_addr.ip6 = ip6_addr;
1356 memset(&sdp_mask->dst_addr.ip6, 0xFF,
1357 sizeof sdp_mask->dst_addr.ip6);
1358 } else {
1359 cma_set_ip_ver(cma_data, 6);
1360 cma_set_ip_ver(cma_mask, 0xF);
406b6a25
SH		 1361 if (!cma_any_addr(addr)) {
1362 cma_data->dst_addr.ip6 = ip6_addr;
1363 memset(&cma_mask->dst_addr.ip6, 0xFF,
1364 sizeof cma_mask->dst_addr.ip6);
1365 }
e51060f0
SH		 1366 }
1367 break;
1368 default:
1369 break;
1370 }
1371}
1372
07ebafba
TT		 1373static int cma_iw_handler(struct iw_cm_id *iw_id, struct iw_cm_event *iw_event)
1374{
1375 struct rdma_id_private *id_priv = iw_id->context;
a1b1b61f 1376 struct rdma_cm_event event;
07ebafba
TT		 1377 struct sockaddr_in *sin;
1378 int ret = 0;
1379
550e5ca7 1380 if (cma_disable_callback(id_priv, RDMA_CM_CONNECT))
be65f086 1381 return 0;
07ebafba 1382
be65f086 1383 memset(&event, 0, sizeof event);
07ebafba
TT		 1384 switch (iw_event->event) {
1385 case IW_CM_EVENT_CLOSE:
a1b1b61f 1386 event.event = RDMA_CM_EVENT_DISCONNECTED;
07ebafba
TT		 1387 break;
1388 case IW_CM_EVENT_CONNECT_REPLY:
1389 sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr;
1390 *sin = iw_event->local_addr;
1391 sin = (struct sockaddr_in *) &id_priv->id.route.addr.dst_addr;
1392 *sin = iw_event->remote_addr;
881a045f
SW		 1393 switch (iw_event->status) {
1394 case 0:
a1b1b61f 1395 event.event = RDMA_CM_EVENT_ESTABLISHED;
3ebeebc3
KS		 1396 event.param.conn.initiator_depth = iw_event->ird;
1397 event.param.conn.responder_resources = iw_event->ord;
881a045f
SW		 1398 break;
1399 case -ECONNRESET:
1400 case -ECONNREFUSED:
1401 event.event = RDMA_CM_EVENT_REJECTED;
1402 break;
1403 case -ETIMEDOUT:
1404 event.event = RDMA_CM_EVENT_UNREACHABLE;
1405 break;
1406 default:
1407 event.event = RDMA_CM_EVENT_CONNECT_ERROR;
1408 break;
1409 }
07ebafba
TT		 1410 break;
1411 case IW_CM_EVENT_ESTABLISHED:
a1b1b61f 1412 event.event = RDMA_CM_EVENT_ESTABLISHED;
3ebeebc3
KS		 1413 event.param.conn.initiator_depth = iw_event->ird;
1414 event.param.conn.responder_resources = iw_event->ord;
07ebafba
TT		 1415 break;
1416 default:
1417 BUG_ON(1);
1418 }
1419
a1b1b61f
SH		 1420 event.status = iw_event->status;
1421 event.param.conn.private_data = iw_event->private_data;
1422 event.param.conn.private_data_len = iw_event->private_data_len;
1423 ret = id_priv->id.event_handler(&id_priv->id, &event);
07ebafba
TT		 1424 if (ret) {
1425 /* Destroy the CM ID by returning a non-zero value. */
1426 id_priv->cm_id.iw = NULL;
550e5ca7 1427 cma_exch(id_priv, RDMA_CM_DESTROYING);
de910bd9 1428 mutex_unlock(&id_priv->handler_mutex);
07ebafba
TT		 1429 rdma_destroy_id(&id_priv->id);
1430 return ret;
1431 }
1432
de910bd9 1433 mutex_unlock(&id_priv->handler_mutex);
07ebafba
TT		 1434 return ret;
1435}
1436
1437static int iw_conn_req_handler(struct iw_cm_id *cm_id,
1438 struct iw_cm_event *iw_event)
1439{
1440 struct rdma_cm_id *new_cm_id;
1441 struct rdma_id_private *listen_id, *conn_id;
1442 struct sockaddr_in *sin;
1443 struct net_device *dev = NULL;
a1b1b61f 1444 struct rdma_cm_event event;
07ebafba 1445 int ret;
8d8293cf 1446 struct ib_device_attr attr;
07ebafba
TT		 1447
1448 listen_id = cm_id->context;
550e5ca7 1449 if (cma_disable_callback(listen_id, RDMA_CM_LISTEN))
8aa08602 1450 return -ECONNABORTED;
07ebafba
TT		 1451
1452 /* Create a new RDMA id for the new IW CM ID */
1453 new_cm_id = rdma_create_id(listen_id->id.event_handler,
1454 listen_id->id.context,
b26f9b99 1455 RDMA_PS_TCP, IB_QPT_RC);
10f32065 1456 if (IS_ERR(new_cm_id)) {
07ebafba
TT		 1457 ret = -ENOMEM;
1458 goto out;
1459 }
1460 conn_id = container_of(new_cm_id, struct rdma_id_private, id);
de910bd9 1461 mutex_lock_nested(&conn_id->handler_mutex, SINGLE_DEPTH_NESTING);
550e5ca7 1462 conn_id->state = RDMA_CM_CONNECT;
07ebafba 1463
1ab35276 1464 dev = ip_dev_find(&init_net, iw_event->local_addr.sin_addr.s_addr);
07ebafba
TT		 1465 if (!dev) {
1466 ret = -EADDRNOTAVAIL;
de910bd9 1467 mutex_unlock(&conn_id->handler_mutex);
07ebafba
TT		 1468 rdma_destroy_id(new_cm_id);
1469 goto out;
1470 }
1471 ret = rdma_copy_addr(&conn_id->id.route.addr.dev_addr, dev, NULL);
1472 if (ret) {
de910bd9 1473 mutex_unlock(&conn_id->handler_mutex);
07ebafba
TT		 1474 rdma_destroy_id(new_cm_id);
1475 goto out;
1476 }
1477
1478 ret = cma_acquire_dev(conn_id);
1479 if (ret) {
de910bd9 1480 mutex_unlock(&conn_id->handler_mutex);
07ebafba
TT		 1481 rdma_destroy_id(new_cm_id);
1482 goto out;
1483 }
1484
1485 conn_id->cm_id.iw = cm_id;
1486 cm_id->context = conn_id;
1487 cm_id->cm_handler = cma_iw_handler;
1488
1489 sin = (struct sockaddr_in *) &new_cm_id->route.addr.src_addr;
1490 *sin = iw_event->local_addr;
1491 sin = (struct sockaddr_in *) &new_cm_id->route.addr.dst_addr;
1492 *sin = iw_event->remote_addr;
1493
8d8293cf
SW		 1494 ret = ib_query_device(conn_id->id.device, &attr);
1495 if (ret) {
de910bd9 1496 mutex_unlock(&conn_id->handler_mutex);
8d8293cf
SW		 1497 rdma_destroy_id(new_cm_id);
1498 goto out;
1499 }
1500
a1b1b61f
SH		 1501 memset(&event, 0, sizeof event);
1502 event.event = RDMA_CM_EVENT_CONNECT_REQUEST;
1503 event.param.conn.private_data = iw_event->private_data;
1504 event.param.conn.private_data_len = iw_event->private_data_len;
3ebeebc3
KS		 1505 event.param.conn.initiator_depth = iw_event->ird;
1506 event.param.conn.responder_resources = iw_event->ord;
25ae21a1
SH		 1507
1508 /*
1509 * Protect against the user destroying conn_id from another thread
1510 * until we're done accessing it.
1511 */
1512 atomic_inc(&conn_id->refcount);
a1b1b61f 1513 ret = conn_id->id.event_handler(&conn_id->id, &event);
07ebafba
TT		 1514 if (ret) {
1515 /* User wants to destroy the CM ID */
1516 conn_id->cm_id.iw = NULL;
550e5ca7 1517 cma_exch(conn_id, RDMA_CM_DESTROYING);
de910bd9 1518 mutex_unlock(&conn_id->handler_mutex);
25ae21a1 1519 cma_deref_id(conn_id);
07ebafba 1520 rdma_destroy_id(&conn_id->id);
de910bd9 1521 goto out;
07ebafba
TT		 1522 }
1523
de910bd9 1524 mutex_unlock(&conn_id->handler_mutex);
25ae21a1 1525 cma_deref_id(conn_id);
de910bd9 1526
07ebafba
TT		 1527out:
1528 if (dev)
1529 dev_put(dev);
de910bd9 1530 mutex_unlock(&listen_id->handler_mutex);
07ebafba
TT		 1531 return ret;
1532}
1533
e51060f0
SH		 1534static int cma_ib_listen(struct rdma_id_private *id_priv)
1535{
1536 struct ib_cm_compare_data compare_data;
1537 struct sockaddr *addr;
0c9361fc 1538 struct ib_cm_id *id;
e51060f0
SH		 1539 __be64 svc_id;
1540 int ret;
1541
0c9361fc
JM		 1542 id = ib_create_cm_id(id_priv->id.device, cma_req_handler, id_priv);
1543 if (IS_ERR(id))
1544 return PTR_ERR(id);
1545
1546 id_priv->cm_id.ib = id;
e51060f0 1547
3f446754 1548 addr = (struct sockaddr *) &id_priv->id.route.addr.src_addr;
e51060f0 1549 svc_id = cma_get_service_id(id_priv->id.ps, addr);
406b6a25 1550 if (cma_any_addr(addr) && !id_priv->afonly)
e51060f0
SH		 1551 ret = ib_cm_listen(id_priv->cm_id.ib, svc_id, 0, NULL);
1552 else {
1553 cma_set_compare_data(id_priv->id.ps, addr, &compare_data);
1554 ret = ib_cm_listen(id_priv->cm_id.ib, svc_id, 0, &compare_data);
1555 }
1556
1557 if (ret) {
1558 ib_destroy_cm_id(id_priv->cm_id.ib);
1559 id_priv->cm_id.ib = NULL;
1560 }
1561
1562 return ret;
1563}
1564
07ebafba
TT		 1565static int cma_iw_listen(struct rdma_id_private *id_priv, int backlog)
1566{
1567 int ret;
1568 struct sockaddr_in *sin;
0c9361fc
JM		 1569 struct iw_cm_id *id;
1570
1571 id = iw_create_cm_id(id_priv->id.device,
1572 iw_conn_req_handler,
1573 id_priv);
1574 if (IS_ERR(id))
1575 return PTR_ERR(id);
07ebafba 1576
0c9361fc 1577 id_priv->cm_id.iw = id;
07ebafba
TT		 1578
1579 sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr;
1580 id_priv->cm_id.iw->local_addr = *sin;
1581
1582 ret = iw_cm_listen(id_priv->cm_id.iw, backlog);
1583
1584 if (ret) {
1585 iw_destroy_cm_id(id_priv->cm_id.iw);
1586 id_priv->cm_id.iw = NULL;
1587 }
1588
1589 return ret;
1590}
1591
e51060f0
SH		 1592static int cma_listen_handler(struct rdma_cm_id *id,
1593 struct rdma_cm_event *event)
1594{
1595 struct rdma_id_private *id_priv = id->context;
1596
1597 id->context = id_priv->id.context;
1598 id->event_handler = id_priv->id.event_handler;
1599 return id_priv->id.event_handler(id, event);
1600}
1601
1602static void cma_listen_on_dev(struct rdma_id_private *id_priv,
1603 struct cma_device *cma_dev)
1604{
1605 struct rdma_id_private *dev_id_priv;
1606 struct rdma_cm_id *id;
1607 int ret;
1608
b26f9b99
SH		 1609 id = rdma_create_id(cma_listen_handler, id_priv, id_priv->id.ps,
1610 id_priv->id.qp_type);
e51060f0
SH		 1611 if (IS_ERR(id))
1612 return;
1613
1614 dev_id_priv = container_of(id, struct rdma_id_private, id);
1615
550e5ca7 1616 dev_id_priv->state = RDMA_CM_ADDR_BOUND;
e51060f0 1617 memcpy(&id->route.addr.src_addr, &id_priv->id.route.addr.src_addr,
ef560861 1618 rdma_addr_size((struct sockaddr *) &id_priv->id.route.addr.src_addr));
e51060f0
SH		 1619
1620 cma_attach_to_dev(dev_id_priv, cma_dev);
1621 list_add_tail(&dev_id_priv->listen_list, &id_priv->listen_list);
d02d1f53
SH		 1622 atomic_inc(&id_priv->refcount);
1623 dev_id_priv->internal_id = 1;
5b0ec991 1624 dev_id_priv->afonly = id_priv->afonly;
e51060f0
SH		 1625
1626 ret = rdma_listen(id, id_priv->backlog);
1627 if (ret)
d02d1f53 1628 printk(KERN_WARNING "RDMA CMA: cma_listen_on_dev, error %d, "
468f2239 1629 "listening on device %s\n", ret, cma_dev->device->name);
e51060f0
SH		 1630}
1631
1632static void cma_listen_on_all(struct rdma_id_private *id_priv)
1633{
1634 struct cma_device *cma_dev;
1635
1636 mutex_lock(&lock);
1637 list_add_tail(&id_priv->list, &listen_any_list);
1638 list_for_each_entry(cma_dev, &dev_list, list)
1639 cma_listen_on_dev(id_priv, cma_dev);
1640 mutex_unlock(&lock);
1641}
1642
a81c994d
SH		 1643void rdma_set_service_type(struct rdma_cm_id *id, int tos)
1644{
1645 struct rdma_id_private *id_priv;
1646
1647 id_priv = container_of(id, struct rdma_id_private, id);
1648 id_priv->tos = (u8) tos;
1649}
1650EXPORT_SYMBOL(rdma_set_service_type);
1651
e51060f0
SH		 1652static void cma_query_handler(int status, struct ib_sa_path_rec *path_rec,
1653 void *context)
1654{
1655 struct cma_work *work = context;
1656 struct rdma_route *route;
1657
1658 route = &work->id->id.route;
1659
1660 if (!status) {
1661 route->num_paths = 1;
1662 *route->path_rec = *path_rec;
1663 } else {
550e5ca7
NM		 1664 work->old_state = RDMA_CM_ROUTE_QUERY;
1665 work->new_state = RDMA_CM_ADDR_RESOLVED;
e51060f0 1666 work->event.event = RDMA_CM_EVENT_ROUTE_ERROR;
8f0472d3 1667 work->event.status = status;
e51060f0
SH		 1668 }
1669
1670 queue_work(cma_wq, &work->work);
1671}
1672
1673static int cma_query_ib_route(struct rdma_id_private *id_priv, int timeout_ms,
1674 struct cma_work *work)
1675{
a81c994d 1676 struct rdma_addr *addr = &id_priv->id.route.addr;
e51060f0 1677 struct ib_sa_path_rec path_rec;
a81c994d
SH		 1678 ib_sa_comp_mask comp_mask;
1679 struct sockaddr_in6 *sin6;
e51060f0
SH		 1680
1681 memset(&path_rec, 0, sizeof path_rec);
6f8372b6
SH		 1682 rdma_addr_get_sgid(&addr->dev_addr, &path_rec.sgid);
1683 rdma_addr_get_dgid(&addr->dev_addr, &path_rec.dgid);
a81c994d 1684 path_rec.pkey = cpu_to_be16(ib_addr_get_pkey(&addr->dev_addr));
e51060f0 1685 path_rec.numb_path = 1;
962063e6 1686 path_rec.reversible = 1;
3f446754
RD		 1687 path_rec.service_id = cma_get_service_id(id_priv->id.ps,
1688 (struct sockaddr *) &addr->dst_addr);
a81c994d
SH		 1689
1690 comp_mask = IB_SA_PATH_REC_DGID | IB_SA_PATH_REC_SGID |
1691 IB_SA_PATH_REC_PKEY | IB_SA_PATH_REC_NUMB_PATH |
1692 IB_SA_PATH_REC_REVERSIBLE | IB_SA_PATH_REC_SERVICE_ID;
1693
3f446754 1694 if (addr->src_addr.ss_family == AF_INET) {
a81c994d
SH		 1695 path_rec.qos_class = cpu_to_be16((u16) id_priv->tos);
1696 comp_mask |= IB_SA_PATH_REC_QOS_CLASS;
1697 } else {
1698 sin6 = (struct sockaddr_in6 *) &addr->src_addr;
1699 path_rec.traffic_class = (u8) (be32_to_cpu(sin6->sin6_flowinfo) >> 20);
1700 comp_mask |= IB_SA_PATH_REC_TRAFFIC_CLASS;
1701 }
e51060f0 1702
c1a0b23b 1703 id_priv->query_id = ib_sa_path_rec_get(&sa_client, id_priv->id.device,
a81c994d
SH		 1704 id_priv->id.port_num, &path_rec,
1705 comp_mask, timeout_ms,
1706 GFP_KERNEL, cma_query_handler,
1707 work, &id_priv->query);
e51060f0
SH		 1708
1709 return (id_priv->query_id < 0) ? id_priv->query_id : 0;
1710}
1711
c4028958 1712static void cma_work_handler(struct work_struct *_work)
e51060f0 1713{
c4028958 1714 struct cma_work *work = container_of(_work, struct cma_work, work);
e51060f0
SH		 1715 struct rdma_id_private *id_priv = work->id;
1716 int destroy = 0;
1717
de910bd9 1718 mutex_lock(&id_priv->handler_mutex);
e51060f0
SH		 1719 if (!cma_comp_exch(id_priv, work->old_state, work->new_state))
1720 goto out;
1721
1722 if (id_priv->id.event_handler(&id_priv->id, &work->event)) {
550e5ca7 1723 cma_exch(id_priv, RDMA_CM_DESTROYING);
e51060f0
SH		 1724 destroy = 1;
1725 }
1726out:
de910bd9 1727 mutex_unlock(&id_priv->handler_mutex);
e51060f0
SH		 1728 cma_deref_id(id_priv);
1729 if (destroy)
1730 rdma_destroy_id(&id_priv->id);
1731 kfree(work);
1732}
1733
dd5bdff8
OG		 1734static void cma_ndev_work_handler(struct work_struct *_work)
1735{
1736 struct cma_ndev_work *work = container_of(_work, struct cma_ndev_work, work);
1737 struct rdma_id_private *id_priv = work->id;
1738 int destroy = 0;
1739
1740 mutex_lock(&id_priv->handler_mutex);
550e5ca7
NM		 1741 if (id_priv->state == RDMA_CM_DESTROYING ||
1742 id_priv->state == RDMA_CM_DEVICE_REMOVAL)
dd5bdff8
OG		 1743 goto out;
1744
1745 if (id_priv->id.event_handler(&id_priv->id, &work->event)) {
550e5ca7 1746 cma_exch(id_priv, RDMA_CM_DESTROYING);
dd5bdff8
OG		 1747 destroy = 1;
1748 }
1749
1750out:
1751 mutex_unlock(&id_priv->handler_mutex);
1752 cma_deref_id(id_priv);
1753 if (destroy)
1754 rdma_destroy_id(&id_priv->id);
1755 kfree(work);
1756}
1757
e51060f0
SH		 1758static int cma_resolve_ib_route(struct rdma_id_private *id_priv, int timeout_ms)
1759{
1760 struct rdma_route *route = &id_priv->id.route;
1761 struct cma_work *work;
1762 int ret;
1763
1764 work = kzalloc(sizeof *work, GFP_KERNEL);
1765 if (!work)
1766 return -ENOMEM;
1767
1768 work->id = id_priv;
c4028958 1769 INIT_WORK(&work->work, cma_work_handler);
550e5ca7
NM		 1770 work->old_state = RDMA_CM_ROUTE_QUERY;
1771 work->new_state = RDMA_CM_ROUTE_RESOLVED;
e51060f0
SH		 1772 work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED;
1773
1774 route->path_rec = kmalloc(sizeof *route->path_rec, GFP_KERNEL);
1775 if (!route->path_rec) {
1776 ret = -ENOMEM;
1777 goto err1;
1778 }
1779
1780 ret = cma_query_ib_route(id_priv, timeout_ms, work);
1781 if (ret)
1782 goto err2;
1783
1784 return 0;
1785err2:
1786 kfree(route->path_rec);
1787 route->path_rec = NULL;
1788err1:
1789 kfree(work);
1790 return ret;
1791}
1792
1793int rdma_set_ib_paths(struct rdma_cm_id *id,
1794 struct ib_sa_path_rec *path_rec, int num_paths)
1795{
1796 struct rdma_id_private *id_priv;
1797 int ret;
1798
1799 id_priv = container_of(id, struct rdma_id_private, id);
550e5ca7
NM		 1800 if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED,
1801 RDMA_CM_ROUTE_RESOLVED))
e51060f0
SH		 1802 return -EINVAL;
1803
9893e742
JL		 1804 id->route.path_rec = kmemdup(path_rec, sizeof *path_rec * num_paths,
1805 GFP_KERNEL);
e51060f0
SH		 1806 if (!id->route.path_rec) {
1807 ret = -ENOMEM;
1808 goto err;
1809 }
1810
ae2d9293 1811 id->route.num_paths = num_paths;
e51060f0
SH		 1812 return 0;
1813err:
550e5ca7 1814 cma_comp_exch(id_priv, RDMA_CM_ROUTE_RESOLVED, RDMA_CM_ADDR_RESOLVED);
e51060f0
SH		 1815 return ret;
1816}
1817EXPORT_SYMBOL(rdma_set_ib_paths);
1818
07ebafba
TT		 1819static int cma_resolve_iw_route(struct rdma_id_private *id_priv, int timeout_ms)
1820{
1821 struct cma_work *work;
1822
1823 work = kzalloc(sizeof *work, GFP_KERNEL);
1824 if (!work)
1825 return -ENOMEM;
1826
1827 work->id = id_priv;
c4028958 1828 INIT_WORK(&work->work, cma_work_handler);
550e5ca7
NM		 1829 work->old_state = RDMA_CM_ROUTE_QUERY;
1830 work->new_state = RDMA_CM_ROUTE_RESOLVED;
07ebafba
TT		 1831 work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED;
1832 queue_work(cma_wq, &work->work);
1833 return 0;
1834}
1835
3c86aa70
EC		 1836static int cma_resolve_iboe_route(struct rdma_id_private *id_priv)
1837{
1838 struct rdma_route *route = &id_priv->id.route;
1839 struct rdma_addr *addr = &route->addr;
1840 struct cma_work *work;
1841 int ret;
1842 struct sockaddr_in *src_addr = (struct sockaddr_in *)&route->addr.src_addr;
1843 struct sockaddr_in *dst_addr = (struct sockaddr_in *)&route->addr.dst_addr;
1844 struct net_device *ndev = NULL;
af7bd463 1845 u16 vid;
3c86aa70
EC		 1846
1847 if (src_addr->sin_family != dst_addr->sin_family)
1848 return -EINVAL;
1849
1850 work = kzalloc(sizeof *work, GFP_KERNEL);
1851 if (!work)
1852 return -ENOMEM;
1853
1854 work->id = id_priv;
1855 INIT_WORK(&work->work, cma_work_handler);
1856
1857 route->path_rec = kzalloc(sizeof *route->path_rec, GFP_KERNEL);
1858 if (!route->path_rec) {
1859 ret = -ENOMEM;
1860 goto err1;
1861 }
1862
1863 route->num_paths = 1;
1864
3c86aa70
EC		 1865 if (addr->dev_addr.bound_dev_if)
1866 ndev = dev_get_by_index(&init_net, addr->dev_addr.bound_dev_if);
1867 if (!ndev) {
1868 ret = -ENODEV;
1869 goto err2;
1870 }
1871
af7bd463
EC		 1872 vid = rdma_vlan_dev_vlan_id(ndev);
1873
1874 iboe_mac_vlan_to_ll(&route->path_rec->sgid, addr->dev_addr.src_dev_addr, vid);
1875 iboe_mac_vlan_to_ll(&route->path_rec->dgid, addr->dev_addr.dst_dev_addr, vid);
1876
1877 route->path_rec->hop_limit = 1;
1878 route->path_rec->reversible = 1;
1879 route->path_rec->pkey = cpu_to_be16(0xffff);
1880 route->path_rec->mtu_selector = IB_SA_EQ;
366cddb4
AV		 1881 route->path_rec->sl = netdev_get_prio_tc_map(
1882 ndev->priv_flags & IFF_802_1Q_VLAN ?
1883 vlan_dev_real_dev(ndev) : ndev,
1884 rt_tos2priority(id_priv->tos));
af7bd463 1885
3c86aa70
EC		 1886 route->path_rec->mtu = iboe_get_mtu(ndev->mtu);
1887 route->path_rec->rate_selector = IB_SA_EQ;
1888 route->path_rec->rate = iboe_get_rate(ndev);
1889 dev_put(ndev);
1890 route->path_rec->packet_life_time_selector = IB_SA_EQ;
1891 route->path_rec->packet_life_time = CMA_IBOE_PACKET_LIFETIME;
1892 if (!route->path_rec->mtu) {
1893 ret = -EINVAL;
1894 goto err2;
1895 }
1896
550e5ca7
NM		 1897 work->old_state = RDMA_CM_ROUTE_QUERY;
1898 work->new_state = RDMA_CM_ROUTE_RESOLVED;
3c86aa70
EC		 1899 work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED;
1900 work->event.status = 0;
1901
1902 queue_work(cma_wq, &work->work);
1903
1904 return 0;
1905
1906err2:
1907 kfree(route->path_rec);
1908 route->path_rec = NULL;
1909err1:
1910 kfree(work);
1911 return ret;
1912}
1913
e51060f0
SH		 1914int rdma_resolve_route(struct rdma_cm_id *id, int timeout_ms)
1915{
1916 struct rdma_id_private *id_priv;
1917 int ret;
1918
1919 id_priv = container_of(id, struct rdma_id_private, id);
550e5ca7 1920 if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED, RDMA_CM_ROUTE_QUERY))
e51060f0
SH		 1921 return -EINVAL;
1922
1923 atomic_inc(&id_priv->refcount);
07ebafba
TT		 1924 switch (rdma_node_get_transport(id->device->node_type)) {
1925 case RDMA_TRANSPORT_IB:
3c86aa70
EC		 1926 switch (rdma_port_get_link_layer(id->device, id->port_num)) {
1927 case IB_LINK_LAYER_INFINIBAND:
1928 ret = cma_resolve_ib_route(id_priv, timeout_ms);
1929 break;
1930 case IB_LINK_LAYER_ETHERNET:
1931 ret = cma_resolve_iboe_route(id_priv);
1932 break;
1933 default:
1934 ret = -ENOSYS;
1935 }
e51060f0 1936 break;
07ebafba
TT		 1937 case RDMA_TRANSPORT_IWARP:
1938 ret = cma_resolve_iw_route(id_priv, timeout_ms);
1939 break;
e51060f0
SH		 1940 default:
1941 ret = -ENOSYS;
1942 break;
1943 }
1944 if (ret)
1945 goto err;
1946
1947 return 0;
1948err:
550e5ca7 1949 cma_comp_exch(id_priv, RDMA_CM_ROUTE_QUERY, RDMA_CM_ADDR_RESOLVED);
e51060f0
SH		 1950 cma_deref_id(id_priv);
1951 return ret;
1952}
1953EXPORT_SYMBOL(rdma_resolve_route);
1954
1955static int cma_bind_loopback(struct rdma_id_private *id_priv)
1956{
1957 struct cma_device *cma_dev;
1958 struct ib_port_attr port_attr;
f0ee3404 1959 union ib_gid gid;
e51060f0
SH		 1960 u16 pkey;
1961 int ret;
1962 u8 p;
1963
1964 mutex_lock(&lock);
e82153b5
KK		 1965 if (list_empty(&dev_list)) {
1966 ret = -ENODEV;
1967 goto out;
1968 }
e51060f0
SH		 1969 list_for_each_entry(cma_dev, &dev_list, list)
1970 for (p = 1; p <= cma_dev->device->phys_port_cnt; ++p)
e82153b5 1971 if (!ib_query_port(cma_dev->device, p, &port_attr) &&
e51060f0
SH		 1972 port_attr.state == IB_PORT_ACTIVE)
1973 goto port_found;
1974
e82153b5
KK		 1975 p = 1;
1976 cma_dev = list_entry(dev_list.next, struct cma_device, list);
e51060f0
SH		 1977
1978port_found:
f0ee3404 1979 ret = ib_get_cached_gid(cma_dev->device, p, 0, &gid);
e51060f0
SH		 1980 if (ret)
1981 goto out;
1982
1983 ret = ib_get_cached_pkey(cma_dev->device, p, 0, &pkey);
1984 if (ret)
1985 goto out;
1986
6f8372b6 1987 id_priv->id.route.addr.dev_addr.dev_type =
3c86aa70 1988 (rdma_port_get_link_layer(cma_dev->device, p) == IB_LINK_LAYER_INFINIBAND) ?
6f8372b6
SH		 1989 ARPHRD_INFINIBAND : ARPHRD_ETHER;
1990
1991 rdma_addr_set_sgid(&id_priv->id.route.addr.dev_addr, &gid);
e51060f0
SH		 1992 ib_addr_set_pkey(&id_priv->id.route.addr.dev_addr, pkey);
1993 id_priv->id.port_num = p;
1994 cma_attach_to_dev(id_priv, cma_dev);
1995out:
1996 mutex_unlock(&lock);
1997 return ret;
1998}
1999
2000static void addr_handler(int status, struct sockaddr *src_addr,
2001 struct rdma_dev_addr *dev_addr, void *context)
2002{
2003 struct rdma_id_private *id_priv = context;
a1b1b61f 2004 struct rdma_cm_event event;
e51060f0 2005
a1b1b61f 2006 memset(&event, 0, sizeof event);
de910bd9 2007 mutex_lock(&id_priv->handler_mutex);
550e5ca7
NM		 2008 if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_QUERY,
2009 RDMA_CM_ADDR_RESOLVED))
61a73c70 2010 goto out;
61a73c70
SH		 2011
2012 if (!status && !id_priv->cma_dev)
e51060f0
SH		 2013 status = cma_acquire_dev(id_priv);
2014
2015 if (status) {
550e5ca7
NM		 2016 if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_RESOLVED,
2017 RDMA_CM_ADDR_BOUND))
e51060f0 2018 goto out;
a1b1b61f
SH		 2019 event.event = RDMA_CM_EVENT_ADDR_ERROR;
2020 event.status = status;
e51060f0 2021 } else {
e51060f0 2022 memcpy(&id_priv->id.route.addr.src_addr, src_addr,
ef560861 2023 rdma_addr_size(src_addr));
a1b1b61f 2024 event.event = RDMA_CM_EVENT_ADDR_RESOLVED;
e51060f0
SH		 2025 }
2026
a1b1b61f 2027 if (id_priv->id.event_handler(&id_priv->id, &event)) {
550e5ca7 2028 cma_exch(id_priv, RDMA_CM_DESTROYING);
de910bd9 2029 mutex_unlock(&id_priv->handler_mutex);
e51060f0
SH		 2030 cma_deref_id(id_priv);
2031 rdma_destroy_id(&id_priv->id);
2032 return;
2033 }
2034out:
de910bd9 2035 mutex_unlock(&id_priv->handler_mutex);
e51060f0
SH		 2036 cma_deref_id(id_priv);
2037}
2038
2039static int cma_resolve_loopback(struct rdma_id_private *id_priv)
2040{
2041 struct cma_work *work;
6f8372b6 2042 struct sockaddr *src, *dst;
f0ee3404 2043 union ib_gid gid;
e51060f0
SH		 2044 int ret;
2045
2046 work = kzalloc(sizeof *work, GFP_KERNEL);
2047 if (!work)
2048 return -ENOMEM;
2049
2050 if (!id_priv->cma_dev) {
2051 ret = cma_bind_loopback(id_priv);
2052 if (ret)
2053 goto err;
2054 }
2055
6f8372b6
SH		 2056 rdma_addr_get_sgid(&id_priv->id.route.addr.dev_addr, &gid);
2057 rdma_addr_set_dgid(&id_priv->id.route.addr.dev_addr, &gid);
e51060f0 2058
6f8372b6
SH		 2059 src = (struct sockaddr *) &id_priv->id.route.addr.src_addr;
2060 if (cma_zero_addr(src)) {
2061 dst = (struct sockaddr *) &id_priv->id.route.addr.dst_addr;
2062 if ((src->sa_family = dst->sa_family) == AF_INET) {
4e3fd7a0
AD		 2063 ((struct sockaddr_in *)src)->sin_addr =
2064 ((struct sockaddr_in *)dst)->sin_addr;
6f8372b6 2065 } else {
4e3fd7a0
AD		 2066 ((struct sockaddr_in6 *)src)->sin6_addr =
2067 ((struct sockaddr_in6 *)dst)->sin6_addr;
6f8372b6 2068 }
e51060f0
SH		 2069 }
2070
2071 work->id = id_priv;
c4028958 2072 INIT_WORK(&work->work, cma_work_handler);
550e5ca7
NM		 2073 work->old_state = RDMA_CM_ADDR_QUERY;
2074 work->new_state = RDMA_CM_ADDR_RESOLVED;
e51060f0
SH		 2075 work->event.event = RDMA_CM_EVENT_ADDR_RESOLVED;
2076 queue_work(cma_wq, &work->work);
2077 return 0;
2078err:
2079 kfree(work);
2080 return ret;
2081}
2082
2083static int cma_bind_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
2084 struct sockaddr *dst_addr)
2085{
d14714df
SH		 2086 if (!src_addr || !src_addr->sa_family) {
2087 src_addr = (struct sockaddr *) &id->route.addr.src_addr;
2088 if ((src_addr->sa_family = dst_addr->sa_family) == AF_INET6) {
2089 ((struct sockaddr_in6 *) src_addr)->sin6_scope_id =
2090 ((struct sockaddr_in6 *) dst_addr)->sin6_scope_id;
2091 }
2092 }
2093 return rdma_bind_addr(id, src_addr);
e51060f0
SH		 2094}
2095
2096int rdma_resolve_addr(struct rdma_cm_id *id, struct sockaddr *src_addr,
2097 struct sockaddr *dst_addr, int timeout_ms)
2098{
2099 struct rdma_id_private *id_priv;
2100 int ret;
2101
2102 id_priv = container_of(id, struct rdma_id_private, id);
550e5ca7 2103 if (id_priv->state == RDMA_CM_IDLE) {
e51060f0
SH		 2104 ret = cma_bind_addr(id, src_addr, dst_addr);
2105 if (ret)
2106 return ret;
2107 }
2108
550e5ca7 2109 if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_ADDR_QUERY))
e51060f0
SH		 2110 return -EINVAL;
2111
2112 atomic_inc(&id_priv->refcount);
ef560861 2113 memcpy(&id->route.addr.dst_addr, dst_addr, rdma_addr_size(dst_addr));
e51060f0
SH		 2114 if (cma_any_addr(dst_addr))
2115 ret = cma_resolve_loopback(id_priv);
2116 else
3f446754 2117 ret = rdma_resolve_ip(&addr_client, (struct sockaddr *) &id->route.addr.src_addr,
7a118df3 2118 dst_addr, &id->route.addr.dev_addr,
e51060f0
SH		 2119 timeout_ms, addr_handler, id_priv);
2120 if (ret)
2121 goto err;
2122
2123 return 0;
2124err:
550e5ca7 2125 cma_comp_exch(id_priv, RDMA_CM_ADDR_QUERY, RDMA_CM_ADDR_BOUND);
e51060f0
SH		 2126 cma_deref_id(id_priv);
2127 return ret;
2128}
2129EXPORT_SYMBOL(rdma_resolve_addr);
2130
a9bb7912
HS		 2131int rdma_set_reuseaddr(struct rdma_cm_id *id, int reuse)
2132{
2133 struct rdma_id_private *id_priv;
2134 unsigned long flags;
2135 int ret;
2136
2137 id_priv = container_of(id, struct rdma_id_private, id);
2138 spin_lock_irqsave(&id_priv->lock, flags);
c8dea2f9 2139 if (reuse || id_priv->state == RDMA_CM_IDLE) {
a9bb7912
HS		 2140 id_priv->reuseaddr = reuse;
2141 ret = 0;
2142 } else {
2143 ret = -EINVAL;
2144 }
2145 spin_unlock_irqrestore(&id_priv->lock, flags);
2146 return ret;
2147}
2148EXPORT_SYMBOL(rdma_set_reuseaddr);
2149
68602120
SH		 2150int rdma_set_afonly(struct rdma_cm_id *id, int afonly)
2151{
2152 struct rdma_id_private *id_priv;
2153 unsigned long flags;
2154 int ret;
2155
2156 id_priv = container_of(id, struct rdma_id_private, id);
2157 spin_lock_irqsave(&id_priv->lock, flags);
2158 if (id_priv->state == RDMA_CM_IDLE || id_priv->state == RDMA_CM_ADDR_BOUND) {
2159 id_priv->options |= (1 << CMA_OPTION_AFONLY);
2160 id_priv->afonly = afonly;
2161 ret = 0;
2162 } else {
2163 ret = -EINVAL;
2164 }
2165 spin_unlock_irqrestore(&id_priv->lock, flags);
2166 return ret;
2167}
2168EXPORT_SYMBOL(rdma_set_afonly);
2169
e51060f0
SH		 2170static void cma_bind_port(struct rdma_bind_list *bind_list,
2171 struct rdma_id_private *id_priv)
2172{
58afdcb7
SH		 2173 struct sockaddr *addr;
2174 struct sockaddr_ib *sib;
2175 u64 sid, mask;
2176 __be16 port;
e51060f0 2177
58afdcb7
SH		 2178 addr = (struct sockaddr *) &id_priv->id.route.addr.src_addr;
2179 port = htons(bind_list->port);
2180
2181 switch (addr->sa_family) {
2182 case AF_INET:
2183 ((struct sockaddr_in *) addr)->sin_port = port;
2184 break;
2185 case AF_INET6:
2186 ((struct sockaddr_in6 *) addr)->sin6_port = port;
2187 break;
2188 case AF_IB:
2189 sib = (struct sockaddr_ib *) addr;
2190 sid = be64_to_cpu(sib->sib_sid);
2191 mask = be64_to_cpu(sib->sib_sid_mask);
2192 sib->sib_sid = cpu_to_be64((sid & mask) | (u64) ntohs(port));
2193 sib->sib_sid_mask = cpu_to_be64(~0ULL);
2194 break;
2195 }
e51060f0
SH		 2196 id_priv->bind_list = bind_list;
2197 hlist_add_head(&id_priv->node, &bind_list->owners);
2198}
2199
2200static int cma_alloc_port(struct idr *ps, struct rdma_id_private *id_priv,
2201 unsigned short snum)
2202{
2203 struct rdma_bind_list *bind_list;
3b069c5d 2204 int ret;
e51060f0 2205
cb164b8c 2206 bind_list = kzalloc(sizeof *bind_list, GFP_KERNEL);
e51060f0
SH		 2207 if (!bind_list)
2208 return -ENOMEM;
2209
3b069c5d
TH		 2210 ret = idr_alloc(ps, bind_list, snum, snum + 1, GFP_KERNEL);
2211 if (ret < 0)
2212 goto err;
aedec080
SH		 2213
2214 bind_list->ps = ps;
3b069c5d 2215 bind_list->port = (unsigned short)ret;
aedec080
SH		 2216 cma_bind_port(bind_list, id_priv);
2217 return 0;
3b069c5d 2218err:
aedec080 2219 kfree(bind_list);
3b069c5d 2220 return ret == -ENOSPC ? -EADDRNOTAVAIL : ret;
aedec080 2221}
e51060f0 2222
aedec080
SH		 2223static int cma_alloc_any_port(struct idr *ps, struct rdma_id_private *id_priv)
2224{
5d7220e8
TH		 2225 static unsigned int last_used_port;
2226 int low, high, remaining;
2227 unsigned int rover;
e51060f0 2228
227b60f5 2229 inet_get_local_port_range(&low, &high);
5d7220e8
TH		 2230 remaining = (high - low) + 1;
2231 rover = net_random() % remaining + low;
2232retry:
2233 if (last_used_port != rover &&
2234 !idr_find(ps, (unsigned short) rover)) {
2235 int ret = cma_alloc_port(ps, id_priv, rover);
2236 /*
2237 * Remember previously used port number in order to avoid
2238 * re-using same port immediately after it is closed.
2239 */
2240 if (!ret)
2241 last_used_port = rover;
2242 if (ret != -EADDRNOTAVAIL)
2243 return ret;
e51060f0 2244 }
5d7220e8
TH		 2245 if (--remaining) {
2246 rover++;
2247 if ((rover < low) || (rover > high))
2248 rover = low;
2249 goto retry;
2250 }
2251 return -EADDRNOTAVAIL;
e51060f0
SH		 2252}
2253
a9bb7912
HS		 2254/*
2255 * Check that the requested port is available. This is called when trying to
2256 * bind to a specific port, or when trying to listen on a bound port. In
2257 * the latter case, the provided id_priv may already be on the bind_list, but
2258 * we still need to check that it's okay to start listening.
2259 */
2260static int cma_check_port(struct rdma_bind_list *bind_list,
2261 struct rdma_id_private *id_priv, uint8_t reuseaddr)
e51060f0
SH		 2262{
2263 struct rdma_id_private *cur_id;
43b752da 2264 struct sockaddr *addr, *cur_addr;
e51060f0 2265
43b752da 2266 addr = (struct sockaddr *) &id_priv->id.route.addr.src_addr;
b67bfe0d 2267 hlist_for_each_entry(cur_id, &bind_list->owners, node) {
a9bb7912
HS		 2268 if (id_priv == cur_id)
2269 continue;
3cd96564 2270
5b0ec991
SH		 2271 if ((cur_id->state != RDMA_CM_LISTEN) && reuseaddr &&
2272 cur_id->reuseaddr)
2273 continue;
e51060f0 2274
5b0ec991
SH		 2275 cur_addr = (struct sockaddr *) &cur_id->id.route.addr.src_addr;
2276 if (id_priv->afonly && cur_id->afonly &&
2277 (addr->sa_family != cur_addr->sa_family))
2278 continue;
2279
2280 if (cma_any_addr(addr) || cma_any_addr(cur_addr))
2281 return -EADDRNOTAVAIL;
2282
2283 if (!cma_addr_cmp(addr, cur_addr))
2284 return -EADDRINUSE;
a9bb7912 2285 }
e51060f0
SH		 2286 return 0;
2287}
2288
a9bb7912
HS		 2289static int cma_use_port(struct idr *ps, struct rdma_id_private *id_priv)
2290{
2291 struct rdma_bind_list *bind_list;
2292 unsigned short snum;
2293 int ret;
2294
2295 snum = ntohs(cma_port((struct sockaddr *) &id_priv->id.route.addr.src_addr));
2296 if (snum < PROT_SOCK && !capable(CAP_NET_BIND_SERVICE))
2297 return -EACCES;
2298
2299 bind_list = idr_find(ps, snum);
2300 if (!bind_list) {
2301 ret = cma_alloc_port(ps, id_priv, snum);
2302 } else {
2303 ret = cma_check_port(bind_list, id_priv, id_priv->reuseaddr);
2304 if (!ret)
2305 cma_bind_port(bind_list, id_priv);
2306 }
2307 return ret;
2308}
2309
2310static int cma_bind_listen(struct rdma_id_private *id_priv)
2311{
2312 struct rdma_bind_list *bind_list = id_priv->bind_list;
2313 int ret = 0;
2314
2315 mutex_lock(&lock);
2316 if (bind_list->owners.first->next)
2317 ret = cma_check_port(bind_list, id_priv, 0);
2318 mutex_unlock(&lock);
2319 return ret;
2320}
2321
58afdcb7 2322static struct idr *cma_select_inet_ps(struct rdma_id_private *id_priv)
e51060f0 2323{
e51060f0
SH		 2324 switch (id_priv->id.ps) {
2325 case RDMA_PS_SDP:
58afdcb7 2326 return &sdp_ps;
e51060f0 2327 case RDMA_PS_TCP:
58afdcb7 2328 return &tcp_ps;
628e5f6d 2329 case RDMA_PS_UDP:
58afdcb7 2330 return &udp_ps;
c8f6a362 2331 case RDMA_PS_IPOIB:
58afdcb7 2332 return &ipoib_ps;
2d2e9415 2333 case RDMA_PS_IB:
58afdcb7 2334 return &ib_ps;
e51060f0 2335 default:
58afdcb7
SH		 2336 return NULL;
2337 }
2338}
2339
2340static struct idr *cma_select_ib_ps(struct rdma_id_private *id_priv)
2341{
2342 struct idr *ps = NULL;
2343 struct sockaddr_ib *sib;
2344 u64 sid_ps, mask, sid;
2345
2346 sib = (struct sockaddr_ib *) &id_priv->id.route.addr.src_addr;
2347 mask = be64_to_cpu(sib->sib_sid_mask) & RDMA_IB_IP_PS_MASK;
2348 sid = be64_to_cpu(sib->sib_sid) & mask;
2349
2350 if ((id_priv->id.ps == RDMA_PS_IB) && (sid == (RDMA_IB_IP_PS_IB & mask))) {
2351 sid_ps = RDMA_IB_IP_PS_IB;
2352 ps = &ib_ps;
2353 } else if (((id_priv->id.ps == RDMA_PS_IB) || (id_priv->id.ps == RDMA_PS_TCP)) &&
2354 (sid == (RDMA_IB_IP_PS_TCP & mask))) {
2355 sid_ps = RDMA_IB_IP_PS_TCP;
2356 ps = &tcp_ps;
2357 } else if (((id_priv->id.ps == RDMA_PS_IB) || (id_priv->id.ps == RDMA_PS_UDP)) &&
2358 (sid == (RDMA_IB_IP_PS_UDP & mask))) {
2359 sid_ps = RDMA_IB_IP_PS_UDP;
2360 ps = &udp_ps;
e51060f0
SH		 2361 }
2362
58afdcb7
SH		 2363 if (ps) {
2364 sib->sib_sid = cpu_to_be64(sid_ps | ntohs(cma_port((struct sockaddr *) sib)));
2365 sib->sib_sid_mask = cpu_to_be64(RDMA_IB_IP_PS_MASK |
2366 be64_to_cpu(sib->sib_sid_mask));
2367 }
2368 return ps;
2369}
2370
2371static int cma_get_port(struct rdma_id_private *id_priv)
2372{
2373 struct idr *ps;
2374 int ret;
2375
2376 if (id_priv->id.route.addr.src_addr.ss_family != AF_IB)
2377 ps = cma_select_inet_ps(id_priv);
2378 else
2379 ps = cma_select_ib_ps(id_priv);
2380 if (!ps)
2381 return -EPROTONOSUPPORT;
2382
e51060f0 2383 mutex_lock(&lock);
3f446754 2384 if (cma_any_port((struct sockaddr *) &id_priv->id.route.addr.src_addr))
aedec080 2385 ret = cma_alloc_any_port(ps, id_priv);
e51060f0
SH		 2386 else
2387 ret = cma_use_port(ps, id_priv);
2388 mutex_unlock(&lock);
2389
2390 return ret;
2391}
2392
d14714df
SH		 2393static int cma_check_linklocal(struct rdma_dev_addr *dev_addr,
2394 struct sockaddr *addr)
2395{
d90f9b35 2396#if IS_ENABLED(CONFIG_IPV6)
d14714df
SH		 2397 struct sockaddr_in6 *sin6;
2398
2399 if (addr->sa_family != AF_INET6)
2400 return 0;
2401
2402 sin6 = (struct sockaddr_in6 *) addr;
2403 if ((ipv6_addr_type(&sin6->sin6_addr) & IPV6_ADDR_LINKLOCAL) &&
2404 !sin6->sin6_scope_id)
2405 return -EINVAL;
2406
2407 dev_addr->bound_dev_if = sin6->sin6_scope_id;
2408#endif
2409 return 0;
2410}
2411
a9bb7912
HS		 2412int rdma_listen(struct rdma_cm_id *id, int backlog)
2413{
2414 struct rdma_id_private *id_priv;
2415 int ret;
2416
2417 id_priv = container_of(id, struct rdma_id_private, id);
550e5ca7 2418 if (id_priv->state == RDMA_CM_IDLE) {
a9bb7912
HS		 2419 ((struct sockaddr *) &id->route.addr.src_addr)->sa_family = AF_INET;
2420 ret = rdma_bind_addr(id, (struct sockaddr *) &id->route.addr.src_addr);
2421 if (ret)
2422 return ret;
2423 }
2424
550e5ca7 2425 if (!cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_LISTEN))
a9bb7912
HS		 2426 return -EINVAL;
2427
2428 if (id_priv->reuseaddr) {
2429 ret = cma_bind_listen(id_priv);
2430 if (ret)
2431 goto err;
2432 }
2433
2434 id_priv->backlog = backlog;
2435 if (id->device) {
2436 switch (rdma_node_get_transport(id->device->node_type)) {
2437 case RDMA_TRANSPORT_IB:
2438 ret = cma_ib_listen(id_priv);
2439 if (ret)
2440 goto err;
2441 break;
2442 case RDMA_TRANSPORT_IWARP:
2443 ret = cma_iw_listen(id_priv, backlog);
2444 if (ret)
2445 goto err;
2446 break;
2447 default:
2448 ret = -ENOSYS;
2449 goto err;
2450 }
2451 } else
2452 cma_listen_on_all(id_priv);
2453
2454 return 0;
2455err:
2456 id_priv->backlog = 0;
550e5ca7 2457 cma_comp_exch(id_priv, RDMA_CM_LISTEN, RDMA_CM_ADDR_BOUND);
a9bb7912
HS		 2458 return ret;
2459}
2460EXPORT_SYMBOL(rdma_listen);
2461
e51060f0
SH		 2462int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr)
2463{
2464 struct rdma_id_private *id_priv;
2465 int ret;
2466
680f920a
SH		 2467 if (addr->sa_family != AF_INET && addr->sa_family != AF_INET6 &&
2468 addr->sa_family != AF_IB)
e51060f0
SH		 2469 return -EAFNOSUPPORT;
2470
2471 id_priv = container_of(id, struct rdma_id_private, id);
550e5ca7 2472 if (!cma_comp_exch(id_priv, RDMA_CM_IDLE, RDMA_CM_ADDR_BOUND))
e51060f0
SH		 2473 return -EINVAL;
2474
d14714df
SH		 2475 ret = cma_check_linklocal(&id->route.addr.dev_addr, addr);
2476 if (ret)
2477 goto err1;
2478
8523c048 2479 if (!cma_any_addr(addr)) {
680f920a 2480 ret = cma_translate_addr(addr, &id->route.addr.dev_addr);
e51060f0 2481 if (ret)
255d0c14
KK		 2482 goto err1;
2483
255d0c14 2484 ret = cma_acquire_dev(id_priv);
255d0c14
KK		 2485 if (ret)
2486 goto err1;
e51060f0
SH		 2487 }
2488
ef560861 2489 memcpy(&id->route.addr.src_addr, addr, rdma_addr_size(addr));
68602120
SH		 2490 if (!(id_priv->options & (1 << CMA_OPTION_AFONLY))) {
2491 if (addr->sa_family == AF_INET)
2492 id_priv->afonly = 1;
5b0ec991 2493#if IS_ENABLED(CONFIG_IPV6)
68602120
SH		 2494 else if (addr->sa_family == AF_INET6)
2495 id_priv->afonly = init_net.ipv6.sysctl.bindv6only;
5b0ec991 2496#endif
68602120 2497 }
e51060f0
SH		 2498 ret = cma_get_port(id_priv);
2499 if (ret)
255d0c14 2500 goto err2;
e51060f0
SH		 2501
2502 return 0;
255d0c14 2503err2:
a396d43a
SH		 2504 if (id_priv->cma_dev)
2505 cma_release_dev(id_priv);
255d0c14 2506err1:
550e5ca7 2507 cma_comp_exch(id_priv, RDMA_CM_ADDR_BOUND, RDMA_CM_IDLE);
e51060f0
SH		 2508 return ret;
2509}
2510EXPORT_SYMBOL(rdma_bind_addr);
2511
2512static int cma_format_hdr(void *hdr, enum rdma_port_space ps,
2513 struct rdma_route *route)
2514{
e51060f0
SH		 2515 struct cma_hdr *cma_hdr;
2516 struct sdp_hh *sdp_hdr;
2517
1f5175ad
AS		 2518 if (route->addr.src_addr.ss_family == AF_INET) {
2519 struct sockaddr_in *src4, *dst4;
2520
2521 src4 = (struct sockaddr_in *) &route->addr.src_addr;
2522 dst4 = (struct sockaddr_in *) &route->addr.dst_addr;
2523
2524 switch (ps) {
2525 case RDMA_PS_SDP:
2526 sdp_hdr = hdr;
2527 if (sdp_get_majv(sdp_hdr->sdp_version) != SDP_MAJ_VERSION)
2528 return -EINVAL;
2529 sdp_set_ip_ver(sdp_hdr, 4);
2530 sdp_hdr->src_addr.ip4.addr = src4->sin_addr.s_addr;
2531 sdp_hdr->dst_addr.ip4.addr = dst4->sin_addr.s_addr;
2532 sdp_hdr->port = src4->sin_port;
2533 break;
2534 default:
2535 cma_hdr = hdr;
2536 cma_hdr->cma_version = CMA_VERSION;
2537 cma_set_ip_ver(cma_hdr, 4);
2538 cma_hdr->src_addr.ip4.addr = src4->sin_addr.s_addr;
2539 cma_hdr->dst_addr.ip4.addr = dst4->sin_addr.s_addr;
2540 cma_hdr->port = src4->sin_port;
2541 break;
2542 }
2543 } else {
2544 struct sockaddr_in6 *src6, *dst6;
2545
2546 src6 = (struct sockaddr_in6 *) &route->addr.src_addr;
2547 dst6 = (struct sockaddr_in6 *) &route->addr.dst_addr;
2548
2549 switch (ps) {
2550 case RDMA_PS_SDP:
2551 sdp_hdr = hdr;
2552 if (sdp_get_majv(sdp_hdr->sdp_version) != SDP_MAJ_VERSION)
2553 return -EINVAL;
2554 sdp_set_ip_ver(sdp_hdr, 6);
2555 sdp_hdr->src_addr.ip6 = src6->sin6_addr;
2556 sdp_hdr->dst_addr.ip6 = dst6->sin6_addr;
2557 sdp_hdr->port = src6->sin6_port;
2558 break;
2559 default:
2560 cma_hdr = hdr;
2561 cma_hdr->cma_version = CMA_VERSION;
2562 cma_set_ip_ver(cma_hdr, 6);
2563 cma_hdr->src_addr.ip6 = src6->sin6_addr;
2564 cma_hdr->dst_addr.ip6 = dst6->sin6_addr;
2565 cma_hdr->port = src6->sin6_port;
2566 break;
2567 }
e51060f0
SH		 2568 }
2569 return 0;
2570}
2571
628e5f6d
SH		 2572static int cma_sidr_rep_handler(struct ib_cm_id *cm_id,
2573 struct ib_cm_event *ib_event)
2574{
2575 struct rdma_id_private *id_priv = cm_id->context;
2576 struct rdma_cm_event event;
2577 struct ib_cm_sidr_rep_event_param *rep = &ib_event->param.sidr_rep_rcvd;
2578 int ret = 0;
2579
550e5ca7 2580 if (cma_disable_callback(id_priv, RDMA_CM_CONNECT))
8aa08602 2581 return 0;
628e5f6d 2582
8aa08602 2583 memset(&event, 0, sizeof event);
628e5f6d
SH		 2584 switch (ib_event->event) {
2585 case IB_CM_SIDR_REQ_ERROR:
2586 event.event = RDMA_CM_EVENT_UNREACHABLE;
2587 event.status = -ETIMEDOUT;
2588 break;
2589 case IB_CM_SIDR_REP_RECEIVED:
2590 event.param.ud.private_data = ib_event->private_data;
2591 event.param.ud.private_data_len = IB_CM_SIDR_REP_PRIVATE_DATA_SIZE;
2592 if (rep->status != IB_SIDR_SUCCESS) {
2593 event.event = RDMA_CM_EVENT_UNREACHABLE;
2594 event.status = ib_event->param.sidr_rep_rcvd.status;
2595 break;
2596 }
d2ca39f2
YE		 2597 ret = cma_set_qkey(id_priv);
2598 if (ret) {
2599 event.event = RDMA_CM_EVENT_ADDR_ERROR;
2600 event.status = -EINVAL;
2601 break;
2602 }
c8f6a362 2603 if (id_priv->qkey != rep->qkey) {
628e5f6d
SH		 2604 event.event = RDMA_CM_EVENT_UNREACHABLE;
2605 event.status = -EINVAL;
2606 break;
2607 }
2608 ib_init_ah_from_path(id_priv->id.device, id_priv->id.port_num,
2609 id_priv->id.route.path_rec,
2610 &event.param.ud.ah_attr);
2611 event.param.ud.qp_num = rep->qpn;
2612 event.param.ud.qkey = rep->qkey;
2613 event.event = RDMA_CM_EVENT_ESTABLISHED;
2614 event.status = 0;
2615 break;
2616 default:
468f2239 2617 printk(KERN_ERR "RDMA CMA: unexpected IB CM event: %d\n",
628e5f6d
SH		 2618 ib_event->event);
2619 goto out;
2620 }
2621
2622 ret = id_priv->id.event_handler(&id_priv->id, &event);
2623 if (ret) {
2624 /* Destroy the CM ID by returning a non-zero value. */
2625 id_priv->cm_id.ib = NULL;
550e5ca7 2626 cma_exch(id_priv, RDMA_CM_DESTROYING);
de910bd9 2627 mutex_unlock(&id_priv->handler_mutex);
628e5f6d
SH		 2628 rdma_destroy_id(&id_priv->id);
2629 return ret;
2630 }
2631out:
de910bd9 2632 mutex_unlock(&id_priv->handler_mutex);
628e5f6d
SH		 2633 return ret;
2634}
2635
2636static int cma_resolve_ib_udp(struct rdma_id_private *id_priv,
2637 struct rdma_conn_param *conn_param)
2638{
2639 struct ib_cm_sidr_req_param req;
2640 struct rdma_route *route;
0c9361fc 2641 struct ib_cm_id *id;
628e5f6d
SH		 2642 int ret;
2643
2644 req.private_data_len = sizeof(struct cma_hdr) +
2645 conn_param->private_data_len;
04ded167
SH		 2646 if (req.private_data_len < conn_param->private_data_len)
2647 return -EINVAL;
2648
628e5f6d
SH		 2649 req.private_data = kzalloc(req.private_data_len, GFP_ATOMIC);
2650 if (!req.private_data)
2651 return -ENOMEM;
2652
2653 if (conn_param->private_data && conn_param->private_data_len)
2654 memcpy((void *) req.private_data + sizeof(struct cma_hdr),
2655 conn_param->private_data, conn_param->private_data_len);
2656
2657 route = &id_priv->id.route;
2658 ret = cma_format_hdr((void *) req.private_data, id_priv->id.ps, route);
2659 if (ret)
2660 goto out;
2661
0c9361fc
JM		 2662 id = ib_create_cm_id(id_priv->id.device, cma_sidr_rep_handler,
2663 id_priv);
2664 if (IS_ERR(id)) {
2665 ret = PTR_ERR(id);
628e5f6d
SH		 2666 goto out;
2667 }
0c9361fc 2668 id_priv->cm_id.ib = id;
628e5f6d
SH		 2669
2670 req.path = route->path_rec;
2671 req.service_id = cma_get_service_id(id_priv->id.ps,
3f446754 2672 (struct sockaddr *) &route->addr.dst_addr);
628e5f6d
SH		 2673 req.timeout_ms = 1 << (CMA_CM_RESPONSE_TIMEOUT - 8);
2674 req.max_cm_retries = CMA_MAX_CM_RETRIES;
2675
2676 ret = ib_send_cm_sidr_req(id_priv->cm_id.ib, &req);
2677 if (ret) {
2678 ib_destroy_cm_id(id_priv->cm_id.ib);
2679 id_priv->cm_id.ib = NULL;
2680 }
2681out:
2682 kfree(req.private_data);
2683 return ret;
2684}
2685
e51060f0
SH		 2686static int cma_connect_ib(struct rdma_id_private *id_priv,
2687 struct rdma_conn_param *conn_param)
2688{
2689 struct ib_cm_req_param req;
2690 struct rdma_route *route;
2691 void *private_data;
0c9361fc 2692 struct ib_cm_id *id;
e51060f0
SH		 2693 int offset, ret;
2694
2695 memset(&req, 0, sizeof req);
2696 offset = cma_user_data_offset(id_priv->id.ps);
2697 req.private_data_len = offset + conn_param->private_data_len;
04ded167
SH		 2698 if (req.private_data_len < conn_param->private_data_len)
2699 return -EINVAL;
2700
e51060f0
SH		 2701 private_data = kzalloc(req.private_data_len, GFP_ATOMIC);
2702 if (!private_data)
2703 return -ENOMEM;
2704
2705 if (conn_param->private_data && conn_param->private_data_len)
2706 memcpy(private_data + offset, conn_param->private_data,
2707 conn_param->private_data_len);
2708
0c9361fc
JM		 2709 id = ib_create_cm_id(id_priv->id.device, cma_ib_handler, id_priv);
2710 if (IS_ERR(id)) {
2711 ret = PTR_ERR(id);
e51060f0
SH		 2712 goto out;
2713 }
0c9361fc 2714 id_priv->cm_id.ib = id;
e51060f0
SH		 2715
2716 route = &id_priv->id.route;
2717 ret = cma_format_hdr(private_data, id_priv->id.ps, route);
2718 if (ret)
2719 goto out;
2720 req.private_data = private_data;
2721
2722 req.primary_path = &route->path_rec[0];
2723 if (route->num_paths == 2)
2724 req.alternate_path = &route->path_rec[1];
2725
2726 req.service_id = cma_get_service_id(id_priv->id.ps,
3f446754 2727 (struct sockaddr *) &route->addr.dst_addr);
e51060f0 2728 req.qp_num = id_priv->qp_num;
18c441a6 2729 req.qp_type = id_priv->id.qp_type;
e51060f0
SH		 2730 req.starting_psn = id_priv->seq_num;
2731 req.responder_resources = conn_param->responder_resources;
2732 req.initiator_depth = conn_param->initiator_depth;
2733 req.flow_control = conn_param->flow_control;
4ede178a
SH		 2734 req.retry_count = min_t(u8, 7, conn_param->retry_count);
2735 req.rnr_retry_count = min_t(u8, 7, conn_param->rnr_retry_count);
e51060f0
SH		 2736 req.remote_cm_response_timeout = CMA_CM_RESPONSE_TIMEOUT;
2737 req.local_cm_response_timeout = CMA_CM_RESPONSE_TIMEOUT;
2738 req.max_cm_retries = CMA_MAX_CM_RETRIES;
2739 req.srq = id_priv->srq ? 1 : 0;
2740
2741 ret = ib_send_cm_req(id_priv->cm_id.ib, &req);
2742out:
0c9361fc
JM		 2743 if (ret && !IS_ERR(id)) {
2744 ib_destroy_cm_id(id);
675a027c
KK		 2745 id_priv->cm_id.ib = NULL;
2746 }
2747
e51060f0
SH		 2748 kfree(private_data);
2749 return ret;
2750}
2751
07ebafba
TT		 2752static int cma_connect_iw(struct rdma_id_private *id_priv,
2753 struct rdma_conn_param *conn_param)
2754{
2755 struct iw_cm_id *cm_id;
2756 struct sockaddr_in* sin;
2757 int ret;
2758 struct iw_cm_conn_param iw_param;
2759
2760 cm_id = iw_create_cm_id(id_priv->id.device, cma_iw_handler, id_priv);
0c9361fc
JM		 2761 if (IS_ERR(cm_id))
2762 return PTR_ERR(cm_id);
07ebafba
TT		 2763
2764 id_priv->cm_id.iw = cm_id;
2765
2766 sin = (struct sockaddr_in*) &id_priv->id.route.addr.src_addr;
2767 cm_id->local_addr = *sin;
2768
2769 sin = (struct sockaddr_in*) &id_priv->id.route.addr.dst_addr;
2770 cm_id->remote_addr = *sin;
2771
5851bb89 2772 ret = cma_modify_qp_rtr(id_priv, conn_param);
675a027c
KK		 2773 if (ret)
2774 goto out;
07ebafba 2775
f45ee80e
HS		 2776 if (conn_param) {
2777 iw_param.ord = conn_param->initiator_depth;
2778 iw_param.ird = conn_param->responder_resources;
2779 iw_param.private_data = conn_param->private_data;
2780 iw_param.private_data_len = conn_param->private_data_len;
2781 iw_param.qpn = id_priv->id.qp ? id_priv->qp_num : conn_param->qp_num;
2782 } else {
2783 memset(&iw_param, 0, sizeof iw_param);
07ebafba 2784 iw_param.qpn = id_priv->qp_num;
f45ee80e 2785 }
07ebafba
TT		 2786 ret = iw_cm_connect(cm_id, &iw_param);
2787out:
0c9361fc 2788 if (ret) {
675a027c
KK		 2789 iw_destroy_cm_id(cm_id);
2790 id_priv->cm_id.iw = NULL;
2791 }
07ebafba
TT		 2792 return ret;
2793}
2794
e51060f0
SH		 2795int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
2796{
2797 struct rdma_id_private *id_priv;
2798 int ret;
2799
2800 id_priv = container_of(id, struct rdma_id_private, id);
550e5ca7 2801 if (!cma_comp_exch(id_priv, RDMA_CM_ROUTE_RESOLVED, RDMA_CM_CONNECT))
e51060f0
SH		 2802 return -EINVAL;
2803
2804 if (!id->qp) {
2805 id_priv->qp_num = conn_param->qp_num;
e51060f0
SH		 2806 id_priv->srq = conn_param->srq;
2807 }
2808
07ebafba
TT		 2809 switch (rdma_node_get_transport(id->device->node_type)) {
2810 case RDMA_TRANSPORT_IB:
b26f9b99 2811 if (id->qp_type == IB_QPT_UD)
628e5f6d
SH		 2812 ret = cma_resolve_ib_udp(id_priv, conn_param);
2813 else
2814 ret = cma_connect_ib(id_priv, conn_param);
e51060f0 2815 break;
07ebafba
TT		 2816 case RDMA_TRANSPORT_IWARP:
2817 ret = cma_connect_iw(id_priv, conn_param);
2818 break;
e51060f0
SH		 2819 default:
2820 ret = -ENOSYS;
2821 break;
2822 }
2823 if (ret)
2824 goto err;
2825
2826 return 0;
2827err:
550e5ca7 2828 cma_comp_exch(id_priv, RDMA_CM_CONNECT, RDMA_CM_ROUTE_RESOLVED);
e51060f0
SH		 2829 return ret;
2830}
2831EXPORT_SYMBOL(rdma_connect);
2832
2833static int cma_accept_ib(struct rdma_id_private *id_priv,
2834 struct rdma_conn_param *conn_param)
2835{
2836 struct ib_cm_rep_param rep;
5851bb89 2837 int ret;
0fe313b0 2838
5851bb89
SH		 2839 ret = cma_modify_qp_rtr(id_priv, conn_param);
2840 if (ret)
2841 goto out;
0fe313b0 2842
5851bb89
SH		 2843 ret = cma_modify_qp_rts(id_priv, conn_param);
2844 if (ret)
2845 goto out;
e51060f0
SH		 2846
2847 memset(&rep, 0, sizeof rep);
2848 rep.qp_num = id_priv->qp_num;
2849 rep.starting_psn = id_priv->seq_num;
2850 rep.private_data = conn_param->private_data;
2851 rep.private_data_len = conn_param->private_data_len;
2852 rep.responder_resources = conn_param->responder_resources;
2853 rep.initiator_depth = conn_param->initiator_depth;
e51060f0
SH		 2854 rep.failover_accepted = 0;
2855 rep.flow_control = conn_param->flow_control;
4ede178a 2856 rep.rnr_retry_count = min_t(u8, 7, conn_param->rnr_retry_count);
e51060f0
SH		 2857 rep.srq = id_priv->srq ? 1 : 0;
2858
0fe313b0
SH		 2859 ret = ib_send_cm_rep(id_priv->cm_id.ib, &rep);
2860out:
2861 return ret;
e51060f0
SH		 2862}
2863
07ebafba
TT		 2864static int cma_accept_iw(struct rdma_id_private *id_priv,
2865 struct rdma_conn_param *conn_param)
2866{
2867 struct iw_cm_conn_param iw_param;
2868 int ret;
2869
5851bb89 2870 ret = cma_modify_qp_rtr(id_priv, conn_param);
07ebafba
TT		 2871 if (ret)
2872 return ret;
2873
2874 iw_param.ord = conn_param->initiator_depth;
2875 iw_param.ird = conn_param->responder_resources;
2876 iw_param.private_data = conn_param->private_data;
2877 iw_param.private_data_len = conn_param->private_data_len;
2878 if (id_priv->id.qp) {
2879 iw_param.qpn = id_priv->qp_num;
2880 } else
2881 iw_param.qpn = conn_param->qp_num;
2882
2883 return iw_cm_accept(id_priv->cm_id.iw, &iw_param);
2884}
2885
628e5f6d
SH		 2886static int cma_send_sidr_rep(struct rdma_id_private *id_priv,
2887 enum ib_cm_sidr_status status,
2888 const void *private_data, int private_data_len)
2889{
2890 struct ib_cm_sidr_rep_param rep;
d2ca39f2 2891 int ret;
628e5f6d
SH		 2892
2893 memset(&rep, 0, sizeof rep);
2894 rep.status = status;
2895 if (status == IB_SIDR_SUCCESS) {
d2ca39f2
YE		 2896 ret = cma_set_qkey(id_priv);
2897 if (ret)
2898 return ret;
628e5f6d 2899 rep.qp_num = id_priv->qp_num;
c8f6a362 2900 rep.qkey = id_priv->qkey;
628e5f6d
SH		 2901 }
2902 rep.private_data = private_data;
2903 rep.private_data_len = private_data_len;
2904
2905 return ib_send_cm_sidr_rep(id_priv->cm_id.ib, &rep);
2906}
2907
e51060f0
SH		 2908int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
2909{
2910 struct rdma_id_private *id_priv;
2911 int ret;
2912
2913 id_priv = container_of(id, struct rdma_id_private, id);
83e9502d
NM		 2914
2915 id_priv->owner = task_pid_nr(current);
2916
550e5ca7 2917 if (!cma_comp(id_priv, RDMA_CM_CONNECT))
e51060f0
SH		 2918 return -EINVAL;
2919
2920 if (!id->qp && conn_param) {
2921 id_priv->qp_num = conn_param->qp_num;
e51060f0
SH		 2922 id_priv->srq = conn_param->srq;
2923 }
2924
07ebafba
TT		 2925 switch (rdma_node_get_transport(id->device->node_type)) {
2926 case RDMA_TRANSPORT_IB:
f45ee80e
HS		 2927 if (id->qp_type == IB_QPT_UD) {
2928 if (conn_param)
2929 ret = cma_send_sidr_rep(id_priv, IB_SIDR_SUCCESS,
2930 conn_param->private_data,
2931 conn_param->private_data_len);
2932 else
2933 ret = cma_send_sidr_rep(id_priv, IB_SIDR_SUCCESS,
2934 NULL, 0);
2935 } else {
2936 if (conn_param)
2937 ret = cma_accept_ib(id_priv, conn_param);
2938 else
2939 ret = cma_rep_recv(id_priv);
2940 }
e51060f0 2941 break;
07ebafba
TT		 2942 case RDMA_TRANSPORT_IWARP:
2943 ret = cma_accept_iw(id_priv, conn_param);
2944 break;
e51060f0
SH		 2945 default:
2946 ret = -ENOSYS;
2947 break;
2948 }
2949
2950 if (ret)
2951 goto reject;
2952
2953 return 0;
2954reject:
c5483388 2955 cma_modify_qp_err(id_priv);
e51060f0
SH		 2956 rdma_reject(id, NULL, 0);
2957 return ret;
2958}
2959EXPORT_SYMBOL(rdma_accept);
2960
0fe313b0
SH		 2961int rdma_notify(struct rdma_cm_id *id, enum ib_event_type event)
2962{
2963 struct rdma_id_private *id_priv;
2964 int ret;
2965
2966 id_priv = container_of(id, struct rdma_id_private, id);
0c9361fc 2967 if (!id_priv->cm_id.ib)
0fe313b0
SH		 2968 return -EINVAL;
2969
2970 switch (id->device->node_type) {
2971 case RDMA_NODE_IB_CA:
2972 ret = ib_cm_notify(id_priv->cm_id.ib, event);
2973 break;
2974 default:
2975 ret = 0;
2976 break;
2977 }
2978 return ret;
2979}
2980EXPORT_SYMBOL(rdma_notify);
2981
e51060f0
SH		 2982int rdma_reject(struct rdma_cm_id *id, const void *private_data,
2983 u8 private_data_len)
2984{
2985 struct rdma_id_private *id_priv;
2986 int ret;
2987
2988 id_priv = container_of(id, struct rdma_id_private, id);
0c9361fc 2989 if (!id_priv->cm_id.ib)
e51060f0
SH		 2990 return -EINVAL;
2991
07ebafba
TT		 2992 switch (rdma_node_get_transport(id->device->node_type)) {
2993 case RDMA_TRANSPORT_IB:
b26f9b99 2994 if (id->qp_type == IB_QPT_UD)
628e5f6d
SH		 2995 ret = cma_send_sidr_rep(id_priv, IB_SIDR_REJECT,
2996 private_data, private_data_len);
2997 else
2998 ret = ib_send_cm_rej(id_priv->cm_id.ib,
2999 IB_CM_REJ_CONSUMER_DEFINED, NULL,
3000 0, private_data, private_data_len);
e51060f0 3001 break;
07ebafba
TT		 3002 case RDMA_TRANSPORT_IWARP:
3003 ret = iw_cm_reject(id_priv->cm_id.iw,
3004 private_data, private_data_len);
3005 break;
e51060f0
SH		 3006 default:
3007 ret = -ENOSYS;
3008 break;
3009 }
3010 return ret;
3011}
3012EXPORT_SYMBOL(rdma_reject);
3013
3014int rdma_disconnect(struct rdma_cm_id *id)
3015{
3016 struct rdma_id_private *id_priv;
3017 int ret;
3018
3019 id_priv = container_of(id, struct rdma_id_private, id);
0c9361fc 3020 if (!id_priv->cm_id.ib)
e51060f0
SH		 3021 return -EINVAL;
3022
07ebafba
TT		 3023 switch (rdma_node_get_transport(id->device->node_type)) {
3024 case RDMA_TRANSPORT_IB:
c5483388 3025 ret = cma_modify_qp_err(id_priv);
07ebafba
TT		 3026 if (ret)
3027 goto out;
e51060f0
SH		 3028 /* Initiate or respond to a disconnect. */
3029 if (ib_send_cm_dreq(id_priv->cm_id.ib, NULL, 0))
3030 ib_send_cm_drep(id_priv->cm_id.ib, NULL, 0);
3031 break;
07ebafba
TT		 3032 case RDMA_TRANSPORT_IWARP:
3033 ret = iw_cm_disconnect(id_priv->cm_id.iw, 0);
3034 break;
e51060f0 3035 default:
07ebafba 3036 ret = -EINVAL;
e51060f0
SH		 3037 break;
3038 }
3039out:
3040 return ret;
3041}
3042EXPORT_SYMBOL(rdma_disconnect);
3043
c8f6a362
SH		 3044static int cma_ib_mc_handler(int status, struct ib_sa_multicast *multicast)
3045{
3046 struct rdma_id_private *id_priv;
3047 struct cma_multicast *mc = multicast->context;
3048 struct rdma_cm_event event;
3049 int ret;
3050
3051 id_priv = mc->id_priv;
550e5ca7
NM		 3052 if (cma_disable_callback(id_priv, RDMA_CM_ADDR_BOUND) &&
3053 cma_disable_callback(id_priv, RDMA_CM_ADDR_RESOLVED))
8aa08602 3054 return 0;
c8f6a362 3055
c5483388 3056 mutex_lock(&id_priv->qp_mutex);
c8f6a362
SH		 3057 if (!status && id_priv->id.qp)
3058 status = ib_attach_mcast(id_priv->id.qp, &multicast->rec.mgid,
46ea5061 3059 be16_to_cpu(multicast->rec.mlid));
c5483388 3060 mutex_unlock(&id_priv->qp_mutex);
c8f6a362
SH		 3061
3062 memset(&event, 0, sizeof event);
3063 event.status = status;
3064 event.param.ud.private_data = mc->context;
3065 if (!status) {
3066 event.event = RDMA_CM_EVENT_MULTICAST_JOIN;
3067 ib_init_ah_from_mcmember(id_priv->id.device,
3068 id_priv->id.port_num, &multicast->rec,
3069 &event.param.ud.ah_attr);
3070 event.param.ud.qp_num = 0xFFFFFF;
3071 event.param.ud.qkey = be32_to_cpu(multicast->rec.qkey);
3072 } else
3073 event.event = RDMA_CM_EVENT_MULTICAST_ERROR;
3074
3075 ret = id_priv->id.event_handler(&id_priv->id, &event);
3076 if (ret) {
550e5ca7 3077 cma_exch(id_priv, RDMA_CM_DESTROYING);
de910bd9 3078 mutex_unlock(&id_priv->handler_mutex);
c8f6a362
SH		 3079 rdma_destroy_id(&id_priv->id);
3080 return 0;
3081 }
8aa08602 3082
de910bd9 3083 mutex_unlock(&id_priv->handler_mutex);
c8f6a362
SH		 3084 return 0;
3085}
3086
3087static void cma_set_mgid(struct rdma_id_private *id_priv,
3088 struct sockaddr *addr, union ib_gid *mgid)
3089{
3090 unsigned char mc_map[MAX_ADDR_LEN];
3091 struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
3092 struct sockaddr_in *sin = (struct sockaddr_in *) addr;
3093 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *) addr;
3094
3095 if (cma_any_addr(addr)) {
3096 memset(mgid, 0, sizeof *mgid);
3097 } else if ((addr->sa_family == AF_INET6) &&
1c9b2819 3098 ((be32_to_cpu(sin6->sin6_addr.s6_addr32[0]) & 0xFFF0FFFF) ==
c8f6a362
SH		 3099 0xFF10A01B)) {
3100 /* IPv6 address is an SA assigned MGID. */
3101 memcpy(mgid, &sin6->sin6_addr, sizeof *mgid);
e2e62697
JG		 3102 } else if ((addr->sa_family == AF_INET6)) {
3103 ipv6_ib_mc_map(&sin6->sin6_addr, dev_addr->broadcast, mc_map);
3104 if (id_priv->id.ps == RDMA_PS_UDP)
3105 mc_map[7] = 0x01; /* Use RDMA CM signature */
3106 *mgid = *(union ib_gid *) (mc_map + 4);
c8f6a362 3107 } else {
a9e527e3 3108 ip_ib_mc_map(sin->sin_addr.s_addr, dev_addr->broadcast, mc_map);
c8f6a362
SH		 3109 if (id_priv->id.ps == RDMA_PS_UDP)
3110 mc_map[7] = 0x01; /* Use RDMA CM signature */
c8f6a362
SH		 3111 *mgid = *(union ib_gid *) (mc_map + 4);
3112 }
3113}
3114
3115static int cma_join_ib_multicast(struct rdma_id_private *id_priv,
3116 struct cma_multicast *mc)
3117{
3118 struct ib_sa_mcmember_rec rec;
3119 struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
3120 ib_sa_comp_mask comp_mask;
3121 int ret;
3122
3123 ib_addr_get_mgid(dev_addr, &rec.mgid);
3124 ret = ib_sa_get_mcmember_rec(id_priv->id.device, id_priv->id.port_num,
3125 &rec.mgid, &rec);
3126 if (ret)
3127 return ret;
3128
3f446754 3129 cma_set_mgid(id_priv, (struct sockaddr *) &mc->addr, &rec.mgid);
c8f6a362
SH		 3130 if (id_priv->id.ps == RDMA_PS_UDP)
3131 rec.qkey = cpu_to_be32(RDMA_UDP_QKEY);
6f8372b6 3132 rdma_addr_get_sgid(dev_addr, &rec.port_gid);
c8f6a362
SH		 3133 rec.pkey = cpu_to_be16(ib_addr_get_pkey(dev_addr));
3134 rec.join_state = 1;
3135
3136 comp_mask = IB_SA_MCMEMBER_REC_MGID | IB_SA_MCMEMBER_REC_PORT_GID |
3137 IB_SA_MCMEMBER_REC_PKEY | IB_SA_MCMEMBER_REC_JOIN_STATE |
3138 IB_SA_MCMEMBER_REC_QKEY | IB_SA_MCMEMBER_REC_SL |
3139 IB_SA_MCMEMBER_REC_FLOW_LABEL |
3140 IB_SA_MCMEMBER_REC_TRAFFIC_CLASS;
3141
84adeee9
YE		 3142 if (id_priv->id.ps == RDMA_PS_IPOIB)
3143 comp_mask |= IB_SA_MCMEMBER_REC_RATE |
2a22fb8c
DB		 3144 IB_SA_MCMEMBER_REC_RATE_SELECTOR |
3145 IB_SA_MCMEMBER_REC_MTU_SELECTOR |
3146 IB_SA_MCMEMBER_REC_MTU |
3147 IB_SA_MCMEMBER_REC_HOP_LIMIT;
84adeee9 3148
c8f6a362
SH		 3149 mc->multicast.ib = ib_sa_join_multicast(&sa_client, id_priv->id.device,
3150 id_priv->id.port_num, &rec,
3151 comp_mask, GFP_KERNEL,
3152 cma_ib_mc_handler, mc);
4e289045 3153 return PTR_RET(mc->multicast.ib);
c8f6a362
SH		 3154}
3155
3c86aa70
EC		 3156static void iboe_mcast_work_handler(struct work_struct *work)
3157{
3158 struct iboe_mcast_work *mw = container_of(work, struct iboe_mcast_work, work);
3159 struct cma_multicast *mc = mw->mc;
3160 struct ib_sa_multicast *m = mc->multicast.ib;
3161
3162 mc->multicast.ib->context = mc;
3163 cma_ib_mc_handler(0, m);
3164 kref_put(&mc->mcref, release_mc);
3165 kfree(mw);
3166}
3167
3168static void cma_iboe_set_mgid(struct sockaddr *addr, union ib_gid *mgid)
3169{
3170 struct sockaddr_in *sin = (struct sockaddr_in *)addr;
3171 struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)addr;
3172
3173 if (cma_any_addr(addr)) {
3174 memset(mgid, 0, sizeof *mgid);
3175 } else if (addr->sa_family == AF_INET6) {
3176 memcpy(mgid, &sin6->sin6_addr, sizeof *mgid);
3177 } else {
3178 mgid->raw[0] = 0xff;
3179 mgid->raw[1] = 0x0e;
3180 mgid->raw[2] = 0;
3181 mgid->raw[3] = 0;
3182 mgid->raw[4] = 0;
3183 mgid->raw[5] = 0;
3184 mgid->raw[6] = 0;
3185 mgid->raw[7] = 0;
3186 mgid->raw[8] = 0;
3187 mgid->raw[9] = 0;
3188 mgid->raw[10] = 0xff;
3189 mgid->raw[11] = 0xff;
3190 *(__be32 *)(&mgid->raw[12]) = sin->sin_addr.s_addr;
3191 }
3192}
3193
3194static int cma_iboe_join_multicast(struct rdma_id_private *id_priv,
3195 struct cma_multicast *mc)
3196{
3197 struct iboe_mcast_work *work;
3198 struct rdma_dev_addr *dev_addr = &id_priv->id.route.addr.dev_addr;
3199 int err;
3200 struct sockaddr *addr = (struct sockaddr *)&mc->addr;
3201 struct net_device *ndev = NULL;
3202
3203 if (cma_zero_addr((struct sockaddr *)&mc->addr))
3204 return -EINVAL;
3205
3206 work = kzalloc(sizeof *work, GFP_KERNEL);
3207 if (!work)
3208 return -ENOMEM;
3209
3210 mc->multicast.ib = kzalloc(sizeof(struct ib_sa_multicast), GFP_KERNEL);
3211 if (!mc->multicast.ib) {
3212 err = -ENOMEM;
3213 goto out1;
3214 }
3215
3216 cma_iboe_set_mgid(addr, &mc->multicast.ib->rec.mgid);
3217
3218 mc->multicast.ib->rec.pkey = cpu_to_be16(0xffff);
3219 if (id_priv->id.ps == RDMA_PS_UDP)
3220 mc->multicast.ib->rec.qkey = cpu_to_be32(RDMA_UDP_QKEY);
3221
3222 if (dev_addr->bound_dev_if)
3223 ndev = dev_get_by_index(&init_net, dev_addr->bound_dev_if);
3224 if (!ndev) {
3225 err = -ENODEV;
3226 goto out2;
3227 }
3228 mc->multicast.ib->rec.rate = iboe_get_rate(ndev);
3229 mc->multicast.ib->rec.hop_limit = 1;
3230 mc->multicast.ib->rec.mtu = iboe_get_mtu(ndev->mtu);
3231 dev_put(ndev);
3232 if (!mc->multicast.ib->rec.mtu) {
3233 err = -EINVAL;
3234 goto out2;
3235 }
3236 iboe_addr_get_sgid(dev_addr, &mc->multicast.ib->rec.port_gid);
3237 work->id = id_priv;
3238 work->mc = mc;
3239 INIT_WORK(&work->work, iboe_mcast_work_handler);
3240 kref_get(&mc->mcref);
3241 queue_work(cma_wq, &work->work);
3242
3243 return 0;
3244
3245out2:
3246 kfree(mc->multicast.ib);
3247out1:
3248 kfree(work);
3249 return err;
3250}
3251
c8f6a362
SH		 3252int rdma_join_multicast(struct rdma_cm_id *id, struct sockaddr *addr,
3253 void *context)
3254{
3255 struct rdma_id_private *id_priv;
3256 struct cma_multicast *mc;
3257 int ret;
3258
3259 id_priv = container_of(id, struct rdma_id_private, id);
550e5ca7
NM		 3260 if (!cma_comp(id_priv, RDMA_CM_ADDR_BOUND) &&
3261 !cma_comp(id_priv, RDMA_CM_ADDR_RESOLVED))
c8f6a362
SH		 3262 return -EINVAL;
3263
3264 mc = kmalloc(sizeof *mc, GFP_KERNEL);
3265 if (!mc)
3266 return -ENOMEM;
3267
ef560861 3268 memcpy(&mc->addr, addr, rdma_addr_size(addr));
c8f6a362
SH		 3269 mc->context = context;
3270 mc->id_priv = id_priv;
3271
3272 spin_lock(&id_priv->lock);
3273 list_add(&mc->list, &id_priv->mc_list);
3274 spin_unlock(&id_priv->lock);
3275
3276 switch (rdma_node_get_transport(id->device->node_type)) {
3277 case RDMA_TRANSPORT_IB:
3c86aa70
EC		 3278 switch (rdma_port_get_link_layer(id->device, id->port_num)) {
3279 case IB_LINK_LAYER_INFINIBAND:
3280 ret = cma_join_ib_multicast(id_priv, mc);
3281 break;
3282 case IB_LINK_LAYER_ETHERNET:
3283 kref_init(&mc->mcref);
3284 ret = cma_iboe_join_multicast(id_priv, mc);
3285 break;
3286 default:
3287 ret = -EINVAL;
3288 }
c8f6a362
SH		 3289 break;
3290 default:
3291 ret = -ENOSYS;
3292 break;
3293 }
3294
3295 if (ret) {
3296 spin_lock_irq(&id_priv->lock);
3297 list_del(&mc->list);
3298 spin_unlock_irq(&id_priv->lock);
3299 kfree(mc);
3300 }
3301 return ret;
3302}
3303EXPORT_SYMBOL(rdma_join_multicast);
3304
3305void rdma_leave_multicast(struct rdma_cm_id *id, struct sockaddr *addr)
3306{
3307 struct rdma_id_private *id_priv;
3308 struct cma_multicast *mc;
3309
3310 id_priv = container_of(id, struct rdma_id_private, id);
3311 spin_lock_irq(&id_priv->lock);
3312 list_for_each_entry(mc, &id_priv->mc_list, list) {
ef560861 3313 if (!memcmp(&mc->addr, addr, rdma_addr_size(addr))) {
c8f6a362
SH		 3314 list_del(&mc->list);
3315 spin_unlock_irq(&id_priv->lock);
3316
3317 if (id->qp)
3318 ib_detach_mcast(id->qp,
3319 &mc->multicast.ib->rec.mgid,
46ea5061 3320 be16_to_cpu(mc->multicast.ib->rec.mlid));
3c86aa70
EC		 3321 if (rdma_node_get_transport(id_priv->cma_dev->device->node_type) == RDMA_TRANSPORT_IB) {
3322 switch (rdma_port_get_link_layer(id->device, id->port_num)) {
3323 case IB_LINK_LAYER_INFINIBAND:
3324 ib_sa_free_multicast(mc->multicast.ib);
3325 kfree(mc);
3326 break;
3327 case IB_LINK_LAYER_ETHERNET:
3328 kref_put(&mc->mcref, release_mc);
3329 break;
3330 default:
3331 break;
3332 }
3333 }
c8f6a362
SH		 3334 return;
3335 }
3336 }
3337 spin_unlock_irq(&id_priv->lock);
3338}
3339EXPORT_SYMBOL(rdma_leave_multicast);
3340
dd5bdff8
OG		 3341static int cma_netdev_change(struct net_device *ndev, struct rdma_id_private *id_priv)
3342{
3343 struct rdma_dev_addr *dev_addr;
3344 struct cma_ndev_work *work;
3345
3346 dev_addr = &id_priv->id.route.addr.dev_addr;
3347
6266ed6e 3348 if ((dev_addr->bound_dev_if == ndev->ifindex) &&
dd5bdff8
OG		 3349 memcmp(dev_addr->src_dev_addr, ndev->dev_addr, ndev->addr_len)) {
3350 printk(KERN_INFO "RDMA CM addr change for ndev %s used by id %p\n",
3351 ndev->name, &id_priv->id);
3352 work = kzalloc(sizeof *work, GFP_KERNEL);
3353 if (!work)
3354 return -ENOMEM;
3355
3356 INIT_WORK(&work->work, cma_ndev_work_handler);
3357 work->id = id_priv;
3358 work->event.event = RDMA_CM_EVENT_ADDR_CHANGE;
3359 atomic_inc(&id_priv->refcount);
3360 queue_work(cma_wq, &work->work);
3361 }
3362
3363 return 0;
3364}
3365
3366static int cma_netdev_callback(struct notifier_block *self, unsigned long event,
351638e7 3367 void *ptr)
dd5bdff8 3368{
351638e7 3369 struct net_device *ndev = netdev_notifier_info_to_dev(ptr);
dd5bdff8
OG		 3370 struct cma_device *cma_dev;
3371 struct rdma_id_private *id_priv;
3372 int ret = NOTIFY_DONE;
3373
3374 if (dev_net(ndev) != &init_net)
3375 return NOTIFY_DONE;
3376
3377 if (event != NETDEV_BONDING_FAILOVER)
3378 return NOTIFY_DONE;
3379
3380 if (!(ndev->flags & IFF_MASTER) || !(ndev->priv_flags & IFF_BONDING))
3381 return NOTIFY_DONE;
3382
3383 mutex_lock(&lock);
3384 list_for_each_entry(cma_dev, &dev_list, list)
3385 list_for_each_entry(id_priv, &cma_dev->id_list, list) {
3386 ret = cma_netdev_change(ndev, id_priv);
3387 if (ret)
3388 goto out;
3389 }
3390
3391out:
3392 mutex_unlock(&lock);
3393 return ret;
3394}
3395
3396static struct notifier_block cma_nb = {
3397 .notifier_call = cma_netdev_callback
3398};
3399
e51060f0
SH		 3400static void cma_add_one(struct ib_device *device)
3401{
3402 struct cma_device *cma_dev;
3403 struct rdma_id_private *id_priv;
3404
3405 cma_dev = kmalloc(sizeof *cma_dev, GFP_KERNEL);
3406 if (!cma_dev)
3407 return;
3408
3409 cma_dev->device = device;
e51060f0
SH		 3410
3411 init_completion(&cma_dev->comp);
3412 atomic_set(&cma_dev->refcount, 1);
3413 INIT_LIST_HEAD(&cma_dev->id_list);
3414 ib_set_client_data(device, &cma_client, cma_dev);
3415
3416 mutex_lock(&lock);
3417 list_add_tail(&cma_dev->list, &dev_list);
3418 list_for_each_entry(id_priv, &listen_any_list, list)
3419 cma_listen_on_dev(id_priv, cma_dev);
3420 mutex_unlock(&lock);
e51060f0
SH		 3421}
3422
3423static int cma_remove_id_dev(struct rdma_id_private *id_priv)
3424{
a1b1b61f 3425 struct rdma_cm_event event;
550e5ca7 3426 enum rdma_cm_state state;
de910bd9 3427 int ret = 0;
e51060f0
SH		 3428
3429 /* Record that we want to remove the device */
550e5ca7
NM		 3430 state = cma_exch(id_priv, RDMA_CM_DEVICE_REMOVAL);
3431 if (state == RDMA_CM_DESTROYING)
e51060f0
SH		 3432 return 0;
3433
3434 cma_cancel_operation(id_priv, state);
de910bd9 3435 mutex_lock(&id_priv->handler_mutex);
e51060f0
SH		 3436
3437 /* Check for destruction from another callback. */
550e5ca7 3438 if (!cma_comp(id_priv, RDMA_CM_DEVICE_REMOVAL))
de910bd9 3439 goto out;
e51060f0 3440
a1b1b61f
SH		 3441 memset(&event, 0, sizeof event);
3442 event.event = RDMA_CM_EVENT_DEVICE_REMOVAL;
de910bd9
OG		 3443 ret = id_priv->id.event_handler(&id_priv->id, &event);
3444out:
3445 mutex_unlock(&id_priv->handler_mutex);
3446 return ret;
e51060f0
SH		 3447}
3448
3449static void cma_process_remove(struct cma_device *cma_dev)
3450{
e51060f0
SH		 3451 struct rdma_id_private *id_priv;
3452 int ret;
3453
e51060f0
SH		 3454 mutex_lock(&lock);
3455 while (!list_empty(&cma_dev->id_list)) {
3456 id_priv = list_entry(cma_dev->id_list.next,
3457 struct rdma_id_private, list);
3458
d02d1f53 3459 list_del(&id_priv->listen_list);
94de178a 3460 list_del_init(&id_priv->list);
e51060f0
SH		 3461 atomic_inc(&id_priv->refcount);
3462 mutex_unlock(&lock);
3463
d02d1f53 3464 ret = id_priv->internal_id ? 1 : cma_remove_id_dev(id_priv);
e51060f0
SH		 3465 cma_deref_id(id_priv);
3466 if (ret)
3467 rdma_destroy_id(&id_priv->id);
3468
3469 mutex_lock(&lock);
3470 }
3471 mutex_unlock(&lock);
3472
3473 cma_deref_dev(cma_dev);
3474 wait_for_completion(&cma_dev->comp);
3475}
3476
3477static void cma_remove_one(struct ib_device *device)
3478{
3479 struct cma_device *cma_dev;
3480
3481 cma_dev = ib_get_client_data(device, &cma_client);
3482 if (!cma_dev)
3483 return;
3484
3485 mutex_lock(&lock);
3486 list_del(&cma_dev->list);
3487 mutex_unlock(&lock);
3488
3489 cma_process_remove(cma_dev);
3490 kfree(cma_dev);
3491}
3492
753f618a
NM		 3493static int cma_get_id_stats(struct sk_buff *skb, struct netlink_callback *cb)
3494{
3495 struct nlmsghdr *nlh;
3496 struct rdma_cm_id_stats *id_stats;
3497 struct rdma_id_private *id_priv;
3498 struct rdma_cm_id *id = NULL;
3499 struct cma_device *cma_dev;
3500 int i_dev = 0, i_id = 0;
3501
3502 /*
3503 * We export all of the IDs as a sequence of messages. Each
3504 * ID gets its own netlink message.
3505 */
3506 mutex_lock(&lock);
3507
3508 list_for_each_entry(cma_dev, &dev_list, list) {
3509 if (i_dev < cb->args[0]) {
3510 i_dev++;
3511 continue;
3512 }
3513
3514 i_id = 0;
3515 list_for_each_entry(id_priv, &cma_dev->id_list, list) {
3516 if (i_id < cb->args[1]) {
3517 i_id++;
3518 continue;
3519 }
3520
3521 id_stats = ibnl_put_msg(skb, &nlh, cb->nlh->nlmsg_seq,
3522 sizeof *id_stats, RDMA_NL_RDMA_CM,
3523 RDMA_NL_RDMA_CM_ID_STATS);
3524 if (!id_stats)
3525 goto out;
3526
3527 memset(id_stats, 0, sizeof *id_stats);
3528 id = &id_priv->id;
3529 id_stats->node_type = id->route.addr.dev_addr.dev_type;
3530 id_stats->port_num = id->port_num;
3531 id_stats->bound_dev_if =
3532 id->route.addr.dev_addr.bound_dev_if;
3533
3534 if (id->route.addr.src_addr.ss_family == AF_INET) {
3535 if (ibnl_put_attr(skb, nlh,
3536 sizeof(struct sockaddr_in),
3537 &id->route.addr.src_addr,
3538 RDMA_NL_RDMA_CM_ATTR_SRC_ADDR)) {
3539 goto out;
3540 }
3541 if (ibnl_put_attr(skb, nlh,
3542 sizeof(struct sockaddr_in),
3543 &id->route.addr.dst_addr,
3544 RDMA_NL_RDMA_CM_ATTR_DST_ADDR)) {
3545 goto out;
3546 }
3547 } else if (id->route.addr.src_addr.ss_family == AF_INET6) {
3548 if (ibnl_put_attr(skb, nlh,
3549 sizeof(struct sockaddr_in6),
3550 &id->route.addr.src_addr,
3551 RDMA_NL_RDMA_CM_ATTR_SRC_ADDR)) {
3552 goto out;
3553 }
3554 if (ibnl_put_attr(skb, nlh,
3555 sizeof(struct sockaddr_in6),
3556 &id->route.addr.dst_addr,
3557 RDMA_NL_RDMA_CM_ATTR_DST_ADDR)) {
3558 goto out;
3559 }
3560 }
3561
83e9502d 3562 id_stats->pid = id_priv->owner;
753f618a
NM		 3563 id_stats->port_space = id->ps;
3564 id_stats->cm_state = id_priv->state;
3565 id_stats->qp_num = id_priv->qp_num;
3566 id_stats->qp_type = id->qp_type;
3567
3568 i_id++;
3569 }
3570
3571 cb->args[1] = 0;
3572 i_dev++;
3573 }
3574
3575out:
3576 mutex_unlock(&lock);
3577 cb->args[0] = i_dev;
3578 cb->args[1] = i_id;
3579
3580 return skb->len;
3581}
3582
3583static const struct ibnl_client_cbs cma_cb_table[] = {
809d5fc9
G		 3584 [RDMA_NL_RDMA_CM_ID_STATS] = { .dump = cma_get_id_stats,
3585 .module = THIS_MODULE },
753f618a
NM		 3586};
3587
716abb1f 3588static int __init cma_init(void)
e51060f0 3589{
5d7220e8 3590 int ret;
227b60f5 3591
c7f743a6 3592 cma_wq = create_singlethread_workqueue("rdma_cm");
e51060f0
SH		 3593 if (!cma_wq)
3594 return -ENOMEM;
3595
c1a0b23b 3596 ib_sa_register_client(&sa_client);
7a118df3 3597 rdma_addr_register_client(&addr_client);
dd5bdff8 3598 register_netdevice_notifier(&cma_nb);
c1a0b23b 3599
e51060f0
SH		 3600 ret = ib_register_client(&cma_client);
3601 if (ret)
3602 goto err;
753f618a
NM		 3603
3604 if (ibnl_add_client(RDMA_NL_RDMA_CM, RDMA_NL_RDMA_CM_NUM_OPS, cma_cb_table))
3605 printk(KERN_WARNING "RDMA CMA: failed to add netlink callback\n");
3606
e51060f0
SH		 3607 return 0;
3608
3609err:
dd5bdff8 3610 unregister_netdevice_notifier(&cma_nb);
7a118df3 3611 rdma_addr_unregister_client(&addr_client);
c1a0b23b 3612 ib_sa_unregister_client(&sa_client);
e51060f0
SH		 3613 destroy_workqueue(cma_wq);
3614 return ret;
3615}
3616
716abb1f 3617static void __exit cma_cleanup(void)
e51060f0 3618{
753f618a 3619 ibnl_remove_client(RDMA_NL_RDMA_CM);
e51060f0 3620 ib_unregister_client(&cma_client);
dd5bdff8 3621 unregister_netdevice_notifier(&cma_nb);
7a118df3 3622 rdma_addr_unregister_client(&addr_client);
c1a0b23b 3623 ib_sa_unregister_client(&sa_client);
e51060f0
SH		 3624 destroy_workqueue(cma_wq);
3625 idr_destroy(&sdp_ps);
3626 idr_destroy(&tcp_ps);
628e5f6d 3627 idr_destroy(&udp_ps);
c8f6a362 3628 idr_destroy(&ipoib_ps);
2d2e9415 3629 idr_destroy(&ib_ps);
e51060f0
SH		 3630}
3631
3632module_init(cma_init);
3633module_exit(cma_cleanup);
Linux 6.x block layer and io_uring tree(s)