projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'arm64-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/arm64/linux
[linux-block.git] / drivers / infiniband / ulp / ipoib / ipoib_main.c
CommitLineData
1da177e4
LT		 1/*
2 * Copyright (c) 2004 Topspin Communications. All rights reserved.
2a1d9b7f
RD		 3 * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
4 * Copyright (c) 2004 Voltaire, Inc. All rights reserved.
1da177e4
LT		 5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
1da177e4
LT		 33 */
34
35#include "ipoib.h"
36
1da177e4
LT		 37#include <linux/module.h>
38
39#include <linux/init.h>
40#include <linux/slab.h>
0f485251 41#include <linux/kernel.h>
10313cbb 42#include <linux/vmalloc.h>
1da177e4
LT		 43
44#include <linux/if_arp.h> /* For ARPHRD_xxx */
45
46#include <linux/ip.h>
47#include <linux/in.h>
48
b63b70d8
SP		 49#include <linux/jhash.h>
50#include <net/arp.h>
ddde896e
GS		 51#include <net/addrconf.h>
52#include <linux/inetdevice.h>
53#include <rdma/ib_cache.h>
14c85021 54
1da177e4
LT		 55MODULE_AUTHOR("Roland Dreier");
56MODULE_DESCRIPTION("IP-over-InfiniBand net driver");
57MODULE_LICENSE("Dual BSD/GPL");
58
0f485251
SM		 59int ipoib_sendq_size __read_mostly = IPOIB_TX_RING_SIZE;
60int ipoib_recvq_size __read_mostly = IPOIB_RX_RING_SIZE;
61
62module_param_named(send_queue_size, ipoib_sendq_size, int, 0444);
63MODULE_PARM_DESC(send_queue_size, "Number of descriptors in send queue");
64module_param_named(recv_queue_size, ipoib_recvq_size, int, 0444);
65MODULE_PARM_DESC(recv_queue_size, "Number of descriptors in receive queue");
66
1da177e4
LT		 67#ifdef CONFIG_INFINIBAND_IPOIB_DEBUG
68int ipoib_debug_level;
69
70module_param_named(debug_level, ipoib_debug_level, int, 0644);
71MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0");
72#endif
73
1732b0ef
RD		 74struct ipoib_path_iter {
75 struct net_device *dev;
76 struct ipoib_path path;
77};
78
1da177e4
LT		 79static const u8 ipv4_bcast_addr[] = {
80 0x00, 0xff, 0xff, 0xff,
81 0xff, 0x12, 0x40, 0x1b, 0x00, 0x00, 0x00, 0x00,
82 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff
83};
84
85struct workqueue_struct *ipoib_workqueue;
86
c1a0b23b
MT		 87struct ib_sa_client ipoib_sa_client;
88
11a0ae4c 89static int ipoib_add_one(struct ib_device *device);
7c1eb45a 90static void ipoib_remove_one(struct ib_device *device, void *client_data);
b63b70d8 91static void ipoib_neigh_reclaim(struct rcu_head *rp);
ddde896e 92static struct net_device *ipoib_get_net_dev_by_params(
1fb7f897 93 struct ib_device *dev, u32 port, u16 pkey,
ddde896e
GS		 94 const union ib_gid *gid, const struct sockaddr *addr,
95 void *client_data);
492a7e67 96static int ipoib_set_mac(struct net_device *dev, void *addr);
31a82362
FD		 97static int ipoib_ioctl(struct net_device *dev, struct ifreq *ifr,
98 int cmd);
1da177e4
LT		 99
100static struct ib_client ipoib_client = {
101 .name = "ipoib",
102 .add = ipoib_add_one,
ddde896e
GS		 103 .remove = ipoib_remove_one,
104 .get_net_dev_by_params = ipoib_get_net_dev_by_params,
1da177e4
LT		 105};
106
771a5258
SR		 107#ifdef CONFIG_INFINIBAND_IPOIB_DEBUG
108static int ipoib_netdev_event(struct notifier_block *this,
109 unsigned long event, void *ptr)
110{
111 struct netdev_notifier_info *ni = ptr;
112 struct net_device *dev = ni->dev;
113
114 if (dev->netdev_ops->ndo_open != ipoib_open)
115 return NOTIFY_DONE;
116
117 switch (event) {
118 case NETDEV_REGISTER:
119 ipoib_create_debug_files(dev);
120 break;
121 case NETDEV_CHANGENAME:
122 ipoib_delete_debug_files(dev);
123 ipoib_create_debug_files(dev);
124 break;
125 case NETDEV_UNREGISTER:
126 ipoib_delete_debug_files(dev);
127 break;
128 }
129
130 return NOTIFY_DONE;
131}
132#endif
133
1da177e4
LT		 134int ipoib_open(struct net_device *dev)
135{
c1048aff 136 struct ipoib_dev_priv *priv = ipoib_priv(dev);
1da177e4
LT		 137
138 ipoib_dbg(priv, "bringing up interface\n");
139
437708c4
MS		 140 netif_carrier_off(dev);
141
e028cc55 142 set_bit(IPOIB_FLAG_ADMIN_UP, &priv->flags);
1da177e4 143
efc82eee 144 if (ipoib_ib_dev_open(dev)) {
dd57c930
AE		 145 if (!test_bit(IPOIB_PKEY_ASSIGNED, &priv->flags))
146 return 0;
b8a1b1ce 147 goto err_disable;
dd57c930 148 }
fe25c561 149
5c37077f 150 ipoib_ib_dev_up(dev);
1da177e4
LT		 151
152 if (!test_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags)) {
153 struct ipoib_dev_priv *cpriv;
154
155 /* Bring up any child interfaces too */
f47944cc 156 down_read(&priv->vlan_rwsem);
1da177e4
LT		 157 list_for_each_entry(cpriv, &priv->child_intfs, list) {
158 int flags;
159
160 flags = cpriv->dev->flags;
161 if (flags & IFF_UP)
162 continue;
163
567c5e13 164 dev_change_flags(cpriv->dev, flags | IFF_UP, NULL);
1da177e4 165 }
f47944cc 166 up_read(&priv->vlan_rwsem);
3ccbd933
JW		 167 } else if (priv->parent) {
168 struct ipoib_dev_priv *ppriv = ipoib_priv(priv->parent);
1da177e4 169
3ccbd933
JW		 170 if (!test_bit(IPOIB_FLAG_ADMIN_UP, &ppriv->flags))
171 ipoib_dbg(priv, "parent device %s is not up, so child device may be not functioning.\n",
172 ppriv->dev->name);
173 }
1da177e4
LT		 174 netif_start_queue(dev);
175
176 return 0;
b8a1b1ce 177
b8a1b1ce 178err_disable:
b8a1b1ce
RD		 179 clear_bit(IPOIB_FLAG_ADMIN_UP, &priv->flags);
180
181 return -EINVAL;
1da177e4
LT		 182}
183
184static int ipoib_stop(struct net_device *dev)
185{
c1048aff 186 struct ipoib_dev_priv *priv = ipoib_priv(dev);
1da177e4
LT		 187
188 ipoib_dbg(priv, "stopping interface\n");
189
190 clear_bit(IPOIB_FLAG_ADMIN_UP, &priv->flags);
191
192 netif_stop_queue(dev);
193
efc82eee 194 ipoib_ib_dev_down(dev);
cd565b4b 195 ipoib_ib_dev_stop(dev);
1da177e4
LT		 196
197 if (!test_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags)) {
198 struct ipoib_dev_priv *cpriv;
199
200 /* Bring down any child interfaces too */
f47944cc 201 down_read(&priv->vlan_rwsem);
1da177e4
LT		 202 list_for_each_entry(cpriv, &priv->child_intfs, list) {
203 int flags;
204
205 flags = cpriv->dev->flags;
206 if (!(flags & IFF_UP))
207 continue;
208
567c5e13 209 dev_change_flags(cpriv->dev, flags & ~IFF_UP, NULL);
1da177e4 210 }
f47944cc 211 up_read(&priv->vlan_rwsem);
1da177e4
LT		 212 }
213
214 return 0;
215}
216
9ca36f7d 217static netdev_features_t ipoib_fix_features(struct net_device *dev, netdev_features_t features)
3d96c74d 218{
c1048aff 219 struct ipoib_dev_priv *priv = ipoib_priv(dev);
3d96c74d
MM		 220
221 if (test_bit(IPOIB_FLAG_ADMIN_CM, &priv->flags))
c4268778 222 features &= ~(NETIF_F_IP_CSUM | NETIF_F_TSO);
3d96c74d
MM		 223
224 return features;
225}
226
1da177e4
LT		 227static int ipoib_change_mtu(struct net_device *dev, int new_mtu)
228{
c1048aff 229 struct ipoib_dev_priv *priv = ipoib_priv(dev);
ed7b521d 230 int ret = 0;
1da177e4 231
839fcaba 232 /* dev->mtu > 2K ==> connected mode */
586a6934
PS		 233 if (ipoib_cm_admin_enabled(dev)) {
234 if (new_mtu > ipoib_cm_max_mtu(dev))
235 return -EINVAL;
236
839fcaba
MT		 237 if (new_mtu > priv->mcast_mtu)
238 ipoib_warn(priv, "mtu > %d will cause multicast packet drops.\n",
239 priv->mcast_mtu);
586a6934 240
1eb2cded 241 WRITE_ONCE(dev->mtu, new_mtu);
839fcaba
MT		 242 return 0;
243 }
244
142a9c28
MS		 245 if (new_mtu < (ETH_MIN_MTU + IPOIB_ENCAP_LEN) ||
246 new_mtu > IPOIB_UD_MTU(priv->max_ib_mtu))
1da177e4
LT		 247 return -EINVAL;
248
249 priv->admin_mtu = new_mtu;
250
29da686d
FD		 251 if (priv->mcast_mtu < priv->admin_mtu)
252 ipoib_dbg(priv, "MTU must be smaller than the underlying "
253 "link layer MTU - 4 (%u)\n", priv->mcast_mtu);
254
ed7b521d 255 new_mtu = min(priv->mcast_mtu, priv->admin_mtu);
1da177e4 256
ed7b521d
ES		 257 if (priv->rn_ops->ndo_change_mtu) {
258 bool carrier_status = netif_carrier_ok(dev);
259
260 netif_carrier_off(dev);
261
262 /* notify lower level on the real mtu */
263 ret = priv->rn_ops->ndo_change_mtu(dev, new_mtu);
264
265 if (carrier_status)
266 netif_carrier_on(dev);
267 } else {
1eb2cded 268 WRITE_ONCE(dev->mtu, new_mtu);
ed7b521d
ES		 269 }
270
271 return ret;
1da177e4
LT		 272}
273
b6c871e5
ES		 274static void ipoib_get_stats(struct net_device *dev,
275 struct rtnl_link_stats64 *stats)
276{
277 struct ipoib_dev_priv *priv = ipoib_priv(dev);
278
279 if (priv->rn_ops->ndo_get_stats64)
280 priv->rn_ops->ndo_get_stats64(dev, stats);
281 else
282 netdev_stats_to_stats64(stats, &dev->stats);
283}
284
ddde896e
GS		 285/* Called with an RCU read lock taken */
286static bool ipoib_is_dev_match_addr_rcu(const struct sockaddr *addr,
287 struct net_device *dev)
288{
289 struct net *net = dev_net(dev);
290 struct in_device *in_dev;
291 struct sockaddr_in *addr_in = (struct sockaddr_in *)addr;
292 struct sockaddr_in6 *addr_in6 = (struct sockaddr_in6 *)addr;
293 __be32 ret_addr;
294
295 switch (addr->sa_family) {
296 case AF_INET:
297 in_dev = in_dev_get(dev);
298 if (!in_dev)
299 return false;
300
301 ret_addr = inet_confirm_addr(net, in_dev, 0,
302 addr_in->sin_addr.s_addr,
303 RT_SCOPE_HOST);
304 in_dev_put(in_dev);
305 if (ret_addr)
306 return true;
307
308 break;
309 case AF_INET6:
310 if (IS_ENABLED(CONFIG_IPV6) &&
311 ipv6_chk_addr(net, &addr_in6->sin6_addr, dev, 1))
312 return true;
313
314 break;
315 }
316 return false;
317}
318
bf194997 319/*
ddde896e
GS		 320 * Find the master net_device on top of the given net_device.
321 * @dev: base IPoIB net_device
322 *
323 * Returns the master net_device with a reference held, or the same net_device
324 * if no master exists.
325 */
326static struct net_device *ipoib_get_master_net_dev(struct net_device *dev)
327{
328 struct net_device *master;
329
330 rcu_read_lock();
331 master = netdev_master_upper_dev_get_rcu(dev);
e4e40a87 332 dev_hold(master);
ddde896e
GS		 333 rcu_read_unlock();
334
335 if (master)
336 return master;
337
338 dev_hold(dev);
339 return dev;
340}
341
e0e79c8e
DA		 342struct ipoib_walk_data {
343 const struct sockaddr *addr;
344 struct net_device *result;
345};
346
eff74233
TY		 347static int ipoib_upper_walk(struct net_device *upper,
348 struct netdev_nested_priv *priv)
e0e79c8e 349{
eff74233 350 struct ipoib_walk_data *data = (struct ipoib_walk_data *)priv->data;
e0e79c8e
DA		 351 int ret = 0;
352
353 if (ipoib_is_dev_match_addr_rcu(data->addr, upper)) {
354 dev_hold(upper);
355 data->result = upper;
356 ret = 1;
357 }
358
359 return ret;
360}
361
ddde896e 362/**
bf194997
LR		 363 * ipoib_get_net_dev_match_addr - Find a net_device matching
364 * the given address, which is an upper device of the given net_device.
365 *
ddde896e
GS		 366 * @addr: IP address to look for.
367 * @dev: base IPoIB net_device
368 *
369 * If found, returns the net_device with a reference held. Otherwise return
370 * NULL.
371 */
372static struct net_device *ipoib_get_net_dev_match_addr(
373 const struct sockaddr *addr, struct net_device *dev)
374{
eff74233 375 struct netdev_nested_priv priv;
e0e79c8e
DA		 376 struct ipoib_walk_data data = {
377 .addr = addr,
378 };
ddde896e 379
eff74233 380 priv.data = (void *)&data;
ddde896e
GS		 381 rcu_read_lock();
382 if (ipoib_is_dev_match_addr_rcu(addr, dev)) {
383 dev_hold(dev);
e0e79c8e 384 data.result = dev;
ddde896e
GS		 385 goto out;
386 }
387
eff74233 388 netdev_walk_all_upper_dev_rcu(dev, ipoib_upper_walk, &priv);
ddde896e
GS		 389out:
390 rcu_read_unlock();
e0e79c8e 391 return data.result;
ddde896e
GS		 392}
393
394/* returns the number of IPoIB netdevs on top a given ipoib device matching a
395 * pkey_index and address, if one exists.
396 *
397 * @found_net_dev: contains a matching net_device if the return value >= 1,
398 * with a reference held. */
399static int ipoib_match_gid_pkey_addr(struct ipoib_dev_priv *priv,
400 const union ib_gid *gid,
401 u16 pkey_index,
402 const struct sockaddr *addr,
403 int nesting,
404 struct net_device **found_net_dev)
405{
406 struct ipoib_dev_priv *child_priv;
407 struct net_device *net_dev = NULL;
408 int matches = 0;
409
410 if (priv->pkey_index == pkey_index &&
411 (!gid || !memcmp(gid, &priv->local_gid, sizeof(*gid)))) {
412 if (!addr) {
413 net_dev = ipoib_get_master_net_dev(priv->dev);
414 } else {
415 /* Verify the net_device matches the IP address, as
416 * IPoIB child devices currently share a GID. */
417 net_dev = ipoib_get_net_dev_match_addr(addr, priv->dev);
418 }
419 if (net_dev) {
420 if (!*found_net_dev)
421 *found_net_dev = net_dev;
422 else
423 dev_put(net_dev);
424 ++matches;
425 }
426 }
427
428 /* Check child interfaces */
429 down_read_nested(&priv->vlan_rwsem, nesting);
430 list_for_each_entry(child_priv, &priv->child_intfs, list) {
431 matches += ipoib_match_gid_pkey_addr(child_priv, gid,
432 pkey_index, addr,
433 nesting + 1,
434 found_net_dev);
435 if (matches > 1)
436 break;
437 }
438 up_read(&priv->vlan_rwsem);
439
440 return matches;
441}
442
443/* Returns the number of matching net_devs found (between 0 and 2). Also
444 * return the matching net_device in the @net_dev parameter, holding a
445 * reference to the net_device, if the number of matches >= 1 */
1fb7f897 446static int __ipoib_get_net_dev_by_params(struct list_head *dev_list, u32 port,
ddde896e
GS		 447 u16 pkey_index,
448 const union ib_gid *gid,
449 const struct sockaddr *addr,
450 struct net_device **net_dev)
451{
452 struct ipoib_dev_priv *priv;
453 int matches = 0;
454
455 *net_dev = NULL;
456
457 list_for_each_entry(priv, dev_list, list) {
458 if (priv->port != port)
459 continue;
460
461 matches += ipoib_match_gid_pkey_addr(priv, gid, pkey_index,
462 addr, 0, net_dev);
463 if (matches > 1)
464 break;
465 }
466
467 return matches;
468}
469
470static struct net_device *ipoib_get_net_dev_by_params(
1fb7f897 471 struct ib_device *dev, u32 port, u16 pkey,
ddde896e
GS		 472 const union ib_gid *gid, const struct sockaddr *addr,
473 void *client_data)
474{
475 struct net_device *net_dev;
476 struct list_head *dev_list = client_data;
477 u16 pkey_index;
478 int matches;
479 int ret;
480
481 if (!rdma_protocol_ib(dev, port))
482 return NULL;
483
484 ret = ib_find_cached_pkey(dev, port, pkey, &pkey_index);
485 if (ret)
486 return NULL;
487
ddde896e
GS		 488 /* See if we can find a unique device matching the L2 parameters */
489 matches = __ipoib_get_net_dev_by_params(dev_list, port, pkey_index,
490 gid, NULL, &net_dev);
491
492 switch (matches) {
493 case 0:
494 return NULL;
495 case 1:
496 return net_dev;
497 }
498
499 dev_put(net_dev);
500
501 /* Couldn't find a unique device with L2 parameters only. Use L3
502 * address to uniquely match the net device */
503 matches = __ipoib_get_net_dev_by_params(dev_list, port, pkey_index,
504 gid, addr, &net_dev);
505 switch (matches) {
506 case 0:
507 return NULL;
508 default:
509 dev_warn_ratelimited(&dev->dev,
510 "duplicate IP address detected\n");
df561f66 511 fallthrough;
ddde896e
GS		 512 case 1:
513 return net_dev;
514 }
515}
516
71d9c5f9
RD		 517int ipoib_set_mode(struct net_device *dev, const char *buf)
518{
c1048aff 519 struct ipoib_dev_priv *priv = ipoib_priv(dev);
71d9c5f9 520
80b5b35a
FD		 521 if ((test_bit(IPOIB_FLAG_ADMIN_CM, &priv->flags) &&
522 !strcmp(buf, "connected\n")) ||
523 (!test_bit(IPOIB_FLAG_ADMIN_CM, &priv->flags) &&
524 !strcmp(buf, "datagram\n"))) {
525 return 0;
526 }
527
71d9c5f9
RD		 528 /* flush paths if we switch modes so that connections are restarted */
529 if (IPOIB_CM_SUPPORTED(dev->dev_addr) && !strcmp(buf, "connected\n")) {
530 set_bit(IPOIB_FLAG_ADMIN_CM, &priv->flags);
531 ipoib_warn(priv, "enabling connected mode "
532 "will cause multicast packet drops\n");
533 netdev_update_features(dev);
edcd2a74 534 dev_set_mtu(dev, ipoib_cm_max_mtu(dev));
8f149b68 535 netif_set_real_num_tx_queues(dev, 1);
71d9c5f9 536 rtnl_unlock();
e622f2f4 537 priv->tx_wr.wr.send_flags &= ~IB_SEND_IP_CSUM;
71d9c5f9
RD		 538
539 ipoib_flush_paths(dev);
0a0007f2 540 return (!rtnl_trylock()) ? -EBUSY : 0;
71d9c5f9
RD		 541 }
542
543 if (!strcmp(buf, "datagram\n")) {
544 clear_bit(IPOIB_FLAG_ADMIN_CM, &priv->flags);
545 netdev_update_features(dev);
546 dev_set_mtu(dev, min(priv->mcast_mtu, dev->mtu));
8f149b68 547 netif_set_real_num_tx_queues(dev, dev->num_tx_queues);
71d9c5f9
RD		 548 rtnl_unlock();
549 ipoib_flush_paths(dev);
0a0007f2 550 return (!rtnl_trylock()) ? -EBUSY : 0;
71d9c5f9
RD		 551 }
552
553 return -EINVAL;
554}
555
546481c2 556struct ipoib_path *__path_find(struct net_device *dev, void *gid)
1da177e4 557{
c1048aff 558 struct ipoib_dev_priv *priv = ipoib_priv(dev);
1da177e4
LT		 559 struct rb_node *n = priv->path_tree.rb_node;
560 struct ipoib_path *path;
561 int ret;
562
563 while (n) {
564 path = rb_entry(n, struct ipoib_path, rb_node);
565
37c22a77 566 ret = memcmp(gid, path->pathrec.dgid.raw,
1da177e4
LT		 567 sizeof (union ib_gid));
568
569 if (ret < 0)
570 n = n->rb_left;
571 else if (ret > 0)
572 n = n->rb_right;
573 else
574 return path;
575 }
576
577 return NULL;
578}
579
580static int __path_add(struct net_device *dev, struct ipoib_path *path)
581{
c1048aff 582 struct ipoib_dev_priv *priv = ipoib_priv(dev);
1da177e4
LT		 583 struct rb_node **n = &priv->path_tree.rb_node;
584 struct rb_node *pn = NULL;
585 struct ipoib_path *tpath;
586 int ret;
587
588 while (*n) {
589 pn = *n;
590 tpath = rb_entry(pn, struct ipoib_path, rb_node);
591
592 ret = memcmp(path->pathrec.dgid.raw, tpath->pathrec.dgid.raw,
593 sizeof (union ib_gid));
594 if (ret < 0)
595 n = &pn->rb_left;
596 else if (ret > 0)
597 n = &pn->rb_right;
598 else
599 return -EEXIST;
600 }
601
602 rb_link_node(&path->rb_node, pn, n);
603 rb_insert_color(&path->rb_node, &priv->path_tree);
604
605 list_add_tail(&path->list, &priv->path_list);
606
607 return 0;
608}
609
610static void path_free(struct net_device *dev, struct ipoib_path *path)
611{
1da177e4 612 struct sk_buff *skb;
1da177e4
LT		 613
614 while ((skb = __skb_dequeue(&path->queue)))
615 dev_kfree_skb_irq(skb);
616
0dd9ce18 617 ipoib_dbg(ipoib_priv(dev), "%s\n", __func__);
1da177e4 618
b63b70d8
SP		 619 /* remove all neigh connected to this path */
620 ipoib_del_neighs_by_gid(dev, path->pathrec.dgid.raw);
1da177e4
LT		 621
622 if (path->ah)
623 ipoib_put_ah(path->ah);
624
625 kfree(path);
626}
627
1732b0ef
RD		 628#ifdef CONFIG_INFINIBAND_IPOIB_DEBUG
629
630struct ipoib_path_iter *ipoib_path_iter_init(struct net_device *dev)
631{
632 struct ipoib_path_iter *iter;
633
b1b63970 634 iter = kmalloc(sizeof(*iter), GFP_KERNEL);
1732b0ef
RD		 635 if (!iter)
636 return NULL;
637
638 iter->dev = dev;
639 memset(iter->path.pathrec.dgid.raw, 0, 16);
640
641 if (ipoib_path_iter_next(iter)) {
642 kfree(iter);
643 return NULL;
644 }
645
646 return iter;
647}
648
649int ipoib_path_iter_next(struct ipoib_path_iter *iter)
650{
c1048aff 651 struct ipoib_dev_priv *priv = ipoib_priv(iter->dev);
1732b0ef
RD		 652 struct rb_node *n;
653 struct ipoib_path *path;
654 int ret = 1;
655
656 spin_lock_irq(&priv->lock);
657
658 n = rb_first(&priv->path_tree);
659
660 while (n) {
661 path = rb_entry(n, struct ipoib_path, rb_node);
662
663 if (memcmp(iter->path.pathrec.dgid.raw, path->pathrec.dgid.raw,
664 sizeof (union ib_gid)) < 0) {
665 iter->path = *path;
666 ret = 0;
667 break;
668 }
669
670 n = rb_next(n);
671 }
672
673 spin_unlock_irq(&priv->lock);
674
675 return ret;
676}
677
678void ipoib_path_iter_read(struct ipoib_path_iter *iter,
679 struct ipoib_path *path)
680{
681 *path = iter->path;
682}
683
684#endif /* CONFIG_INFINIBAND_IPOIB_DEBUG */
685
ee1e2c82
MS		 686void ipoib_mark_paths_invalid(struct net_device *dev)
687{
c1048aff 688 struct ipoib_dev_priv *priv = ipoib_priv(dev);
ee1e2c82
MS		 689 struct ipoib_path *path, *tp;
690
691 spin_lock_irq(&priv->lock);
692
693 list_for_each_entry_safe(path, tp, &priv->path_list, list) {
57520751
DC		 694 ipoib_dbg(priv, "mark path LID 0x%08x GID %pI6 invalid\n",
695 be32_to_cpu(sa_path_get_dlid(&path->pathrec)),
696 path->pathrec.dgid.raw);
fa9391db
DL		 697 if (path->ah)
698 path->ah->valid = 0;
ee1e2c82
MS		 699 }
700
701 spin_unlock_irq(&priv->lock);
702}
703
2b084176
ES		 704static void push_pseudo_header(struct sk_buff *skb, const char *daddr)
705{
706 struct ipoib_pseudo_header *phdr;
707
d58ff351 708 phdr = skb_push(skb, sizeof(*phdr));
2b084176
ES		 709 memcpy(phdr->hwaddr, daddr, INFINIBAND_ALEN);
710}
711
1da177e4
LT		 712void ipoib_flush_paths(struct net_device *dev)
713{
c1048aff 714 struct ipoib_dev_priv *priv = ipoib_priv(dev);
1da177e4
LT		 715 struct ipoib_path *path, *tp;
716 LIST_HEAD(remove_list);
943c246e 717 unsigned long flags;
1da177e4 718
943c246e
RD		 719 netif_tx_lock_bh(dev);
720 spin_lock_irqsave(&priv->lock, flags);
1da177e4 721
157de229 722 list_splice_init(&priv->path_list, &remove_list);
1da177e4
LT		 723
724 list_for_each_entry(path, &remove_list, list)
725 rb_erase(&path->rb_node, &priv->path_tree);
726
1da177e4
LT		 727 list_for_each_entry_safe(path, tp, &remove_list, list) {
728 if (path->query)
729 ib_sa_cancel_query(path->query_id, path->query);
943c246e
RD		 730 spin_unlock_irqrestore(&priv->lock, flags);
731 netif_tx_unlock_bh(dev);
1da177e4
LT		 732 wait_for_completion(&path->done);
733 path_free(dev, path);
943c246e
RD		 734 netif_tx_lock_bh(dev);
735 spin_lock_irqsave(&priv->lock, flags);
1da177e4 736 }
943c246e
RD		 737
738 spin_unlock_irqrestore(&priv->lock, flags);
739 netif_tx_unlock_bh(dev);
1da177e4
LT		 740}
741
742static void path_rec_completion(int status,
c2f8fc4e 743 struct sa_path_rec *pathrec,
ccae0447 744 unsigned int num_prs, void *path_ptr)
1da177e4
LT		 745{
746 struct ipoib_path *path = path_ptr;
747 struct net_device *dev = path->dev;
c1048aff 748 struct ipoib_dev_priv *priv = ipoib_priv(dev);
1da177e4 749 struct ipoib_ah *ah = NULL;
c9da4bad 750 struct ipoib_ah *old_ah = NULL;
d04d01b1 751 struct ipoib_neigh *neigh, *tn;
1da177e4
LT		 752 struct sk_buff_head skqueue;
753 struct sk_buff *skb;
754 unsigned long flags;
755
843613b0 756 if (!status)
5b095d98 757 ipoib_dbg(priv, "PathRec LID 0x%04x for GID %pI6\n",
57520751 758 be32_to_cpu(sa_path_get_dlid(pathrec)),
9fdca4da 759 pathrec->dgid.raw);
1da177e4 760 else
5b095d98 761 ipoib_dbg(priv, "PathRec status %d for GID %pI6\n",
fcace2fe 762 status, path->pathrec.dgid.raw);
1da177e4
LT		 763
764 skb_queue_head_init(&skqueue);
765
766 if (!status) {
90898850 767 struct rdma_ah_attr av;
46f1b3d7 768
4ad6a024 769 if (!ib_init_ah_attr_from_path(priv->ca, priv->port,
39839107 770 pathrec, &av, NULL)) {
46f1b3d7 771 ah = ipoib_create_ah(dev, priv->pd, &av);
aa74f487
PP		 772 rdma_destroy_ah_attr(&av);
773 }
1da177e4
LT		 774 }
775
776 spin_lock_irqsave(&priv->lock, flags);
777
3874397c 778 if (!IS_ERR_OR_NULL(ah)) {
43900089
ES		 779 /*
780 * pathrec.dgid is used as the database key from the LLADDR,
781 * it must remain unchanged even if the SA returns a different
782 * GID to use in the AH.
783 */
784 if (memcmp(pathrec->dgid.raw, path->pathrec.dgid.raw,
785 sizeof(union ib_gid))) {
786 ipoib_dbg(
787 priv,
788 "%s got PathRec for gid %pI6 while asked for %pI6\n",
789 dev->name, pathrec->dgid.raw,
790 path->pathrec.dgid.raw);
791 memcpy(pathrec->dgid.raw, path->pathrec.dgid.raw,
792 sizeof(union ib_gid));
793 }
794
1da177e4
LT		 795 path->pathrec = *pathrec;
796
c9da4bad
RD		 797 old_ah = path->ah;
798 path->ah = ah;
799
1da177e4 800 ipoib_dbg(priv, "created address handle %p for LID 0x%04x, SL %d\n",
57520751 801 ah, be32_to_cpu(sa_path_get_dlid(pathrec)),
9fdca4da 802 pathrec->sl);
1da177e4
LT		 803
804 while ((skb = __skb_dequeue(&path->queue)))
805 __skb_queue_tail(&skqueue, skb);
806
d04d01b1 807 list_for_each_entry_safe(neigh, tn, &path->neigh_list, list) {
ee1e2c82
MS		 808 if (neigh->ah) {
809 WARN_ON(neigh->ah != old_ah);
810 /*
811 * Dropping the ah reference inside
812 * priv->lock is safe here, because we
813 * will hold one more reference from
814 * the original value of path->ah (ie
815 * old_ah).
816 */
817 ipoib_put_ah(neigh->ah);
818 }
1da177e4
LT		 819 kref_get(&path->ah->ref);
820 neigh->ah = path->ah;
821
b63b70d8 822 if (ipoib_cm_enabled(dev, neigh->daddr)) {
839fcaba
MT		 823 if (!ipoib_cm_get(neigh))
824 ipoib_cm_set(neigh, ipoib_cm_create_tx(dev,
825 path,
826 neigh));
827 if (!ipoib_cm_get(neigh)) {
b63b70d8 828 ipoib_neigh_free(neigh);
839fcaba
MT		 829 continue;
830 }
831 }
832
1da177e4
LT		 833 while ((skb = __skb_dequeue(&neigh->queue)))
834 __skb_queue_tail(&skqueue, skb);
835 }
fa9391db 836 path->ah->valid = 1;
5872a9fc 837 }
1da177e4 838
5872a9fc 839 path->query = NULL;
1da177e4
LT		 840 complete(&path->done);
841
842 spin_unlock_irqrestore(&priv->lock, flags);
843
f72dd566
RD		 844 if (IS_ERR_OR_NULL(ah))
845 ipoib_del_neighs_by_gid(dev, path->pathrec.dgid.raw);
846
ee1e2c82
MS		 847 if (old_ah)
848 ipoib_put_ah(old_ah);
849
1da177e4 850 while ((skb = __skb_dequeue(&skqueue))) {
d32b9a81 851 int ret;
1da177e4 852 skb->dev = dev;
d32b9a81
FD		 853 ret = dev_queue_xmit(skb);
854 if (ret)
855 ipoib_warn(priv, "%s: dev_queue_xmit failed to re-queue packet, ret:%d\n",
856 __func__, ret);
1da177e4
LT		 857 }
858}
859
98aebc55
ES		 860static void init_path_rec(struct ipoib_dev_priv *priv, struct ipoib_path *path,
861 void *gid)
862{
863 path->dev = priv->dev;
864
865 if (rdma_cap_opa_ah(priv->ca, priv->port))
866 path->pathrec.rec_type = SA_PATH_REC_TYPE_OPA;
867 else
868 path->pathrec.rec_type = SA_PATH_REC_TYPE_IB;
869
870 memcpy(path->pathrec.dgid.raw, gid, sizeof(union ib_gid));
871 path->pathrec.sgid = priv->local_gid;
872 path->pathrec.pkey = cpu_to_be16(priv->pkey);
873 path->pathrec.numb_path = 1;
874 path->pathrec.traffic_class = priv->broadcast->mcmember.traffic_class;
875}
876
37c22a77 877static struct ipoib_path *path_rec_create(struct net_device *dev, void *gid)
1da177e4 878{
c1048aff 879 struct ipoib_dev_priv *priv = ipoib_priv(dev);
1da177e4
LT		 880 struct ipoib_path *path;
881
1401b53a
JM		 882 if (!priv->broadcast)
883 return NULL;
884
b1b63970 885 path = kzalloc(sizeof(*path), GFP_ATOMIC);
1da177e4
LT		 886 if (!path)
887 return NULL;
888
1da177e4
LT		 889 skb_queue_head_init(&path->queue);
890
891 INIT_LIST_HEAD(&path->neigh_list);
1da177e4 892
98aebc55 893 init_path_rec(priv, path, gid);
1da177e4
LT		 894
895 return path;
896}
897
898static int path_rec_start(struct net_device *dev,
899 struct ipoib_path *path)
900{
c1048aff 901 struct ipoib_dev_priv *priv = ipoib_priv(dev);
1da177e4 902
5b095d98 903 ipoib_dbg(priv, "Start path record lookup for %pI6\n",
fcace2fe 904 path->pathrec.dgid.raw);
1da177e4 905
65c7edda
RD		 906 init_completion(&path->done);
907
1da177e4 908 path->query_id =
c1a0b23b 909 ib_sa_path_rec_get(&ipoib_sa_client, priv->ca, priv->port,
1da177e4
LT		 910 &path->pathrec,
911 IB_SA_PATH_REC_DGID |
912 IB_SA_PATH_REC_SGID |
913 IB_SA_PATH_REC_NUMB_PATH |
81668838 914 IB_SA_PATH_REC_TRAFFIC_CLASS |
1da177e4
LT		 915 IB_SA_PATH_REC_PKEY,
916 1000, GFP_ATOMIC,
917 path_rec_completion,
918 path, &path->query);
919 if (path->query_id < 0) {
01b3fc8b 920 ipoib_warn(priv, "ib_sa_path_rec_get failed: %d\n", path->query_id);
1da177e4 921 path->query = NULL;
93a3ab93 922 complete(&path->done);
1da177e4
LT		 923 return path->query_id;
924 }
925
926 return 0;
927}
928
fa9391db
DL		 929static void neigh_refresh_path(struct ipoib_neigh *neigh, u8 *daddr,
930 struct net_device *dev)
931{
932 struct ipoib_dev_priv *priv = ipoib_priv(dev);
933 struct ipoib_path *path;
934 unsigned long flags;
935
936 spin_lock_irqsave(&priv->lock, flags);
937
938 path = __path_find(dev, daddr + 4);
939 if (!path)
940 goto out;
941 if (!path->query)
942 path_rec_start(dev, path);
943out:
944 spin_unlock_irqrestore(&priv->lock, flags);
945}
946
16ba3def
ES		 947static struct ipoib_neigh *neigh_add_path(struct sk_buff *skb, u8 *daddr,
948 struct net_device *dev)
1da177e4 949{
c1048aff 950 struct ipoib_dev_priv *priv = ipoib_priv(dev);
cd565b4b 951 struct rdma_netdev *rn = netdev_priv(dev);
1da177e4
LT		 952 struct ipoib_path *path;
953 struct ipoib_neigh *neigh;
943c246e 954 unsigned long flags;
1da177e4 955
b5120a6e 956 spin_lock_irqsave(&priv->lock, flags);
b63b70d8 957 neigh = ipoib_neigh_alloc(daddr, dev);
1da177e4 958 if (!neigh) {
b5120a6e 959 spin_unlock_irqrestore(&priv->lock, flags);
de903512 960 ++dev->stats.tx_dropped;
1da177e4 961 dev_kfree_skb_any(skb);
16ba3def
ES		 962 return NULL;
963 }
964
965 /* To avoid race condition, make sure that the
966 * neigh will be added only once.
967 */
968 if (unlikely(!list_empty(&neigh->list))) {
969 spin_unlock_irqrestore(&priv->lock, flags);
970 return neigh;
1da177e4
LT		 971 }
972
b63b70d8 973 path = __path_find(dev, daddr + 4);
1da177e4 974 if (!path) {
b63b70d8 975 path = path_rec_create(dev, daddr + 4);
1da177e4 976 if (!path)
d2e0655e 977 goto err_path;
1da177e4
LT		 978
979 __path_add(dev, path);
980 }
981
982 list_add_tail(&neigh->list, &path->neigh_list);
983
fa9391db 984 if (path->ah && path->ah->valid) {
1da177e4
LT		 985 kref_get(&path->ah->ref);
986 neigh->ah = path->ah;
987
b63b70d8 988 if (ipoib_cm_enabled(dev, neigh->daddr)) {
839fcaba
MT		 989 if (!ipoib_cm_get(neigh))
990 ipoib_cm_set(neigh, ipoib_cm_create_tx(dev, path, neigh));
991 if (!ipoib_cm_get(neigh)) {
b63b70d8 992 ipoib_neigh_free(neigh);
839fcaba
MT		 993 goto err_drop;
994 }
fc791b63
PA		 995 if (skb_queue_len(&neigh->queue) <
996 IPOIB_MAX_PATH_REC_QUEUE) {
2b084176 997 push_pseudo_header(skb, neigh->daddr);
839fcaba 998 __skb_queue_tail(&neigh->queue, skb);
fc791b63 999 } else {
839fcaba
MT		 1000 ipoib_warn(priv, "queue length limit %d. Packet drop.\n",
1001 skb_queue_len(&neigh->queue));
1002 goto err_drop;
1003 }
721d67cd
RD		 1004 } else {
1005 spin_unlock_irqrestore(&priv->lock, flags);
cd565b4b
ES		 1006 path->ah->last_send = rn->send(dev, skb, path->ah->ah,
1007 IPOIB_QPN(daddr));
b63b70d8 1008 ipoib_neigh_put(neigh);
16ba3def 1009 return NULL;
721d67cd 1010 }
1da177e4
LT		 1011 } else {
1012 neigh->ah = NULL;
1da177e4
LT		 1013
1014 if (!path->query && path_rec_start(dev, path))
49b8e744 1015 goto err_path;
2b084176
ES		 1016 if (skb_queue_len(&neigh->queue) < IPOIB_MAX_PATH_REC_QUEUE) {
1017 push_pseudo_header(skb, neigh->daddr);
1e85b806 1018 __skb_queue_tail(&neigh->queue, skb);
2b084176 1019 } else {
1e85b806 1020 goto err_drop;
2b084176 1021 }
1da177e4
LT		 1022 }
1023
943c246e 1024 spin_unlock_irqrestore(&priv->lock, flags);
b63b70d8 1025 ipoib_neigh_put(neigh);
16ba3def 1026 return NULL;
1da177e4 1027
d2e0655e 1028err_path:
b63b70d8 1029 ipoib_neigh_free(neigh);
839fcaba 1030err_drop:
de903512 1031 ++dev->stats.tx_dropped;
1da177e4
LT		 1032 dev_kfree_skb_any(skb);
1033
943c246e 1034 spin_unlock_irqrestore(&priv->lock, flags);
b63b70d8 1035 ipoib_neigh_put(neigh);
16ba3def
ES		 1036
1037 return NULL;
1da177e4
LT		 1038}
1039
1040static void unicast_arp_send(struct sk_buff *skb, struct net_device *dev,
fc791b63 1041 struct ipoib_pseudo_header *phdr)
1da177e4 1042{
c1048aff 1043 struct ipoib_dev_priv *priv = ipoib_priv(dev);
cd565b4b 1044 struct rdma_netdev *rn = netdev_priv(dev);
1da177e4 1045 struct ipoib_path *path;
943c246e 1046 unsigned long flags;
1da177e4 1047
943c246e 1048 spin_lock_irqsave(&priv->lock, flags);
1da177e4 1049
98aebc55
ES		 1050 /* no broadcast means that all paths are (going to be) not valid */
1051 if (!priv->broadcast)
1052 goto drop_and_unlock;
1053
fc791b63 1054 path = __path_find(dev, phdr->hwaddr + 4);
fa9391db 1055 if (!path || !path->ah || !path->ah->valid) {
71d98b46 1056 if (!path) {
fc791b63 1057 path = path_rec_create(dev, phdr->hwaddr + 4);
15517080
ES		 1058 if (!path)
1059 goto drop_and_unlock;
1060 __path_add(dev, path);
1061 } else {
1062 /*
1063 * make sure there are no changes in the existing
1064 * path record
1065 */
1066 init_path_rec(priv, path, phdr->hwaddr + 4);
1067 }
1068 if (!path->query && path_rec_start(dev, path)) {
1069 goto drop_and_unlock;
71d98b46 1070 }
1da177e4 1071
15517080
ES		 1072 if (skb_queue_len(&path->queue) < IPOIB_MAX_PATH_REC_QUEUE) {
1073 push_pseudo_header(skb, phdr->hwaddr);
1074 __skb_queue_tail(&path->queue, skb);
1075 goto unlock;
1da177e4 1076 } else {
98aebc55 1077 goto drop_and_unlock;
1da177e4 1078 }
1da177e4
LT		 1079 }
1080
943c246e 1081 spin_unlock_irqrestore(&priv->lock, flags);
15517080
ES		 1082 ipoib_dbg(priv, "Send unicast ARP to %08x\n",
1083 be32_to_cpu(sa_path_get_dlid(&path->pathrec)));
1084 path->ah->last_send = rn->send(dev, skb, path->ah->ah,
1085 IPOIB_QPN(phdr->hwaddr));
98aebc55
ES		 1086 return;
1087
1088drop_and_unlock:
1089 ++dev->stats.tx_dropped;
1090 dev_kfree_skb_any(skb);
15517080 1091unlock:
98aebc55 1092 spin_unlock_irqrestore(&priv->lock, flags);
1da177e4
LT		 1093}
1094
47a3968a 1095static netdev_tx_t ipoib_start_xmit(struct sk_buff *skb, struct net_device *dev)
1da177e4 1096{
c1048aff 1097 struct ipoib_dev_priv *priv = ipoib_priv(dev);
cd565b4b 1098 struct rdma_netdev *rn = netdev_priv(dev);
1da177e4 1099 struct ipoib_neigh *neigh;
fc791b63 1100 struct ipoib_pseudo_header *phdr;
b63b70d8 1101 struct ipoib_header *header;
1da177e4
LT		 1102 unsigned long flags;
1103
fc791b63
PA		 1104 phdr = (struct ipoib_pseudo_header *) skb->data;
1105 skb_pull(skb, sizeof(*phdr));
b63b70d8
SP		 1106 header = (struct ipoib_header *) skb->data;
1107
fc791b63 1108 if (unlikely(phdr->hwaddr[4] == 0xff)) {
b63b70d8
SP		 1109 /* multicast, arrange "if" according to probability */
1110 if ((header->proto != htons(ETH_P_IP)) &&
1111 (header->proto != htons(ETH_P_IPV6)) &&
1112 (header->proto != htons(ETH_P_ARP)) &&
dc850b0e
PM		 1113 (header->proto != htons(ETH_P_RARP)) &&
1114 (header->proto != htons(ETH_P_TIPC))) {
b63b70d8 1115 /* ethertype not supported by IPoIB */
17e6abee
DM		 1116 ++dev->stats.tx_dropped;
1117 dev_kfree_skb_any(skb);
b63b70d8 1118 return NETDEV_TX_OK;
17e6abee 1119 }
b63b70d8 1120 /* Add in the P_Key for multicast*/
fc791b63
PA		 1121 phdr->hwaddr[8] = (priv->pkey >> 8) & 0xff;
1122 phdr->hwaddr[9] = priv->pkey & 0xff;
b63b70d8 1123
fc791b63 1124 neigh = ipoib_neigh_get(dev, phdr->hwaddr);
b63b70d8
SP		 1125 if (likely(neigh))
1126 goto send_using_neigh;
fc791b63 1127 ipoib_mcast_send(dev, phdr->hwaddr, skb);
b63b70d8 1128 return NETDEV_TX_OK;
17e6abee 1129 }
1da177e4 1130
b63b70d8
SP		 1131 /* unicast, arrange "switch" according to probability */
1132 switch (header->proto) {
1133 case htons(ETH_P_IP):
1134 case htons(ETH_P_IPV6):
dc850b0e 1135 case htons(ETH_P_TIPC):
fc791b63 1136 neigh = ipoib_neigh_get(dev, phdr->hwaddr);
b63b70d8 1137 if (unlikely(!neigh)) {
16ba3def
ES		 1138 neigh = neigh_add_path(skb, phdr->hwaddr, dev);
1139 if (likely(!neigh))
1140 return NETDEV_TX_OK;
a50df398 1141 }
b63b70d8
SP		 1142 break;
1143 case htons(ETH_P_ARP):
1144 case htons(ETH_P_RARP):
1145 /* for unicast ARP and RARP should always perform path find */
fc791b63 1146 unicast_arp_send(skb, dev, phdr);
b63b70d8
SP		 1147 return NETDEV_TX_OK;
1148 default:
1149 /* ethertype not supported by IPoIB */
1150 ++dev->stats.tx_dropped;
1151 dev_kfree_skb_any(skb);
1152 return NETDEV_TX_OK;
1153 }
8a7f7521 1154
b63b70d8
SP		 1155send_using_neigh:
1156 /* note we now hold a ref to neigh */
1157 if (ipoib_cm_get(neigh)) {
1158 if (ipoib_cm_up(neigh)) {
1159 ipoib_cm_send(dev, skb, ipoib_cm_get(neigh));
1160 goto unref;
1da177e4 1161 }
fa9391db 1162 } else if (neigh->ah && neigh->ah->valid) {
cd565b4b
ES		 1163 neigh->ah->last_send = rn->send(dev, skb, neigh->ah->ah,
1164 IPOIB_QPN(phdr->hwaddr));
b63b70d8 1165 goto unref;
fa9391db
DL		 1166 } else if (neigh->ah) {
1167 neigh_refresh_path(neigh, phdr->hwaddr, dev);
b63b70d8 1168 }
1da177e4 1169
b63b70d8 1170 if (skb_queue_len(&neigh->queue) < IPOIB_MAX_PATH_REC_QUEUE) {
2b084176 1171 push_pseudo_header(skb, phdr->hwaddr);
b63b70d8
SP		 1172 spin_lock_irqsave(&priv->lock, flags);
1173 __skb_queue_tail(&neigh->queue, skb);
1174 spin_unlock_irqrestore(&priv->lock, flags);
1da177e4 1175 } else {
b63b70d8
SP		 1176 ++dev->stats.tx_dropped;
1177 dev_kfree_skb_any(skb);
1178 }
1da177e4 1179
b63b70d8
SP		 1180unref:
1181 ipoib_neigh_put(neigh);
1da177e4 1182
1da177e4
LT		 1183 return NETDEV_TX_OK;
1184}
1185
0290bd29 1186static void ipoib_timeout(struct net_device *dev, unsigned int txqueue)
1da177e4 1187{
c1048aff 1188 struct ipoib_dev_priv *priv = ipoib_priv(dev);
042a00f9 1189 struct rdma_netdev *rn = netdev_priv(dev);
1da177e4 1190
042a00f9
MM		 1191 if (rn->tx_timeout) {
1192 rn->tx_timeout(dev, txqueue);
1193 return;
1194 }
4b2d319b 1195 ipoib_warn(priv, "transmit timeout: latency %d msecs\n",
4d0e9657 1196 jiffies_to_msecs(jiffies - dev_trans_start(dev)));
1acba6a8
VF		 1197 ipoib_warn(priv,
1198 "queue stopped %d, tx_head %u, tx_tail %u, global_tx_head %u, global_tx_tail %u\n",
1199 netif_queue_stopped(dev), priv->tx_head, priv->tx_tail,
1200 priv->global_tx_head, priv->global_tx_tail);
1201
50af5d12
JW		 1202
1203 schedule_work(&priv->tx_timeout_work);
1204}
1205
1206void ipoib_ib_tx_timeout_work(struct work_struct *work)
1207{
1208 struct ipoib_dev_priv *priv = container_of(work,
1209 struct ipoib_dev_priv,
1210 tx_timeout_work);
1211 int err;
1212
1213 rtnl_lock();
1214
1215 if (!test_bit(IPOIB_FLAG_ADMIN_UP, &priv->flags))
1216 goto unlock;
1217
1218 ipoib_stop(priv->dev);
1219 err = ipoib_open(priv->dev);
1220 if (err) {
1221 ipoib_warn(priv, "ipoib_open failed recovering from a tx_timeout, err(%d).\n",
1222 err);
1223 goto unlock;
1224 }
1225
1226 netif_tx_wake_all_queues(priv->dev);
1227unlock:
1228 rtnl_unlock();
1229
1da177e4
LT		 1230}
1231
1232static int ipoib_hard_header(struct sk_buff *skb,
1233 struct net_device *dev,
1234 unsigned short type,
0578cdad
KH		 1235 const void *daddr,
1236 const void *saddr,
1237 unsigned int len)
1da177e4
LT		 1238{
1239 struct ipoib_header *header;
1240
b1b63970 1241 header = skb_push(skb, sizeof(*header));
1da177e4
LT		 1242
1243 header->proto = htons(type);
1244 header->reserved = 0;
1245
1246 /*
b63b70d8 1247 * we don't rely on dst_entry structure, always stuff the
fc791b63 1248 * destination address into skb hard header so we can figure out where
936d7de3 1249 * to send the packet later.
1da177e4 1250 */
2b084176 1251 push_pseudo_header(skb, daddr);
1da177e4 1252
fc791b63 1253 return IPOIB_HARD_LEN;
1da177e4
LT		 1254}
1255
1256static void ipoib_set_mcast_list(struct net_device *dev)
1257{
c1048aff 1258 struct ipoib_dev_priv *priv = ipoib_priv(dev);
1da177e4 1259
7a343d4c
LA		 1260 if (!test_bit(IPOIB_FLAG_OPER_UP, &priv->flags)) {
1261 ipoib_dbg(priv, "IPOIB_FLAG_OPER_UP not set");
1262 return;
1263 }
1264
0b39578b 1265 queue_work(priv->wq, &priv->restart_task);
1da177e4
LT		 1266}
1267
5aa7add8
ND		 1268static int ipoib_get_iflink(const struct net_device *dev)
1269{
c1048aff 1270 struct ipoib_dev_priv *priv = ipoib_priv(dev);
5aa7add8 1271
2c153959
ES		 1272 /* parent interface */
1273 if (!test_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags))
e353ea9c 1274 return READ_ONCE(dev->ifindex);
2c153959
ES		 1275
1276 /* child/vlan interface */
e353ea9c 1277 return READ_ONCE(priv->parent->ifindex);
5aa7add8
ND		 1278}
1279
b63b70d8 1280static u32 ipoib_addr_hash(struct ipoib_neigh_hash *htbl, u8 *daddr)
1da177e4 1281{
b63b70d8
SP		 1282 /*
1283 * Use only the address parts that contributes to spreading
1284 * The subnet prefix is not used as one can not connect to
1285 * same remote port (GUID) using the same remote QPN via two
1286 * different subnets.
1287 */
1288 /* qpn octets[1:4) & port GUID octets[12:20) */
9d1ad66e 1289 u32 *d32 = (u32 *) daddr;
b63b70d8
SP		 1290 u32 hv;
1291
9d1ad66e 1292 hv = jhash_3words(d32[3], d32[4], IPOIB_QPN_MASK & d32[0], 0);
b63b70d8
SP		 1293 return hv & htbl->mask;
1294}
1295
1296struct ipoib_neigh *ipoib_neigh_get(struct net_device *dev, u8 *daddr)
1297{
c1048aff 1298 struct ipoib_dev_priv *priv = ipoib_priv(dev);
b63b70d8
SP		 1299 struct ipoib_neigh_table *ntbl = &priv->ntbl;
1300 struct ipoib_neigh_hash *htbl;
1301 struct ipoib_neigh *neigh = NULL;
1302 u32 hash_val;
1303
1304 rcu_read_lock_bh();
1305
1306 htbl = rcu_dereference_bh(ntbl->htbl);
1307
1308 if (!htbl)
1309 goto out_unlock;
1310
1311 hash_val = ipoib_addr_hash(htbl, daddr);
1312 for (neigh = rcu_dereference_bh(htbl->buckets[hash_val]);
1313 neigh != NULL;
1314 neigh = rcu_dereference_bh(neigh->hnext)) {
1315 if (memcmp(daddr, neigh->daddr, INFINIBAND_ALEN) == 0) {
1316 /* found, take one ref on behalf of the caller */
a5e27fb6 1317 if (!refcount_inc_not_zero(&neigh->refcnt)) {
b63b70d8
SP		 1318 /* deleted */
1319 neigh = NULL;
1320 goto out_unlock;
1321 }
61c78eea
ES		 1322
1323 if (likely(skb_queue_len(&neigh->queue) < IPOIB_MAX_PATH_REC_QUEUE))
1324 neigh->alive = jiffies;
b63b70d8
SP		 1325 goto out_unlock;
1326 }
1327 }
1328
1329out_unlock:
1330 rcu_read_unlock_bh();
1331 return neigh;
1332}
1333
1334static void __ipoib_reap_neigh(struct ipoib_dev_priv *priv)
1335{
1336 struct ipoib_neigh_table *ntbl = &priv->ntbl;
1337 struct ipoib_neigh_hash *htbl;
1338 unsigned long neigh_obsolete;
1339 unsigned long dt;
1da177e4 1340 unsigned long flags;
b63b70d8 1341 int i;
bd99b2e0 1342 LIST_HEAD(remove_list);
1da177e4 1343
b5120a6e 1344 spin_lock_irqsave(&priv->lock, flags);
b63b70d8
SP		 1345
1346 htbl = rcu_dereference_protected(ntbl->htbl,
b5120a6e 1347 lockdep_is_held(&priv->lock));
b63b70d8
SP		 1348
1349 if (!htbl)
1350 goto out_unlock;
1351
1352 /* neigh is obsolete if it was idle for two GC periods */
1353 dt = 2 * arp_tbl.gc_interval;
1354 neigh_obsolete = jiffies - dt;
b63b70d8
SP		 1355
1356 for (i = 0; i < htbl->size; i++) {
1357 struct ipoib_neigh *neigh;
1358 struct ipoib_neigh __rcu **np = &htbl->buckets[i];
1359
1360 while ((neigh = rcu_dereference_protected(*np,
b5120a6e 1361 lockdep_is_held(&priv->lock))) != NULL) {
b63b70d8
SP		 1362 /* was the neigh idle for two GC periods */
1363 if (time_after(neigh_obsolete, neigh->alive)) {
bd99b2e0 1364
432c55ff 1365 ipoib_check_and_add_mcast_sendonly(priv, neigh->daddr + 4, &remove_list);
bd99b2e0 1366
b63b70d8
SP		 1367 rcu_assign_pointer(*np,
1368 rcu_dereference_protected(neigh->hnext,
b5120a6e 1369 lockdep_is_held(&priv->lock)));
b63b70d8 1370 /* remove from path/mc list */
c586071d 1371 list_del_init(&neigh->list);
b63b70d8
SP		 1372 call_rcu(&neigh->rcu, ipoib_neigh_reclaim);
1373 } else {
1374 np = &neigh->hnext;
1375 }
1da177e4 1376
b63b70d8
SP		 1377 }
1378 }
1da177e4 1379
b63b70d8 1380out_unlock:
b5120a6e 1381 spin_unlock_irqrestore(&priv->lock, flags);
50be28de 1382 ipoib_mcast_remove_list(&remove_list);
b63b70d8 1383}
1da177e4 1384
b63b70d8
SP		 1385static void ipoib_reap_neigh(struct work_struct *work)
1386{
1387 struct ipoib_dev_priv *priv =
1388 container_of(work, struct ipoib_dev_priv, neigh_reap_task.work);
1389
1390 __ipoib_reap_neigh(priv);
1391
cda8daf1
ES		 1392 queue_delayed_work(priv->wq, &priv->neigh_reap_task,
1393 arp_tbl.gc_interval);
1da177e4
LT		 1394}
1395
b63b70d8
SP		 1396
1397static struct ipoib_neigh *ipoib_neigh_ctor(u8 *daddr,
732a2170 1398 struct net_device *dev)
d2e0655e
MT		 1399{
1400 struct ipoib_neigh *neigh;
1401
b1b63970 1402 neigh = kzalloc(sizeof(*neigh), GFP_ATOMIC);
d2e0655e
MT		 1403 if (!neigh)
1404 return NULL;
1405
732a2170 1406 neigh->dev = dev;
b63b70d8 1407 memcpy(&neigh->daddr, daddr, sizeof(neigh->daddr));
82b39913 1408 skb_queue_head_init(&neigh->queue);
b63b70d8 1409 INIT_LIST_HEAD(&neigh->list);
839fcaba 1410 ipoib_cm_set(neigh, NULL);
b63b70d8 1411 /* one ref on behalf of the caller */
a5e27fb6 1412 refcount_set(&neigh->refcnt, 1);
b63b70d8
SP		 1413
1414 return neigh;
1415}
1416
1417struct ipoib_neigh *ipoib_neigh_alloc(u8 *daddr,
1418 struct net_device *dev)
1419{
c1048aff 1420 struct ipoib_dev_priv *priv = ipoib_priv(dev);
b63b70d8
SP		 1421 struct ipoib_neigh_table *ntbl = &priv->ntbl;
1422 struct ipoib_neigh_hash *htbl;
1423 struct ipoib_neigh *neigh;
1424 u32 hash_val;
1425
b63b70d8 1426 htbl = rcu_dereference_protected(ntbl->htbl,
b5120a6e 1427 lockdep_is_held(&priv->lock));
b63b70d8
SP		 1428 if (!htbl) {
1429 neigh = NULL;
1430 goto out_unlock;
1431 }
1432
1433 /* need to add a new neigh, but maybe some other thread succeeded?
1434 * recalc hash, maybe hash resize took place so we do a search
1435 */
1436 hash_val = ipoib_addr_hash(htbl, daddr);
1437 for (neigh = rcu_dereference_protected(htbl->buckets[hash_val],
b5120a6e 1438 lockdep_is_held(&priv->lock));
b63b70d8
SP		 1439 neigh != NULL;
1440 neigh = rcu_dereference_protected(neigh->hnext,
b5120a6e 1441 lockdep_is_held(&priv->lock))) {
b63b70d8
SP		 1442 if (memcmp(daddr, neigh->daddr, INFINIBAND_ALEN) == 0) {
1443 /* found, take one ref on behalf of the caller */
a5e27fb6 1444 if (!refcount_inc_not_zero(&neigh->refcnt)) {
b63b70d8
SP		 1445 /* deleted */
1446 neigh = NULL;
1447 break;
1448 }
1449 neigh->alive = jiffies;
1450 goto out_unlock;
1451 }
1452 }
1453
1454 neigh = ipoib_neigh_ctor(daddr, dev);
1455 if (!neigh)
1456 goto out_unlock;
1457
1458 /* one ref on behalf of the hash table */
a5e27fb6 1459 refcount_inc(&neigh->refcnt);
b63b70d8
SP		 1460 neigh->alive = jiffies;
1461 /* put in hash */
1462 rcu_assign_pointer(neigh->hnext,
1463 rcu_dereference_protected(htbl->buckets[hash_val],
b5120a6e 1464 lockdep_is_held(&priv->lock)));
b63b70d8
SP		 1465 rcu_assign_pointer(htbl->buckets[hash_val], neigh);
1466 atomic_inc(&ntbl->entries);
1467
1468out_unlock:
d2e0655e
MT		 1469
1470 return neigh;
1471}
1472
b63b70d8 1473void ipoib_neigh_dtor(struct ipoib_neigh *neigh)
d2e0655e 1474{
b63b70d8
SP		 1475 /* neigh reference count was dropprd to zero */
1476 struct net_device *dev = neigh->dev;
c1048aff 1477 struct ipoib_dev_priv *priv = ipoib_priv(dev);
2745b5b7 1478 struct sk_buff *skb;
b63b70d8
SP		 1479 if (neigh->ah)
1480 ipoib_put_ah(neigh->ah);
2745b5b7 1481 while ((skb = __skb_dequeue(&neigh->queue))) {
de903512 1482 ++dev->stats.tx_dropped;
2745b5b7
MT		 1483 dev_kfree_skb_any(skb);
1484 }
839fcaba
MT		 1485 if (ipoib_cm_get(neigh))
1486 ipoib_cm_destroy_tx(ipoib_cm_get(neigh));
c1048aff 1487 ipoib_dbg(ipoib_priv(dev),
b63b70d8
SP		 1488 "neigh free for %06x %pI6\n",
1489 IPOIB_QPN(neigh->daddr),
1490 neigh->daddr + 4);
d2e0655e 1491 kfree(neigh);
b63b70d8
SP		 1492 if (atomic_dec_and_test(&priv->ntbl.entries)) {
1493 if (test_bit(IPOIB_NEIGH_TBL_FLUSH, &priv->flags))
1494 complete(&priv->ntbl.flushed);
1495 }
1496}
1497
1498static void ipoib_neigh_reclaim(struct rcu_head *rp)
1499{
1500 /* Called as a result of removal from hash table */
1501 struct ipoib_neigh *neigh = container_of(rp, struct ipoib_neigh, rcu);
1502 /* note TX context may hold another ref */
1503 ipoib_neigh_put(neigh);
d2e0655e
MT		 1504}
1505
b63b70d8 1506void ipoib_neigh_free(struct ipoib_neigh *neigh)
1da177e4 1507{
b63b70d8 1508 struct net_device *dev = neigh->dev;
c1048aff 1509 struct ipoib_dev_priv *priv = ipoib_priv(dev);
b63b70d8
SP		 1510 struct ipoib_neigh_table *ntbl = &priv->ntbl;
1511 struct ipoib_neigh_hash *htbl;
1512 struct ipoib_neigh __rcu **np;
1513 struct ipoib_neigh *n;
1514 u32 hash_val;
1515
b63b70d8 1516 htbl = rcu_dereference_protected(ntbl->htbl,
b5120a6e 1517 lockdep_is_held(&priv->lock));
b63b70d8 1518 if (!htbl)
b5120a6e 1519 return;
b63b70d8
SP		 1520
1521 hash_val = ipoib_addr_hash(htbl, neigh->daddr);
1522 np = &htbl->buckets[hash_val];
1523 for (n = rcu_dereference_protected(*np,
b5120a6e 1524 lockdep_is_held(&priv->lock));
b63b70d8 1525 n != NULL;
6c723a68 1526 n = rcu_dereference_protected(*np,
b5120a6e 1527 lockdep_is_held(&priv->lock))) {
b63b70d8
SP		 1528 if (n == neigh) {
1529 /* found */
1530 rcu_assign_pointer(*np,
1531 rcu_dereference_protected(neigh->hnext,
b5120a6e 1532 lockdep_is_held(&priv->lock)));
49b8e744 1533 /* remove from parent list */
c586071d 1534 list_del_init(&neigh->list);
b63b70d8 1535 call_rcu(&neigh->rcu, ipoib_neigh_reclaim);
b5120a6e 1536 return;
b63b70d8
SP		 1537 } else {
1538 np = &n->hnext;
1539 }
1540 }
b63b70d8
SP		 1541}
1542
1543static int ipoib_neigh_hash_init(struct ipoib_dev_priv *priv)
1544{
1545 struct ipoib_neigh_table *ntbl = &priv->ntbl;
1546 struct ipoib_neigh_hash *htbl;
52374967 1547 struct ipoib_neigh __rcu **buckets;
b63b70d8
SP		 1548 u32 size;
1549
1550 clear_bit(IPOIB_NEIGH_TBL_FLUSH, &priv->flags);
1551 ntbl->htbl = NULL;
b63b70d8
SP		 1552 htbl = kzalloc(sizeof(*htbl), GFP_KERNEL);
1553 if (!htbl)
1554 return -ENOMEM;
b63b70d8 1555 size = roundup_pow_of_two(arp_tbl.gc_thresh3);
259e1914 1556 buckets = kvcalloc(size, sizeof(*buckets), GFP_KERNEL);
b63b70d8
SP		 1557 if (!buckets) {
1558 kfree(htbl);
1559 return -ENOMEM;
1560 }
1561 htbl->size = size;
1562 htbl->mask = (size - 1);
1563 htbl->buckets = buckets;
52374967 1564 RCU_INIT_POINTER(ntbl->htbl, htbl);
66172c09 1565 htbl->ntbl = ntbl;
b63b70d8
SP		 1566 atomic_set(&ntbl->entries, 0);
1567
1568 /* start garbage collection */
0b39578b 1569 queue_delayed_work(priv->wq, &priv->neigh_reap_task,
b63b70d8 1570 arp_tbl.gc_interval);
1da177e4
LT		 1571
1572 return 0;
1573}
1574
b63b70d8
SP		 1575static void neigh_hash_free_rcu(struct rcu_head *head)
1576{
1577 struct ipoib_neigh_hash *htbl = container_of(head,
1578 struct ipoib_neigh_hash,
1579 rcu);
1580 struct ipoib_neigh __rcu **buckets = htbl->buckets;
66172c09 1581 struct ipoib_neigh_table *ntbl = htbl->ntbl;
b63b70d8 1582
259e1914 1583 kvfree(buckets);
b63b70d8 1584 kfree(htbl);
66172c09 1585 complete(&ntbl->deleted);
b63b70d8
SP		 1586}
1587
1588void ipoib_del_neighs_by_gid(struct net_device *dev, u8 *gid)
1589{
c1048aff 1590 struct ipoib_dev_priv *priv = ipoib_priv(dev);
b63b70d8
SP		 1591 struct ipoib_neigh_table *ntbl = &priv->ntbl;
1592 struct ipoib_neigh_hash *htbl;
1593 unsigned long flags;
1594 int i;
1595
1596 /* remove all neigh connected to a given path or mcast */
b5120a6e 1597 spin_lock_irqsave(&priv->lock, flags);
b63b70d8
SP		 1598
1599 htbl = rcu_dereference_protected(ntbl->htbl,
b5120a6e 1600 lockdep_is_held(&priv->lock));
b63b70d8
SP		 1601
1602 if (!htbl)
1603 goto out_unlock;
1604
1605 for (i = 0; i < htbl->size; i++) {
1606 struct ipoib_neigh *neigh;
1607 struct ipoib_neigh __rcu **np = &htbl->buckets[i];
1608
1609 while ((neigh = rcu_dereference_protected(*np,
b5120a6e 1610 lockdep_is_held(&priv->lock))) != NULL) {
b63b70d8
SP		 1611 /* delete neighs belong to this parent */
1612 if (!memcmp(gid, neigh->daddr + 4, sizeof (union ib_gid))) {
1613 rcu_assign_pointer(*np,
1614 rcu_dereference_protected(neigh->hnext,
b5120a6e 1615 lockdep_is_held(&priv->lock)));
b63b70d8 1616 /* remove from parent list */
c586071d 1617 list_del_init(&neigh->list);
b63b70d8
SP		 1618 call_rcu(&neigh->rcu, ipoib_neigh_reclaim);
1619 } else {
1620 np = &neigh->hnext;
1621 }
1622
1623 }
1624 }
1625out_unlock:
b5120a6e 1626 spin_unlock_irqrestore(&priv->lock, flags);
b63b70d8
SP		 1627}
1628
1629static void ipoib_flush_neighs(struct ipoib_dev_priv *priv)
1630{
1631 struct ipoib_neigh_table *ntbl = &priv->ntbl;
1632 struct ipoib_neigh_hash *htbl;
1633 unsigned long flags;
66172c09 1634 int i, wait_flushed = 0;
b63b70d8 1635
66172c09 1636 init_completion(&priv->ntbl.flushed);
d2e46fcc 1637 set_bit(IPOIB_NEIGH_TBL_FLUSH, &priv->flags);
b63b70d8 1638
b5120a6e 1639 spin_lock_irqsave(&priv->lock, flags);
b63b70d8
SP		 1640
1641 htbl = rcu_dereference_protected(ntbl->htbl,
b5120a6e 1642 lockdep_is_held(&priv->lock));
b63b70d8
SP		 1643 if (!htbl)
1644 goto out_unlock;
1645
66172c09
SP		 1646 wait_flushed = atomic_read(&priv->ntbl.entries);
1647 if (!wait_flushed)
1648 goto free_htbl;
1649
b63b70d8
SP		 1650 for (i = 0; i < htbl->size; i++) {
1651 struct ipoib_neigh *neigh;
1652 struct ipoib_neigh __rcu **np = &htbl->buckets[i];
1653
1654 while ((neigh = rcu_dereference_protected(*np,
b5120a6e 1655 lockdep_is_held(&priv->lock))) != NULL) {
b63b70d8
SP		 1656 rcu_assign_pointer(*np,
1657 rcu_dereference_protected(neigh->hnext,
b5120a6e 1658 lockdep_is_held(&priv->lock)));
b63b70d8 1659 /* remove from path/mc list */
c586071d 1660 list_del_init(&neigh->list);
b63b70d8
SP		 1661 call_rcu(&neigh->rcu, ipoib_neigh_reclaim);
1662 }
1663 }
1664
66172c09 1665free_htbl:
b63b70d8
SP		 1666 rcu_assign_pointer(ntbl->htbl, NULL);
1667 call_rcu(&htbl->rcu, neigh_hash_free_rcu);
1668
1669out_unlock:
b5120a6e 1670 spin_unlock_irqrestore(&priv->lock, flags);
66172c09
SP		 1671 if (wait_flushed)
1672 wait_for_completion(&priv->ntbl.flushed);
b63b70d8
SP		 1673}
1674
1675static void ipoib_neigh_hash_uninit(struct net_device *dev)
1676{
c1048aff 1677 struct ipoib_dev_priv *priv = ipoib_priv(dev);
b63b70d8 1678
0dd9ce18 1679 ipoib_dbg(priv, "%s\n", __func__);
66172c09 1680 init_completion(&priv->ntbl.deleted);
b63b70d8 1681
cda8daf1 1682 cancel_delayed_work_sync(&priv->neigh_reap_task);
b63b70d8 1683
66172c09
SP		 1684 ipoib_flush_neighs(priv);
1685
1686 wait_for_completion(&priv->ntbl.deleted);
b63b70d8
SP		 1687}
1688
8966e28d
ES		 1689static void ipoib_napi_add(struct net_device *dev)
1690{
1691 struct ipoib_dev_priv *priv = ipoib_priv(dev);
1692
2157f5ca
JK		 1693 netif_napi_add_weight(dev, &priv->recv_napi, ipoib_rx_poll,
1694 IPOIB_NUM_WC);
1695 netif_napi_add_weight(dev, &priv->send_napi, ipoib_tx_poll,
1696 MAX_SEND_CQE);
8966e28d
ES		 1697}
1698
1699static void ipoib_napi_del(struct net_device *dev)
1700{
1701 struct ipoib_dev_priv *priv = ipoib_priv(dev);
1702
1703 netif_napi_del(&priv->recv_napi);
1704 netif_napi_del(&priv->send_napi);
1705}
1706
0a1a9726 1707static void ipoib_dev_uninit_default(struct net_device *dev)
515ed4f3 1708{
c1048aff 1709 struct ipoib_dev_priv *priv = ipoib_priv(dev);
b63b70d8 1710
515ed4f3
ES		 1711 ipoib_transport_dev_cleanup(dev);
1712
8966e28d 1713 ipoib_napi_del(dev);
b53d4566 1714
515ed4f3
ES		 1715 ipoib_cm_dev_cleanup(dev);
1716
1717 kfree(priv->rx_ring);
1718 vfree(priv->tx_ring);
1719
1720 priv->rx_ring = NULL;
1721 priv->tx_ring = NULL;
1722}
1723
cd565b4b 1724static int ipoib_dev_init_default(struct net_device *dev)
1da177e4 1725{
c1048aff 1726 struct ipoib_dev_priv *priv = ipoib_priv(dev);
10f7b9bc 1727 u8 addr_mod[3];
1da177e4 1728
8966e28d 1729 ipoib_napi_add(dev);
cd565b4b 1730
1da177e4 1731 /* Allocate RX/TX "rings" to hold queued skbs */
6396bb22
KC		 1732 priv->rx_ring = kcalloc(ipoib_recvq_size,
1733 sizeof(*priv->rx_ring),
1734 GFP_KERNEL);
74226649 1735 if (!priv->rx_ring)
be7aa663 1736 goto out;
1da177e4 1737
fad953ce
KC		 1738 priv->tx_ring = vzalloc(array_size(ipoib_sendq_size,
1739 sizeof(*priv->tx_ring)));
1da177e4 1740 if (!priv->tx_ring) {
c55359a2
YS		 1741 pr_warn("%s: failed to allocate TX ring (%d entries)\n",
1742 priv->ca->name, ipoib_sendq_size);
1da177e4
LT		 1743 goto out_rx_ring_cleanup;
1744 }
1da177e4 1745
1acba6a8 1746 /* priv->tx_head, tx_tail and global_tx_tail/head are already 0 */
1da177e4 1747
cd565b4b
ES		 1748 if (ipoib_transport_dev_init(dev, priv->ca)) {
1749 pr_warn("%s: ipoib_transport_dev_init failed\n",
1750 priv->ca->name);
1da177e4 1751 goto out_tx_ring_cleanup;
515ed4f3
ES		 1752 }
1753
cd565b4b 1754 /* after qp created set dev address */
10f7b9bc
JK		 1755 addr_mod[0] = (priv->qp->qp_num >> 16) & 0xff;
1756 addr_mod[1] = (priv->qp->qp_num >> 8) & 0xff;
1757 addr_mod[2] = (priv->qp->qp_num) & 0xff;
1758 dev_addr_mod(priv->dev, 1, addr_mod, sizeof(addr_mod));
cd565b4b 1759
515ed4f3
ES		 1760 return 0;
1761
1762out_tx_ring_cleanup:
1763 vfree(priv->tx_ring);
1764
1765out_rx_ring_cleanup:
1766 kfree(priv->rx_ring);
1767
1768out:
8966e28d 1769 ipoib_napi_del(dev);
515ed4f3
ES		 1770 return -ENOMEM;
1771}
1772
31a82362
FD		 1773static int ipoib_ioctl(struct net_device *dev, struct ifreq *ifr,
1774 int cmd)
1775{
1776 struct ipoib_dev_priv *priv = ipoib_priv(dev);
1777
a7605370 1778 if (!priv->rn_ops->ndo_eth_ioctl)
31a82362
FD		 1779 return -EOPNOTSUPP;
1780
a7605370 1781 return priv->rn_ops->ndo_eth_ioctl(dev, ifr, cmd);
31a82362
FD		 1782}
1783
eaeb3984 1784static int ipoib_dev_init(struct net_device *dev)
515ed4f3 1785{
c1048aff 1786 struct ipoib_dev_priv *priv = ipoib_priv(dev);
515ed4f3
ES		 1787 int ret = -ENOMEM;
1788
515ed4f3 1789 priv->qp = NULL;
1da177e4 1790
be7aa663 1791 /*
515ed4f3
ES		 1792 * the various IPoIB tasks assume they will never race against
1793 * themselves, so always use a single thread workqueue
be7aa663 1794 */
515ed4f3
ES		 1795 priv->wq = alloc_ordered_workqueue("ipoib_wq", WQ_MEM_RECLAIM);
1796 if (!priv->wq) {
1797 pr_warn("%s: failed to allocate device WQ\n", dev->name);
1798 goto out;
1799 }
1800
1801 /* create pd, which used both for control and datapath*/
1802 priv->pd = ib_alloc_pd(priv->ca, 0);
1803 if (IS_ERR(priv->pd)) {
eaeb3984 1804 pr_warn("%s: failed to allocate PD\n", priv->ca->name);
515ed4f3
ES		 1805 goto clean_wq;
1806 }
1807
cd565b4b 1808 ret = priv->rn_ops->ndo_init(dev);
515ed4f3
ES		 1809 if (ret) {
1810 pr_warn("%s failed to init HW resource\n", dev->name);
1811 goto out_free_pd;
1812 }
1813
99a7e2bf
WY		 1814 ret = ipoib_neigh_hash_init(priv);
1815 if (ret) {
515ed4f3 1816 pr_warn("%s failed to init neigh hash\n", dev->name);
be7aa663 1817 goto out_dev_uninit;
515ed4f3
ES		 1818 }
1819
1820 if (dev->flags & IFF_UP) {
1821 if (ipoib_ib_dev_open(dev)) {
1822 pr_warn("%s failed to open device\n", dev->name);
1823 ret = -ENODEV;
cda8daf1 1824 goto out_hash_uninit;
515ed4f3
ES		 1825 }
1826 }
be7aa663 1827
1da177e4
LT		 1828 return 0;
1829
cda8daf1
ES		 1830out_hash_uninit:
1831 ipoib_neigh_hash_uninit(dev);
1832
be7aa663
DL		 1833out_dev_uninit:
1834 ipoib_ib_dev_cleanup(dev);
1835
515ed4f3
ES		 1836out_free_pd:
1837 if (priv->pd) {
1838 ib_dealloc_pd(priv->pd);
1839 priv->pd = NULL;
1840 }
1da177e4 1841
515ed4f3
ES		 1842clean_wq:
1843 if (priv->wq) {
1844 destroy_workqueue(priv->wq);
1845 priv->wq = NULL;
1846 }
1da177e4
LT		 1847
1848out:
515ed4f3 1849 return ret;
1da177e4
LT		 1850}
1851
7cbee87c
JG		 1852/*
1853 * This must be called before doing an unregister_netdev on a parent device to
1854 * shutdown the IB event handler.
1855 */
1856static void ipoib_parent_unregister_pre(struct net_device *ndev)
1857{
1858 struct ipoib_dev_priv *priv = ipoib_priv(ndev);
1859
1860 /*
1861 * ipoib_set_mac checks netif_running before pushing work, clearing
1862 * running ensures the it will not add more work.
1863 */
1864 rtnl_lock();
567c5e13 1865 dev_change_flags(priv->dev, priv->dev->flags & ~IFF_UP, NULL);
7cbee87c
JG		 1866 rtnl_unlock();
1867
1868 /* ipoib_event() cannot be running once this returns */
1869 ib_unregister_event_handler(&priv->event_handler);
1870
1871 /*
1872 * Work on the queue grabs the rtnl lock, so this cannot be done while
1873 * also holding it.
1874 */
1875 flush_workqueue(ipoib_workqueue);
1876}
1877
eaeb3984
JG		 1878static void ipoib_set_dev_features(struct ipoib_dev_priv *priv)
1879{
1880 priv->hca_caps = priv->ca->attrs.device_cap_flags;
e945c653 1881 priv->kernel_caps = priv->ca->attrs.kernel_cap_flags;
eaeb3984
JG		 1882
1883 if (priv->hca_caps & IB_DEVICE_UD_IP_CSUM) {
1884 priv->dev->hw_features |= NETIF_F_IP_CSUM | NETIF_F_RXCSUM;
1885
e945c653 1886 if (priv->kernel_caps & IBK_UD_TSO)
eaeb3984
JG		 1887 priv->dev->hw_features |= NETIF_F_TSO;
1888
1889 priv->dev->features |= priv->dev->hw_features;
1890 }
1891}
1892
1893static int ipoib_parent_init(struct net_device *ndev)
1894{
1895 struct ipoib_dev_priv *priv = ipoib_priv(ndev);
1896 struct ib_port_attr attr;
1897 int result;
1898
1899 result = ib_query_port(priv->ca, priv->port, &attr);
1900 if (result) {
1901 pr_warn("%s: ib_query_port %d failed\n", priv->ca->name,
1902 priv->port);
1903 return result;
1904 }
6d72344c 1905 priv->max_ib_mtu = rdma_mtu_from_attr(priv->ca, priv->port, &attr);
eaeb3984
JG		 1906
1907 result = ib_query_pkey(priv->ca, priv->port, 0, &priv->pkey);
1908 if (result) {
1909 pr_warn("%s: ib_query_pkey port %d failed (ret = %d)\n",
1910 priv->ca->name, priv->port, result);
1911 return result;
1912 }
1913
1914 result = rdma_query_gid(priv->ca, priv->port, 0, &priv->local_gid);
1915 if (result) {
1916 pr_warn("%s: rdma_query_gid port %d failed (ret = %d)\n",
1917 priv->ca->name, priv->port, result);
1918 return result;
1919 }
10f7b9bc 1920 dev_addr_mod(priv->dev, 4, priv->local_gid.raw, sizeof(union ib_gid));
eaeb3984
JG		 1921
1922 SET_NETDEV_DEV(priv->dev, priv->ca->dev.parent);
9b8b2a32
AM		 1923 priv->dev->dev_port = priv->port - 1;
1924 /* Let's set this one too for backwards compatibility. */
eaeb3984
JG		 1925 priv->dev->dev_id = priv->port - 1;
1926
1927 return 0;
1928}
1929
1930static void ipoib_child_init(struct net_device *ndev)
1931{
1932 struct ipoib_dev_priv *priv = ipoib_priv(ndev);
1933 struct ipoib_dev_priv *ppriv = ipoib_priv(priv->parent);
1934
1935 priv->max_ib_mtu = ppriv->max_ib_mtu;
1936 set_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags);
87fb5c1c
MG		 1937 if (memchr_inv(priv->dev->dev_addr, 0, INFINIBAND_ALEN))
1938 memcpy(&priv->local_gid, priv->dev->dev_addr + 4,
1939 sizeof(priv->local_gid));
1940 else {
10f7b9bc
JK		 1941 __dev_addr_set(priv->dev, ppriv->dev->dev_addr,
1942 INFINIBAND_ALEN);
87fb5c1c
MG		 1943 memcpy(&priv->local_gid, &ppriv->local_gid,
1944 sizeof(priv->local_gid));
1945 }
eaeb3984
JG		 1946}
1947
1948static int ipoib_ndo_init(struct net_device *ndev)
1949{
1950 struct ipoib_dev_priv *priv = ipoib_priv(ndev);
1951 int rc;
b7e159eb 1952 struct rdma_netdev *rn = netdev_priv(ndev);
eaeb3984
JG		 1953
1954 if (priv->parent) {
1955 ipoib_child_init(ndev);
1956 } else {
1957 rc = ipoib_parent_init(ndev);
1958 if (rc)
1959 return rc;
1960 }
1961
1962 /* MTU will be reset when mcast join happens */
1963 ndev->mtu = IPOIB_UD_MTU(priv->max_ib_mtu);
1964 priv->mcast_mtu = priv->admin_mtu = ndev->mtu;
b7e159eb 1965 rn->mtu = priv->mcast_mtu;
eaeb3984
JG		 1966 ndev->max_mtu = IPOIB_CM_MTU;
1967
1968 ndev->neigh_priv_len = sizeof(struct ipoib_neigh);
1969
1970 /*
1971 * Set the full membership bit, so that we join the right
1972 * broadcast group, etc.
1973 */
1974 priv->pkey |= 0x8000;
1975
1976 ndev->broadcast[8] = priv->pkey >> 8;
1977 ndev->broadcast[9] = priv->pkey & 0xff;
1978 set_bit(IPOIB_FLAG_DEV_ADDR_SET, &priv->flags);
1979
1980 ipoib_set_dev_features(priv);
1981
1982 rc = ipoib_dev_init(ndev);
1983 if (rc) {
1984 pr_warn("%s: failed to initialize device: %s port %d (ret = %d)\n",
1985 priv->ca->name, priv->dev->name, priv->port, rc);
91b01061
VF		 1986 return rc;
1987 }
1988
1989 if (priv->parent) {
1990 struct ipoib_dev_priv *ppriv = ipoib_priv(priv->parent);
1991
1992 dev_hold(priv->parent);
1993
1994 down_write(&ppriv->vlan_rwsem);
1995 list_add_tail(&priv->list, &ppriv->child_intfs);
1996 up_write(&ppriv->vlan_rwsem);
eaeb3984
JG		 1997 }
1998
1999 return 0;
2000}
2001
7cbee87c 2002static void ipoib_ndo_uninit(struct net_device *dev)
1da177e4 2003{
25405d98 2004 struct ipoib_dev_priv *priv = ipoib_priv(dev);
9baa0b03
OG		 2005
2006 ASSERT_RTNL();
1da177e4 2007
25405d98
JG		 2008 /*
2009 * ipoib_remove_one guarantees the children are removed before the
2010 * parent, and that is the only place where a parent can be removed.
2011 */
2012 WARN_ON(!list_empty(&priv->child_intfs));
1da177e4 2013
91b01061
VF		 2014 if (priv->parent) {
2015 struct ipoib_dev_priv *ppriv = ipoib_priv(priv->parent);
2016
2017 down_write(&ppriv->vlan_rwsem);
2018 list_del(&priv->list);
2019 up_write(&ppriv->vlan_rwsem);
2020 }
2021
be7aa663
DL		 2022 ipoib_neigh_hash_uninit(dev);
2023
1da177e4
LT		 2024 ipoib_ib_dev_cleanup(dev);
2025
515ed4f3
ES		 2026 /* no more works over the priv->wq */
2027 if (priv->wq) {
65936bf2
JG		 2028 /* See ipoib_mcast_carrier_on_task() */
2029 WARN_ON(test_bit(IPOIB_FLAG_OPER_UP, &priv->flags));
515ed4f3
ES		 2030 destroy_workqueue(priv->wq);
2031 priv->wq = NULL;
2032 }
13476d35 2033
7a1c2abf 2034 dev_put(priv->parent);
1da177e4
LT		 2035}
2036
9c3c5f8e
EC		 2037static int ipoib_set_vf_link_state(struct net_device *dev, int vf, int link_state)
2038{
c1048aff 2039 struct ipoib_dev_priv *priv = ipoib_priv(dev);
9c3c5f8e
EC		 2040
2041 return ib_set_vf_link_state(priv->ca, vf, priv->port, link_state);
2042}
2043
2044static int ipoib_get_vf_config(struct net_device *dev, int vf,
2045 struct ifla_vf_info *ivf)
2046{
c1048aff 2047 struct ipoib_dev_priv *priv = ipoib_priv(dev);
9c3c5f8e
EC		 2048 int err;
2049
2050 err = ib_get_vf_config(priv->ca, vf, priv->port, ivf);
2051 if (err)
2052 return err;
2053
2054 ivf->vf = vf;
64d701c6 2055 memcpy(ivf->mac, dev->dev_addr, dev->addr_len);
9c3c5f8e
EC		 2056
2057 return 0;
2058}
2059
2060static int ipoib_set_vf_guid(struct net_device *dev, int vf, u64 guid, int type)
2061{
c1048aff 2062 struct ipoib_dev_priv *priv = ipoib_priv(dev);
9c3c5f8e
EC		 2063
2064 if (type != IFLA_VF_IB_NODE_GUID && type != IFLA_VF_IB_PORT_GUID)
2065 return -EINVAL;
2066
2067 return ib_set_vf_guid(priv->ca, vf, priv->port, guid, type);
2068}
2069
2446887e
DG		 2070static int ipoib_get_vf_guid(struct net_device *dev, int vf,
2071 struct ifla_vf_guid *node_guid,
2072 struct ifla_vf_guid *port_guid)
2073{
2074 struct ipoib_dev_priv *priv = ipoib_priv(dev);
2075
2076 return ib_get_vf_guid(priv->ca, vf, priv->port, node_guid, port_guid);
2077}
2078
9c3c5f8e
EC		 2079static int ipoib_get_vf_stats(struct net_device *dev, int vf,
2080 struct ifla_vf_stats *vf_stats)
2081{
c1048aff 2082 struct ipoib_dev_priv *priv = ipoib_priv(dev);
9c3c5f8e
EC		 2083
2084 return ib_get_vf_stats(priv->ca, vf, priv->port, vf_stats);
2085}
2086
3b04ddde
SH		 2087static const struct header_ops ipoib_header_ops = {
2088 .create = ipoib_hard_header,
2089};
2090
9c3c5f8e 2091static const struct net_device_ops ipoib_netdev_ops_pf = {
eaeb3984 2092 .ndo_init = ipoib_ndo_init,
7cbee87c 2093 .ndo_uninit = ipoib_ndo_uninit,
9c3c5f8e
EC		 2094 .ndo_open = ipoib_open,
2095 .ndo_stop = ipoib_stop,
2096 .ndo_change_mtu = ipoib_change_mtu,
2097 .ndo_fix_features = ipoib_fix_features,
2098 .ndo_start_xmit = ipoib_start_xmit,
2099 .ndo_tx_timeout = ipoib_timeout,
2100 .ndo_set_rx_mode = ipoib_set_mcast_list,
2101 .ndo_get_iflink = ipoib_get_iflink,
2102 .ndo_set_vf_link_state = ipoib_set_vf_link_state,
2103 .ndo_get_vf_config = ipoib_get_vf_config,
2104 .ndo_get_vf_stats = ipoib_get_vf_stats,
2446887e 2105 .ndo_get_vf_guid = ipoib_get_vf_guid,
9c3c5f8e 2106 .ndo_set_vf_guid = ipoib_set_vf_guid,
492a7e67 2107 .ndo_set_mac_address = ipoib_set_mac,
b6c871e5 2108 .ndo_get_stats64 = ipoib_get_stats,
a7605370 2109 .ndo_eth_ioctl = ipoib_ioctl,
9c3c5f8e
EC		 2110};
2111
2112static const struct net_device_ops ipoib_netdev_ops_vf = {
eaeb3984 2113 .ndo_init = ipoib_ndo_init,
7cbee87c 2114 .ndo_uninit = ipoib_ndo_uninit,
fe8114e8
SH		 2115 .ndo_open = ipoib_open,
2116 .ndo_stop = ipoib_stop,
2117 .ndo_change_mtu = ipoib_change_mtu,
3d96c74d 2118 .ndo_fix_features = ipoib_fix_features,
fe8114e8
SH		 2119 .ndo_start_xmit = ipoib_start_xmit,
2120 .ndo_tx_timeout = ipoib_timeout,
afc4b13d 2121 .ndo_set_rx_mode = ipoib_set_mcast_list,
5aa7add8 2122 .ndo_get_iflink = ipoib_get_iflink,
eb54714d 2123 .ndo_get_stats64 = ipoib_get_stats,
a7605370 2124 .ndo_eth_ioctl = ipoib_ioctl,
fe8114e8
SH		 2125};
2126
8f149b68
GL		 2127static const struct net_device_ops ipoib_netdev_default_pf = {
2128 .ndo_init = ipoib_dev_init_default,
2129 .ndo_uninit = ipoib_dev_uninit_default,
2130 .ndo_open = ipoib_ib_dev_open_default,
2131 .ndo_stop = ipoib_ib_dev_stop_default,
2132};
2133
cd565b4b 2134void ipoib_setup_common(struct net_device *dev)
1da177e4 2135{
2337f809 2136 dev->header_ops = &ipoib_header_ops;
8f149b68 2137 dev->netdev_ops = &ipoib_netdev_default_pf;
bea3348e 2138
82c24c18
EC		 2139 ipoib_set_ethtool_ops(dev);
2140
50af5d12 2141 dev->watchdog_timeo = 10 * HZ;
1da177e4 2142
2337f809 2143 dev->flags |= IFF_BROADCAST | IFF_MULTICAST;
1da177e4 2144
fc791b63 2145 dev->hard_header_len = IPOIB_HARD_LEN;
2337f809
RD		 2146 dev->addr_len = INFINIBAND_ALEN;
2147 dev->type = ARPHRD_INFINIBAND;
2148 dev->tx_queue_len = ipoib_sendq_size * 2;
eb14032f 2149 dev->features = (NETIF_F_VLAN_CHALLENGED |
eb14032f 2150 NETIF_F_HIGHDMA);
02875878 2151 netif_keep_dst(dev);
1da177e4 2152
1da177e4 2153 memcpy(dev->broadcast, ipv4_bcast_addr, INFINIBAND_ALEN);
9f49a5b5
JG		 2154
2155 /*
2156 * unregister_netdev always frees the netdev, we use this mode
2157 * consistently to unify all the various unregister paths, including
2158 * those connected to rtnl_link_ops which require it.
2159 */
2160 dev->needs_free_netdev = true;
cd565b4b 2161}
1da177e4 2162
cd565b4b
ES		 2163static void ipoib_build_priv(struct net_device *dev)
2164{
2165 struct ipoib_dev_priv *priv = ipoib_priv(dev);
1da177e4 2166
cd565b4b 2167 priv->dev = dev;
1da177e4 2168 spin_lock_init(&priv->lock);
f47944cc 2169 init_rwsem(&priv->vlan_rwsem);
edf3f301 2170 mutex_init(&priv->mcast_mutex);
1da177e4
LT		 2171
2172 INIT_LIST_HEAD(&priv->path_list);
2173 INIT_LIST_HEAD(&priv->child_intfs);
2174 INIT_LIST_HEAD(&priv->dead_ahs);
2175 INIT_LIST_HEAD(&priv->multicast_list);
2176
c4028958 2177 INIT_DELAYED_WORK(&priv->mcast_task, ipoib_mcast_join_task);
e8224e4b 2178 INIT_WORK(&priv->carrier_on_task, ipoib_mcast_carrier_on_task);
50af5d12 2179 INIT_WORK(&priv->reschedule_napi_work, ipoib_napi_schedule_work);
ee1e2c82
MS		 2180 INIT_WORK(&priv->flush_light, ipoib_ib_dev_flush_light);
2181 INIT_WORK(&priv->flush_normal, ipoib_ib_dev_flush_normal);
2182 INIT_WORK(&priv->flush_heavy, ipoib_ib_dev_flush_heavy);
c4028958 2183 INIT_WORK(&priv->restart_task, ipoib_mcast_restart_task);
50af5d12 2184 INIT_WORK(&priv->tx_timeout_work, ipoib_ib_tx_timeout_work);
c4028958 2185 INIT_DELAYED_WORK(&priv->ah_reap_task, ipoib_reap_ah);
b63b70d8 2186 INIT_DELAYED_WORK(&priv->neigh_reap_task, ipoib_reap_neigh);
1da177e4
LT		 2187}
2188
1fb7f897 2189static struct net_device *ipoib_alloc_netdev(struct ib_device *hca, u32 port,
5d6b0cb3 2190 const char *name)
cd565b4b
ES		 2191{
2192 struct net_device *dev;
2193
f6a8a19b
DD		 2194 dev = rdma_alloc_netdev(hca, port, RDMA_NETDEV_IPOIB, name,
2195 NET_NAME_UNKNOWN, ipoib_setup_common);
5d6b0cb3 2196 if (!IS_ERR(dev) || PTR_ERR(dev) != -EOPNOTSUPP)
f6a8a19b 2197 return dev;
cd565b4b 2198
5d6b0cb3
DD		 2199 dev = alloc_netdev(sizeof(struct rdma_netdev), name, NET_NAME_UNKNOWN,
2200 ipoib_setup_common);
2201 if (!dev)
2202 return ERR_PTR(-ENOMEM);
2203 return dev;
cd565b4b
ES		 2204}
2205
1fb7f897 2206int ipoib_intf_init(struct ib_device *hca, u32 port, const char *name,
5d6b0cb3 2207 struct net_device *dev)
cd565b4b 2208{
5d6b0cb3 2209 struct rdma_netdev *rn = netdev_priv(dev);
cd565b4b 2210 struct ipoib_dev_priv *priv;
5d6b0cb3 2211 int rc;
cd565b4b
ES		 2212
2213 priv = kzalloc(sizeof(*priv), GFP_KERNEL);
2214 if (!priv)
5d6b0cb3 2215 return -ENOMEM;
cd565b4b 2216
eaeb3984
JG		 2217 priv->ca = hca;
2218 priv->port = port;
2219
5d6b0cb3
DD		 2220 rc = rdma_init_netdev(hca, port, RDMA_NETDEV_IPOIB, name,
2221 NET_NAME_UNKNOWN, ipoib_setup_common, dev);
2222 if (rc) {
2223 if (rc != -EOPNOTSUPP)
2224 goto out;
2225
5d6b0cb3
DD		 2226 rn->send = ipoib_send;
2227 rn->attach_mcast = ipoib_mcast_attach;
2228 rn->detach_mcast = ipoib_mcast_detach;
2229 rn->hca = hca;
e632291a
DT		 2230
2231 rc = netif_set_real_num_tx_queues(dev, 1);
2232 if (rc)
2233 goto out;
2234
2235 rc = netif_set_real_num_rx_queues(dev, 1);
2236 if (rc)
2237 goto out;
5d6b0cb3 2238 }
cd565b4b
ES		 2239
2240 priv->rn_ops = dev->netdev_ops;
2241
e945c653 2242 if (hca->attrs.kernel_cap_flags & IBK_VIRTUAL_FUNCTION)
cd565b4b
ES		 2243 dev->netdev_ops = &ipoib_netdev_ops_vf;
2244 else
2245 dev->netdev_ops = &ipoib_netdev_ops_pf;
2246
cd565b4b 2247 rn->clnt_priv = priv;
9f49a5b5
JG		 2248 /*
2249 * Only the child register_netdev flows can handle priv_destructor
2250 * being set, so we force it to NULL here and handle manually until it
2251 * is safe to turn on.
2252 */
2253 priv->next_priv_destructor = dev->priv_destructor;
2254 dev->priv_destructor = NULL;
2255
cd565b4b
ES		 2256 ipoib_build_priv(dev);
2257
5d6b0cb3
DD		 2258 return 0;
2259
2260out:
cd565b4b 2261 kfree(priv);
5d6b0cb3
DD		 2262 return rc;
2263}
2264
1fb7f897 2265struct net_device *ipoib_intf_alloc(struct ib_device *hca, u32 port,
5d6b0cb3
DD		 2266 const char *name)
2267{
2268 struct net_device *dev;
2269 int rc;
2270
2271 dev = ipoib_alloc_netdev(hca, port, name);
2272 if (IS_ERR(dev))
2273 return dev;
2274
2275 rc = ipoib_intf_init(hca, port, name, dev);
2276 if (rc) {
2277 free_netdev(dev);
2278 return ERR_PTR(rc);
2279 }
2280
2281 /*
2282 * Upon success the caller must ensure ipoib_intf_free is called or
2283 * register_netdevice succeed'd and priv_destructor is set to
2284 * ipoib_intf_free.
2285 */
2286 return dev;
1da177e4
LT		 2287}
2288
9f49a5b5
JG		 2289void ipoib_intf_free(struct net_device *dev)
2290{
2291 struct ipoib_dev_priv *priv = ipoib_priv(dev);
2292 struct rdma_netdev *rn = netdev_priv(dev);
2293
2294 dev->priv_destructor = priv->next_priv_destructor;
2295 if (dev->priv_destructor)
2296 dev->priv_destructor(dev);
2297
2298 /*
2299 * There are some error flows around register_netdev failing that may
2300 * attempt to call priv_destructor twice, prevent that from happening.
2301 */
2302 dev->priv_destructor = NULL;
2303
2304 /* unregister/destroy is very complicated. Make bugs more obvious. */
2305 rn->clnt_priv = NULL;
2306
2307 kfree(priv);
2308}
2309
1f8f60f3
Y		 2310static ssize_t pkey_show(struct device *dev, struct device_attribute *attr,
2311 char *buf)
1da177e4 2312{
c1048aff
ES		 2313 struct net_device *ndev = to_net_dev(dev);
2314 struct ipoib_dev_priv *priv = ipoib_priv(ndev);
1da177e4 2315
1c7fd726 2316 return sysfs_emit(buf, "0x%04x\n", priv->pkey);
1da177e4 2317}
1f8f60f3 2318static DEVICE_ATTR_RO(pkey);
1da177e4 2319
1f8f60f3
Y		 2320static ssize_t umcast_show(struct device *dev, struct device_attribute *attr,
2321 char *buf)
335a64a5 2322{
c1048aff
ES		 2323 struct net_device *ndev = to_net_dev(dev);
2324 struct ipoib_dev_priv *priv = ipoib_priv(ndev);
335a64a5 2325
1c7fd726
JP		 2326 return sysfs_emit(buf, "%d\n",
2327 test_bit(IPOIB_FLAG_UMCAST, &priv->flags));
335a64a5
OG		 2328}
2329
862096a8 2330void ipoib_set_umcast(struct net_device *ndev, int umcast_val)
335a64a5 2331{
c1048aff 2332 struct ipoib_dev_priv *priv = ipoib_priv(ndev);
335a64a5
OG		 2333
2334 if (umcast_val > 0) {
2335 set_bit(IPOIB_FLAG_UMCAST, &priv->flags);
2336 ipoib_warn(priv, "ignoring multicast groups joined directly "
2337 "by userspace\n");
2338 } else
2339 clear_bit(IPOIB_FLAG_UMCAST, &priv->flags);
862096a8
OG		 2340}
2341
1f8f60f3
Y		 2342static ssize_t umcast_store(struct device *dev, struct device_attribute *attr,
2343 const char *buf, size_t count)
862096a8
OG		 2344{
2345 unsigned long umcast_val = simple_strtoul(buf, NULL, 0);
2346
2347 ipoib_set_umcast(to_net_dev(dev), umcast_val);
335a64a5
OG		 2348
2349 return count;
2350}
1f8f60f3 2351static DEVICE_ATTR_RW(umcast);
335a64a5
OG		 2352
2353int ipoib_add_umcast_attr(struct net_device *dev)
2354{
2355 return device_create_file(&dev->dev, &dev_attr_umcast);
2356}
2357
492a7e67
MB		 2358static void set_base_guid(struct ipoib_dev_priv *priv, union ib_gid *gid)
2359{
2360 struct ipoib_dev_priv *child_priv;
2361 struct net_device *netdev = priv->dev;
2362
9b29953b 2363 netif_addr_lock_bh(netdev);
492a7e67
MB		 2364
2365 memcpy(&priv->local_gid.global.interface_id,
2366 &gid->global.interface_id,
2367 sizeof(gid->global.interface_id));
10f7b9bc 2368 dev_addr_mod(netdev, 4, (u8 *)&priv->local_gid, sizeof(priv->local_gid));
492a7e67
MB		 2369 clear_bit(IPOIB_FLAG_DEV_ADDR_SET, &priv->flags);
2370
9b29953b 2371 netif_addr_unlock_bh(netdev);
492a7e67
MB		 2372
2373 if (!test_bit(IPOIB_FLAG_SUBINTERFACE, &priv->flags)) {
2374 down_read(&priv->vlan_rwsem);
2375 list_for_each_entry(child_priv, &priv->child_intfs, list)
2376 set_base_guid(child_priv, gid);
2377 up_read(&priv->vlan_rwsem);
2378 }
2379}
2380
2381static int ipoib_check_lladdr(struct net_device *dev,
2382 struct sockaddr_storage *ss)
2383{
2384 union ib_gid *gid = (union ib_gid *)(ss->__data + 4);
2385 int ret = 0;
2386
9b29953b 2387 netif_addr_lock_bh(dev);
492a7e67
MB		 2388
2389 /* Make sure the QPN, reserved and subnet prefix match the current
2390 * lladdr, it also makes sure the lladdr is unicast.
2391 */
2392 if (memcmp(dev->dev_addr, ss->__data,
2393 4 + sizeof(gid->global.subnet_prefix)) ||
2394 gid->global.interface_id == 0)
2395 ret = -EINVAL;
2396
9b29953b 2397 netif_addr_unlock_bh(dev);
492a7e67
MB		 2398
2399 return ret;
2400}
2401
2402static int ipoib_set_mac(struct net_device *dev, void *addr)
2403{
c1048aff 2404 struct ipoib_dev_priv *priv = ipoib_priv(dev);
492a7e67
MB		 2405 struct sockaddr_storage *ss = addr;
2406 int ret;
2407
2408 if (!(dev->priv_flags & IFF_LIVE_ADDR_CHANGE) && netif_running(dev))
2409 return -EBUSY;
2410
2411 ret = ipoib_check_lladdr(dev, ss);
2412 if (ret)
2413 return ret;
2414
2415 set_base_guid(priv, (union ib_gid *)(ss->__data + 4));
2416
2417 queue_work(ipoib_workqueue, &priv->flush_light);
2418
2419 return 0;
2420}
2421
1f8f60f3
Y		 2422static ssize_t create_child_store(struct device *dev,
2423 struct device_attribute *attr,
2424 const char *buf, size_t count)
1da177e4
LT		 2425{
2426 int pkey;
2427 int ret;
2428
2429 if (sscanf(buf, "%i", &pkey) != 1)
2430 return -EINVAL;
2431
3d790a4c 2432 if (pkey <= 0 || pkey > 0xffff || pkey == 0x8000)
1da177e4
LT		 2433 return -EINVAL;
2434
43cb76d9 2435 ret = ipoib_vlan_add(to_net_dev(dev), pkey);
1da177e4
LT		 2436
2437 return ret ? ret : count;
2438}
1f8f60f3 2439static DEVICE_ATTR_WO(create_child);
1da177e4 2440
1f8f60f3
Y		 2441static ssize_t delete_child_store(struct device *dev,
2442 struct device_attribute *attr,
2443 const char *buf, size_t count)
1da177e4
LT		 2444{
2445 int pkey;
2446 int ret;
2447
2448 if (sscanf(buf, "%i", &pkey) != 1)
2449 return -EINVAL;
2450
2451 if (pkey < 0 || pkey > 0xffff)
2452 return -EINVAL;
2453
43cb76d9 2454 ret = ipoib_vlan_delete(to_net_dev(dev), pkey);
1da177e4
LT		 2455
2456 return ret ? ret : count;
2457
2458}
1f8f60f3 2459static DEVICE_ATTR_WO(delete_child);
1da177e4
LT		 2460
2461int ipoib_add_pkey_attr(struct net_device *dev)
2462{
43cb76d9 2463 return device_create_file(&dev->dev, &dev_attr_pkey);
1da177e4
LT		 2464}
2465
f6350da4
AM		 2466/*
2467 * We erroneously exposed the iface's port number in the dev_id
2468 * sysfs field long after dev_port was introduced for that purpose[1],
2469 * and we need to stop everyone from relying on that.
2470 * Let's overload the shower routine for the dev_id file here
2471 * to gently bring the issue up.
2472 *
2473 * [1] https://www.spinics.net/lists/netdev/msg272123.html
2474 */
2475static ssize_t dev_id_show(struct device *dev,
2476 struct device_attribute *attr, char *buf)
2477{
2478 struct net_device *ndev = to_net_dev(dev);
2479
b79656ed
LR		 2480 /*
2481 * ndev->dev_port will be equal to 0 in old kernel prior to commit
2482 * 9b8b2a323008 ("IB/ipoib: Use dev_port to expose network interface
2483 * port numbers") Zero was chosen as special case for user space
2484 * applications to fallback and query dev_id to check if it has
2485 * different value or not.
2486 *
2487 * Don't print warning in such scenario.
2488 *
2489 * https://github.com/systemd/systemd/blob/master/src/udev/udev-builtin-net_id.c#L358
2490 */
2491 if (ndev->dev_port && ndev->dev_id == ndev->dev_port)
f6350da4
AM		 2492 netdev_info_once(ndev,
2493 "\"%s\" wants to know my dev_id. Should it look at dev_port instead? See Documentation/ABI/testing/sysfs-class-net for more info.\n",
2494 current->comm);
2495
1c7fd726 2496 return sysfs_emit(buf, "%#x\n", ndev->dev_id);
f6350da4
AM		 2497}
2498static DEVICE_ATTR_RO(dev_id);
2499
ed0bc265 2500static int ipoib_intercept_dev_id_attr(struct net_device *dev)
f6350da4
AM		 2501{
2502 device_remove_file(&dev->dev, &dev_attr_dev_id);
2503 return device_create_file(&dev->dev, &dev_attr_dev_id);
2504}
2505
1da177e4 2506static struct net_device *ipoib_add_port(const char *format,
1fb7f897 2507 struct ib_device *hca, u32 port)
1da177e4 2508{
5d6b0cb3
DD		 2509 struct rtnl_link_ops *ops = ipoib_get_link_ops();
2510 struct rdma_netdev_alloc_params params;
1da177e4 2511 struct ipoib_dev_priv *priv;
9f49a5b5 2512 struct net_device *ndev;
eaeb3984 2513 int result;
1da177e4 2514
5d6b0cb3
DD		 2515 ndev = ipoib_intf_alloc(hca, port, format);
2516 if (IS_ERR(ndev)) {
2517 pr_warn("%s, %d: ipoib_intf_alloc failed %ld\n", hca->name, port,
2518 PTR_ERR(ndev));
2519 return ndev;
1da177e4 2520 }
5d6b0cb3 2521 priv = ipoib_priv(ndev);
1da177e4
LT		 2522
2523 INIT_IB_EVENT_HANDLER(&priv->event_handler,
2524 priv->ca, ipoib_event);
dcc9881e 2525 ib_register_event_handler(&priv->event_handler);
1da177e4 2526
10293610
AE		 2527 /* call event handler to ensure pkey in sync */
2528 queue_work(ipoib_workqueue, &priv->flush_heavy);
2529
5ce2dced
KH		 2530 ndev->rtnl_link_ops = ipoib_get_link_ops();
2531
9f49a5b5 2532 result = register_netdev(ndev);
1da177e4 2533 if (result) {
c55359a2
YS		 2534 pr_warn("%s: couldn't register ipoib port %d; error %d\n",
2535 hca->name, port, result);
9f49a5b5
JG		 2536
2537 ipoib_parent_unregister_pre(ndev);
2538 ipoib_intf_free(ndev);
2539 free_netdev(ndev);
2540
2541 return ERR_PTR(result);
1da177e4
LT		 2542 }
2543
3023a1e9
KH		 2544 if (hca->ops.rdma_netdev_get_params) {
2545 int rc = hca->ops.rdma_netdev_get_params(hca, port,
5d6b0cb3
DD		 2546 RDMA_NETDEV_IPOIB,
2547 &params);
2548
2549 if (!rc && ops->priv_size < params.sizeof_priv)
2550 ops->priv_size = params.sizeof_priv;
2551 }
9f49a5b5
JG		 2552 /*
2553 * We cannot set priv_destructor before register_netdev because we
2554 * need priv to be always valid during the error flow to execute
2555 * ipoib_parent_unregister_pre(). Instead handle it manually and only
2556 * enter priv_destructor mode once we are completely registered.
2557 */
2558 ndev->priv_destructor = ipoib_intf_free;
2559
f6350da4
AM		 2560 if (ipoib_intercept_dev_id_attr(ndev))
2561 goto sysfs_failed;
9f49a5b5 2562 if (ipoib_cm_add_mode_attr(ndev))
839fcaba 2563 goto sysfs_failed;
9f49a5b5 2564 if (ipoib_add_pkey_attr(ndev))
1da177e4 2565 goto sysfs_failed;
9f49a5b5 2566 if (ipoib_add_umcast_attr(ndev))
335a64a5 2567 goto sysfs_failed;
9f49a5b5 2568 if (device_create_file(&ndev->dev, &dev_attr_create_child))
1da177e4 2569 goto sysfs_failed;
9f49a5b5 2570 if (device_create_file(&ndev->dev, &dev_attr_delete_child))
1da177e4
LT		 2571 goto sysfs_failed;
2572
9f49a5b5 2573 return ndev;
1da177e4
LT		 2574
2575sysfs_failed:
9f49a5b5
JG		 2576 ipoib_parent_unregister_pre(ndev);
2577 unregister_netdev(ndev);
2578 return ERR_PTR(-ENOMEM);
1da177e4
LT		 2579}
2580
11a0ae4c 2581static int ipoib_add_one(struct ib_device *device)
1da177e4
LT		 2582{
2583 struct list_head *dev_list;
2584 struct net_device *dev;
2585 struct ipoib_dev_priv *priv;
ea1075ed 2586 unsigned int p;
8e37ab68 2587 int count = 0;
07ebafba 2588
b1b63970 2589 dev_list = kmalloc(sizeof(*dev_list), GFP_KERNEL);
1da177e4 2590 if (!dev_list)
11a0ae4c 2591 return -ENOMEM;
1da177e4
LT		 2592
2593 INIT_LIST_HEAD(dev_list);
2594
ea1075ed 2595 rdma_for_each_port (device, p) {
8e37ab68 2596 if (!rdma_protocol_ib(device, p))
7b4c8769 2597 continue;
1da177e4
LT		 2598 dev = ipoib_add_port("ib%d", device, p);
2599 if (!IS_ERR(dev)) {
c1048aff 2600 priv = ipoib_priv(dev);
1da177e4 2601 list_add_tail(&priv->list, dev_list);
8e37ab68 2602 count++;
1da177e4
LT		 2603 }
2604 }
2605
8e37ab68 2606 if (!count) {
ac6dbf7f 2607 kfree(dev_list);
11a0ae4c 2608 return -EOPNOTSUPP;
8e37ab68
MW		 2609 }
2610
1da177e4 2611 ib_set_client_data(device, &ipoib_client, dev_list);
11a0ae4c 2612 return 0;
1da177e4
LT		 2613}
2614
7c1eb45a 2615static void ipoib_remove_one(struct ib_device *device, void *client_data)
1da177e4 2616{
25405d98 2617 struct ipoib_dev_priv *priv, *tmp, *cpriv, *tcpriv;
7c1eb45a 2618 struct list_head *dev_list = client_data;
1da177e4 2619
1da177e4 2620 list_for_each_entry_safe(priv, tmp, dev_list, list) {
25405d98 2621 LIST_HEAD(head);
7cbee87c 2622 ipoib_parent_unregister_pre(priv->dev);
1da177e4 2623
25405d98
JG		 2624 rtnl_lock();
2625
2626 list_for_each_entry_safe(cpriv, tcpriv, &priv->child_intfs,
2627 list)
2628 unregister_netdevice_queue(cpriv->dev, &head);
2629 unregister_netdevice_queue(priv->dev, &head);
2630 unregister_netdevice_many(&head);
2631
2632 rtnl_unlock();
1da177e4 2633 }
06c56e44
MT		 2634
2635 kfree(dev_list);
1da177e4
LT		 2636}
2637
771a5258
SR		 2638#ifdef CONFIG_INFINIBAND_IPOIB_DEBUG
2639static struct notifier_block ipoib_netdev_notifier = {
2640 .notifier_call = ipoib_netdev_event,
2641};
2642#endif
2643
1da177e4
LT		 2644static int __init ipoib_init_module(void)
2645{
2646 int ret;
2647
0f485251
SM		 2648 ipoib_recvq_size = roundup_pow_of_two(ipoib_recvq_size);
2649 ipoib_recvq_size = min(ipoib_recvq_size, IPOIB_MAX_QUEUE_SIZE);
2650 ipoib_recvq_size = max(ipoib_recvq_size, IPOIB_MIN_QUEUE_SIZE);
2651
2652 ipoib_sendq_size = roundup_pow_of_two(ipoib_sendq_size);
2653 ipoib_sendq_size = min(ipoib_sendq_size, IPOIB_MAX_QUEUE_SIZE);
732eacc0 2654 ipoib_sendq_size = max3(ipoib_sendq_size, 2 * MAX_SEND_CQE, IPOIB_MIN_QUEUE_SIZE);
68e995a2
PS		 2655#ifdef CONFIG_INFINIBAND_IPOIB_CM
2656 ipoib_max_conn_qp = min(ipoib_max_conn_qp, IPOIB_CM_MAX_CONN_QP);
11f74b40 2657 ipoib_max_conn_qp = max(ipoib_max_conn_qp, 0);
68e995a2 2658#endif
0f485251 2659
f89271da
EC		 2660 /*
2661 * When copying small received packets, we only copy from the
2662 * linear data part of the SKB, so we rely on this condition.
2663 */
2664 BUILD_BUG_ON(IPOIB_CM_COPYBREAK > IPOIB_CM_HEAD_SIZE);
2665
2e061c69 2666 ipoib_register_debugfs();
1da177e4
LT		 2667
2668 /*
0b39578b
DL		 2669 * We create a global workqueue here that is used for all flush
2670 * operations. However, if you attempt to flush a workqueue
2671 * from a task on that same workqueue, it deadlocks the system.
2672 * We want to be able to flush the tasks associated with a
2673 * specific net device, so we also create a workqueue for each
2674 * netdevice. We queue up the tasks for that device only on
2675 * its private workqueue, and we only queue up flush events
2676 * on our global flush workqueue. This avoids the deadlocks.
1da177e4 2677 */
ee190ab7 2678 ipoib_workqueue = alloc_ordered_workqueue("ipoib_flush", 0);
1da177e4
LT		 2679 if (!ipoib_workqueue) {
2680 ret = -ENOMEM;
2681 goto err_fs;
2682 }
2683
c1a0b23b
MT		 2684 ib_sa_register_client(&ipoib_sa_client);
2685
1da177e4
LT		 2686 ret = ib_register_client(&ipoib_client);
2687 if (ret)
c1a0b23b 2688 goto err_sa;
1da177e4 2689
9baa0b03
OG		 2690 ret = ipoib_netlink_init();
2691 if (ret)
2692 goto err_client;
2693
771a5258
SR		 2694#ifdef CONFIG_INFINIBAND_IPOIB_DEBUG
2695 register_netdevice_notifier(&ipoib_netdev_notifier);
2696#endif
1da177e4
LT		 2697 return 0;
2698
9baa0b03
OG		 2699err_client:
2700 ib_unregister_client(&ipoib_client);
2701
c1a0b23b
MT		 2702err_sa:
2703 ib_sa_unregister_client(&ipoib_sa_client);
1da177e4
LT		 2704 destroy_workqueue(ipoib_workqueue);
2705
9adec1a8
RD		 2706err_fs:
2707 ipoib_unregister_debugfs();
2708
1da177e4
LT		 2709 return ret;
2710}
2711
2712static void __exit ipoib_cleanup_module(void)
2713{
771a5258
SR		 2714#ifdef CONFIG_INFINIBAND_IPOIB_DEBUG
2715 unregister_netdevice_notifier(&ipoib_netdev_notifier);
2716#endif
9baa0b03 2717 ipoib_netlink_fini();
1da177e4 2718 ib_unregister_client(&ipoib_client);
c1a0b23b 2719 ib_sa_unregister_client(&ipoib_sa_client);
9adec1a8 2720 ipoib_unregister_debugfs();
1da177e4
LT		 2721 destroy_workqueue(ipoib_workqueue);
2722}
2723
2724module_init(ipoib_init_module);
2725module_exit(ipoib_cleanup_module);
Linux 6.x block layer and io_uring tree(s)