projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
net/smc: Clear memory when release and reuse buffer
[linux-block.git] / net / smc / smc_core.c
CommitLineData
b2441318 1// SPDX-License-Identifier: GPL-2.0
0cfdd8f9
UB		 2/*
3 * Shared Memory Communications over RDMA (SMC-R) and RoCE
4 *
5 * Basic Transport Functions exploiting Infiniband API
6 *
7 * Copyright IBM Corp. 2016
8 *
9 * Author(s): Ursula Braun <ubraun@linux.vnet.ibm.com>
10 */
11
12#include <linux/socket.h>
13#include <linux/if_vlan.h>
14#include <linux/random.h>
15#include <linux/workqueue.h>
6dabd405 16#include <linux/wait.h>
a33a803c 17#include <linux/reboot.h>
92f3cb0e 18#include <linux/mutex.h>
099b990b
GG		 19#include <linux/list.h>
20#include <linux/smc.h>
0cfdd8f9
UB		 21#include <net/tcp.h>
22#include <net/sock.h>
23#include <rdma/ib_verbs.h>
ddb457c6 24#include <rdma/ib_cache.h>
0cfdd8f9
UB		 25
26#include "smc.h"
27#include "smc_clc.h"
28#include "smc_core.h"
29#include "smc_ib.h"
f38ba179 30#include "smc_wr.h"
9bf9abea 31#include "smc_llc.h"
5f08318f 32#include "smc_cdc.h"
b38d7324 33#include "smc_close.h"
c6ba7c9b 34#include "smc_ism.h"
099b990b 35#include "smc_netlink.h"
e0e4b8fa 36#include "smc_stats.h"
a3a0e81b 37#include "smc_tracepoint.h"
0cfdd8f9 38
5bc11ddb
UB		 39#define SMC_LGR_NUM_INCR 256
40#define SMC_LGR_FREE_DELAY_SERV (600 * HZ)
7f58a1ad 41#define SMC_LGR_FREE_DELAY_CLNT (SMC_LGR_FREE_DELAY_SERV + 10 * HZ)
0cfdd8f9 42
a3db10ef 43struct smc_lgr_list smc_lgr_list = { /* established link groups */
9fda3510
HW		 44 .lock = __SPIN_LOCK_UNLOCKED(smc_lgr_list.lock),
45 .list = LIST_HEAD_INIT(smc_lgr_list.list),
46 .num = 0,
47};
9bf9abea 48
29115cef 49static atomic_t lgr_cnt = ATOMIC_INIT(0); /* number of existing link groups */
6dabd405
UB		 50static DECLARE_WAIT_QUEUE_HEAD(lgrs_deleted);
51
6511aad3
HW		 52static void smc_buf_free(struct smc_link_group *lgr, bool is_rmb,
53 struct smc_buf_desc *buf_desc);
5f78fe96 54static void __smc_lgr_terminate(struct smc_link_group *lgr, bool soft);
a6920d1d 55
541afa10 56static void smc_link_down_work(struct work_struct *work);
1f90a05d 57
a0a62ee1
UB		 58/* return head of link group list and its lock for a given link group */
59static inline struct list_head *smc_lgr_list_head(struct smc_link_group *lgr,
60 spinlock_t **lgr_lock)
61{
62 if (lgr->is_smcd) {
63 *lgr_lock = &lgr->smcd->lgr_lock;
64 return &lgr->smcd->lgr_list;
65 }
66
67 *lgr_lock = &smc_lgr_list.lock;
68 return &smc_lgr_list.list;
69}
70
ddc99286
GG		 71static void smc_ibdev_cnt_inc(struct smc_link *lnk)
72{
73 atomic_inc(&lnk->smcibdev->lnk_cnt_by_port[lnk->ibport - 1]);
74}
75
76static void smc_ibdev_cnt_dec(struct smc_link *lnk)
77{
78 atomic_dec(&lnk->smcibdev->lnk_cnt_by_port[lnk->ibport - 1]);
79}
80
97cdbc42
KG		 81static void smc_lgr_schedule_free_work(struct smc_link_group *lgr)
82{
83 /* client link group creation always follows the server link group
84 * creation. For client use a somewhat higher removal delay time,
85 * otherwise there is a risk of out-of-sync link groups.
86 */
f9aab6f2 87 if (!lgr->freeing) {
8e316b9e
UB		 88 mod_delayed_work(system_wq, &lgr->free_work,
89 (!lgr->is_smcd && lgr->role == SMC_CLNT) ?
90 SMC_LGR_FREE_DELAY_CLNT :
91 SMC_LGR_FREE_DELAY_SERV);
92 }
97cdbc42
KG		 93}
94
0cfdd8f9
UB		 95/* Register connection's alert token in our lookup structure.
96 * To use rbtrees we have to implement our own insert core.
97 * Requires @conns_lock
98 * @smc connection to register
99 * Returns 0 on success, != otherwise.
100 */
101static void smc_lgr_add_alert_token(struct smc_connection *conn)
102{
103 struct rb_node **link, *parent = NULL;
104 u32 token = conn->alert_token_local;
105
106 link = &conn->lgr->conns_all.rb_node;
107 while (*link) {
108 struct smc_connection *cur = rb_entry(*link,
109 struct smc_connection, alert_node);
110
111 parent = *link;
112 if (cur->alert_token_local > token)
113 link = &parent->rb_left;
114 else
115 link = &parent->rb_right;
116 }
117 /* Put the new node there */
118 rb_link_node(&conn->alert_node, parent, link);
119 rb_insert_color(&conn->alert_node, &conn->lgr->conns_all);
120}
121
56bc3b20
KG		 122/* assign an SMC-R link to the connection */
123static int smcr_lgr_conn_assign_link(struct smc_connection *conn, bool first)
124{
125 enum smc_link_state expected = first ? SMC_LNK_ACTIVATING :
126 SMC_LNK_ACTIVE;
127 int i, j;
128
129 /* do link balancing */
130 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
131 struct smc_link *lnk = &conn->lgr->lnk[i];
132
ad6c111b 133 if (lnk->state != expected || lnk->link_is_asym)
56bc3b20
KG		 134 continue;
135 if (conn->lgr->role == SMC_CLNT) {
136 conn->lnk = lnk; /* temporary, SMC server assigns link*/
137 break;
138 }
139 if (conn->lgr->conns_num % 2) {
140 for (j = i + 1; j < SMC_LINKS_PER_LGR_MAX; j++) {
141 struct smc_link *lnk2;
142
143 lnk2 = &conn->lgr->lnk[j];
ad6c111b
KG		 144 if (lnk2->state == expected &&
145 !lnk2->link_is_asym) {
56bc3b20
KG		 146 conn->lnk = lnk2;
147 break;
148 }
149 }
150 }
151 if (!conn->lnk)
152 conn->lnk = lnk;
153 break;
154 }
155 if (!conn->lnk)
156 return SMC_CLC_DECL_NOACTLINK;
07d51580 157 atomic_inc(&conn->lnk->conn_cnt);
56bc3b20
KG		 158 return 0;
159}
160
0cfdd8f9
UB		 161/* Register connection in link group by assigning an alert token
162 * registered in a search tree.
163 * Requires @conns_lock
164 * Note that '0' is a reserved value and not assigned.
165 */
56bc3b20 166static int smc_lgr_register_conn(struct smc_connection *conn, bool first)
0cfdd8f9
UB		 167{
168 struct smc_sock *smc = container_of(conn, struct smc_sock, conn);
169 static atomic_t nexttoken = ATOMIC_INIT(0);
56bc3b20 170 int rc;
0cfdd8f9 171
56bc3b20
KG		 172 if (!conn->lgr->is_smcd) {
173 rc = smcr_lgr_conn_assign_link(conn, first);
174 if (rc)
175 return rc;
176 }
0cfdd8f9
UB		 177 /* find a new alert_token_local value not yet used by some connection
178 * in this link group
179 */
180 sock_hold(&smc->sk); /* sock_put in smc_lgr_unregister_conn() */
181 while (!conn->alert_token_local) {
182 conn->alert_token_local = atomic_inc_return(&nexttoken);
183 if (smc_lgr_find_conn(conn->alert_token_local, conn->lgr))
184 conn->alert_token_local = 0;
185 }
186 smc_lgr_add_alert_token(conn);
187 conn->lgr->conns_num++;
b9247544 188 return 0;
0cfdd8f9
UB		 189}
190
191/* Unregister connection and reset the alert token of the given connection<
192 */
193static void __smc_lgr_unregister_conn(struct smc_connection *conn)
194{
195 struct smc_sock *smc = container_of(conn, struct smc_sock, conn);
196 struct smc_link_group *lgr = conn->lgr;
197
198 rb_erase(&conn->alert_node, &lgr->conns_all);
07d51580
GG		 199 if (conn->lnk)
200 atomic_dec(&conn->lnk->conn_cnt);
0cfdd8f9
UB		 201 lgr->conns_num--;
202 conn->alert_token_local = 0;
0cfdd8f9
UB		 203 sock_put(&smc->sk); /* sock_hold in smc_lgr_register_conn() */
204}
205
fb692ec4 206/* Unregister connection from lgr
0cfdd8f9
UB		 207 */
208static void smc_lgr_unregister_conn(struct smc_connection *conn)
209{
210 struct smc_link_group *lgr = conn->lgr;
0cfdd8f9 211
77f838ac
KG		 212 if (!lgr)
213 return;
0cfdd8f9
UB		 214 write_lock_bh(&lgr->conns_lock);
215 if (conn->alert_token_local) {
0cfdd8f9
UB		 216 __smc_lgr_unregister_conn(conn);
217 }
218 write_unlock_bh(&lgr->conns_lock);
2a0674ff 219 conn->lgr = NULL;
0cfdd8f9
UB		 220}
221
099b990b
GG		 222int smc_nl_get_sys_info(struct sk_buff *skb, struct netlink_callback *cb)
223{
224 struct smc_nl_dmp_ctx *cb_ctx = smc_nl_dmp_ctx(cb);
225 char hostname[SMC_MAX_HOSTNAME_LEN + 1];
226 char smc_seid[SMC_MAX_EID_LEN + 1];
099b990b
GG		 227 struct nlattr *attrs;
228 u8 *seid = NULL;
229 u8 *host = NULL;
230 void *nlh;
231
232 nlh = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq,
233 &smc_gen_nl_family, NLM_F_MULTI,
234 SMC_NETLINK_GET_SYS_INFO);
235 if (!nlh)
236 goto errmsg;
237 if (cb_ctx->pos[0])
238 goto errout;
239 attrs = nla_nest_start(skb, SMC_GEN_SYS_INFO);
240 if (!attrs)
241 goto errout;
242 if (nla_put_u8(skb, SMC_NLA_SYS_VER, SMC_V2))
243 goto errattr;
244 if (nla_put_u8(skb, SMC_NLA_SYS_REL, SMC_RELEASE))
245 goto errattr;
246 if (nla_put_u8(skb, SMC_NLA_SYS_IS_ISM_V2, smc_ism_is_v2_capable()))
247 goto errattr;
b0539f5e
KG		 248 if (nla_put_u8(skb, SMC_NLA_SYS_IS_SMCR_V2, true))
249 goto errattr;
099b990b
GG		 250 smc_clc_get_hostname(&host);
251 if (host) {
25fe2c9c
JK		 252 memcpy(hostname, host, SMC_MAX_HOSTNAME_LEN);
253 hostname[SMC_MAX_HOSTNAME_LEN] = 0;
099b990b
GG		 254 if (nla_put_string(skb, SMC_NLA_SYS_LOCAL_HOST, hostname))
255 goto errattr;
256 }
11a26c59
KG		 257 if (smc_ism_is_v2_capable()) {
258 smc_ism_get_system_eid(&seid);
8a446536
GG		 259 memcpy(smc_seid, seid, SMC_MAX_EID_LEN);
260 smc_seid[SMC_MAX_EID_LEN] = 0;
099b990b
GG		 261 if (nla_put_string(skb, SMC_NLA_SYS_SEID, smc_seid))
262 goto errattr;
263 }
264 nla_nest_end(skb, attrs);
265 genlmsg_end(skb, nlh);
266 cb_ctx->pos[0] = 1;
267 return skb->len;
268
269errattr:
270 nla_nest_cancel(skb, attrs);
271errout:
272 genlmsg_cancel(skb, nlh);
273errmsg:
274 return skb->len;
275}
276
b0539f5e
KG		 277/* Fill SMC_NLA_LGR_D_V2_COMMON/SMC_NLA_LGR_R_V2_COMMON nested attributes */
278static int smc_nl_fill_lgr_v2_common(struct smc_link_group *lgr,
279 struct sk_buff *skb,
280 struct netlink_callback *cb,
281 struct nlattr *v2_attrs)
282{
283 char smc_host[SMC_MAX_HOSTNAME_LEN + 1];
284 char smc_eid[SMC_MAX_EID_LEN + 1];
285
286 if (nla_put_u8(skb, SMC_NLA_LGR_V2_VER, lgr->smc_version))
287 goto errv2attr;
288 if (nla_put_u8(skb, SMC_NLA_LGR_V2_REL, lgr->peer_smc_release))
289 goto errv2attr;
290 if (nla_put_u8(skb, SMC_NLA_LGR_V2_OS, lgr->peer_os))
291 goto errv2attr;
292 memcpy(smc_host, lgr->peer_hostname, SMC_MAX_HOSTNAME_LEN);
293 smc_host[SMC_MAX_HOSTNAME_LEN] = 0;
294 if (nla_put_string(skb, SMC_NLA_LGR_V2_PEER_HOST, smc_host))
295 goto errv2attr;
296 memcpy(smc_eid, lgr->negotiated_eid, SMC_MAX_EID_LEN);
297 smc_eid[SMC_MAX_EID_LEN] = 0;
298 if (nla_put_string(skb, SMC_NLA_LGR_V2_NEG_EID, smc_eid))
299 goto errv2attr;
300
301 nla_nest_end(skb, v2_attrs);
302 return 0;
303
304errv2attr:
305 nla_nest_cancel(skb, v2_attrs);
306 return -EMSGSIZE;
307}
308
309static int smc_nl_fill_smcr_lgr_v2(struct smc_link_group *lgr,
310 struct sk_buff *skb,
311 struct netlink_callback *cb)
312{
313 struct nlattr *v2_attrs;
314
315 v2_attrs = nla_nest_start(skb, SMC_NLA_LGR_R_V2);
316 if (!v2_attrs)
317 goto errattr;
318 if (nla_put_u8(skb, SMC_NLA_LGR_R_V2_DIRECT, !lgr->uses_gateway))
319 goto errv2attr;
320
321 nla_nest_end(skb, v2_attrs);
322 return 0;
323
324errv2attr:
325 nla_nest_cancel(skb, v2_attrs);
326errattr:
327 return -EMSGSIZE;
328}
329
e9b8c845
GG		 330static int smc_nl_fill_lgr(struct smc_link_group *lgr,
331 struct sk_buff *skb,
332 struct netlink_callback *cb)
333{
334 char smc_target[SMC_MAX_PNETID_LEN + 1];
b0539f5e 335 struct nlattr *attrs, *v2_attrs;
e9b8c845
GG		 336
337 attrs = nla_nest_start(skb, SMC_GEN_LGR_SMCR);
338 if (!attrs)
339 goto errout;
340
341 if (nla_put_u32(skb, SMC_NLA_LGR_R_ID, *((u32 *)&lgr->id)))
342 goto errattr;
343 if (nla_put_u32(skb, SMC_NLA_LGR_R_CONNS_NUM, lgr->conns_num))
344 goto errattr;
345 if (nla_put_u8(skb, SMC_NLA_LGR_R_ROLE, lgr->role))
346 goto errattr;
347 if (nla_put_u8(skb, SMC_NLA_LGR_R_TYPE, lgr->type))
348 goto errattr;
349 if (nla_put_u8(skb, SMC_NLA_LGR_R_VLAN_ID, lgr->vlan_id))
350 goto errattr;
8a446536
GG		 351 memcpy(smc_target, lgr->pnet_id, SMC_MAX_PNETID_LEN);
352 smc_target[SMC_MAX_PNETID_LEN] = 0;
e9b8c845
GG		 353 if (nla_put_string(skb, SMC_NLA_LGR_R_PNETID, smc_target))
354 goto errattr;
b0539f5e
KG		 355 if (lgr->smc_version > SMC_V1) {
356 v2_attrs = nla_nest_start(skb, SMC_NLA_LGR_R_V2_COMMON);
357 if (!v2_attrs)
358 goto errattr;
359 if (smc_nl_fill_lgr_v2_common(lgr, skb, cb, v2_attrs))
360 goto errattr;
361 if (smc_nl_fill_smcr_lgr_v2(lgr, skb, cb))
362 goto errattr;
363 }
e9b8c845
GG		 364
365 nla_nest_end(skb, attrs);
366 return 0;
367errattr:
368 nla_nest_cancel(skb, attrs);
369errout:
370 return -EMSGSIZE;
371}
372
5a7e09d5
GG		 373static int smc_nl_fill_lgr_link(struct smc_link_group *lgr,
374 struct smc_link *link,
375 struct sk_buff *skb,
376 struct netlink_callback *cb)
377{
8a446536 378 char smc_ibname[IB_DEVICE_NAME_MAX];
5a7e09d5
GG		 379 u8 smc_gid_target[41];
380 struct nlattr *attrs;
381 u32 link_uid = 0;
382 void *nlh;
383
384 nlh = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq,
385 &smc_gen_nl_family, NLM_F_MULTI,
386 SMC_NETLINK_GET_LINK_SMCR);
387 if (!nlh)
388 goto errmsg;
389
390 attrs = nla_nest_start(skb, SMC_GEN_LINK_SMCR);
391 if (!attrs)
392 goto errout;
393
394 if (nla_put_u8(skb, SMC_NLA_LINK_ID, link->link_id))
395 goto errattr;
396 if (nla_put_u32(skb, SMC_NLA_LINK_STATE, link->state))
397 goto errattr;
398 if (nla_put_u32(skb, SMC_NLA_LINK_CONN_CNT,
399 atomic_read(&link->conn_cnt)))
400 goto errattr;
401 if (nla_put_u8(skb, SMC_NLA_LINK_IB_PORT, link->ibport))
402 goto errattr;
403 if (nla_put_u32(skb, SMC_NLA_LINK_NET_DEV, link->ndev_ifidx))
404 goto errattr;
405 snprintf(smc_ibname, sizeof(smc_ibname), "%s", link->ibname);
406 if (nla_put_string(skb, SMC_NLA_LINK_IB_DEV, smc_ibname))
407 goto errattr;
408 memcpy(&link_uid, link->link_uid, sizeof(link_uid));
409 if (nla_put_u32(skb, SMC_NLA_LINK_UID, link_uid))
410 goto errattr;
411 memcpy(&link_uid, link->peer_link_uid, sizeof(link_uid));
412 if (nla_put_u32(skb, SMC_NLA_LINK_PEER_UID, link_uid))
413 goto errattr;
414 memset(smc_gid_target, 0, sizeof(smc_gid_target));
415 smc_gid_be16_convert(smc_gid_target, link->gid);
416 if (nla_put_string(skb, SMC_NLA_LINK_GID, smc_gid_target))
417 goto errattr;
418 memset(smc_gid_target, 0, sizeof(smc_gid_target));
419 smc_gid_be16_convert(smc_gid_target, link->peer_gid);
420 if (nla_put_string(skb, SMC_NLA_LINK_PEER_GID, smc_gid_target))
421 goto errattr;
422
423 nla_nest_end(skb, attrs);
424 genlmsg_end(skb, nlh);
425 return 0;
426errattr:
427 nla_nest_cancel(skb, attrs);
428errout:
429 genlmsg_cancel(skb, nlh);
430errmsg:
431 return -EMSGSIZE;
432}
433
e9b8c845
GG		 434static int smc_nl_handle_lgr(struct smc_link_group *lgr,
435 struct sk_buff *skb,
5a7e09d5
GG		 436 struct netlink_callback *cb,
437 bool list_links)
e9b8c845
GG		 438{
439 void *nlh;
5a7e09d5 440 int i;
e9b8c845
GG		 441
442 nlh = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq,
443 &smc_gen_nl_family, NLM_F_MULTI,
444 SMC_NETLINK_GET_LGR_SMCR);
445 if (!nlh)
446 goto errmsg;
447 if (smc_nl_fill_lgr(lgr, skb, cb))
448 goto errout;
449
450 genlmsg_end(skb, nlh);
5a7e09d5
GG		 451 if (!list_links)
452 goto out;
453 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
454 if (!smc_link_usable(&lgr->lnk[i]))
455 continue;
456 if (smc_nl_fill_lgr_link(lgr, &lgr->lnk[i], skb, cb))
457 goto errout;
458 }
459out:
e9b8c845
GG		 460 return 0;
461
462errout:
463 genlmsg_cancel(skb, nlh);
464errmsg:
465 return -EMSGSIZE;
466}
467
468static void smc_nl_fill_lgr_list(struct smc_lgr_list *smc_lgr,
469 struct sk_buff *skb,
5a7e09d5
GG		 470 struct netlink_callback *cb,
471 bool list_links)
e9b8c845
GG		 472{
473 struct smc_nl_dmp_ctx *cb_ctx = smc_nl_dmp_ctx(cb);
474 struct smc_link_group *lgr;
475 int snum = cb_ctx->pos[0];
476 int num = 0;
477
478 spin_lock_bh(&smc_lgr->lock);
479 list_for_each_entry(lgr, &smc_lgr->list, list) {
480 if (num < snum)
481 goto next;
5a7e09d5 482 if (smc_nl_handle_lgr(lgr, skb, cb, list_links))
e9b8c845
GG		 483 goto errout;
484next:
485 num++;
486 }
487errout:
488 spin_unlock_bh(&smc_lgr->lock);
489 cb_ctx->pos[0] = num;
490}
491
8f9dde4b
GG		 492static int smc_nl_fill_smcd_lgr(struct smc_link_group *lgr,
493 struct sk_buff *skb,
494 struct netlink_callback *cb)
495{
8f9dde4b 496 char smc_pnet[SMC_MAX_PNETID_LEN + 1];
8f9dde4b
GG		 497 struct nlattr *attrs;
498 void *nlh;
499
500 nlh = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq,
501 &smc_gen_nl_family, NLM_F_MULTI,
502 SMC_NETLINK_GET_LGR_SMCD);
503 if (!nlh)
504 goto errmsg;
505
506 attrs = nla_nest_start(skb, SMC_GEN_LGR_SMCD);
507 if (!attrs)
508 goto errout;
509
510 if (nla_put_u32(skb, SMC_NLA_LGR_D_ID, *((u32 *)&lgr->id)))
511 goto errattr;
512 if (nla_put_u64_64bit(skb, SMC_NLA_LGR_D_GID, lgr->smcd->local_gid,
513 SMC_NLA_LGR_D_PAD))
514 goto errattr;
515 if (nla_put_u64_64bit(skb, SMC_NLA_LGR_D_PEER_GID, lgr->peer_gid,
516 SMC_NLA_LGR_D_PAD))
517 goto errattr;
518 if (nla_put_u8(skb, SMC_NLA_LGR_D_VLAN_ID, lgr->vlan_id))
519 goto errattr;
520 if (nla_put_u32(skb, SMC_NLA_LGR_D_CONNS_NUM, lgr->conns_num))
521 goto errattr;
522 if (nla_put_u32(skb, SMC_NLA_LGR_D_CHID, smc_ism_get_chid(lgr->smcd)))
523 goto errattr;
8a446536
GG		 524 memcpy(smc_pnet, lgr->smcd->pnetid, SMC_MAX_PNETID_LEN);
525 smc_pnet[SMC_MAX_PNETID_LEN] = 0;
8f9dde4b
GG		 526 if (nla_put_string(skb, SMC_NLA_LGR_D_PNETID, smc_pnet))
527 goto errattr;
b0539f5e
KG		 528 if (lgr->smc_version > SMC_V1) {
529 struct nlattr *v2_attrs;
8f9dde4b 530
b0539f5e
KG		 531 v2_attrs = nla_nest_start(skb, SMC_NLA_LGR_D_V2_COMMON);
532 if (!v2_attrs)
533 goto errattr;
534 if (smc_nl_fill_lgr_v2_common(lgr, skb, cb, v2_attrs))
535 goto errattr;
536 }
8f9dde4b
GG		 537 nla_nest_end(skb, attrs);
538 genlmsg_end(skb, nlh);
539 return 0;
540
8f9dde4b
GG		 541errattr:
542 nla_nest_cancel(skb, attrs);
543errout:
544 genlmsg_cancel(skb, nlh);
545errmsg:
546 return -EMSGSIZE;
547}
548
549static int smc_nl_handle_smcd_lgr(struct smcd_dev *dev,
550 struct sk_buff *skb,
551 struct netlink_callback *cb)
552{
553 struct smc_nl_dmp_ctx *cb_ctx = smc_nl_dmp_ctx(cb);
554 struct smc_link_group *lgr;
555 int snum = cb_ctx->pos[1];
556 int rc = 0, num = 0;
557
558 spin_lock_bh(&dev->lgr_lock);
559 list_for_each_entry(lgr, &dev->lgr_list, list) {
560 if (!lgr->is_smcd)
561 continue;
562 if (num < snum)
563 goto next;
564 rc = smc_nl_fill_smcd_lgr(lgr, skb, cb);
565 if (rc)
566 goto errout;
567next:
568 num++;
569 }
570errout:
571 spin_unlock_bh(&dev->lgr_lock);
572 cb_ctx->pos[1] = num;
573 return rc;
574}
575
576static int smc_nl_fill_smcd_dev(struct smcd_dev_list *dev_list,
577 struct sk_buff *skb,
578 struct netlink_callback *cb)
579{
580 struct smc_nl_dmp_ctx *cb_ctx = smc_nl_dmp_ctx(cb);
581 struct smcd_dev *smcd_dev;
582 int snum = cb_ctx->pos[0];
583 int rc = 0, num = 0;
584
585 mutex_lock(&dev_list->mutex);
586 list_for_each_entry(smcd_dev, &dev_list->list, list) {
587 if (list_empty(&smcd_dev->lgr_list))
588 continue;
589 if (num < snum)
590 goto next;
591 rc = smc_nl_handle_smcd_lgr(smcd_dev, skb, cb);
592 if (rc)
593 goto errout;
594next:
595 num++;
596 }
597errout:
598 mutex_unlock(&dev_list->mutex);
599 cb_ctx->pos[0] = num;
600 return rc;
601}
602
e9b8c845
GG		 603int smcr_nl_get_lgr(struct sk_buff *skb, struct netlink_callback *cb)
604{
5a7e09d5
GG		 605 bool list_links = false;
606
607 smc_nl_fill_lgr_list(&smc_lgr_list, skb, cb, list_links);
608 return skb->len;
609}
610
611int smcr_nl_get_link(struct sk_buff *skb, struct netlink_callback *cb)
612{
613 bool list_links = true;
614
615 smc_nl_fill_lgr_list(&smc_lgr_list, skb, cb, list_links);
e9b8c845
GG		 616 return skb->len;
617}
618
8f9dde4b
GG		 619int smcd_nl_get_lgr(struct sk_buff *skb, struct netlink_callback *cb)
620{
621 smc_nl_fill_smcd_dev(&smcd_dev_list, skb, cb);
622 return skb->len;
623}
624
51e3dfa8
UB		 625void smc_lgr_cleanup_early(struct smc_connection *conn)
626{
627 struct smc_link_group *lgr = conn->lgr;
9ec6bf19 628 spinlock_t *lgr_lock;
51e3dfa8
UB		 629
630 if (!lgr)
631 return;
632
633 smc_conn_free(conn);
789b6cc2 634 smc_lgr_list_head(lgr, &lgr_lock);
9ec6bf19
KG		 635 spin_lock_bh(lgr_lock);
636 /* do not use this link group for new connections */
789b6cc2
DL		 637 if (!list_empty(&lgr->list))
638 list_del_init(&lgr->list);
9ec6bf19 639 spin_unlock_bh(lgr_lock);
f9aab6f2 640 __smc_lgr_terminate(lgr, true);
51e3dfa8
UB		 641}
642
a52bcc91
KG		 643static void smcr_lgr_link_deactivate_all(struct smc_link_group *lgr)
644{
645 int i;
646
647 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
648 struct smc_link *lnk = &lgr->lnk[i];
649
650 if (smc_link_usable(lnk))
651 lnk->state = SMC_LNK_INACTIVE;
652 }
6778a6be
KG		 653 wake_up_all(&lgr->llc_msg_waiter);
654 wake_up_all(&lgr->llc_flow_waiter);
a52bcc91
KG		 655}
656
3f3f0e36
UB		 657static void smc_lgr_free(struct smc_link_group *lgr);
658
0cfdd8f9
UB		 659static void smc_lgr_free_work(struct work_struct *work)
660{
661 struct smc_link_group *lgr = container_of(to_delayed_work(work),
662 struct smc_link_group,
663 free_work);
a0a62ee1 664 spinlock_t *lgr_lock;
0cfdd8f9
UB		 665 bool conns;
666
a0a62ee1
UB		 667 smc_lgr_list_head(lgr, &lgr_lock);
668 spin_lock_bh(lgr_lock);
8e316b9e
UB		 669 if (lgr->freeing) {
670 spin_unlock_bh(lgr_lock);
671 return;
672 }
0cfdd8f9
UB		 673 read_lock_bh(&lgr->conns_lock);
674 conns = RB_EMPTY_ROOT(&lgr->conns_all);
675 read_unlock_bh(&lgr->conns_lock);
676 if (!conns) { /* number of lgr connections is no longer zero */
a0a62ee1 677 spin_unlock_bh(lgr_lock);
0cfdd8f9
UB		 678 return;
679 }
8caa6544 680 list_del_init(&lgr->list); /* remove from smc_lgr_list */
8e316b9e
UB		 681 lgr->freeing = 1; /* this instance does the freeing, no new schedule */
682 spin_unlock_bh(lgr_lock);
683 cancel_delayed_work(&lgr->free_work);
0d18a0cb 684
f3811fd7
KG		 685 if (!lgr->is_smcd && !lgr->terminating)
686 smc_llc_send_link_delete_all(lgr, true,
687 SMC_LLC_DEL_PROG_INIT_TERM);
42bfba9e 688 if (lgr->is_smcd && !lgr->terminating)
8e316b9e 689 smc_ism_signal_shutdown(lgr);
a52bcc91
KG		 690 if (!lgr->is_smcd)
691 smcr_lgr_link_deactivate_all(lgr);
8e316b9e 692 smc_lgr_free(lgr);
0cfdd8f9
UB		 693}
694
f528ba24
UB		 695static void smc_lgr_terminate_work(struct work_struct *work)
696{
697 struct smc_link_group *lgr = container_of(work, struct smc_link_group,
698 terminate_work);
699
5f78fe96 700 __smc_lgr_terminate(lgr, true);
f528ba24
UB		 701}
702
026c381f
KG		 703/* return next unique link id for the lgr */
704static u8 smcr_next_link_id(struct smc_link_group *lgr)
705{
706 u8 link_id;
707 int i;
708
709 while (1) {
cf4f5530 710again:
026c381f
KG		 711 link_id = ++lgr->next_link_id;
712 if (!link_id) /* skip zero as link_id */
713 link_id = ++lgr->next_link_id;
714 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
d854fcbf 715 if (smc_link_usable(&lgr->lnk[i]) &&
026c381f 716 lgr->lnk[i].link_id == link_id)
cf4f5530 717 goto again;
026c381f
KG		 718 }
719 break;
720 }
721 return link_id;
722}
723
6443b2f6
GG		 724static void smcr_copy_dev_info_to_link(struct smc_link *link)
725{
726 struct smc_ib_device *smcibdev = link->smcibdev;
727
728 snprintf(link->ibname, sizeof(link->ibname), "%s",
729 smcibdev->ibdev->name);
730 link->ndev_ifidx = smcibdev->ndev_ifidx[link->ibport - 1];
731}
732
336ba09f
KG		 733int smcr_link_init(struct smc_link_group *lgr, struct smc_link *lnk,
734 u8 link_idx, struct smc_init_info *ini)
f3c1dedd 735{
e49300a6 736 struct smc_ib_device *smcibdev;
f3c1dedd
KG		 737 u8 rndvec[3];
738 int rc;
739
e49300a6
KG		 740 if (lgr->smc_version == SMC_V2) {
741 lnk->smcibdev = ini->smcrv2.ib_dev_v2;
742 lnk->ibport = ini->smcrv2.ib_port_v2;
743 } else {
744 lnk->smcibdev = ini->ib_dev;
745 lnk->ibport = ini->ib_port;
746 }
747 get_device(&lnk->smcibdev->ibdev->dev);
748 atomic_inc(&lnk->smcibdev->lnk_cnt);
749 lnk->path_mtu = lnk->smcibdev->pattr[lnk->ibport - 1].active_mtu;
026c381f 750 lnk->link_id = smcr_next_link_id(lgr);
387707fd 751 lnk->lgr = lgr;
026c381f 752 lnk->link_idx = link_idx;
ddc99286 753 smc_ibdev_cnt_inc(lnk);
6443b2f6 754 smcr_copy_dev_info_to_link(lnk);
07d51580 755 atomic_set(&lnk->conn_cnt, 0);
45fa8da0 756 smc_llc_link_set_uid(lnk);
541afa10 757 INIT_WORK(&lnk->link_down_wrk, smc_link_down_work);
e49300a6
KG		 758 if (!lnk->smcibdev->initialized) {
759 rc = (int)smc_ib_setup_per_ibdev(lnk->smcibdev);
f3c1dedd
KG		 760 if (rc)
761 goto out;
762 }
763 get_random_bytes(rndvec, sizeof(rndvec));
764 lnk->psn_initial = rndvec[0] + (rndvec[1] << 8) +
765 (rndvec[2] << 16);
766 rc = smc_ib_determine_gid(lnk->smcibdev, lnk->ibport,
24fb6811
KG		 767 ini->vlan_id, lnk->gid, &lnk->sgid_index,
768 lgr->smc_version == SMC_V2 ?
769 &ini->smcrv2 : NULL);
f3c1dedd
KG		 770 if (rc)
771 goto out;
772 rc = smc_llc_link_init(lnk);
773 if (rc)
774 goto out;
775 rc = smc_wr_alloc_link_mem(lnk);
776 if (rc)
777 goto clear_llc_lnk;
778 rc = smc_ib_create_protection_domain(lnk);
779 if (rc)
780 goto free_link_mem;
781 rc = smc_ib_create_queue_pair(lnk);
782 if (rc)
783 goto dealloc_pd;
784 rc = smc_wr_create_link(lnk);
785 if (rc)
786 goto destroy_qp;
741a49a4 787 lnk->state = SMC_LNK_ACTIVATING;
f3c1dedd
KG		 788 return 0;
789
790destroy_qp:
791 smc_ib_destroy_queue_pair(lnk);
792dealloc_pd:
793 smc_ib_dealloc_protection_domain(lnk);
794free_link_mem:
795 smc_wr_free_link_mem(lnk);
796clear_llc_lnk:
0a99be43 797 smc_llc_link_clear(lnk, false);
f3c1dedd 798out:
ddc99286 799 smc_ibdev_cnt_dec(lnk);
e49300a6
KG		 800 put_device(&lnk->smcibdev->ibdev->dev);
801 smcibdev = lnk->smcibdev;
f3c1dedd 802 memset(lnk, 0, sizeof(struct smc_link));
d854fcbf 803 lnk->state = SMC_LNK_UNUSED;
e49300a6
KG		 804 if (!atomic_dec_return(&smcibdev->lnk_cnt))
805 wake_up(&smcibdev->lnks_deleted);
f3c1dedd
KG		 806 return rc;
807}
808
0cfdd8f9 809/* create a new SMC link group */
bc36d2fc 810static int smc_lgr_create(struct smc_sock *smc, struct smc_init_info *ini)
0cfdd8f9
UB		 811{
812 struct smc_link_group *lgr;
a2351c5d 813 struct list_head *lgr_list;
0cfdd8f9 814 struct smc_link *lnk;
a0a62ee1 815 spinlock_t *lgr_lock;
026c381f 816 u8 link_idx;
0cfdd8f9 817 int rc = 0;
cd6851f3 818 int i;
0cfdd8f9 819
bc36d2fc 820 if (ini->is_smcd && ini->vlan_id) {
5c21c4cc
UB		 821 if (smc_ism_get_vlan(ini->ism_dev[ini->ism_selected],
822 ini->vlan_id)) {
7a62725a 823 rc = SMC_CLC_DECL_ISMVLANERR;
c6ba7c9b 824 goto out;
7a62725a 825 }
c6ba7c9b
HW		 826 }
827
0cfdd8f9
UB		 828 lgr = kzalloc(sizeof(*lgr), GFP_KERNEL);
829 if (!lgr) {
7a62725a 830 rc = SMC_CLC_DECL_MEM;
29ee2701 831 goto ism_put_vlan;
0cfdd8f9 832 }
22ef473d
KG		 833 lgr->tx_wq = alloc_workqueue("smc_tx_wq-%*phN", 0, 0,
834 SMC_LGR_ID_SIZE, &lgr->id);
835 if (!lgr->tx_wq) {
836 rc = -ENOMEM;
837 goto free_lgr;
838 }
bc36d2fc 839 lgr->is_smcd = ini->is_smcd;
517c300e 840 lgr->sync_err = 0;
8e316b9e 841 lgr->terminating = 0;
8e316b9e 842 lgr->freeing = 0;
bc36d2fc 843 lgr->vlan_id = ini->vlan_id;
faca5360
KG		 844 mutex_init(&lgr->sndbufs_lock);
845 mutex_init(&lgr->rmbs_lock);
c6ba7c9b 846 rwlock_init(&lgr->conns_lock);
cd6851f3
UB		 847 for (i = 0; i < SMC_RMBE_SIZES; i++) {
848 INIT_LIST_HEAD(&lgr->sndbufs[i]);
849 INIT_LIST_HEAD(&lgr->rmbs[i]);
850 }
026c381f 851 lgr->next_link_id = 0;
9fda3510
HW		 852 smc_lgr_list.num += SMC_LGR_NUM_INCR;
853 memcpy(&lgr->id, (u8 *)&smc_lgr_list.num, SMC_LGR_ID_SIZE);
0cfdd8f9 854 INIT_DELAYED_WORK(&lgr->free_work, smc_lgr_free_work);
f528ba24 855 INIT_WORK(&lgr->terminate_work, smc_lgr_terminate_work);
0cfdd8f9 856 lgr->conns_all = RB_ROOT;
bc36d2fc 857 if (ini->is_smcd) {
c6ba7c9b 858 /* SMC-D specific settings */
5c21c4cc
UB		 859 get_device(&ini->ism_dev[ini->ism_selected]->dev);
860 lgr->peer_gid = ini->ism_peer_gid[ini->ism_selected];
861 lgr->smcd = ini->ism_dev[ini->ism_selected];
862 lgr_list = &ini->ism_dev[ini->ism_selected]->lgr_list;
a0a62ee1 863 lgr_lock = &lgr->smcd->lgr_lock;
b81a5eb7 864 lgr->smc_version = ini->smcd_version;
50c6b20e 865 lgr->peer_shutdown = 0;
5c21c4cc 866 atomic_inc(&ini->ism_dev[ini->ism_selected]->lgr_cnt);
c6ba7c9b
HW		 867 } else {
868 /* SMC-R specific settings */
e49300a6
KG		 869 struct smc_ib_device *ibdev;
870 int ibport;
871
c6ba7c9b 872 lgr->role = smc->listen_smc ? SMC_SERV : SMC_CLNT;
e49300a6
KG		 873 lgr->smc_version = ini->smcr_version;
874 memcpy(lgr->peer_systemid, ini->peer_systemid,
bc36d2fc 875 SMC_SYSTEMID_LEN);
e49300a6
KG		 876 if (lgr->smc_version == SMC_V2) {
877 ibdev = ini->smcrv2.ib_dev_v2;
878 ibport = ini->smcrv2.ib_port_v2;
879 lgr->saddr = ini->smcrv2.saddr;
880 lgr->uses_gateway = ini->smcrv2.uses_gateway;
881 memcpy(lgr->nexthop_mac, ini->smcrv2.nexthop_mac,
882 ETH_ALEN);
883 } else {
884 ibdev = ini->ib_dev;
885 ibport = ini->ib_port;
886 }
887 memcpy(lgr->pnet_id, ibdev->pnetid[ibport - 1],
35dcf7ec 888 SMC_MAX_PNETID_LEN);
8799e310
KG		 889 if (smc_wr_alloc_lgr_mem(lgr))
890 goto free_wq;
00a049cf
KG		 891 smc_llc_lgr_init(lgr, smc);
892
026c381f
KG		 893 link_idx = SMC_SINGLE_LINK;
894 lnk = &lgr->lnk[link_idx];
895 rc = smcr_link_init(lgr, lnk, link_idx, ini);
8799e310
KG		 896 if (rc) {
897 smc_wr_free_lgr_mem(lgr);
22ef473d 898 goto free_wq;
8799e310 899 }
f3c1dedd
KG		 900 lgr_list = &smc_lgr_list.list;
901 lgr_lock = &smc_lgr_list.lock;
6dabd405 902 atomic_inc(&lgr_cnt);
c6ba7c9b 903 }
0cfdd8f9 904 smc->conn.lgr = lgr;
a0a62ee1 905 spin_lock_bh(lgr_lock);
a9e44502 906 list_add_tail(&lgr->list, lgr_list);
a0a62ee1 907 spin_unlock_bh(lgr_lock);
f38ba179
UB		 908 return 0;
909
22ef473d
KG		 910free_wq:
911 destroy_workqueue(lgr->tx_wq);
f38ba179
UB		 912free_lgr:
913 kfree(lgr);
29ee2701
UB		 914ism_put_vlan:
915 if (ini->is_smcd && ini->vlan_id)
5c21c4cc 916 smc_ism_put_vlan(ini->ism_dev[ini->ism_selected], ini->vlan_id);
0cfdd8f9 917out:
7a62725a
KG		 918 if (rc < 0) {
919 if (rc == -ENOMEM)
920 rc = SMC_CLC_DECL_MEM;
921 else
922 rc = SMC_CLC_DECL_INTERR;
923 }
0cfdd8f9
UB		 924 return rc;
925}
926
c6f02ebe
KG		 927static int smc_write_space(struct smc_connection *conn)
928{
929 int buffer_len = conn->peer_rmbe_size;
930 union smc_host_cursor prod;
931 union smc_host_cursor cons;
932 int space;
933
934 smc_curs_copy(&prod, &conn->local_tx_ctrl.prod, conn);
935 smc_curs_copy(&cons, &conn->local_rx_ctrl.cons, conn);
936 /* determine rx_buf space */
937 space = buffer_len - smc_curs_diff(buffer_len, &cons, &prod);
938 return space;
939}
940
b8ded9de
KG		 941static int smc_switch_cursor(struct smc_sock *smc, struct smc_cdc_tx_pend *pend,
942 struct smc_wr_buf *wr_buf)
c6f02ebe
KG		 943{
944 struct smc_connection *conn = &smc->conn;
945 union smc_host_cursor cons, fin;
946 int rc = 0;
947 int diff;
948
949 smc_curs_copy(&conn->tx_curs_sent, &conn->tx_curs_fin, conn);
950 smc_curs_copy(&fin, &conn->local_tx_ctrl_fin, conn);
951 /* set prod cursor to old state, enforce tx_rdma_writes() */
952 smc_curs_copy(&conn->local_tx_ctrl.prod, &fin, conn);
953 smc_curs_copy(&cons, &conn->local_rx_ctrl.cons, conn);
954
955 if (smc_curs_comp(conn->peer_rmbe_size, &cons, &fin) < 0) {
956 /* cons cursor advanced more than fin, and prod was set
957 * fin above, so now prod is smaller than cons. Fix that.
958 */
959 diff = smc_curs_diff(conn->peer_rmbe_size, &fin, &cons);
960 smc_curs_add(conn->sndbuf_desc->len,
961 &conn->tx_curs_sent, diff);
962 smc_curs_add(conn->sndbuf_desc->len,
963 &conn->tx_curs_fin, diff);
964
965 smp_mb__before_atomic();
966 atomic_add(diff, &conn->sndbuf_space);
967 smp_mb__after_atomic();
968
969 smc_curs_add(conn->peer_rmbe_size,
970 &conn->local_tx_ctrl.prod, diff);
971 smc_curs_add(conn->peer_rmbe_size,
972 &conn->local_tx_ctrl_fin, diff);
973 }
974 /* recalculate, value is used by tx_rdma_writes() */
975 atomic_set(&smc->conn.peer_rmbe_space, smc_write_space(conn));
976
977 if (smc->sk.sk_state != SMC_INIT &&
978 smc->sk.sk_state != SMC_CLOSED) {
b8ded9de 979 rc = smcr_cdc_msg_send_validation(conn, pend, wr_buf);
c6f02ebe 980 if (!rc) {
22ef473d 981 queue_delayed_work(conn->lgr->tx_wq, &conn->tx_work, 0);
c6f02ebe
KG		 982 smc->sk.sk_data_ready(&smc->sk);
983 }
b8ded9de
KG		 984 } else {
985 smc_wr_tx_put_slot(conn->lnk,
986 (struct smc_wr_tx_pend_priv *)pend);
c6f02ebe
KG		 987 }
988 return rc;
989}
990
64513d26
GG		 991void smc_switch_link_and_count(struct smc_connection *conn,
992 struct smc_link *to_lnk)
07d51580
GG		 993{
994 atomic_dec(&conn->lnk->conn_cnt);
995 conn->lnk = to_lnk;
996 atomic_inc(&conn->lnk->conn_cnt);
997}
998
c6f02ebe
KG		 999struct smc_link *smc_switch_conns(struct smc_link_group *lgr,
1000 struct smc_link *from_lnk, bool is_dev_err)
1001{
1002 struct smc_link *to_lnk = NULL;
b8ded9de 1003 struct smc_cdc_tx_pend *pend;
c6f02ebe 1004 struct smc_connection *conn;
b8ded9de 1005 struct smc_wr_buf *wr_buf;
c6f02ebe
KG		 1006 struct smc_sock *smc;
1007 struct rb_node *node;
1008 int i, rc = 0;
1009
1010 /* link is inactive, wake up tx waiters */
1011 smc_wr_wakeup_tx_wait(from_lnk);
1012
1013 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
741a49a4 1014 if (!smc_link_active(&lgr->lnk[i]) || i == from_lnk->link_idx)
c6f02ebe
KG		 1015 continue;
1016 if (is_dev_err && from_lnk->smcibdev == lgr->lnk[i].smcibdev &&
1017 from_lnk->ibport == lgr->lnk[i].ibport) {
1018 continue;
1019 }
1020 to_lnk = &lgr->lnk[i];
1021 break;
1022 }
95f7f3e7 1023 if (!to_lnk || !smc_wr_tx_link_hold(to_lnk)) {
c6f02ebe
KG		 1024 smc_lgr_terminate_sched(lgr);
1025 return NULL;
1026 }
1027again:
1028 read_lock_bh(&lgr->conns_lock);
1029 for (node = rb_first(&lgr->conns_all); node; node = rb_next(node)) {
1030 conn = rb_entry(node, struct smc_connection, alert_node);
1031 if (conn->lnk != from_lnk)
1032 continue;
1033 smc = container_of(conn, struct smc_sock, conn);
1034 /* conn->lnk not yet set in SMC_INIT state */
1035 if (smc->sk.sk_state == SMC_INIT)
1036 continue;
1037 if (smc->sk.sk_state == SMC_CLOSED ||
1038 smc->sk.sk_state == SMC_PEERCLOSEWAIT1 ||
1039 smc->sk.sk_state == SMC_PEERCLOSEWAIT2 ||
1040 smc->sk.sk_state == SMC_APPFINCLOSEWAIT ||
1041 smc->sk.sk_state == SMC_APPCLOSEWAIT1 ||
1042 smc->sk.sk_state == SMC_APPCLOSEWAIT2 ||
1043 smc->sk.sk_state == SMC_PEERFINCLOSEWAIT ||
1044 smc->sk.sk_state == SMC_PEERABORTWAIT ||
1045 smc->sk.sk_state == SMC_PROCESSABORT) {
1046 spin_lock_bh(&conn->send_lock);
07d51580 1047 smc_switch_link_and_count(conn, to_lnk);
c6f02ebe
KG		 1048 spin_unlock_bh(&conn->send_lock);
1049 continue;
1050 }
1051 sock_hold(&smc->sk);
1052 read_unlock_bh(&lgr->conns_lock);
b8ded9de
KG		 1053 /* pre-fetch buffer outside of send_lock, might sleep */
1054 rc = smc_cdc_get_free_slot(conn, to_lnk, &wr_buf, NULL, &pend);
95f7f3e7
KG		 1055 if (rc)
1056 goto err_out;
c6f02ebe
KG		 1057 /* avoid race with smcr_tx_sndbuf_nonempty() */
1058 spin_lock_bh(&conn->send_lock);
07d51580 1059 smc_switch_link_and_count(conn, to_lnk);
b8ded9de 1060 rc = smc_switch_cursor(smc, pend, wr_buf);
c6f02ebe
KG		 1061 spin_unlock_bh(&conn->send_lock);
1062 sock_put(&smc->sk);
95f7f3e7
KG		 1063 if (rc)
1064 goto err_out;
c6f02ebe
KG		 1065 goto again;
1066 }
1067 read_unlock_bh(&lgr->conns_lock);
95f7f3e7 1068 smc_wr_tx_link_put(to_lnk);
c6f02ebe 1069 return to_lnk;
95f7f3e7
KG		 1070
1071err_out:
1072 smcr_link_down_cond_sched(to_lnk);
1073 smc_wr_tx_link_put(to_lnk);
1074 return NULL;
c6f02ebe
KG		 1075}
1076
b9247544 1077static void smcr_buf_unuse(struct smc_buf_desc *rmb_desc,
6d74c3a8 1078 struct smc_link_group *lgr)
b9247544 1079{
d5500667
KG		 1080 int rc;
1081
b9247544
KG		 1082 if (rmb_desc->is_conf_rkey && !list_empty(&lgr->list)) {
1083 /* unregister rmb with peer */
d5500667
KG		 1084 rc = smc_llc_flow_initiate(lgr, SMC_LLC_FLOW_RKEY);
1085 if (!rc) {
1086 /* protect against smc_llc_cli_rkey_exchange() */
1087 mutex_lock(&lgr->llc_conf_mutex);
1088 smc_llc_do_delete_rkey(lgr, rmb_desc);
1089 rmb_desc->is_conf_rkey = false;
1090 mutex_unlock(&lgr->llc_conf_mutex);
1091 smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl);
1092 }
b9247544 1093 }
d5500667 1094
b9247544
KG		 1095 if (rmb_desc->is_reg_err) {
1096 /* buf registration failed, reuse not possible */
faca5360 1097 mutex_lock(&lgr->rmbs_lock);
b9247544 1098 list_del(&rmb_desc->list);
faca5360 1099 mutex_unlock(&lgr->rmbs_lock);
b9247544
KG		 1100
1101 smc_buf_free(lgr, true, rmb_desc);
1102 } else {
1103 rmb_desc->used = 0;
1c552696 1104 memset(rmb_desc->cpu_addr, 0, rmb_desc->len);
b9247544
KG		 1105 }
1106}
1107
fb692ec4
KG		 1108static void smc_buf_unuse(struct smc_connection *conn,
1109 struct smc_link_group *lgr)
cd6851f3 1110{
1c552696 1111 if (conn->sndbuf_desc) {
cd6851f3 1112 conn->sndbuf_desc->used = 0;
1c552696
TL		 1113 memset(conn->sndbuf_desc->cpu_addr, 0, conn->sndbuf_desc->len);
1114 }
1115 if (conn->rmb_desc && lgr->is_smcd) {
b9247544 1116 conn->rmb_desc->used = 0;
1c552696
TL		 1117 memset(conn->rmb_desc->cpu_addr, 0, conn->rmb_desc->len +
1118 sizeof(struct smcd_cdc_msg));
1119 } else if (conn->rmb_desc) {
6d74c3a8 1120 smcr_buf_unuse(conn->rmb_desc, lgr);
1c552696 1121 }
cd6851f3
UB		 1122}
1123
0cfdd8f9
UB		 1124/* remove a finished connection from its link group */
1125void smc_conn_free(struct smc_connection *conn)
1126{
fb692ec4
KG		 1127 struct smc_link_group *lgr = conn->lgr;
1128
1129 if (!lgr)
0cfdd8f9 1130 return;
fb692ec4 1131 if (lgr->is_smcd) {
42bfba9e
UB		 1132 if (!list_empty(&lgr->list))
1133 smc_ism_unset_conn(conn);
be244f28
HW		 1134 tasklet_kill(&conn->rx_tsklet);
1135 } else {
c6ba7c9b 1136 smc_cdc_tx_dismiss_slots(conn);
b286a065
KG		 1137 if (current_work() != &conn->abort_work)
1138 cancel_work_sync(&conn->abort_work);
be244f28 1139 }
2a0674ff
UB		 1140 if (!list_empty(&lgr->list)) {
1141 smc_lgr_unregister_conn(conn);
1142 smc_buf_unuse(conn, lgr); /* allow buffer reuse */
1143 }
fb692ec4
KG		 1144
1145 if (!lgr->conns_num)
1146 smc_lgr_schedule_free_work(lgr);
0cfdd8f9
UB		 1147}
1148
4a3641c1
KG		 1149/* unregister a link from a buf_desc */
1150static void smcr_buf_unmap_link(struct smc_buf_desc *buf_desc, bool is_rmb,
1151 struct smc_link *lnk)
1152{
1153 if (is_rmb)
1154 buf_desc->is_reg_mr[lnk->link_idx] = false;
1155 if (!buf_desc->is_map_ib[lnk->link_idx])
1156 return;
1157 if (is_rmb) {
1158 if (buf_desc->mr_rx[lnk->link_idx]) {
1159 smc_ib_put_memory_region(
1160 buf_desc->mr_rx[lnk->link_idx]);
1161 buf_desc->mr_rx[lnk->link_idx] = NULL;
1162 }
1163 smc_ib_buf_unmap_sg(lnk, buf_desc, DMA_FROM_DEVICE);
1164 } else {
1165 smc_ib_buf_unmap_sg(lnk, buf_desc, DMA_TO_DEVICE);
1166 }
1167 sg_free_table(&buf_desc->sgt[lnk->link_idx]);
1168 buf_desc->is_map_ib[lnk->link_idx] = false;
1169}
1170
1171/* unmap all buffers of lgr for a deleted link */
1172static void smcr_buf_unmap_lgr(struct smc_link *lnk)
1173{
1174 struct smc_link_group *lgr = lnk->lgr;
1175 struct smc_buf_desc *buf_desc, *bf;
1176 int i;
1177
1178 for (i = 0; i < SMC_RMBE_SIZES; i++) {
1179 mutex_lock(&lgr->rmbs_lock);
1180 list_for_each_entry_safe(buf_desc, bf, &lgr->rmbs[i], list)
1181 smcr_buf_unmap_link(buf_desc, true, lnk);
1182 mutex_unlock(&lgr->rmbs_lock);
1183 mutex_lock(&lgr->sndbufs_lock);
1184 list_for_each_entry_safe(buf_desc, bf, &lgr->sndbufs[i],
1185 list)
1186 smcr_buf_unmap_link(buf_desc, false, lnk);
1187 mutex_unlock(&lgr->sndbufs_lock);
1188 }
1189}
1190
1191static void smcr_rtoken_clear_link(struct smc_link *lnk)
1192{
1193 struct smc_link_group *lgr = lnk->lgr;
1194 int i;
1195
1196 for (i = 0; i < SMC_RMBS_PER_LGR_MAX; i++) {
1197 lgr->rtokens[i][lnk->link_idx].rkey = 0;
1198 lgr->rtokens[i][lnk->link_idx].dma_addr = 0;
1199 }
1200}
1201
d5500667 1202/* must be called under lgr->llc_conf_mutex lock */
0a99be43 1203void smcr_link_clear(struct smc_link *lnk, bool log)
0cfdd8f9 1204{
d854fcbf
KG		 1205 struct smc_ib_device *smcibdev;
1206
4a3641c1 1207 if (!lnk->lgr || lnk->state == SMC_LNK_UNUSED)
b9247544 1208 return;
0cfdd8f9 1209 lnk->peer_qpn = 0;
0a99be43 1210 smc_llc_link_clear(lnk, log);
4a3641c1
KG		 1211 smcr_buf_unmap_lgr(lnk);
1212 smcr_rtoken_clear_link(lnk);
bd4ad577 1213 smc_ib_modify_qp_reset(lnk);
f38ba179 1214 smc_wr_free_link(lnk);
bd4ad577
UB		 1215 smc_ib_destroy_queue_pair(lnk);
1216 smc_ib_dealloc_protection_domain(lnk);
f38ba179 1217 smc_wr_free_link_mem(lnk);
ddc99286 1218 smc_ibdev_cnt_dec(lnk);
f3c1dedd 1219 put_device(&lnk->smcibdev->ibdev->dev);
d854fcbf
KG		 1220 smcibdev = lnk->smcibdev;
1221 memset(lnk, 0, sizeof(struct smc_link));
1222 lnk->state = SMC_LNK_UNUSED;
1223 if (!atomic_dec_return(&smcibdev->lnk_cnt))
1224 wake_up(&smcibdev->lnks_deleted);
0cfdd8f9
UB		 1225}
1226
c6ba7c9b
HW		 1227static void smcr_buf_free(struct smc_link_group *lgr, bool is_rmb,
1228 struct smc_buf_desc *buf_desc)
cd6851f3 1229{
b9247544 1230 int i;
6511aad3 1231
4a3641c1
KG		 1232 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++)
1233 smcr_buf_unmap_link(buf_desc, is_rmb, &lgr->lnk[i]);
387707fd 1234
2ef4f27a
SR		 1235 if (buf_desc->pages)
1236 __free_pages(buf_desc->pages, buf_desc->order);
3e034725 1237 kfree(buf_desc);
cd6851f3
UB		 1238}
1239
c6ba7c9b
HW		 1240static void smcd_buf_free(struct smc_link_group *lgr, bool is_dmb,
1241 struct smc_buf_desc *buf_desc)
1242{
be244f28
HW		 1243 if (is_dmb) {
1244 /* restore original buf len */
1245 buf_desc->len += sizeof(struct smcd_cdc_msg);
c6ba7c9b 1246 smc_ism_unregister_dmb(lgr->smcd, buf_desc);
be244f28 1247 } else {
c6ba7c9b 1248 kfree(buf_desc->cpu_addr);
be244f28 1249 }
c6ba7c9b
HW		 1250 kfree(buf_desc);
1251}
1252
1253static void smc_buf_free(struct smc_link_group *lgr, bool is_rmb,
1254 struct smc_buf_desc *buf_desc)
1255{
1256 if (lgr->is_smcd)
1257 smcd_buf_free(lgr, is_rmb, buf_desc);
1258 else
1259 smcr_buf_free(lgr, is_rmb, buf_desc);
1260}
1261
3e034725 1262static void __smc_lgr_free_bufs(struct smc_link_group *lgr, bool is_rmb)
cd6851f3 1263{
3e034725
UB		 1264 struct smc_buf_desc *buf_desc, *bf_desc;
1265 struct list_head *buf_list;
cd6851f3
UB		 1266 int i;
1267
1268 for (i = 0; i < SMC_RMBE_SIZES; i++) {
3e034725
UB		 1269 if (is_rmb)
1270 buf_list = &lgr->rmbs[i];
1271 else
1272 buf_list = &lgr->sndbufs[i];
1273 list_for_each_entry_safe(buf_desc, bf_desc, buf_list,
cd6851f3 1274 list) {
3e034725 1275 list_del(&buf_desc->list);
6511aad3 1276 smc_buf_free(lgr, is_rmb, buf_desc);
cd6851f3
UB		 1277 }
1278 }
1279}
1280
3e034725
UB		 1281static void smc_lgr_free_bufs(struct smc_link_group *lgr)
1282{
1283 /* free send buffers */
1284 __smc_lgr_free_bufs(lgr, false);
1285 /* free rmbs */
1286 __smc_lgr_free_bufs(lgr, true);
1287}
1288
0cfdd8f9 1289/* remove a link group */
3f3f0e36 1290static void smc_lgr_free(struct smc_link_group *lgr)
0cfdd8f9 1291{
b9247544
KG		 1292 int i;
1293
a52bcc91
KG		 1294 if (!lgr->is_smcd) {
1295 mutex_lock(&lgr->llc_conf_mutex);
1296 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
1297 if (lgr->lnk[i].state != SMC_LNK_UNUSED)
0a99be43 1298 smcr_link_clear(&lgr->lnk[i], false);
a52bcc91
KG		 1299 }
1300 mutex_unlock(&lgr->llc_conf_mutex);
1301 smc_llc_lgr_clear(lgr);
1302 }
1303
3e034725 1304 smc_lgr_free_bufs(lgr);
22ef473d 1305 destroy_workqueue(lgr->tx_wq);
b3cb53c0 1306 if (lgr->is_smcd) {
f9aab6f2
UB		 1307 smc_ism_put_vlan(lgr->smcd, lgr->vlan_id);
1308 put_device(&lgr->smcd->dev);
5edd6b9c
UB		 1309 if (!atomic_dec_return(&lgr->smcd->lgr_cnt))
1310 wake_up(&lgr->smcd->lgrs_deleted);
b3cb53c0 1311 } else {
8799e310 1312 smc_wr_free_lgr_mem(lgr);
6dabd405
UB		 1313 if (!atomic_dec_return(&lgr_cnt))
1314 wake_up(&lgrs_deleted);
b3cb53c0 1315 }
0cfdd8f9
UB		 1316 kfree(lgr);
1317}
1318
2a0674ff
UB		 1319static void smc_sk_wake_ups(struct smc_sock *smc)
1320{
1321 smc->sk.sk_write_space(&smc->sk);
1322 smc->sk.sk_data_ready(&smc->sk);
1323 smc->sk.sk_state_change(&smc->sk);
1324}
1325
1326/* kill a connection */
5421ec28 1327static void smc_conn_kill(struct smc_connection *conn, bool soft)
2a0674ff
UB		 1328{
1329 struct smc_sock *smc = container_of(conn, struct smc_sock, conn);
1330
50c6b20e
UB		 1331 if (conn->lgr->is_smcd && conn->lgr->peer_shutdown)
1332 conn->local_tx_ctrl.conn_state_flags.peer_conn_abort = 1;
1333 else
1334 smc_close_abort(conn);
2a0674ff 1335 conn->killed = 1;
50c6b20e 1336 smc->sk.sk_err = ECONNABORTED;
2a0674ff 1337 smc_sk_wake_ups(smc);
42bfba9e
UB		 1338 if (conn->lgr->is_smcd) {
1339 smc_ism_unset_conn(conn);
5421ec28
UB		 1340 if (soft)
1341 tasklet_kill(&conn->rx_tsklet);
1342 else
1343 tasklet_unlock_wait(&conn->rx_tsklet);
6a37ad3d
UB		 1344 } else {
1345 smc_cdc_tx_dismiss_slots(conn);
42bfba9e 1346 }
2a0674ff 1347 smc_lgr_unregister_conn(conn);
81cf4f47 1348 smc_close_active_abort(smc);
2a0674ff
UB		 1349}
1350
42bfba9e
UB		 1351static void smc_lgr_cleanup(struct smc_link_group *lgr)
1352{
1353 if (lgr->is_smcd) {
1354 smc_ism_signal_shutdown(lgr);
42bfba9e 1355 } else {
3e0c40af
KG		 1356 u32 rsn = lgr->llc_termination_rsn;
1357
1358 if (!rsn)
1359 rsn = SMC_LLC_DEL_PROG_INIT_TERM;
1360 smc_llc_send_link_delete_all(lgr, false, rsn);
a52bcc91 1361 smcr_lgr_link_deactivate_all(lgr);
42bfba9e
UB		 1362 }
1363}
1364
ba952060
KG		 1365/* terminate link group
1366 * @soft: true if link group shutdown can take its time
1367 * false if immediate link group shutdown is required
1368 */
5421ec28 1369static void __smc_lgr_terminate(struct smc_link_group *lgr, bool soft)
0cfdd8f9
UB		 1370{
1371 struct smc_connection *conn;
b38d7324 1372 struct smc_sock *smc;
0cfdd8f9
UB		 1373 struct rb_node *node;
1374
517c300e
KG		 1375 if (lgr->terminating)
1376 return; /* lgr already terminating */
a52bcc91
KG		 1377 /* cancel free_work sync, will terminate when lgr->freeing is set */
1378 cancel_delayed_work_sync(&lgr->free_work);
517c300e 1379 lgr->terminating = 1;
0cfdd8f9 1380
69318b52
UB		 1381 /* kill remaining link group connections */
1382 read_lock_bh(&lgr->conns_lock);
0cfdd8f9
UB		 1383 node = rb_first(&lgr->conns_all);
1384 while (node) {
69318b52 1385 read_unlock_bh(&lgr->conns_lock);
0cfdd8f9 1386 conn = rb_entry(node, struct smc_connection, alert_node);
b38d7324 1387 smc = container_of(conn, struct smc_sock, conn);
81cf4f47 1388 sock_hold(&smc->sk); /* sock_put below */
69318b52 1389 lock_sock(&smc->sk);
5421ec28 1390 smc_conn_kill(conn, soft);
69318b52 1391 release_sock(&smc->sk);
81cf4f47 1392 sock_put(&smc->sk); /* sock_hold above */
69318b52 1393 read_lock_bh(&lgr->conns_lock);
0cfdd8f9
UB		 1394 node = rb_first(&lgr->conns_all);
1395 }
69318b52 1396 read_unlock_bh(&lgr->conns_lock);
42bfba9e 1397 smc_lgr_cleanup(lgr);
a52bcc91 1398 smc_lgr_free(lgr);
0cfdd8f9
UB		 1399}
1400
5f78fe96
KG		 1401/* unlink link group and schedule termination */
1402void smc_lgr_terminate_sched(struct smc_link_group *lgr)
b9f227c3 1403{
a0a62ee1
UB		 1404 spinlock_t *lgr_lock;
1405
1406 smc_lgr_list_head(lgr, &lgr_lock);
1407 spin_lock_bh(lgr_lock);
3739707c 1408 if (list_empty(&lgr->list) || lgr->terminating || lgr->freeing) {
8caa6544
UB		 1409 spin_unlock_bh(lgr_lock);
1410 return; /* lgr already terminating */
1411 }
1412 list_del_init(&lgr->list);
a52bcc91 1413 lgr->freeing = 1;
a0a62ee1 1414 spin_unlock_bh(lgr_lock);
5f78fe96 1415 schedule_work(&lgr->terminate_work);
b9f227c3
HW		 1416}
1417
5421ec28 1418/* Called when peer lgr shutdown (regularly or abnormally) is received */
0512f69e 1419void smc_smcd_terminate(struct smcd_dev *dev, u64 peer_gid, unsigned short vlan)
c6ba7c9b
HW		 1420{
1421 struct smc_link_group *lgr, *l;
1422 LIST_HEAD(lgr_free_list);
1423
1424 /* run common cleanup function and build free list */
a0a62ee1 1425 spin_lock_bh(&dev->lgr_lock);
a2351c5d
UB		 1426 list_for_each_entry_safe(lgr, l, &dev->lgr_list, list) {
1427 if ((!peer_gid || lgr->peer_gid == peer_gid) &&
0512f69e 1428 (vlan == VLAN_VID_MASK || lgr->vlan_id == vlan)) {
50c6b20e
UB		 1429 if (peer_gid) /* peer triggered termination */
1430 lgr->peer_shutdown = 1;
c6ba7c9b 1431 list_move(&lgr->list, &lgr_free_list);
a52bcc91 1432 lgr->freeing = 1;
c6ba7c9b
HW		 1433 }
1434 }
a0a62ee1 1435 spin_unlock_bh(&dev->lgr_lock);
c6ba7c9b
HW		 1436
1437 /* cancel the regular free workers and actually free lgrs */
1438 list_for_each_entry_safe(lgr, l, &lgr_free_list, list) {
1439 list_del_init(&lgr->list);
50c6b20e 1440 schedule_work(&lgr->terminate_work);
c6ba7c9b
HW		 1441 }
1442}
1443
5421ec28
UB		 1444/* Called when an SMCD device is removed or the smc module is unloaded */
1445void smc_smcd_terminate_all(struct smcd_dev *smcd)
1446{
1447 struct smc_link_group *lgr, *lg;
1448 LIST_HEAD(lgr_free_list);
1449
1450 spin_lock_bh(&smcd->lgr_lock);
1451 list_splice_init(&smcd->lgr_list, &lgr_free_list);
1452 list_for_each_entry(lgr, &lgr_free_list, list)
1453 lgr->freeing = 1;
1454 spin_unlock_bh(&smcd->lgr_lock);
1455
1456 list_for_each_entry_safe(lgr, lg, &lgr_free_list, list) {
1457 list_del_init(&lgr->list);
1458 __smc_lgr_terminate(lgr, false);
1459 }
5edd6b9c
UB		 1460
1461 if (atomic_read(&smcd->lgr_cnt))
1462 wait_event(smcd->lgrs_deleted, !atomic_read(&smcd->lgr_cnt));
5421ec28
UB		 1463}
1464
0b29ec64
UB		 1465/* Called when an SMCR device is removed or the smc module is unloaded.
1466 * If smcibdev is given, all SMCR link groups using this device are terminated.
1467 * If smcibdev is NULL, all SMCR link groups are terminated.
1468 */
1469void smc_smcr_terminate_all(struct smc_ib_device *smcibdev)
1470{
1471 struct smc_link_group *lgr, *lg;
1472 LIST_HEAD(lgr_free_list);
b9247544 1473 int i;
0b29ec64
UB		 1474
1475 spin_lock_bh(&smc_lgr_list.lock);
1476 if (!smcibdev) {
1477 list_splice_init(&smc_lgr_list.list, &lgr_free_list);
1478 list_for_each_entry(lgr, &lgr_free_list, list)
1479 lgr->freeing = 1;
1480 } else {
1481 list_for_each_entry_safe(lgr, lg, &smc_lgr_list.list, list) {
b9247544 1482 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
87523930
KG		 1483 if (lgr->lnk[i].smcibdev == smcibdev)
1484 smcr_link_down_cond_sched(&lgr->lnk[i]);
0b29ec64
UB		 1485 }
1486 }
1487 }
1488 spin_unlock_bh(&smc_lgr_list.lock);
1489
1490 list_for_each_entry_safe(lgr, lg, &lgr_free_list, list) {
1491 list_del_init(&lgr->list);
3e0c40af 1492 smc_llc_set_termination_rsn(lgr, SMC_LLC_DEL_OP_INIT_TERM);
0b29ec64
UB		 1493 __smc_lgr_terminate(lgr, false);
1494 }
6dabd405
UB		 1495
1496 if (smcibdev) {
1497 if (atomic_read(&smcibdev->lnk_cnt))
1498 wait_event(smcibdev->lnks_deleted,
1499 !atomic_read(&smcibdev->lnk_cnt));
1500 } else {
1501 if (atomic_read(&lgr_cnt))
1502 wait_event(lgrs_deleted, !atomic_read(&lgr_cnt));
1503 }
0b29ec64
UB		 1504}
1505
ad6c111b
KG		 1506/* set new lgr type and clear all asymmetric link tagging */
1507void smcr_lgr_set_type(struct smc_link_group *lgr, enum smc_lgr_type new_type)
1508{
0a99be43 1509 char *lgr_type = "";
ad6c111b
KG		 1510 int i;
1511
1512 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++)
1513 if (smc_link_usable(&lgr->lnk[i]))
1514 lgr->lnk[i].link_is_asym = false;
0a99be43
KG		 1515 if (lgr->type == new_type)
1516 return;
ad6c111b 1517 lgr->type = new_type;
0a99be43
KG		 1518
1519 switch (lgr->type) {
1520 case SMC_LGR_NONE:
1521 lgr_type = "NONE";
1522 break;
1523 case SMC_LGR_SINGLE:
1524 lgr_type = "SINGLE";
1525 break;
1526 case SMC_LGR_SYMMETRIC:
1527 lgr_type = "SYMMETRIC";
1528 break;
1529 case SMC_LGR_ASYMMETRIC_PEER:
1530 lgr_type = "ASYMMETRIC_PEER";
1531 break;
1532 case SMC_LGR_ASYMMETRIC_LOCAL:
1533 lgr_type = "ASYMMETRIC_LOCAL";
1534 break;
1535 }
1536 pr_warn_ratelimited("smc: SMC-R lg %*phN state changed: "
1537 "%s, pnetid %.16s\n", SMC_LGR_ID_SIZE, &lgr->id,
1538 lgr_type, lgr->pnet_id);
ad6c111b
KG		 1539}
1540
1541/* set new lgr type and tag a link as asymmetric */
1542void smcr_lgr_set_type_asym(struct smc_link_group *lgr,
1543 enum smc_lgr_type new_type, int asym_lnk_idx)
1544{
1545 smcr_lgr_set_type(lgr, new_type);
1546 lgr->lnk[asym_lnk_idx].link_is_asym = true;
1547}
1548
b286a065
KG		 1549/* abort connection, abort_work scheduled from tasklet context */
1550static void smc_conn_abort_work(struct work_struct *work)
1551{
1552 struct smc_connection *conn = container_of(work,
1553 struct smc_connection,
1554 abort_work);
1555 struct smc_sock *smc = container_of(conn, struct smc_sock, conn);
1556
a18cee47 1557 lock_sock(&smc->sk);
b286a065 1558 smc_conn_kill(conn, true);
a18cee47 1559 release_sock(&smc->sk);
b286a065
KG		 1560 sock_put(&smc->sk); /* sock_hold done by schedulers of abort_work */
1561}
1562
1f90a05d
KG		 1563void smcr_port_add(struct smc_ib_device *smcibdev, u8 ibport)
1564{
1f90a05d
KG		 1565 struct smc_link_group *lgr, *n;
1566
1567 list_for_each_entry_safe(lgr, n, &smc_lgr_list.list, list) {
c48254fa
KG		 1568 struct smc_link *link;
1569
1f90a05d
KG		 1570 if (strncmp(smcibdev->pnetid[ibport - 1], lgr->pnet_id,
1571 SMC_MAX_PNETID_LEN) ||
1572 lgr->type == SMC_LGR_SYMMETRIC ||
1573 lgr->type == SMC_LGR_ASYMMETRIC_PEER)
1574 continue;
c48254fa
KG		 1575
1576 /* trigger local add link processing */
1577 link = smc_llc_usable_link(lgr);
1578 if (link)
1579 smc_llc_add_link_local(link);
1f90a05d
KG		 1580 }
1581}
1582
541afa10
KG		 1583/* link is down - switch connections to alternate link,
1584 * must be called under lgr->llc_conf_mutex lock
1585 */
1586static void smcr_link_down(struct smc_link *lnk)
1587{
1588 struct smc_link_group *lgr = lnk->lgr;
1589 struct smc_link *to_lnk;
1590 int del_link_id;
1591
1592 if (!lgr || lnk->state == SMC_LNK_UNUSED || list_empty(&lgr->list))
1593 return;
1594
1595 smc_ib_modify_qp_reset(lnk);
c6f02ebe 1596 to_lnk = smc_switch_conns(lgr, lnk, true);
541afa10 1597 if (!to_lnk) { /* no backup link available */
0a99be43 1598 smcr_link_clear(lnk, true);
541afa10
KG		 1599 return;
1600 }
ad6c111b 1601 smcr_lgr_set_type(lgr, SMC_LGR_SINGLE);
541afa10
KG		 1602 del_link_id = lnk->link_id;
1603
1604 if (lgr->role == SMC_SERV) {
1605 /* trigger local delete link processing */
4dadd151 1606 smc_llc_srv_delete_link_local(to_lnk, del_link_id);
541afa10
KG		 1607 } else {
1608 if (lgr->llc_flow_lcl.type != SMC_LLC_FLOW_NONE) {
1609 /* another llc task is ongoing */
1610 mutex_unlock(&lgr->llc_conf_mutex);
6778a6be
KG		 1611 wait_event_timeout(lgr->llc_flow_waiter,
1612 (list_empty(&lgr->list) ||
1613 lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE),
541afa10
KG		 1614 SMC_LLC_WAIT_TIME);
1615 mutex_lock(&lgr->llc_conf_mutex);
1616 }
68fd8942 1617 if (!list_empty(&lgr->list)) {
6778a6be
KG		 1618 smc_llc_send_delete_link(to_lnk, del_link_id,
1619 SMC_LLC_REQ, true,
1620 SMC_LLC_DEL_LOST_PATH);
68fd8942
KG		 1621 smcr_link_clear(lnk, true);
1622 }
6778a6be 1623 wake_up(&lgr->llc_flow_waiter); /* wake up next waiter */
541afa10
KG		 1624 }
1625}
1626
1627/* must be called under lgr->llc_conf_mutex lock */
1628void smcr_link_down_cond(struct smc_link *lnk)
1629{
a3a0e81b
TL		 1630 if (smc_link_downing(&lnk->state)) {
1631 trace_smcr_link_down(lnk, __builtin_return_address(0));
541afa10 1632 smcr_link_down(lnk);
a3a0e81b 1633 }
541afa10
KG		 1634}
1635
1636/* will get the lgr->llc_conf_mutex lock */
1637void smcr_link_down_cond_sched(struct smc_link *lnk)
1638{
a3a0e81b
TL		 1639 if (smc_link_downing(&lnk->state)) {
1640 trace_smcr_link_down(lnk, __builtin_return_address(0));
541afa10 1641 schedule_work(&lnk->link_down_wrk);
a3a0e81b 1642 }
541afa10
KG		 1643}
1644
1645void smcr_port_err(struct smc_ib_device *smcibdev, u8 ibport)
1646{
1647 struct smc_link_group *lgr, *n;
1648 int i;
1649
1650 list_for_each_entry_safe(lgr, n, &smc_lgr_list.list, list) {
1651 if (strncmp(smcibdev->pnetid[ibport - 1], lgr->pnet_id,
1652 SMC_MAX_PNETID_LEN))
1653 continue; /* lgr is not affected */
1654 if (list_empty(&lgr->list))
1655 continue;
1656 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
1657 struct smc_link *lnk = &lgr->lnk[i];
1658
1659 if (smc_link_usable(lnk) &&
1660 lnk->smcibdev == smcibdev && lnk->ibport == ibport)
1661 smcr_link_down_cond_sched(lnk);
1662 }
1663 }
1664}
1665
541afa10
KG		 1666static void smc_link_down_work(struct work_struct *work)
1667{
1668 struct smc_link *link = container_of(work, struct smc_link,
1669 link_down_wrk);
1670 struct smc_link_group *lgr = link->lgr;
1671
1672 if (list_empty(&lgr->list))
1673 return;
6778a6be 1674 wake_up_all(&lgr->llc_msg_waiter);
541afa10
KG		 1675 mutex_lock(&lgr->llc_conf_mutex);
1676 smcr_link_down(link);
1677 mutex_unlock(&lgr->llc_conf_mutex);
1678}
1679
587acad4
KG		 1680static int smc_vlan_by_tcpsk_walk(struct net_device *lower_dev,
1681 struct netdev_nested_priv *priv)
1682{
1683 unsigned short *vlan_id = (unsigned short *)priv->data;
1684
1685 if (is_vlan_dev(lower_dev)) {
1686 *vlan_id = vlan_dev_vlan_id(lower_dev);
1687 return 1;
1688 }
1689
1690 return 0;
1691}
1692
1693/* Determine vlan of internal TCP socket. */
bc36d2fc 1694int smc_vlan_by_tcpsk(struct socket *clcsock, struct smc_init_info *ini)
0cfdd8f9
UB		 1695{
1696 struct dst_entry *dst = sk_dst_get(clcsock->sk);
587acad4 1697 struct netdev_nested_priv priv;
cb9d43f6 1698 struct net_device *ndev;
587acad4 1699 int rc = 0;
0cfdd8f9 1700
bc36d2fc 1701 ini->vlan_id = 0;
0cfdd8f9
UB		 1702 if (!dst) {
1703 rc = -ENOTCONN;
1704 goto out;
1705 }
1706 if (!dst->dev) {
1707 rc = -ENODEV;
1708 goto out_rel;
1709 }
1710
cb9d43f6
UB		 1711 ndev = dst->dev;
1712 if (is_vlan_dev(ndev)) {
bc36d2fc 1713 ini->vlan_id = vlan_dev_vlan_id(ndev);
cb9d43f6
UB		 1714 goto out_rel;
1715 }
1716
587acad4 1717 priv.data = (void *)&ini->vlan_id;
cb9d43f6 1718 rtnl_lock();
587acad4 1719 netdev_walk_all_lower_dev(ndev, smc_vlan_by_tcpsk_walk, &priv);
cb9d43f6 1720 rtnl_unlock();
0cfdd8f9
UB		 1721
1722out_rel:
1723 dst_release(dst);
1724out:
1725 return rc;
1726}
1727
e49300a6
KG		 1728static bool smcr_lgr_match(struct smc_link_group *lgr, u8 smcr_version,
1729 u8 peer_systemid[],
1730 u8 peer_gid[],
1731 u8 peer_mac_v1[],
ee05ff7a 1732 enum smc_lgr_role role, u32 clcqpn)
0cfdd8f9 1733{
b9247544
KG		 1734 int i;
1735
e49300a6 1736 if (memcmp(lgr->peer_systemid, peer_systemid, SMC_SYSTEMID_LEN) ||
b9247544
KG		 1737 lgr->role != role)
1738 return false;
1739
1740 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
741a49a4 1741 if (!smc_link_active(&lgr->lnk[i]))
b9247544
KG		 1742 continue;
1743 if ((lgr->role == SMC_SERV || lgr->lnk[i].peer_qpn == clcqpn) &&
e49300a6
KG		 1744 !memcmp(lgr->lnk[i].peer_gid, peer_gid, SMC_GID_SIZE) &&
1745 (smcr_version == SMC_V2 ||
1746 !memcmp(lgr->lnk[i].peer_mac, peer_mac_v1, ETH_ALEN)))
b9247544
KG		 1747 return true;
1748 }
1749 return false;
c6ba7c9b 1750}
0cfdd8f9 1751
c6ba7c9b
HW		 1752static bool smcd_lgr_match(struct smc_link_group *lgr,
1753 struct smcd_dev *smcismdev, u64 peer_gid)
1754{
1755 return lgr->peer_gid == peer_gid && lgr->smcd == smcismdev;
0cfdd8f9
UB		 1756}
1757
1758/* create a new SMC connection (and a new link group if necessary) */
bc36d2fc 1759int smc_conn_create(struct smc_sock *smc, struct smc_init_info *ini)
0cfdd8f9
UB		 1760{
1761 struct smc_connection *conn = &smc->conn;
a2351c5d 1762 struct list_head *lgr_list;
0cfdd8f9 1763 struct smc_link_group *lgr;
0cfdd8f9 1764 enum smc_lgr_role role;
a0a62ee1 1765 spinlock_t *lgr_lock;
0cfdd8f9
UB		 1766 int rc = 0;
1767
5c21c4cc 1768 lgr_list = ini->is_smcd ? &ini->ism_dev[ini->ism_selected]->lgr_list :
3fc64937 1769 &smc_lgr_list.list;
5c21c4cc 1770 lgr_lock = ini->is_smcd ? &ini->ism_dev[ini->ism_selected]->lgr_lock :
3fc64937 1771 &smc_lgr_list.lock;
5ac54d87 1772 ini->first_contact_local = 1;
0cfdd8f9 1773 role = smc->listen_smc ? SMC_SERV : SMC_CLNT;
5ac54d87 1774 if (role == SMC_CLNT && ini->first_contact_peer)
0cfdd8f9
UB		 1775 /* create new link group as well */
1776 goto create;
1777
1778 /* determine if an existing link group can be reused */
a0a62ee1 1779 spin_lock_bh(lgr_lock);
a2351c5d 1780 list_for_each_entry(lgr, lgr_list, list) {
0cfdd8f9 1781 write_lock_bh(&lgr->conns_lock);
bc36d2fc 1782 if ((ini->is_smcd ?
5c21c4cc
UB		 1783 smcd_lgr_match(lgr, ini->ism_dev[ini->ism_selected],
1784 ini->ism_peer_gid[ini->ism_selected]) :
e49300a6
KG		 1785 smcr_lgr_match(lgr, ini->smcr_version,
1786 ini->peer_systemid,
1787 ini->peer_gid, ini->peer_mac, role,
1788 ini->ib_clcqpn)) &&
0cfdd8f9 1789 !lgr->sync_err &&
0530bd6e
KG		 1790 (ini->smcd_version == SMC_V2 ||
1791 lgr->vlan_id == ini->vlan_id) &&
a9e44502 1792 (role == SMC_CLNT || ini->is_smcd ||
c6ba7c9b 1793 lgr->conns_num < SMC_RMBS_PER_LGR_MAX)) {
0cfdd8f9 1794 /* link group found */
5ac54d87 1795 ini->first_contact_local = 0;
0cfdd8f9 1796 conn->lgr = lgr;
56bc3b20 1797 rc = smc_lgr_register_conn(conn, false);
0cfdd8f9 1798 write_unlock_bh(&lgr->conns_lock);
b9247544
KG		 1799 if (!rc && delayed_work_pending(&lgr->free_work))
1800 cancel_delayed_work(&lgr->free_work);
0cfdd8f9
UB		 1801 break;
1802 }
1803 write_unlock_bh(&lgr->conns_lock);
1804 }
a0a62ee1 1805 spin_unlock_bh(lgr_lock);
b9247544
KG		 1806 if (rc)
1807 return rc;
0cfdd8f9 1808
5ac54d87
UB		 1809 if (role == SMC_CLNT && !ini->first_contact_peer &&
1810 ini->first_contact_local) {
0cfdd8f9
UB		 1811 /* Server reuses a link group, but Client wants to start
1812 * a new one
1813 * send out_of_sync decline, reason synchr. error
1814 */
7a62725a 1815 return SMC_CLC_DECL_SYNCERR;
0cfdd8f9
UB		 1816 }
1817
1818create:
5ac54d87 1819 if (ini->first_contact_local) {
bc36d2fc 1820 rc = smc_lgr_create(smc, ini);
0cfdd8f9
UB		 1821 if (rc)
1822 goto out;
44808792
HZ		 1823 lgr = conn->lgr;
1824 write_lock_bh(&lgr->conns_lock);
56bc3b20 1825 rc = smc_lgr_register_conn(conn, true);
44808792 1826 write_unlock_bh(&lgr->conns_lock);
b9247544
KG		 1827 if (rc)
1828 goto out;
0cfdd8f9 1829 }
5f08318f 1830 conn->local_tx_ctrl.common.type = SMC_CDC_MSG_TYPE;
cbba07a7 1831 conn->local_tx_ctrl.len = SMC_WR_TX_SIZE;
de8474eb 1832 conn->urg_state = SMC_URG_READ;
b286a065 1833 INIT_WORK(&smc->conn.abort_work, smc_conn_abort_work);
bc36d2fc 1834 if (ini->is_smcd) {
be244f28
HW		 1835 conn->rx_off = sizeof(struct smcd_cdc_msg);
1836 smcd_cdc_rx_init(conn); /* init tasklet for this conn */
2d2bfeb8
UB		 1837 } else {
1838 conn->rx_off = 0;
be244f28 1839 }
5f08318f
UB		 1840#ifndef KERNEL_HAS_ATOMIC64
1841 spin_lock_init(&conn->acurs_lock);
1842#endif
0cfdd8f9
UB		 1843
1844out:
7a62725a 1845 return rc;
0cfdd8f9 1846}
cd6851f3 1847
67161779
SR		 1848#define SMCD_DMBE_SIZES 6 /* 0 -> 16KB, 1 -> 32KB, .. 6 -> 1MB */
1849#define SMCR_RMBE_SIZES 5 /* 0 -> 16KB, 1 -> 32KB, .. 5 -> 512KB */
1850
1851/* convert the RMB size into the compressed notation (minimum 16K, see
1852 * SMCD/R_DMBE_SIZES.
2f6becaf
HW		 1853 * In contrast to plain ilog2, this rounds towards the next power of 2,
1854 * so the socket application gets at least its desired sndbuf / rcvbuf size.
1855 */
67161779 1856static u8 smc_compress_bufsize(int size, bool is_smcd, bool is_rmb)
2f6becaf 1857{
67161779 1858 const unsigned int max_scat = SG_MAX_SINGLE_ALLOC * PAGE_SIZE;
2f6becaf
HW		 1859 u8 compressed;
1860
1861 if (size <= SMC_BUF_MIN_SIZE)
1862 return 0;
1863
67161779
SR		 1864 size = (size - 1) >> 14; /* convert to 16K multiple */
1865 compressed = min_t(u8, ilog2(size) + 1,
1866 is_smcd ? SMCD_DMBE_SIZES : SMCR_RMBE_SIZES);
1867
1868 if (!is_smcd && is_rmb)
1869 /* RMBs are backed by & limited to max size of scatterlists */
1870 compressed = min_t(u8, compressed, ilog2(max_scat >> 14));
1871
2f6becaf
HW		 1872 return compressed;
1873}
1874
1875/* convert the RMB size from compressed notation into integer */
1876int smc_uncompress_bufsize(u8 compressed)
1877{
1878 u32 size;
1879
1880 size = 0x00000001 << (((int)compressed) + 14);
1881 return (int)size;
1882}
1883
3e034725
UB		 1884/* try to reuse a sndbuf or rmb description slot for a certain
1885 * buffer size; if not available, return NULL
cd6851f3 1886 */
8437bda0 1887static struct smc_buf_desc *smc_buf_get_slot(int compressed_bufsize,
faca5360 1888 struct mutex *lock,
8437bda0 1889 struct list_head *buf_list)
cd6851f3 1890{
3e034725 1891 struct smc_buf_desc *buf_slot;
cd6851f3 1892
faca5360 1893 mutex_lock(lock);
3e034725
UB		 1894 list_for_each_entry(buf_slot, buf_list, list) {
1895 if (cmpxchg(&buf_slot->used, 0, 1) == 0) {
faca5360 1896 mutex_unlock(lock);
3e034725 1897 return buf_slot;
cd6851f3
UB		 1898 }
1899 }
faca5360 1900 mutex_unlock(lock);
cd6851f3
UB		 1901 return NULL;
1902}
1903
952310cc
UB		 1904/* one of the conditions for announcing a receiver's current window size is
1905 * that it "results in a minimum increase in the window size of 10% of the
1906 * receive buffer space" [RFC7609]
1907 */
1908static inline int smc_rmb_wnd_update_limit(int rmbe_size)
1909{
1910 return min_t(int, rmbe_size / 10, SOCK_MIN_SNDBUF / 2);
1911}
1912
b9247544
KG		 1913/* map an rmb buf to a link */
1914static int smcr_buf_map_link(struct smc_buf_desc *buf_desc, bool is_rmb,
1915 struct smc_link *lnk)
1916{
1917 int rc;
1918
1919 if (buf_desc->is_map_ib[lnk->link_idx])
1920 return 0;
1921
1922 rc = sg_alloc_table(&buf_desc->sgt[lnk->link_idx], 1, GFP_KERNEL);
1923 if (rc)
1924 return rc;
1925 sg_set_buf(buf_desc->sgt[lnk->link_idx].sgl,
1926 buf_desc->cpu_addr, buf_desc->len);
1927
1928 /* map sg table to DMA address */
1929 rc = smc_ib_buf_map_sg(lnk, buf_desc,
1930 is_rmb ? DMA_FROM_DEVICE : DMA_TO_DEVICE);
1931 /* SMC protocol depends on mapping to one DMA address only */
1932 if (rc != 1) {
1933 rc = -EAGAIN;
1934 goto free_table;
1935 }
1936
1937 /* create a new memory region for the RMB */
1938 if (is_rmb) {
1939 rc = smc_ib_get_memory_region(lnk->roce_pd,
1940 IB_ACCESS_REMOTE_WRITE |
1941 IB_ACCESS_LOCAL_WRITE,
1942 buf_desc, lnk->link_idx);
1943 if (rc)
1944 goto buf_unmap;
1945 smc_ib_sync_sg_for_device(lnk, buf_desc, DMA_FROM_DEVICE);
1946 }
1947 buf_desc->is_map_ib[lnk->link_idx] = true;
1948 return 0;
1949
1950buf_unmap:
1951 smc_ib_buf_unmap_sg(lnk, buf_desc,
1952 is_rmb ? DMA_FROM_DEVICE : DMA_TO_DEVICE);
1953free_table:
1954 sg_free_table(&buf_desc->sgt[lnk->link_idx]);
1955 return rc;
1956}
1957
d5500667
KG		 1958/* register a new rmb on IB device,
1959 * must be called under lgr->llc_conf_mutex lock
1960 */
7562a13d
KG		 1961int smcr_link_reg_rmb(struct smc_link *link, struct smc_buf_desc *rmb_desc)
1962{
1963 if (list_empty(&link->lgr->list))
1964 return -ENOLINK;
1965 if (!rmb_desc->is_reg_mr[link->link_idx]) {
1966 /* register memory region for new rmb */
1967 if (smc_wr_reg_send(link, rmb_desc->mr_rx[link->link_idx])) {
1968 rmb_desc->is_reg_err = true;
1969 return -EFAULT;
1970 }
1971 rmb_desc->is_reg_mr[link->link_idx] = true;
1972 }
1973 return 0;
1974}
1975
fb33d277
KG		 1976static int _smcr_buf_map_lgr(struct smc_link *lnk, struct mutex *lock,
1977 struct list_head *lst, bool is_rmb)
1978{
1979 struct smc_buf_desc *buf_desc, *bf;
1980 int rc = 0;
1981
1982 mutex_lock(lock);
1983 list_for_each_entry_safe(buf_desc, bf, lst, list) {
1984 if (!buf_desc->used)
1985 continue;
1986 rc = smcr_buf_map_link(buf_desc, is_rmb, lnk);
1987 if (rc)
1988 goto out;
1989 }
1990out:
1991 mutex_unlock(lock);
1992 return rc;
1993}
1994
1995/* map all used buffers of lgr for a new link */
1996int smcr_buf_map_lgr(struct smc_link *lnk)
1997{
1998 struct smc_link_group *lgr = lnk->lgr;
1999 int i, rc = 0;
2000
2001 for (i = 0; i < SMC_RMBE_SIZES; i++) {
2002 rc = _smcr_buf_map_lgr(lnk, &lgr->rmbs_lock,
2003 &lgr->rmbs[i], true);
2004 if (rc)
2005 return rc;
2006 rc = _smcr_buf_map_lgr(lnk, &lgr->sndbufs_lock,
2007 &lgr->sndbufs[i], false);
2008 if (rc)
2009 return rc;
2010 }
2011 return 0;
2012}
2013
d5500667
KG		 2014/* register all used buffers of lgr for a new link,
2015 * must be called under lgr->llc_conf_mutex lock
2016 */
fb33d277
KG		 2017int smcr_buf_reg_lgr(struct smc_link *lnk)
2018{
2019 struct smc_link_group *lgr = lnk->lgr;
2020 struct smc_buf_desc *buf_desc, *bf;
2021 int i, rc = 0;
2022
2023 mutex_lock(&lgr->rmbs_lock);
2024 for (i = 0; i < SMC_RMBE_SIZES; i++) {
2025 list_for_each_entry_safe(buf_desc, bf, &lgr->rmbs[i], list) {
2026 if (!buf_desc->used)
2027 continue;
2028 rc = smcr_link_reg_rmb(lnk, buf_desc);
2029 if (rc)
2030 goto out;
2031 }
2032 }
2033out:
2034 mutex_unlock(&lgr->rmbs_lock);
2035 return rc;
2036}
2037
c6ba7c9b
HW		 2038static struct smc_buf_desc *smcr_new_buf_create(struct smc_link_group *lgr,
2039 bool is_rmb, int bufsize)
b33982c3
UB		 2040{
2041 struct smc_buf_desc *buf_desc;
b33982c3
UB		 2042
2043 /* try to alloc a new buffer */
2044 buf_desc = kzalloc(sizeof(*buf_desc), GFP_KERNEL);
2045 if (!buf_desc)
2046 return ERR_PTR(-ENOMEM);
2047
2ef4f27a
SR		 2048 buf_desc->order = get_order(bufsize);
2049 buf_desc->pages = alloc_pages(GFP_KERNEL | __GFP_NOWARN |
2050 __GFP_NOMEMALLOC | __GFP_COMP |
2051 __GFP_NORETRY | __GFP_ZERO,
2052 buf_desc->order);
2053 if (!buf_desc->pages) {
b33982c3
UB		 2054 kfree(buf_desc);
2055 return ERR_PTR(-EAGAIN);
2056 }
2ef4f27a 2057 buf_desc->cpu_addr = (void *)page_address(buf_desc->pages);
b9247544
KG		 2058 buf_desc->len = bufsize;
2059 return buf_desc;
2060}
b33982c3 2061
b9247544
KG		 2062/* map buf_desc on all usable links,
2063 * unused buffers stay mapped as long as the link is up
2064 */
2065static int smcr_buf_map_usable_links(struct smc_link_group *lgr,
2066 struct smc_buf_desc *buf_desc, bool is_rmb)
2067{
2068 int i, rc = 0;
b33982c3 2069
d5500667
KG		 2070 /* protect against parallel link reconfiguration */
2071 mutex_lock(&lgr->llc_conf_mutex);
b9247544
KG		 2072 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
2073 struct smc_link *lnk = &lgr->lnk[i];
b33982c3 2074
d854fcbf 2075 if (!smc_link_usable(lnk))
b9247544
KG		 2076 continue;
2077 if (smcr_buf_map_link(buf_desc, is_rmb, lnk)) {
b9247544
KG		 2078 rc = -ENOMEM;
2079 goto out;
b33982c3
UB		 2080 }
2081 }
b9247544 2082out:
d5500667 2083 mutex_unlock(&lgr->llc_conf_mutex);
b9247544 2084 return rc;
b33982c3
UB		 2085}
2086
c6ba7c9b
HW		 2087static struct smc_buf_desc *smcd_new_buf_create(struct smc_link_group *lgr,
2088 bool is_dmb, int bufsize)
2089{
2090 struct smc_buf_desc *buf_desc;
2091 int rc;
2092
c6ba7c9b
HW		 2093 /* try to alloc a new DMB */
2094 buf_desc = kzalloc(sizeof(*buf_desc), GFP_KERNEL);
2095 if (!buf_desc)
2096 return ERR_PTR(-ENOMEM);
2097 if (is_dmb) {
2098 rc = smc_ism_register_dmb(lgr, bufsize, buf_desc);
2099 if (rc) {
2100 kfree(buf_desc);
96d6fded
KG		 2101 if (rc == -ENOMEM)
2102 return ERR_PTR(-EAGAIN);
2103 if (rc == -ENOSPC)
2104 return ERR_PTR(-ENOSPC);
2105 return ERR_PTR(-EIO);
c6ba7c9b 2106 }
be244f28
HW		 2107 buf_desc->pages = virt_to_page(buf_desc->cpu_addr);
2108 /* CDC header stored in buf. So, pretend it was smaller */
2109 buf_desc->len = bufsize - sizeof(struct smcd_cdc_msg);
c6ba7c9b
HW		 2110 } else {
2111 buf_desc->cpu_addr = kzalloc(bufsize, GFP_KERNEL |
2112 __GFP_NOWARN | __GFP_NORETRY |
2113 __GFP_NOMEMALLOC);
2114 if (!buf_desc->cpu_addr) {
2115 kfree(buf_desc);
2116 return ERR_PTR(-EAGAIN);
2117 }
2118 buf_desc->len = bufsize;
2119 }
2120 return buf_desc;
2121}
2122
2123static int __smc_buf_create(struct smc_sock *smc, bool is_smcd, bool is_rmb)
cd6851f3 2124{
8437bda0 2125 struct smc_buf_desc *buf_desc = ERR_PTR(-ENOMEM);
cd6851f3
UB		 2126 struct smc_connection *conn = &smc->conn;
2127 struct smc_link_group *lgr = conn->lgr;
3e034725 2128 struct list_head *buf_list;
c45abf31 2129 int bufsize, bufsize_short;
e0e4b8fa 2130 bool is_dgraded = false;
faca5360 2131 struct mutex *lock; /* lock buffer list */
3e034725 2132 int sk_buf_size;
cd6851f3 2133
3e034725
UB		 2134 if (is_rmb)
2135 /* use socket recv buffer size (w/o overhead) as start value */
2136 sk_buf_size = smc->sk.sk_rcvbuf / 2;
2137 else
2138 /* use socket send buffer size (w/o overhead) as start value */
2139 sk_buf_size = smc->sk.sk_sndbuf / 2;
2140
67161779 2141 for (bufsize_short = smc_compress_bufsize(sk_buf_size, is_smcd, is_rmb);
c45abf31 2142 bufsize_short >= 0; bufsize_short--) {
3e034725
UB		 2143 if (is_rmb) {
2144 lock = &lgr->rmbs_lock;
2145 buf_list = &lgr->rmbs[bufsize_short];
2146 } else {
2147 lock = &lgr->sndbufs_lock;
2148 buf_list = &lgr->sndbufs[bufsize_short];
9d8fb617 2149 }
c45abf31 2150 bufsize = smc_uncompress_bufsize(bufsize_short);
a3fe3d01 2151
3e034725 2152 /* check for reusable slot in the link group */
8437bda0 2153 buf_desc = smc_buf_get_slot(bufsize_short, lock, buf_list);
3e034725 2154 if (buf_desc) {
194730a9
GG		 2155 SMC_STAT_RMB_SIZE(smc, is_smcd, is_rmb, bufsize);
2156 SMC_STAT_BUF_REUSE(smc, is_smcd, is_rmb);
cd6851f3
UB		 2157 break; /* found reusable slot */
2158 }
a3fe3d01 2159
c6ba7c9b
HW		 2160 if (is_smcd)
2161 buf_desc = smcd_new_buf_create(lgr, is_rmb, bufsize);
2162 else
2163 buf_desc = smcr_new_buf_create(lgr, is_rmb, bufsize);
2164
b33982c3
UB		 2165 if (PTR_ERR(buf_desc) == -ENOMEM)
2166 break;
e0e4b8fa
GG		 2167 if (IS_ERR(buf_desc)) {
2168 if (!is_dgraded) {
2169 is_dgraded = true;
194730a9 2170 SMC_STAT_RMB_DOWNGRADED(smc, is_smcd, is_rmb);
e0e4b8fa 2171 }
a3fe3d01 2172 continue;
e0e4b8fa 2173 }
897e1c24 2174
194730a9
GG		 2175 SMC_STAT_RMB_ALLOC(smc, is_smcd, is_rmb);
2176 SMC_STAT_RMB_SIZE(smc, is_smcd, is_rmb, bufsize);
3e034725 2177 buf_desc->used = 1;
faca5360 2178 mutex_lock(lock);
3e034725 2179 list_add(&buf_desc->list, buf_list);
faca5360 2180 mutex_unlock(lock);
3e034725 2181 break; /* found */
cd6851f3 2182 }
3e034725 2183
b33982c3 2184 if (IS_ERR(buf_desc))
72b7f6c4 2185 return PTR_ERR(buf_desc);
3e034725 2186
b9247544
KG		 2187 if (!is_smcd) {
2188 if (smcr_buf_map_usable_links(lgr, buf_desc, is_rmb)) {
6d74c3a8 2189 smcr_buf_unuse(buf_desc, lgr);
b9247544
KG		 2190 return -ENOMEM;
2191 }
2192 }
2193
3e034725
UB		 2194 if (is_rmb) {
2195 conn->rmb_desc = buf_desc;
c45abf31
UB		 2196 conn->rmbe_size_short = bufsize_short;
2197 smc->sk.sk_rcvbuf = bufsize * 2;
5f08318f 2198 atomic_set(&conn->bytes_to_rcv, 0);
be244f28
HW		 2199 conn->rmbe_update_limit =
2200 smc_rmb_wnd_update_limit(buf_desc->len);
c6ba7c9b
HW		 2201 if (is_smcd)
2202 smc_ism_set_conn(conn); /* map RMB/smcd_dev to conn */
cd6851f3 2203 } else {
3e034725 2204 conn->sndbuf_desc = buf_desc;
3e034725
UB		 2205 smc->sk.sk_sndbuf = bufsize * 2;
2206 atomic_set(&conn->sndbuf_space, bufsize);
cd6851f3 2207 }
3e034725
UB		 2208 return 0;
2209}
2210
10428dd8
UB		 2211void smc_sndbuf_sync_sg_for_cpu(struct smc_connection *conn)
2212{
741a49a4 2213 if (!conn->lgr || conn->lgr->is_smcd || !smc_link_active(conn->lnk))
c6ba7c9b 2214 return;
387707fd 2215 smc_ib_sync_sg_for_cpu(conn->lnk, conn->sndbuf_desc, DMA_TO_DEVICE);
10428dd8
UB		 2216}
2217
2218void smc_sndbuf_sync_sg_for_device(struct smc_connection *conn)
2219{
741a49a4 2220 if (!conn->lgr || conn->lgr->is_smcd || !smc_link_active(conn->lnk))
c6ba7c9b 2221 return;
387707fd 2222 smc_ib_sync_sg_for_device(conn->lnk, conn->sndbuf_desc, DMA_TO_DEVICE);
10428dd8
UB		 2223}
2224
2225void smc_rmb_sync_sg_for_cpu(struct smc_connection *conn)
2226{
b9247544 2227 int i;
10428dd8 2228
c6ba7c9b
HW		 2229 if (!conn->lgr || conn->lgr->is_smcd)
2230 return;
b9247544 2231 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
741a49a4 2232 if (!smc_link_active(&conn->lgr->lnk[i]))
b9247544
KG		 2233 continue;
2234 smc_ib_sync_sg_for_cpu(&conn->lgr->lnk[i], conn->rmb_desc,
2235 DMA_FROM_DEVICE);
2236 }
10428dd8
UB		 2237}
2238
2239void smc_rmb_sync_sg_for_device(struct smc_connection *conn)
2240{
b9247544 2241 int i;
10428dd8 2242
c6ba7c9b
HW		 2243 if (!conn->lgr || conn->lgr->is_smcd)
2244 return;
b9247544 2245 for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) {
741a49a4 2246 if (!smc_link_active(&conn->lgr->lnk[i]))
b9247544
KG		 2247 continue;
2248 smc_ib_sync_sg_for_device(&conn->lgr->lnk[i], conn->rmb_desc,
2249 DMA_FROM_DEVICE);
2250 }
10428dd8
UB		 2251}
2252
3e034725
UB		 2253/* create the send and receive buffer for an SMC socket;
2254 * receive buffers are called RMBs;
2255 * (even though the SMC protocol allows more than one RMB-element per RMB,
2256 * the Linux implementation uses just one RMB-element per RMB, i.e. uses an
2257 * extra RMB for every connection in a link group
2258 */
c6ba7c9b 2259int smc_buf_create(struct smc_sock *smc, bool is_smcd)
3e034725
UB		 2260{
2261 int rc;
2262
2263 /* create send buffer */
c6ba7c9b 2264 rc = __smc_buf_create(smc, is_smcd, false);
3e034725
UB		 2265 if (rc)
2266 return rc;
2267 /* create rmb */
c6ba7c9b 2268 rc = __smc_buf_create(smc, is_smcd, true);
fd7f3a74
KG		 2269 if (rc) {
2270 mutex_lock(&smc->conn.lgr->sndbufs_lock);
2271 list_del(&smc->conn.sndbuf_desc->list);
2272 mutex_unlock(&smc->conn.lgr->sndbufs_lock);
6511aad3 2273 smc_buf_free(smc->conn.lgr, false, smc->conn.sndbuf_desc);
1d8df41d 2274 smc->conn.sndbuf_desc = NULL;
fd7f3a74 2275 }
3e034725 2276 return rc;
cd6851f3 2277}
bd4ad577
UB		 2278
2279static inline int smc_rmb_reserve_rtoken_idx(struct smc_link_group *lgr)
2280{
2281 int i;
2282
2283 for_each_clear_bit(i, lgr->rtokens_used_mask, SMC_RMBS_PER_LGR_MAX) {
2284 if (!test_and_set_bit(i, lgr->rtokens_used_mask))
2285 return i;
2286 }
2287 return -ENOSPC;
2288}
2289
ba21abd2
KG		 2290static int smc_rtoken_find_by_link(struct smc_link_group *lgr, int lnk_idx,
2291 u32 rkey)
2292{
2293 int i;
2294
2295 for (i = 0; i < SMC_RMBS_PER_LGR_MAX; i++) {
2296 if (test_bit(i, lgr->rtokens_used_mask) &&
2297 lgr->rtokens[i][lnk_idx].rkey == rkey)
2298 return i;
2299 }
2300 return -ENOENT;
2301}
2302
2303/* set rtoken for a new link to an existing rmb */
2304void smc_rtoken_set(struct smc_link_group *lgr, int link_idx, int link_idx_new,
2305 __be32 nw_rkey_known, __be64 nw_vaddr, __be32 nw_rkey)
2306{
2307 int rtok_idx;
2308
2309 rtok_idx = smc_rtoken_find_by_link(lgr, link_idx, ntohl(nw_rkey_known));
2310 if (rtok_idx == -ENOENT)
2311 return;
2312 lgr->rtokens[rtok_idx][link_idx_new].rkey = ntohl(nw_rkey);
2313 lgr->rtokens[rtok_idx][link_idx_new].dma_addr = be64_to_cpu(nw_vaddr);
2314}
2315
2316/* set rtoken for a new link whose link_id is given */
2317void smc_rtoken_set2(struct smc_link_group *lgr, int rtok_idx, int link_id,
2318 __be64 nw_vaddr, __be32 nw_rkey)
2319{
2320 u64 dma_addr = be64_to_cpu(nw_vaddr);
2321 u32 rkey = ntohl(nw_rkey);
2322 bool found = false;
2323 int link_idx;
2324
2325 for (link_idx = 0; link_idx < SMC_LINKS_PER_LGR_MAX; link_idx++) {
2326 if (lgr->lnk[link_idx].link_id == link_id) {
2327 found = true;
2328 break;
2329 }
2330 }
2331 if (!found)
2332 return;
2333 lgr->rtokens[rtok_idx][link_idx].rkey = rkey;
2334 lgr->rtokens[rtok_idx][link_idx].dma_addr = dma_addr;
2335}
2336
4ed75de5 2337/* add a new rtoken from peer */
387707fd 2338int smc_rtoken_add(struct smc_link *lnk, __be64 nw_vaddr, __be32 nw_rkey)
bd4ad577 2339{
387707fd 2340 struct smc_link_group *lgr = smc_get_lgr(lnk);
4ed75de5
KG		 2341 u64 dma_addr = be64_to_cpu(nw_vaddr);
2342 u32 rkey = ntohl(nw_rkey);
bd4ad577
UB		 2343 int i;
2344
2345 for (i = 0; i < SMC_RMBS_PER_LGR_MAX; i++) {
387707fd
KG		 2346 if (lgr->rtokens[i][lnk->link_idx].rkey == rkey &&
2347 lgr->rtokens[i][lnk->link_idx].dma_addr == dma_addr &&
bd4ad577 2348 test_bit(i, lgr->rtokens_used_mask)) {
4ed75de5
KG		 2349 /* already in list */
2350 return i;
2351 }
2352 }
2353 i = smc_rmb_reserve_rtoken_idx(lgr);
2354 if (i < 0)
2355 return i;
387707fd
KG		 2356 lgr->rtokens[i][lnk->link_idx].rkey = rkey;
2357 lgr->rtokens[i][lnk->link_idx].dma_addr = dma_addr;
4ed75de5
KG		 2358 return i;
2359}
2360
e07d31dc 2361/* delete an rtoken from all links */
387707fd 2362int smc_rtoken_delete(struct smc_link *lnk, __be32 nw_rkey)
4ed75de5 2363{
387707fd 2364 struct smc_link_group *lgr = smc_get_lgr(lnk);
4ed75de5 2365 u32 rkey = ntohl(nw_rkey);
e07d31dc 2366 int i, j;
4ed75de5
KG		 2367
2368 for (i = 0; i < SMC_RMBS_PER_LGR_MAX; i++) {
387707fd 2369 if (lgr->rtokens[i][lnk->link_idx].rkey == rkey &&
4ed75de5 2370 test_bit(i, lgr->rtokens_used_mask)) {
e07d31dc
KG		 2371 for (j = 0; j < SMC_LINKS_PER_LGR_MAX; j++) {
2372 lgr->rtokens[i][j].rkey = 0;
2373 lgr->rtokens[i][j].dma_addr = 0;
2374 }
4ed75de5 2375 clear_bit(i, lgr->rtokens_used_mask);
bd4ad577
UB		 2376 return 0;
2377 }
2378 }
4ed75de5
KG		 2379 return -ENOENT;
2380}
2381
2382/* save rkey and dma_addr received from peer during clc handshake */
2383int smc_rmb_rtoken_handling(struct smc_connection *conn,
e07d31dc 2384 struct smc_link *lnk,
4ed75de5
KG		 2385 struct smc_clc_msg_accept_confirm *clc)
2386{
3d9725a6
UB		 2387 conn->rtoken_idx = smc_rtoken_add(lnk, clc->r0.rmb_dma_addr,
2388 clc->r0.rmb_rkey);
bd4ad577
UB		 2389 if (conn->rtoken_idx < 0)
2390 return conn->rtoken_idx;
bd4ad577
UB		 2391 return 0;
2392}
9fda3510 2393
c3d9494e
UB		 2394static void smc_core_going_away(void)
2395{
2396 struct smc_ib_device *smcibdev;
2397 struct smcd_dev *smcd;
2398
92f3cb0e 2399 mutex_lock(&smc_ib_devices.mutex);
c3d9494e
UB		 2400 list_for_each_entry(smcibdev, &smc_ib_devices.list, list) {
2401 int i;
2402
2403 for (i = 0; i < SMC_MAX_PORTS; i++)
2404 set_bit(i, smcibdev->ports_going_away);
2405 }
92f3cb0e 2406 mutex_unlock(&smc_ib_devices.mutex);
c3d9494e 2407
82087c03 2408 mutex_lock(&smcd_dev_list.mutex);
c3d9494e
UB		 2409 list_for_each_entry(smcd, &smcd_dev_list.list, list) {
2410 smcd->going_away = 1;
2411 }
82087c03 2412 mutex_unlock(&smcd_dev_list.mutex);
c3d9494e
UB		 2413}
2414
5421ec28
UB		 2415/* Clean up all SMC link groups */
2416static void smc_lgrs_shutdown(void)
9fda3510 2417{
a2351c5d 2418 struct smcd_dev *smcd;
9fda3510 2419
c3d9494e
UB		 2420 smc_core_going_away();
2421
0b29ec64 2422 smc_smcr_terminate_all(NULL);
a2351c5d 2423
82087c03 2424 mutex_lock(&smcd_dev_list.mutex);
a2351c5d 2425 list_for_each_entry(smcd, &smcd_dev_list.list, list)
5421ec28 2426 smc_smcd_terminate_all(smcd);
82087c03 2427 mutex_unlock(&smcd_dev_list.mutex);
9fda3510 2428}
5421ec28 2429
a33a803c
UB		 2430static int smc_core_reboot_event(struct notifier_block *this,
2431 unsigned long event, void *ptr)
2432{
2433 smc_lgrs_shutdown();
28a3b840 2434 smc_ib_unregister_client();
a33a803c
UB		 2435 return 0;
2436}
2437
2438static struct notifier_block smc_reboot_notifier = {
2439 .notifier_call = smc_core_reboot_event,
2440};
2441
6dabd405
UB		 2442int __init smc_core_init(void)
2443{
a33a803c 2444 return register_reboot_notifier(&smc_reboot_notifier);
6dabd405
UB		 2445}
2446
5421ec28
UB		 2447/* Called (from smc_exit) when module is removed */
2448void smc_core_exit(void)
2449{
a33a803c 2450 unregister_reboot_notifier(&smc_reboot_notifier);
5421ec28
UB		 2451 smc_lgrs_shutdown();
2452}
Linux 6.x block layer and io_uring tree(s)