Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
0cfdd8f9 UB |
2 | /* |
3 | * Shared Memory Communications over RDMA (SMC-R) and RoCE | |
4 | * | |
5 | * Basic Transport Functions exploiting Infiniband API | |
6 | * | |
7 | * Copyright IBM Corp. 2016 | |
8 | * | |
9 | * Author(s): Ursula Braun <ubraun@linux.vnet.ibm.com> | |
10 | */ | |
11 | ||
12 | #include <linux/socket.h> | |
13 | #include <linux/if_vlan.h> | |
14 | #include <linux/random.h> | |
15 | #include <linux/workqueue.h> | |
6dabd405 | 16 | #include <linux/wait.h> |
a33a803c | 17 | #include <linux/reboot.h> |
92f3cb0e | 18 | #include <linux/mutex.h> |
099b990b GG |
19 | #include <linux/list.h> |
20 | #include <linux/smc.h> | |
0cfdd8f9 UB |
21 | #include <net/tcp.h> |
22 | #include <net/sock.h> | |
23 | #include <rdma/ib_verbs.h> | |
ddb457c6 | 24 | #include <rdma/ib_cache.h> |
0cfdd8f9 UB |
25 | |
26 | #include "smc.h" | |
27 | #include "smc_clc.h" | |
28 | #include "smc_core.h" | |
29 | #include "smc_ib.h" | |
f38ba179 | 30 | #include "smc_wr.h" |
9bf9abea | 31 | #include "smc_llc.h" |
5f08318f | 32 | #include "smc_cdc.h" |
b38d7324 | 33 | #include "smc_close.h" |
c6ba7c9b | 34 | #include "smc_ism.h" |
099b990b | 35 | #include "smc_netlink.h" |
e0e4b8fa | 36 | #include "smc_stats.h" |
a3a0e81b | 37 | #include "smc_tracepoint.h" |
0cfdd8f9 | 38 | |
5bc11ddb UB |
39 | #define SMC_LGR_NUM_INCR 256 |
40 | #define SMC_LGR_FREE_DELAY_SERV (600 * HZ) | |
7f58a1ad | 41 | #define SMC_LGR_FREE_DELAY_CLNT (SMC_LGR_FREE_DELAY_SERV + 10 * HZ) |
0cfdd8f9 | 42 | |
a3db10ef | 43 | struct smc_lgr_list smc_lgr_list = { /* established link groups */ |
9fda3510 HW |
44 | .lock = __SPIN_LOCK_UNLOCKED(smc_lgr_list.lock), |
45 | .list = LIST_HEAD_INIT(smc_lgr_list.list), | |
46 | .num = 0, | |
47 | }; | |
9bf9abea | 48 | |
29115cef | 49 | static atomic_t lgr_cnt = ATOMIC_INIT(0); /* number of existing link groups */ |
6dabd405 UB |
50 | static DECLARE_WAIT_QUEUE_HEAD(lgrs_deleted); |
51 | ||
6511aad3 HW |
52 | static void smc_buf_free(struct smc_link_group *lgr, bool is_rmb, |
53 | struct smc_buf_desc *buf_desc); | |
5f78fe96 | 54 | static void __smc_lgr_terminate(struct smc_link_group *lgr, bool soft); |
a6920d1d | 55 | |
541afa10 | 56 | static void smc_link_down_work(struct work_struct *work); |
1f90a05d | 57 | |
a0a62ee1 UB |
58 | /* return head of link group list and its lock for a given link group */ |
59 | static inline struct list_head *smc_lgr_list_head(struct smc_link_group *lgr, | |
60 | spinlock_t **lgr_lock) | |
61 | { | |
62 | if (lgr->is_smcd) { | |
63 | *lgr_lock = &lgr->smcd->lgr_lock; | |
64 | return &lgr->smcd->lgr_list; | |
65 | } | |
66 | ||
67 | *lgr_lock = &smc_lgr_list.lock; | |
68 | return &smc_lgr_list.list; | |
69 | } | |
70 | ||
ddc99286 GG |
71 | static void smc_ibdev_cnt_inc(struct smc_link *lnk) |
72 | { | |
73 | atomic_inc(&lnk->smcibdev->lnk_cnt_by_port[lnk->ibport - 1]); | |
74 | } | |
75 | ||
76 | static void smc_ibdev_cnt_dec(struct smc_link *lnk) | |
77 | { | |
78 | atomic_dec(&lnk->smcibdev->lnk_cnt_by_port[lnk->ibport - 1]); | |
79 | } | |
80 | ||
97cdbc42 KG |
81 | static void smc_lgr_schedule_free_work(struct smc_link_group *lgr) |
82 | { | |
83 | /* client link group creation always follows the server link group | |
84 | * creation. For client use a somewhat higher removal delay time, | |
85 | * otherwise there is a risk of out-of-sync link groups. | |
86 | */ | |
f9aab6f2 | 87 | if (!lgr->freeing) { |
8e316b9e UB |
88 | mod_delayed_work(system_wq, &lgr->free_work, |
89 | (!lgr->is_smcd && lgr->role == SMC_CLNT) ? | |
90 | SMC_LGR_FREE_DELAY_CLNT : | |
91 | SMC_LGR_FREE_DELAY_SERV); | |
92 | } | |
97cdbc42 KG |
93 | } |
94 | ||
0cfdd8f9 UB |
95 | /* Register connection's alert token in our lookup structure. |
96 | * To use rbtrees we have to implement our own insert core. | |
97 | * Requires @conns_lock | |
98 | * @smc connection to register | |
99 | * Returns 0 on success, != otherwise. | |
100 | */ | |
101 | static void smc_lgr_add_alert_token(struct smc_connection *conn) | |
102 | { | |
103 | struct rb_node **link, *parent = NULL; | |
104 | u32 token = conn->alert_token_local; | |
105 | ||
106 | link = &conn->lgr->conns_all.rb_node; | |
107 | while (*link) { | |
108 | struct smc_connection *cur = rb_entry(*link, | |
109 | struct smc_connection, alert_node); | |
110 | ||
111 | parent = *link; | |
112 | if (cur->alert_token_local > token) | |
113 | link = &parent->rb_left; | |
114 | else | |
115 | link = &parent->rb_right; | |
116 | } | |
117 | /* Put the new node there */ | |
118 | rb_link_node(&conn->alert_node, parent, link); | |
119 | rb_insert_color(&conn->alert_node, &conn->lgr->conns_all); | |
120 | } | |
121 | ||
56bc3b20 KG |
122 | /* assign an SMC-R link to the connection */ |
123 | static int smcr_lgr_conn_assign_link(struct smc_connection *conn, bool first) | |
124 | { | |
125 | enum smc_link_state expected = first ? SMC_LNK_ACTIVATING : | |
126 | SMC_LNK_ACTIVE; | |
127 | int i, j; | |
128 | ||
129 | /* do link balancing */ | |
35112271 | 130 | conn->lnk = NULL; /* reset conn->lnk first */ |
56bc3b20 KG |
131 | for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { |
132 | struct smc_link *lnk = &conn->lgr->lnk[i]; | |
133 | ||
ad6c111b | 134 | if (lnk->state != expected || lnk->link_is_asym) |
56bc3b20 KG |
135 | continue; |
136 | if (conn->lgr->role == SMC_CLNT) { | |
137 | conn->lnk = lnk; /* temporary, SMC server assigns link*/ | |
138 | break; | |
139 | } | |
140 | if (conn->lgr->conns_num % 2) { | |
141 | for (j = i + 1; j < SMC_LINKS_PER_LGR_MAX; j++) { | |
142 | struct smc_link *lnk2; | |
143 | ||
144 | lnk2 = &conn->lgr->lnk[j]; | |
ad6c111b KG |
145 | if (lnk2->state == expected && |
146 | !lnk2->link_is_asym) { | |
56bc3b20 KG |
147 | conn->lnk = lnk2; |
148 | break; | |
149 | } | |
150 | } | |
151 | } | |
152 | if (!conn->lnk) | |
153 | conn->lnk = lnk; | |
154 | break; | |
155 | } | |
156 | if (!conn->lnk) | |
157 | return SMC_CLC_DECL_NOACTLINK; | |
07d51580 | 158 | atomic_inc(&conn->lnk->conn_cnt); |
56bc3b20 KG |
159 | return 0; |
160 | } | |
161 | ||
0cfdd8f9 UB |
162 | /* Register connection in link group by assigning an alert token |
163 | * registered in a search tree. | |
164 | * Requires @conns_lock | |
165 | * Note that '0' is a reserved value and not assigned. | |
166 | */ | |
56bc3b20 | 167 | static int smc_lgr_register_conn(struct smc_connection *conn, bool first) |
0cfdd8f9 UB |
168 | { |
169 | struct smc_sock *smc = container_of(conn, struct smc_sock, conn); | |
170 | static atomic_t nexttoken = ATOMIC_INIT(0); | |
56bc3b20 | 171 | int rc; |
0cfdd8f9 | 172 | |
56bc3b20 KG |
173 | if (!conn->lgr->is_smcd) { |
174 | rc = smcr_lgr_conn_assign_link(conn, first); | |
36595d8a WG |
175 | if (rc) { |
176 | conn->lgr = NULL; | |
56bc3b20 | 177 | return rc; |
36595d8a | 178 | } |
56bc3b20 | 179 | } |
0cfdd8f9 UB |
180 | /* find a new alert_token_local value not yet used by some connection |
181 | * in this link group | |
182 | */ | |
183 | sock_hold(&smc->sk); /* sock_put in smc_lgr_unregister_conn() */ | |
184 | while (!conn->alert_token_local) { | |
185 | conn->alert_token_local = atomic_inc_return(&nexttoken); | |
186 | if (smc_lgr_find_conn(conn->alert_token_local, conn->lgr)) | |
187 | conn->alert_token_local = 0; | |
188 | } | |
189 | smc_lgr_add_alert_token(conn); | |
190 | conn->lgr->conns_num++; | |
b9247544 | 191 | return 0; |
0cfdd8f9 UB |
192 | } |
193 | ||
194 | /* Unregister connection and reset the alert token of the given connection< | |
195 | */ | |
196 | static void __smc_lgr_unregister_conn(struct smc_connection *conn) | |
197 | { | |
198 | struct smc_sock *smc = container_of(conn, struct smc_sock, conn); | |
199 | struct smc_link_group *lgr = conn->lgr; | |
200 | ||
201 | rb_erase(&conn->alert_node, &lgr->conns_all); | |
07d51580 GG |
202 | if (conn->lnk) |
203 | atomic_dec(&conn->lnk->conn_cnt); | |
0cfdd8f9 UB |
204 | lgr->conns_num--; |
205 | conn->alert_token_local = 0; | |
0cfdd8f9 UB |
206 | sock_put(&smc->sk); /* sock_hold in smc_lgr_register_conn() */ |
207 | } | |
208 | ||
fb692ec4 | 209 | /* Unregister connection from lgr |
0cfdd8f9 UB |
210 | */ |
211 | static void smc_lgr_unregister_conn(struct smc_connection *conn) | |
212 | { | |
213 | struct smc_link_group *lgr = conn->lgr; | |
0cfdd8f9 | 214 | |
ea89c6c0 | 215 | if (!smc_conn_lgr_valid(conn)) |
77f838ac | 216 | return; |
0cfdd8f9 UB |
217 | write_lock_bh(&lgr->conns_lock); |
218 | if (conn->alert_token_local) { | |
0cfdd8f9 UB |
219 | __smc_lgr_unregister_conn(conn); |
220 | } | |
221 | write_unlock_bh(&lgr->conns_lock); | |
0cfdd8f9 UB |
222 | } |
223 | ||
d386d59b WG |
224 | static void smc_lgr_buf_list_add(struct smc_link_group *lgr, |
225 | bool is_rmb, | |
226 | struct list_head *buf_list, | |
227 | struct smc_buf_desc *buf_desc) | |
228 | { | |
229 | list_add(&buf_desc->list, buf_list); | |
230 | if (is_rmb) { | |
231 | lgr->alloc_rmbs += buf_desc->len; | |
232 | lgr->alloc_rmbs += | |
233 | lgr->is_smcd ? sizeof(struct smcd_cdc_msg) : 0; | |
234 | } else { | |
235 | lgr->alloc_sndbufs += buf_desc->len; | |
236 | } | |
237 | } | |
238 | ||
239 | static void smc_lgr_buf_list_del(struct smc_link_group *lgr, | |
240 | bool is_rmb, | |
241 | struct smc_buf_desc *buf_desc) | |
242 | { | |
243 | list_del(&buf_desc->list); | |
244 | if (is_rmb) { | |
245 | lgr->alloc_rmbs -= buf_desc->len; | |
246 | lgr->alloc_rmbs -= | |
247 | lgr->is_smcd ? sizeof(struct smcd_cdc_msg) : 0; | |
248 | } else { | |
249 | lgr->alloc_sndbufs -= buf_desc->len; | |
250 | } | |
251 | } | |
252 | ||
099b990b GG |
253 | int smc_nl_get_sys_info(struct sk_buff *skb, struct netlink_callback *cb) |
254 | { | |
255 | struct smc_nl_dmp_ctx *cb_ctx = smc_nl_dmp_ctx(cb); | |
256 | char hostname[SMC_MAX_HOSTNAME_LEN + 1]; | |
257 | char smc_seid[SMC_MAX_EID_LEN + 1]; | |
099b990b GG |
258 | struct nlattr *attrs; |
259 | u8 *seid = NULL; | |
260 | u8 *host = NULL; | |
261 | void *nlh; | |
262 | ||
263 | nlh = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq, | |
264 | &smc_gen_nl_family, NLM_F_MULTI, | |
265 | SMC_NETLINK_GET_SYS_INFO); | |
266 | if (!nlh) | |
267 | goto errmsg; | |
268 | if (cb_ctx->pos[0]) | |
269 | goto errout; | |
270 | attrs = nla_nest_start(skb, SMC_GEN_SYS_INFO); | |
271 | if (!attrs) | |
272 | goto errout; | |
273 | if (nla_put_u8(skb, SMC_NLA_SYS_VER, SMC_V2)) | |
274 | goto errattr; | |
275 | if (nla_put_u8(skb, SMC_NLA_SYS_REL, SMC_RELEASE)) | |
276 | goto errattr; | |
277 | if (nla_put_u8(skb, SMC_NLA_SYS_IS_ISM_V2, smc_ism_is_v2_capable())) | |
278 | goto errattr; | |
b0539f5e KG |
279 | if (nla_put_u8(skb, SMC_NLA_SYS_IS_SMCR_V2, true)) |
280 | goto errattr; | |
099b990b GG |
281 | smc_clc_get_hostname(&host); |
282 | if (host) { | |
25fe2c9c JK |
283 | memcpy(hostname, host, SMC_MAX_HOSTNAME_LEN); |
284 | hostname[SMC_MAX_HOSTNAME_LEN] = 0; | |
099b990b GG |
285 | if (nla_put_string(skb, SMC_NLA_SYS_LOCAL_HOST, hostname)) |
286 | goto errattr; | |
287 | } | |
11a26c59 KG |
288 | if (smc_ism_is_v2_capable()) { |
289 | smc_ism_get_system_eid(&seid); | |
8a446536 GG |
290 | memcpy(smc_seid, seid, SMC_MAX_EID_LEN); |
291 | smc_seid[SMC_MAX_EID_LEN] = 0; | |
099b990b GG |
292 | if (nla_put_string(skb, SMC_NLA_SYS_SEID, smc_seid)) |
293 | goto errattr; | |
294 | } | |
295 | nla_nest_end(skb, attrs); | |
296 | genlmsg_end(skb, nlh); | |
297 | cb_ctx->pos[0] = 1; | |
298 | return skb->len; | |
299 | ||
300 | errattr: | |
301 | nla_nest_cancel(skb, attrs); | |
302 | errout: | |
303 | genlmsg_cancel(skb, nlh); | |
304 | errmsg: | |
305 | return skb->len; | |
306 | } | |
307 | ||
b0539f5e KG |
308 | /* Fill SMC_NLA_LGR_D_V2_COMMON/SMC_NLA_LGR_R_V2_COMMON nested attributes */ |
309 | static int smc_nl_fill_lgr_v2_common(struct smc_link_group *lgr, | |
310 | struct sk_buff *skb, | |
311 | struct netlink_callback *cb, | |
312 | struct nlattr *v2_attrs) | |
313 | { | |
314 | char smc_host[SMC_MAX_HOSTNAME_LEN + 1]; | |
315 | char smc_eid[SMC_MAX_EID_LEN + 1]; | |
316 | ||
317 | if (nla_put_u8(skb, SMC_NLA_LGR_V2_VER, lgr->smc_version)) | |
318 | goto errv2attr; | |
319 | if (nla_put_u8(skb, SMC_NLA_LGR_V2_REL, lgr->peer_smc_release)) | |
320 | goto errv2attr; | |
321 | if (nla_put_u8(skb, SMC_NLA_LGR_V2_OS, lgr->peer_os)) | |
322 | goto errv2attr; | |
323 | memcpy(smc_host, lgr->peer_hostname, SMC_MAX_HOSTNAME_LEN); | |
324 | smc_host[SMC_MAX_HOSTNAME_LEN] = 0; | |
325 | if (nla_put_string(skb, SMC_NLA_LGR_V2_PEER_HOST, smc_host)) | |
326 | goto errv2attr; | |
327 | memcpy(smc_eid, lgr->negotiated_eid, SMC_MAX_EID_LEN); | |
328 | smc_eid[SMC_MAX_EID_LEN] = 0; | |
329 | if (nla_put_string(skb, SMC_NLA_LGR_V2_NEG_EID, smc_eid)) | |
330 | goto errv2attr; | |
331 | ||
332 | nla_nest_end(skb, v2_attrs); | |
333 | return 0; | |
334 | ||
335 | errv2attr: | |
336 | nla_nest_cancel(skb, v2_attrs); | |
337 | return -EMSGSIZE; | |
338 | } | |
339 | ||
340 | static int smc_nl_fill_smcr_lgr_v2(struct smc_link_group *lgr, | |
341 | struct sk_buff *skb, | |
342 | struct netlink_callback *cb) | |
343 | { | |
344 | struct nlattr *v2_attrs; | |
345 | ||
346 | v2_attrs = nla_nest_start(skb, SMC_NLA_LGR_R_V2); | |
347 | if (!v2_attrs) | |
348 | goto errattr; | |
349 | if (nla_put_u8(skb, SMC_NLA_LGR_R_V2_DIRECT, !lgr->uses_gateway)) | |
350 | goto errv2attr; | |
bbed596c GW |
351 | if (nla_put_u8(skb, SMC_NLA_LGR_R_V2_MAX_CONNS, lgr->max_conns)) |
352 | goto errv2attr; | |
353 | if (nla_put_u8(skb, SMC_NLA_LGR_R_V2_MAX_LINKS, lgr->max_links)) | |
354 | goto errv2attr; | |
b0539f5e KG |
355 | |
356 | nla_nest_end(skb, v2_attrs); | |
357 | return 0; | |
358 | ||
359 | errv2attr: | |
360 | nla_nest_cancel(skb, v2_attrs); | |
361 | errattr: | |
362 | return -EMSGSIZE; | |
363 | } | |
364 | ||
e9b8c845 GG |
365 | static int smc_nl_fill_lgr(struct smc_link_group *lgr, |
366 | struct sk_buff *skb, | |
367 | struct netlink_callback *cb) | |
368 | { | |
369 | char smc_target[SMC_MAX_PNETID_LEN + 1]; | |
b0539f5e | 370 | struct nlattr *attrs, *v2_attrs; |
e9b8c845 GG |
371 | |
372 | attrs = nla_nest_start(skb, SMC_GEN_LGR_SMCR); | |
373 | if (!attrs) | |
374 | goto errout; | |
375 | ||
376 | if (nla_put_u32(skb, SMC_NLA_LGR_R_ID, *((u32 *)&lgr->id))) | |
377 | goto errattr; | |
378 | if (nla_put_u32(skb, SMC_NLA_LGR_R_CONNS_NUM, lgr->conns_num)) | |
379 | goto errattr; | |
380 | if (nla_put_u8(skb, SMC_NLA_LGR_R_ROLE, lgr->role)) | |
381 | goto errattr; | |
382 | if (nla_put_u8(skb, SMC_NLA_LGR_R_TYPE, lgr->type)) | |
383 | goto errattr; | |
ddefb2d2 WG |
384 | if (nla_put_u8(skb, SMC_NLA_LGR_R_BUF_TYPE, lgr->buf_type)) |
385 | goto errattr; | |
e9b8c845 GG |
386 | if (nla_put_u8(skb, SMC_NLA_LGR_R_VLAN_ID, lgr->vlan_id)) |
387 | goto errattr; | |
79d39fc5 TL |
388 | if (nla_put_u64_64bit(skb, SMC_NLA_LGR_R_NET_COOKIE, |
389 | lgr->net->net_cookie, SMC_NLA_LGR_R_PAD)) | |
390 | goto errattr; | |
8a446536 GG |
391 | memcpy(smc_target, lgr->pnet_id, SMC_MAX_PNETID_LEN); |
392 | smc_target[SMC_MAX_PNETID_LEN] = 0; | |
e9b8c845 GG |
393 | if (nla_put_string(skb, SMC_NLA_LGR_R_PNETID, smc_target)) |
394 | goto errattr; | |
d386d59b WG |
395 | if (nla_put_uint(skb, SMC_NLA_LGR_R_SNDBUF_ALLOC, lgr->alloc_sndbufs)) |
396 | goto errattr; | |
397 | if (nla_put_uint(skb, SMC_NLA_LGR_R_RMB_ALLOC, lgr->alloc_rmbs)) | |
398 | goto errattr; | |
b0539f5e KG |
399 | if (lgr->smc_version > SMC_V1) { |
400 | v2_attrs = nla_nest_start(skb, SMC_NLA_LGR_R_V2_COMMON); | |
401 | if (!v2_attrs) | |
402 | goto errattr; | |
403 | if (smc_nl_fill_lgr_v2_common(lgr, skb, cb, v2_attrs)) | |
404 | goto errattr; | |
405 | if (smc_nl_fill_smcr_lgr_v2(lgr, skb, cb)) | |
406 | goto errattr; | |
407 | } | |
e9b8c845 GG |
408 | |
409 | nla_nest_end(skb, attrs); | |
410 | return 0; | |
411 | errattr: | |
412 | nla_nest_cancel(skb, attrs); | |
413 | errout: | |
414 | return -EMSGSIZE; | |
415 | } | |
416 | ||
5a7e09d5 GG |
417 | static int smc_nl_fill_lgr_link(struct smc_link_group *lgr, |
418 | struct smc_link *link, | |
419 | struct sk_buff *skb, | |
420 | struct netlink_callback *cb) | |
421 | { | |
8a446536 | 422 | char smc_ibname[IB_DEVICE_NAME_MAX]; |
5a7e09d5 GG |
423 | u8 smc_gid_target[41]; |
424 | struct nlattr *attrs; | |
425 | u32 link_uid = 0; | |
426 | void *nlh; | |
427 | ||
428 | nlh = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq, | |
429 | &smc_gen_nl_family, NLM_F_MULTI, | |
430 | SMC_NETLINK_GET_LINK_SMCR); | |
431 | if (!nlh) | |
432 | goto errmsg; | |
433 | ||
434 | attrs = nla_nest_start(skb, SMC_GEN_LINK_SMCR); | |
435 | if (!attrs) | |
436 | goto errout; | |
437 | ||
438 | if (nla_put_u8(skb, SMC_NLA_LINK_ID, link->link_id)) | |
439 | goto errattr; | |
440 | if (nla_put_u32(skb, SMC_NLA_LINK_STATE, link->state)) | |
441 | goto errattr; | |
442 | if (nla_put_u32(skb, SMC_NLA_LINK_CONN_CNT, | |
443 | atomic_read(&link->conn_cnt))) | |
444 | goto errattr; | |
445 | if (nla_put_u8(skb, SMC_NLA_LINK_IB_PORT, link->ibport)) | |
446 | goto errattr; | |
447 | if (nla_put_u32(skb, SMC_NLA_LINK_NET_DEV, link->ndev_ifidx)) | |
448 | goto errattr; | |
449 | snprintf(smc_ibname, sizeof(smc_ibname), "%s", link->ibname); | |
450 | if (nla_put_string(skb, SMC_NLA_LINK_IB_DEV, smc_ibname)) | |
451 | goto errattr; | |
452 | memcpy(&link_uid, link->link_uid, sizeof(link_uid)); | |
453 | if (nla_put_u32(skb, SMC_NLA_LINK_UID, link_uid)) | |
454 | goto errattr; | |
455 | memcpy(&link_uid, link->peer_link_uid, sizeof(link_uid)); | |
456 | if (nla_put_u32(skb, SMC_NLA_LINK_PEER_UID, link_uid)) | |
457 | goto errattr; | |
458 | memset(smc_gid_target, 0, sizeof(smc_gid_target)); | |
459 | smc_gid_be16_convert(smc_gid_target, link->gid); | |
460 | if (nla_put_string(skb, SMC_NLA_LINK_GID, smc_gid_target)) | |
461 | goto errattr; | |
462 | memset(smc_gid_target, 0, sizeof(smc_gid_target)); | |
463 | smc_gid_be16_convert(smc_gid_target, link->peer_gid); | |
464 | if (nla_put_string(skb, SMC_NLA_LINK_PEER_GID, smc_gid_target)) | |
465 | goto errattr; | |
466 | ||
467 | nla_nest_end(skb, attrs); | |
468 | genlmsg_end(skb, nlh); | |
469 | return 0; | |
470 | errattr: | |
471 | nla_nest_cancel(skb, attrs); | |
472 | errout: | |
473 | genlmsg_cancel(skb, nlh); | |
474 | errmsg: | |
475 | return -EMSGSIZE; | |
476 | } | |
477 | ||
e9b8c845 GG |
478 | static int smc_nl_handle_lgr(struct smc_link_group *lgr, |
479 | struct sk_buff *skb, | |
5a7e09d5 GG |
480 | struct netlink_callback *cb, |
481 | bool list_links) | |
e9b8c845 GG |
482 | { |
483 | void *nlh; | |
5a7e09d5 | 484 | int i; |
e9b8c845 GG |
485 | |
486 | nlh = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq, | |
487 | &smc_gen_nl_family, NLM_F_MULTI, | |
488 | SMC_NETLINK_GET_LGR_SMCR); | |
489 | if (!nlh) | |
490 | goto errmsg; | |
491 | if (smc_nl_fill_lgr(lgr, skb, cb)) | |
492 | goto errout; | |
493 | ||
494 | genlmsg_end(skb, nlh); | |
5a7e09d5 GG |
495 | if (!list_links) |
496 | goto out; | |
497 | for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { | |
498 | if (!smc_link_usable(&lgr->lnk[i])) | |
499 | continue; | |
500 | if (smc_nl_fill_lgr_link(lgr, &lgr->lnk[i], skb, cb)) | |
501 | goto errout; | |
502 | } | |
503 | out: | |
e9b8c845 GG |
504 | return 0; |
505 | ||
506 | errout: | |
507 | genlmsg_cancel(skb, nlh); | |
508 | errmsg: | |
509 | return -EMSGSIZE; | |
510 | } | |
511 | ||
512 | static void smc_nl_fill_lgr_list(struct smc_lgr_list *smc_lgr, | |
513 | struct sk_buff *skb, | |
5a7e09d5 GG |
514 | struct netlink_callback *cb, |
515 | bool list_links) | |
e9b8c845 GG |
516 | { |
517 | struct smc_nl_dmp_ctx *cb_ctx = smc_nl_dmp_ctx(cb); | |
518 | struct smc_link_group *lgr; | |
519 | int snum = cb_ctx->pos[0]; | |
520 | int num = 0; | |
521 | ||
522 | spin_lock_bh(&smc_lgr->lock); | |
523 | list_for_each_entry(lgr, &smc_lgr->list, list) { | |
524 | if (num < snum) | |
525 | goto next; | |
5a7e09d5 | 526 | if (smc_nl_handle_lgr(lgr, skb, cb, list_links)) |
e9b8c845 GG |
527 | goto errout; |
528 | next: | |
529 | num++; | |
530 | } | |
531 | errout: | |
532 | spin_unlock_bh(&smc_lgr->lock); | |
533 | cb_ctx->pos[0] = num; | |
534 | } | |
535 | ||
8f9dde4b GG |
536 | static int smc_nl_fill_smcd_lgr(struct smc_link_group *lgr, |
537 | struct sk_buff *skb, | |
538 | struct netlink_callback *cb) | |
539 | { | |
8f9dde4b | 540 | char smc_pnet[SMC_MAX_PNETID_LEN + 1]; |
9de4df7b | 541 | struct smcd_dev *smcd = lgr->smcd; |
b40584d1 | 542 | struct smcd_gid smcd_gid; |
8f9dde4b GG |
543 | struct nlattr *attrs; |
544 | void *nlh; | |
545 | ||
546 | nlh = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq, | |
547 | &smc_gen_nl_family, NLM_F_MULTI, | |
548 | SMC_NETLINK_GET_LGR_SMCD); | |
549 | if (!nlh) | |
550 | goto errmsg; | |
551 | ||
552 | attrs = nla_nest_start(skb, SMC_GEN_LGR_SMCD); | |
553 | if (!attrs) | |
554 | goto errout; | |
555 | ||
556 | if (nla_put_u32(skb, SMC_NLA_LGR_D_ID, *((u32 *)&lgr->id))) | |
557 | goto errattr; | |
b40584d1 | 558 | smcd->ops->get_local_gid(smcd, &smcd_gid); |
9de4df7b | 559 | if (nla_put_u64_64bit(skb, SMC_NLA_LGR_D_GID, |
b40584d1 | 560 | smcd_gid.gid, SMC_NLA_LGR_D_PAD)) |
8f9dde4b | 561 | goto errattr; |
01fd1617 WG |
562 | if (nla_put_u64_64bit(skb, SMC_NLA_LGR_D_EXT_GID, |
563 | smcd_gid.gid_ext, SMC_NLA_LGR_D_PAD)) | |
564 | goto errattr; | |
b40584d1 | 565 | if (nla_put_u64_64bit(skb, SMC_NLA_LGR_D_PEER_GID, lgr->peer_gid.gid, |
8f9dde4b GG |
566 | SMC_NLA_LGR_D_PAD)) |
567 | goto errattr; | |
01fd1617 WG |
568 | if (nla_put_u64_64bit(skb, SMC_NLA_LGR_D_PEER_EXT_GID, |
569 | lgr->peer_gid.gid_ext, SMC_NLA_LGR_D_PAD)) | |
570 | goto errattr; | |
8f9dde4b GG |
571 | if (nla_put_u8(skb, SMC_NLA_LGR_D_VLAN_ID, lgr->vlan_id)) |
572 | goto errattr; | |
573 | if (nla_put_u32(skb, SMC_NLA_LGR_D_CONNS_NUM, lgr->conns_num)) | |
574 | goto errattr; | |
575 | if (nla_put_u32(skb, SMC_NLA_LGR_D_CHID, smc_ism_get_chid(lgr->smcd))) | |
576 | goto errattr; | |
d386d59b WG |
577 | if (nla_put_uint(skb, SMC_NLA_LGR_D_SNDBUF_ALLOC, lgr->alloc_sndbufs)) |
578 | goto errattr; | |
579 | if (nla_put_uint(skb, SMC_NLA_LGR_D_DMB_ALLOC, lgr->alloc_rmbs)) | |
580 | goto errattr; | |
8a446536 GG |
581 | memcpy(smc_pnet, lgr->smcd->pnetid, SMC_MAX_PNETID_LEN); |
582 | smc_pnet[SMC_MAX_PNETID_LEN] = 0; | |
8f9dde4b GG |
583 | if (nla_put_string(skb, SMC_NLA_LGR_D_PNETID, smc_pnet)) |
584 | goto errattr; | |
b0539f5e KG |
585 | if (lgr->smc_version > SMC_V1) { |
586 | struct nlattr *v2_attrs; | |
8f9dde4b | 587 | |
b0539f5e KG |
588 | v2_attrs = nla_nest_start(skb, SMC_NLA_LGR_D_V2_COMMON); |
589 | if (!v2_attrs) | |
590 | goto errattr; | |
591 | if (smc_nl_fill_lgr_v2_common(lgr, skb, cb, v2_attrs)) | |
592 | goto errattr; | |
593 | } | |
8f9dde4b GG |
594 | nla_nest_end(skb, attrs); |
595 | genlmsg_end(skb, nlh); | |
596 | return 0; | |
597 | ||
8f9dde4b GG |
598 | errattr: |
599 | nla_nest_cancel(skb, attrs); | |
600 | errout: | |
601 | genlmsg_cancel(skb, nlh); | |
602 | errmsg: | |
603 | return -EMSGSIZE; | |
604 | } | |
605 | ||
606 | static int smc_nl_handle_smcd_lgr(struct smcd_dev *dev, | |
607 | struct sk_buff *skb, | |
608 | struct netlink_callback *cb) | |
609 | { | |
610 | struct smc_nl_dmp_ctx *cb_ctx = smc_nl_dmp_ctx(cb); | |
611 | struct smc_link_group *lgr; | |
612 | int snum = cb_ctx->pos[1]; | |
613 | int rc = 0, num = 0; | |
614 | ||
615 | spin_lock_bh(&dev->lgr_lock); | |
616 | list_for_each_entry(lgr, &dev->lgr_list, list) { | |
617 | if (!lgr->is_smcd) | |
618 | continue; | |
619 | if (num < snum) | |
620 | goto next; | |
621 | rc = smc_nl_fill_smcd_lgr(lgr, skb, cb); | |
622 | if (rc) | |
623 | goto errout; | |
624 | next: | |
625 | num++; | |
626 | } | |
627 | errout: | |
628 | spin_unlock_bh(&dev->lgr_lock); | |
629 | cb_ctx->pos[1] = num; | |
630 | return rc; | |
631 | } | |
632 | ||
633 | static int smc_nl_fill_smcd_dev(struct smcd_dev_list *dev_list, | |
634 | struct sk_buff *skb, | |
635 | struct netlink_callback *cb) | |
636 | { | |
637 | struct smc_nl_dmp_ctx *cb_ctx = smc_nl_dmp_ctx(cb); | |
638 | struct smcd_dev *smcd_dev; | |
639 | int snum = cb_ctx->pos[0]; | |
640 | int rc = 0, num = 0; | |
641 | ||
642 | mutex_lock(&dev_list->mutex); | |
643 | list_for_each_entry(smcd_dev, &dev_list->list, list) { | |
644 | if (list_empty(&smcd_dev->lgr_list)) | |
645 | continue; | |
646 | if (num < snum) | |
647 | goto next; | |
648 | rc = smc_nl_handle_smcd_lgr(smcd_dev, skb, cb); | |
649 | if (rc) | |
650 | goto errout; | |
651 | next: | |
652 | num++; | |
653 | } | |
654 | errout: | |
655 | mutex_unlock(&dev_list->mutex); | |
656 | cb_ctx->pos[0] = num; | |
657 | return rc; | |
658 | } | |
659 | ||
e9b8c845 GG |
660 | int smcr_nl_get_lgr(struct sk_buff *skb, struct netlink_callback *cb) |
661 | { | |
5a7e09d5 GG |
662 | bool list_links = false; |
663 | ||
664 | smc_nl_fill_lgr_list(&smc_lgr_list, skb, cb, list_links); | |
665 | return skb->len; | |
666 | } | |
667 | ||
668 | int smcr_nl_get_link(struct sk_buff *skb, struct netlink_callback *cb) | |
669 | { | |
670 | bool list_links = true; | |
671 | ||
672 | smc_nl_fill_lgr_list(&smc_lgr_list, skb, cb, list_links); | |
e9b8c845 GG |
673 | return skb->len; |
674 | } | |
675 | ||
8f9dde4b GG |
676 | int smcd_nl_get_lgr(struct sk_buff *skb, struct netlink_callback *cb) |
677 | { | |
678 | smc_nl_fill_smcd_dev(&smcd_dev_list, skb, cb); | |
679 | return skb->len; | |
680 | } | |
681 | ||
36595d8a | 682 | void smc_lgr_cleanup_early(struct smc_link_group *lgr) |
51e3dfa8 | 683 | { |
9ec6bf19 | 684 | spinlock_t *lgr_lock; |
51e3dfa8 UB |
685 | |
686 | if (!lgr) | |
687 | return; | |
688 | ||
789b6cc2 | 689 | smc_lgr_list_head(lgr, &lgr_lock); |
9ec6bf19 KG |
690 | spin_lock_bh(lgr_lock); |
691 | /* do not use this link group for new connections */ | |
789b6cc2 DL |
692 | if (!list_empty(&lgr->list)) |
693 | list_del_init(&lgr->list); | |
9ec6bf19 | 694 | spin_unlock_bh(lgr_lock); |
f9aab6f2 | 695 | __smc_lgr_terminate(lgr, true); |
51e3dfa8 UB |
696 | } |
697 | ||
a52bcc91 KG |
698 | static void smcr_lgr_link_deactivate_all(struct smc_link_group *lgr) |
699 | { | |
700 | int i; | |
701 | ||
702 | for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { | |
703 | struct smc_link *lnk = &lgr->lnk[i]; | |
704 | ||
90cee52f | 705 | if (smc_link_sendable(lnk)) |
a52bcc91 KG |
706 | lnk->state = SMC_LNK_INACTIVE; |
707 | } | |
6778a6be KG |
708 | wake_up_all(&lgr->llc_msg_waiter); |
709 | wake_up_all(&lgr->llc_flow_waiter); | |
a52bcc91 KG |
710 | } |
711 | ||
3f3f0e36 UB |
712 | static void smc_lgr_free(struct smc_link_group *lgr); |
713 | ||
0cfdd8f9 UB |
714 | static void smc_lgr_free_work(struct work_struct *work) |
715 | { | |
716 | struct smc_link_group *lgr = container_of(to_delayed_work(work), | |
717 | struct smc_link_group, | |
718 | free_work); | |
a0a62ee1 | 719 | spinlock_t *lgr_lock; |
0cfdd8f9 UB |
720 | bool conns; |
721 | ||
a0a62ee1 UB |
722 | smc_lgr_list_head(lgr, &lgr_lock); |
723 | spin_lock_bh(lgr_lock); | |
8e316b9e UB |
724 | if (lgr->freeing) { |
725 | spin_unlock_bh(lgr_lock); | |
726 | return; | |
727 | } | |
0cfdd8f9 UB |
728 | read_lock_bh(&lgr->conns_lock); |
729 | conns = RB_EMPTY_ROOT(&lgr->conns_all); | |
730 | read_unlock_bh(&lgr->conns_lock); | |
731 | if (!conns) { /* number of lgr connections is no longer zero */ | |
a0a62ee1 | 732 | spin_unlock_bh(lgr_lock); |
0cfdd8f9 UB |
733 | return; |
734 | } | |
8caa6544 | 735 | list_del_init(&lgr->list); /* remove from smc_lgr_list */ |
8e316b9e UB |
736 | lgr->freeing = 1; /* this instance does the freeing, no new schedule */ |
737 | spin_unlock_bh(lgr_lock); | |
738 | cancel_delayed_work(&lgr->free_work); | |
0d18a0cb | 739 | |
f3811fd7 KG |
740 | if (!lgr->is_smcd && !lgr->terminating) |
741 | smc_llc_send_link_delete_all(lgr, true, | |
742 | SMC_LLC_DEL_PROG_INIT_TERM); | |
42bfba9e | 743 | if (lgr->is_smcd && !lgr->terminating) |
8e316b9e | 744 | smc_ism_signal_shutdown(lgr); |
a52bcc91 KG |
745 | if (!lgr->is_smcd) |
746 | smcr_lgr_link_deactivate_all(lgr); | |
8e316b9e | 747 | smc_lgr_free(lgr); |
0cfdd8f9 UB |
748 | } |
749 | ||
f528ba24 UB |
750 | static void smc_lgr_terminate_work(struct work_struct *work) |
751 | { | |
752 | struct smc_link_group *lgr = container_of(work, struct smc_link_group, | |
753 | terminate_work); | |
754 | ||
5f78fe96 | 755 | __smc_lgr_terminate(lgr, true); |
f528ba24 UB |
756 | } |
757 | ||
026c381f KG |
758 | /* return next unique link id for the lgr */ |
759 | static u8 smcr_next_link_id(struct smc_link_group *lgr) | |
760 | { | |
761 | u8 link_id; | |
762 | int i; | |
763 | ||
764 | while (1) { | |
cf4f5530 | 765 | again: |
026c381f KG |
766 | link_id = ++lgr->next_link_id; |
767 | if (!link_id) /* skip zero as link_id */ | |
768 | link_id = ++lgr->next_link_id; | |
769 | for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { | |
d854fcbf | 770 | if (smc_link_usable(&lgr->lnk[i]) && |
026c381f | 771 | lgr->lnk[i].link_id == link_id) |
cf4f5530 | 772 | goto again; |
026c381f KG |
773 | } |
774 | break; | |
775 | } | |
776 | return link_id; | |
777 | } | |
778 | ||
6443b2f6 GG |
779 | static void smcr_copy_dev_info_to_link(struct smc_link *link) |
780 | { | |
781 | struct smc_ib_device *smcibdev = link->smcibdev; | |
782 | ||
783 | snprintf(link->ibname, sizeof(link->ibname), "%s", | |
784 | smcibdev->ibdev->name); | |
785 | link->ndev_ifidx = smcibdev->ndev_ifidx[link->ibport - 1]; | |
786 | } | |
787 | ||
336ba09f KG |
788 | int smcr_link_init(struct smc_link_group *lgr, struct smc_link *lnk, |
789 | u8 link_idx, struct smc_init_info *ini) | |
f3c1dedd | 790 | { |
e49300a6 | 791 | struct smc_ib_device *smcibdev; |
f3c1dedd KG |
792 | u8 rndvec[3]; |
793 | int rc; | |
794 | ||
e49300a6 KG |
795 | if (lgr->smc_version == SMC_V2) { |
796 | lnk->smcibdev = ini->smcrv2.ib_dev_v2; | |
797 | lnk->ibport = ini->smcrv2.ib_port_v2; | |
798 | } else { | |
799 | lnk->smcibdev = ini->ib_dev; | |
800 | lnk->ibport = ini->ib_port; | |
801 | } | |
802 | get_device(&lnk->smcibdev->ibdev->dev); | |
803 | atomic_inc(&lnk->smcibdev->lnk_cnt); | |
20c9398d WG |
804 | refcount_set(&lnk->refcnt, 1); /* link refcnt is set to 1 */ |
805 | lnk->clearing = 0; | |
e49300a6 | 806 | lnk->path_mtu = lnk->smcibdev->pattr[lnk->ibport - 1].active_mtu; |
026c381f | 807 | lnk->link_id = smcr_next_link_id(lgr); |
387707fd | 808 | lnk->lgr = lgr; |
61f434b0 | 809 | smc_lgr_hold(lgr); /* lgr_put in smcr_link_clear() */ |
026c381f | 810 | lnk->link_idx = link_idx; |
e9b1a4f8 | 811 | lnk->wr_rx_id_compl = 0; |
ddc99286 | 812 | smc_ibdev_cnt_inc(lnk); |
6443b2f6 | 813 | smcr_copy_dev_info_to_link(lnk); |
07d51580 | 814 | atomic_set(&lnk->conn_cnt, 0); |
45fa8da0 | 815 | smc_llc_link_set_uid(lnk); |
541afa10 | 816 | INIT_WORK(&lnk->link_down_wrk, smc_link_down_work); |
e49300a6 KG |
817 | if (!lnk->smcibdev->initialized) { |
818 | rc = (int)smc_ib_setup_per_ibdev(lnk->smcibdev); | |
f3c1dedd KG |
819 | if (rc) |
820 | goto out; | |
821 | } | |
822 | get_random_bytes(rndvec, sizeof(rndvec)); | |
823 | lnk->psn_initial = rndvec[0] + (rndvec[1] << 8) + | |
824 | (rndvec[2] << 16); | |
825 | rc = smc_ib_determine_gid(lnk->smcibdev, lnk->ibport, | |
24fb6811 KG |
826 | ini->vlan_id, lnk->gid, &lnk->sgid_index, |
827 | lgr->smc_version == SMC_V2 ? | |
828 | &ini->smcrv2 : NULL); | |
f3c1dedd KG |
829 | if (rc) |
830 | goto out; | |
831 | rc = smc_llc_link_init(lnk); | |
832 | if (rc) | |
833 | goto out; | |
834 | rc = smc_wr_alloc_link_mem(lnk); | |
835 | if (rc) | |
836 | goto clear_llc_lnk; | |
837 | rc = smc_ib_create_protection_domain(lnk); | |
838 | if (rc) | |
839 | goto free_link_mem; | |
840 | rc = smc_ib_create_queue_pair(lnk); | |
841 | if (rc) | |
842 | goto dealloc_pd; | |
843 | rc = smc_wr_create_link(lnk); | |
844 | if (rc) | |
845 | goto destroy_qp; | |
741a49a4 | 846 | lnk->state = SMC_LNK_ACTIVATING; |
f3c1dedd KG |
847 | return 0; |
848 | ||
849 | destroy_qp: | |
850 | smc_ib_destroy_queue_pair(lnk); | |
851 | dealloc_pd: | |
852 | smc_ib_dealloc_protection_domain(lnk); | |
853 | free_link_mem: | |
854 | smc_wr_free_link_mem(lnk); | |
855 | clear_llc_lnk: | |
0a99be43 | 856 | smc_llc_link_clear(lnk, false); |
f3c1dedd | 857 | out: |
ddc99286 | 858 | smc_ibdev_cnt_dec(lnk); |
e49300a6 KG |
859 | put_device(&lnk->smcibdev->ibdev->dev); |
860 | smcibdev = lnk->smcibdev; | |
f3c1dedd | 861 | memset(lnk, 0, sizeof(struct smc_link)); |
d854fcbf | 862 | lnk->state = SMC_LNK_UNUSED; |
e49300a6 KG |
863 | if (!atomic_dec_return(&smcibdev->lnk_cnt)) |
864 | wake_up(&smcibdev->lnks_deleted); | |
61f434b0 | 865 | smc_lgr_put(lgr); /* lgr_hold above */ |
f3c1dedd KG |
866 | return rc; |
867 | } | |
868 | ||
0cfdd8f9 | 869 | /* create a new SMC link group */ |
bc36d2fc | 870 | static int smc_lgr_create(struct smc_sock *smc, struct smc_init_info *ini) |
0cfdd8f9 UB |
871 | { |
872 | struct smc_link_group *lgr; | |
a2351c5d | 873 | struct list_head *lgr_list; |
8c81ba20 | 874 | struct smcd_dev *smcd; |
0cfdd8f9 | 875 | struct smc_link *lnk; |
a0a62ee1 | 876 | spinlock_t *lgr_lock; |
026c381f | 877 | u8 link_idx; |
0cfdd8f9 | 878 | int rc = 0; |
cd6851f3 | 879 | int i; |
0cfdd8f9 | 880 | |
bc36d2fc | 881 | if (ini->is_smcd && ini->vlan_id) { |
5c21c4cc UB |
882 | if (smc_ism_get_vlan(ini->ism_dev[ini->ism_selected], |
883 | ini->vlan_id)) { | |
7a62725a | 884 | rc = SMC_CLC_DECL_ISMVLANERR; |
c6ba7c9b | 885 | goto out; |
7a62725a | 886 | } |
c6ba7c9b HW |
887 | } |
888 | ||
0cfdd8f9 UB |
889 | lgr = kzalloc(sizeof(*lgr), GFP_KERNEL); |
890 | if (!lgr) { | |
7a62725a | 891 | rc = SMC_CLC_DECL_MEM; |
29ee2701 | 892 | goto ism_put_vlan; |
0cfdd8f9 | 893 | } |
22ef473d KG |
894 | lgr->tx_wq = alloc_workqueue("smc_tx_wq-%*phN", 0, 0, |
895 | SMC_LGR_ID_SIZE, &lgr->id); | |
896 | if (!lgr->tx_wq) { | |
897 | rc = -ENOMEM; | |
898 | goto free_lgr; | |
899 | } | |
bc36d2fc | 900 | lgr->is_smcd = ini->is_smcd; |
517c300e | 901 | lgr->sync_err = 0; |
8e316b9e | 902 | lgr->terminating = 0; |
8e316b9e | 903 | lgr->freeing = 0; |
bc36d2fc | 904 | lgr->vlan_id = ini->vlan_id; |
61f434b0 | 905 | refcount_set(&lgr->refcnt, 1); /* set lgr refcnt to 1 */ |
aff7bfed W |
906 | init_rwsem(&lgr->sndbufs_lock); |
907 | init_rwsem(&lgr->rmbs_lock); | |
c6ba7c9b | 908 | rwlock_init(&lgr->conns_lock); |
cd6851f3 UB |
909 | for (i = 0; i < SMC_RMBE_SIZES; i++) { |
910 | INIT_LIST_HEAD(&lgr->sndbufs[i]); | |
911 | INIT_LIST_HEAD(&lgr->rmbs[i]); | |
912 | } | |
026c381f | 913 | lgr->next_link_id = 0; |
9fda3510 HW |
914 | smc_lgr_list.num += SMC_LGR_NUM_INCR; |
915 | memcpy(&lgr->id, (u8 *)&smc_lgr_list.num, SMC_LGR_ID_SIZE); | |
0cfdd8f9 | 916 | INIT_DELAYED_WORK(&lgr->free_work, smc_lgr_free_work); |
f528ba24 | 917 | INIT_WORK(&lgr->terminate_work, smc_lgr_terminate_work); |
0cfdd8f9 | 918 | lgr->conns_all = RB_ROOT; |
bc36d2fc | 919 | if (ini->is_smcd) { |
c6ba7c9b | 920 | /* SMC-D specific settings */ |
8c81ba20 SR |
921 | smcd = ini->ism_dev[ini->ism_selected]; |
922 | get_device(smcd->ops->get_dev(smcd)); | |
b40584d1 WG |
923 | lgr->peer_gid.gid = |
924 | ini->ism_peer_gid[ini->ism_selected].gid; | |
925 | lgr->peer_gid.gid_ext = | |
926 | ini->ism_peer_gid[ini->ism_selected].gid_ext; | |
5c21c4cc UB |
927 | lgr->smcd = ini->ism_dev[ini->ism_selected]; |
928 | lgr_list = &ini->ism_dev[ini->ism_selected]->lgr_list; | |
a0a62ee1 | 929 | lgr_lock = &lgr->smcd->lgr_lock; |
b81a5eb7 | 930 | lgr->smc_version = ini->smcd_version; |
50c6b20e | 931 | lgr->peer_shutdown = 0; |
5c21c4cc | 932 | atomic_inc(&ini->ism_dev[ini->ism_selected]->lgr_cnt); |
c6ba7c9b HW |
933 | } else { |
934 | /* SMC-R specific settings */ | |
e49300a6 KG |
935 | struct smc_ib_device *ibdev; |
936 | int ibport; | |
937 | ||
c6ba7c9b | 938 | lgr->role = smc->listen_smc ? SMC_SERV : SMC_CLNT; |
e49300a6 KG |
939 | lgr->smc_version = ini->smcr_version; |
940 | memcpy(lgr->peer_systemid, ini->peer_systemid, | |
bc36d2fc | 941 | SMC_SYSTEMID_LEN); |
e49300a6 KG |
942 | if (lgr->smc_version == SMC_V2) { |
943 | ibdev = ini->smcrv2.ib_dev_v2; | |
944 | ibport = ini->smcrv2.ib_port_v2; | |
945 | lgr->saddr = ini->smcrv2.saddr; | |
946 | lgr->uses_gateway = ini->smcrv2.uses_gateway; | |
947 | memcpy(lgr->nexthop_mac, ini->smcrv2.nexthop_mac, | |
948 | ETH_ALEN); | |
7f0620b9 | 949 | lgr->max_conns = ini->max_conns; |
69b888e3 | 950 | lgr->max_links = ini->max_links; |
e49300a6 KG |
951 | } else { |
952 | ibdev = ini->ib_dev; | |
953 | ibport = ini->ib_port; | |
7f0620b9 | 954 | lgr->max_conns = SMC_CONN_PER_LGR_MAX; |
69b888e3 | 955 | lgr->max_links = SMC_LINKS_ADD_LNK_MAX; |
e49300a6 KG |
956 | } |
957 | memcpy(lgr->pnet_id, ibdev->pnetid[ibport - 1], | |
35dcf7ec | 958 | SMC_MAX_PNETID_LEN); |
bdee15e8 DC |
959 | rc = smc_wr_alloc_lgr_mem(lgr); |
960 | if (rc) | |
8799e310 | 961 | goto free_wq; |
00a049cf KG |
962 | smc_llc_lgr_init(lgr, smc); |
963 | ||
026c381f KG |
964 | link_idx = SMC_SINGLE_LINK; |
965 | lnk = &lgr->lnk[link_idx]; | |
966 | rc = smcr_link_init(lgr, lnk, link_idx, ini); | |
8799e310 KG |
967 | if (rc) { |
968 | smc_wr_free_lgr_mem(lgr); | |
22ef473d | 969 | goto free_wq; |
8799e310 | 970 | } |
0237a3a6 | 971 | lgr->net = smc_ib_net(lnk->smcibdev); |
f3c1dedd KG |
972 | lgr_list = &smc_lgr_list.list; |
973 | lgr_lock = &smc_lgr_list.lock; | |
b984f370 | 974 | lgr->buf_type = lgr->net->smc.sysctl_smcr_buf_type; |
6dabd405 | 975 | atomic_inc(&lgr_cnt); |
c6ba7c9b | 976 | } |
0cfdd8f9 | 977 | smc->conn.lgr = lgr; |
a0a62ee1 | 978 | spin_lock_bh(lgr_lock); |
a9e44502 | 979 | list_add_tail(&lgr->list, lgr_list); |
a0a62ee1 | 980 | spin_unlock_bh(lgr_lock); |
f38ba179 UB |
981 | return 0; |
982 | ||
22ef473d KG |
983 | free_wq: |
984 | destroy_workqueue(lgr->tx_wq); | |
f38ba179 UB |
985 | free_lgr: |
986 | kfree(lgr); | |
29ee2701 UB |
987 | ism_put_vlan: |
988 | if (ini->is_smcd && ini->vlan_id) | |
5c21c4cc | 989 | smc_ism_put_vlan(ini->ism_dev[ini->ism_selected], ini->vlan_id); |
0cfdd8f9 | 990 | out: |
7a62725a KG |
991 | if (rc < 0) { |
992 | if (rc == -ENOMEM) | |
993 | rc = SMC_CLC_DECL_MEM; | |
994 | else | |
995 | rc = SMC_CLC_DECL_INTERR; | |
996 | } | |
0cfdd8f9 UB |
997 | return rc; |
998 | } | |
999 | ||
c6f02ebe KG |
1000 | static int smc_write_space(struct smc_connection *conn) |
1001 | { | |
1002 | int buffer_len = conn->peer_rmbe_size; | |
1003 | union smc_host_cursor prod; | |
1004 | union smc_host_cursor cons; | |
1005 | int space; | |
1006 | ||
1007 | smc_curs_copy(&prod, &conn->local_tx_ctrl.prod, conn); | |
1008 | smc_curs_copy(&cons, &conn->local_rx_ctrl.cons, conn); | |
1009 | /* determine rx_buf space */ | |
1010 | space = buffer_len - smc_curs_diff(buffer_len, &cons, &prod); | |
1011 | return space; | |
1012 | } | |
1013 | ||
b8ded9de KG |
1014 | static int smc_switch_cursor(struct smc_sock *smc, struct smc_cdc_tx_pend *pend, |
1015 | struct smc_wr_buf *wr_buf) | |
c6f02ebe KG |
1016 | { |
1017 | struct smc_connection *conn = &smc->conn; | |
1018 | union smc_host_cursor cons, fin; | |
1019 | int rc = 0; | |
1020 | int diff; | |
1021 | ||
1022 | smc_curs_copy(&conn->tx_curs_sent, &conn->tx_curs_fin, conn); | |
1023 | smc_curs_copy(&fin, &conn->local_tx_ctrl_fin, conn); | |
1024 | /* set prod cursor to old state, enforce tx_rdma_writes() */ | |
1025 | smc_curs_copy(&conn->local_tx_ctrl.prod, &fin, conn); | |
1026 | smc_curs_copy(&cons, &conn->local_rx_ctrl.cons, conn); | |
1027 | ||
1028 | if (smc_curs_comp(conn->peer_rmbe_size, &cons, &fin) < 0) { | |
1029 | /* cons cursor advanced more than fin, and prod was set | |
1030 | * fin above, so now prod is smaller than cons. Fix that. | |
1031 | */ | |
1032 | diff = smc_curs_diff(conn->peer_rmbe_size, &fin, &cons); | |
1033 | smc_curs_add(conn->sndbuf_desc->len, | |
1034 | &conn->tx_curs_sent, diff); | |
1035 | smc_curs_add(conn->sndbuf_desc->len, | |
1036 | &conn->tx_curs_fin, diff); | |
1037 | ||
1038 | smp_mb__before_atomic(); | |
1039 | atomic_add(diff, &conn->sndbuf_space); | |
1040 | smp_mb__after_atomic(); | |
1041 | ||
1042 | smc_curs_add(conn->peer_rmbe_size, | |
1043 | &conn->local_tx_ctrl.prod, diff); | |
1044 | smc_curs_add(conn->peer_rmbe_size, | |
1045 | &conn->local_tx_ctrl_fin, diff); | |
1046 | } | |
1047 | /* recalculate, value is used by tx_rdma_writes() */ | |
1048 | atomic_set(&smc->conn.peer_rmbe_space, smc_write_space(conn)); | |
1049 | ||
1050 | if (smc->sk.sk_state != SMC_INIT && | |
1051 | smc->sk.sk_state != SMC_CLOSED) { | |
b8ded9de | 1052 | rc = smcr_cdc_msg_send_validation(conn, pend, wr_buf); |
c6f02ebe | 1053 | if (!rc) { |
22ef473d | 1054 | queue_delayed_work(conn->lgr->tx_wq, &conn->tx_work, 0); |
c6f02ebe KG |
1055 | smc->sk.sk_data_ready(&smc->sk); |
1056 | } | |
b8ded9de KG |
1057 | } else { |
1058 | smc_wr_tx_put_slot(conn->lnk, | |
1059 | (struct smc_wr_tx_pend_priv *)pend); | |
c6f02ebe KG |
1060 | } |
1061 | return rc; | |
1062 | } | |
1063 | ||
64513d26 GG |
1064 | void smc_switch_link_and_count(struct smc_connection *conn, |
1065 | struct smc_link *to_lnk) | |
07d51580 GG |
1066 | { |
1067 | atomic_dec(&conn->lnk->conn_cnt); | |
20c9398d WG |
1068 | /* link_hold in smc_conn_create() */ |
1069 | smcr_link_put(conn->lnk); | |
07d51580 GG |
1070 | conn->lnk = to_lnk; |
1071 | atomic_inc(&conn->lnk->conn_cnt); | |
20c9398d WG |
1072 | /* link_put in smc_conn_free() */ |
1073 | smcr_link_hold(conn->lnk); | |
07d51580 GG |
1074 | } |
1075 | ||
c6f02ebe KG |
1076 | struct smc_link *smc_switch_conns(struct smc_link_group *lgr, |
1077 | struct smc_link *from_lnk, bool is_dev_err) | |
1078 | { | |
1079 | struct smc_link *to_lnk = NULL; | |
b8ded9de | 1080 | struct smc_cdc_tx_pend *pend; |
c6f02ebe | 1081 | struct smc_connection *conn; |
b8ded9de | 1082 | struct smc_wr_buf *wr_buf; |
c6f02ebe KG |
1083 | struct smc_sock *smc; |
1084 | struct rb_node *node; | |
1085 | int i, rc = 0; | |
1086 | ||
1087 | /* link is inactive, wake up tx waiters */ | |
1088 | smc_wr_wakeup_tx_wait(from_lnk); | |
1089 | ||
1090 | for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { | |
741a49a4 | 1091 | if (!smc_link_active(&lgr->lnk[i]) || i == from_lnk->link_idx) |
c6f02ebe KG |
1092 | continue; |
1093 | if (is_dev_err && from_lnk->smcibdev == lgr->lnk[i].smcibdev && | |
1094 | from_lnk->ibport == lgr->lnk[i].ibport) { | |
1095 | continue; | |
1096 | } | |
1097 | to_lnk = &lgr->lnk[i]; | |
1098 | break; | |
1099 | } | |
95f7f3e7 | 1100 | if (!to_lnk || !smc_wr_tx_link_hold(to_lnk)) { |
c6f02ebe KG |
1101 | smc_lgr_terminate_sched(lgr); |
1102 | return NULL; | |
1103 | } | |
1104 | again: | |
1105 | read_lock_bh(&lgr->conns_lock); | |
1106 | for (node = rb_first(&lgr->conns_all); node; node = rb_next(node)) { | |
1107 | conn = rb_entry(node, struct smc_connection, alert_node); | |
1108 | if (conn->lnk != from_lnk) | |
1109 | continue; | |
1110 | smc = container_of(conn, struct smc_sock, conn); | |
1111 | /* conn->lnk not yet set in SMC_INIT state */ | |
1112 | if (smc->sk.sk_state == SMC_INIT) | |
1113 | continue; | |
1114 | if (smc->sk.sk_state == SMC_CLOSED || | |
1115 | smc->sk.sk_state == SMC_PEERCLOSEWAIT1 || | |
1116 | smc->sk.sk_state == SMC_PEERCLOSEWAIT2 || | |
1117 | smc->sk.sk_state == SMC_APPFINCLOSEWAIT || | |
1118 | smc->sk.sk_state == SMC_APPCLOSEWAIT1 || | |
1119 | smc->sk.sk_state == SMC_APPCLOSEWAIT2 || | |
1120 | smc->sk.sk_state == SMC_PEERFINCLOSEWAIT || | |
1121 | smc->sk.sk_state == SMC_PEERABORTWAIT || | |
1122 | smc->sk.sk_state == SMC_PROCESSABORT) { | |
1123 | spin_lock_bh(&conn->send_lock); | |
07d51580 | 1124 | smc_switch_link_and_count(conn, to_lnk); |
c6f02ebe KG |
1125 | spin_unlock_bh(&conn->send_lock); |
1126 | continue; | |
1127 | } | |
1128 | sock_hold(&smc->sk); | |
1129 | read_unlock_bh(&lgr->conns_lock); | |
b8ded9de KG |
1130 | /* pre-fetch buffer outside of send_lock, might sleep */ |
1131 | rc = smc_cdc_get_free_slot(conn, to_lnk, &wr_buf, NULL, &pend); | |
95f7f3e7 KG |
1132 | if (rc) |
1133 | goto err_out; | |
c6f02ebe KG |
1134 | /* avoid race with smcr_tx_sndbuf_nonempty() */ |
1135 | spin_lock_bh(&conn->send_lock); | |
07d51580 | 1136 | smc_switch_link_and_count(conn, to_lnk); |
b8ded9de | 1137 | rc = smc_switch_cursor(smc, pend, wr_buf); |
c6f02ebe KG |
1138 | spin_unlock_bh(&conn->send_lock); |
1139 | sock_put(&smc->sk); | |
95f7f3e7 KG |
1140 | if (rc) |
1141 | goto err_out; | |
c6f02ebe KG |
1142 | goto again; |
1143 | } | |
1144 | read_unlock_bh(&lgr->conns_lock); | |
95f7f3e7 | 1145 | smc_wr_tx_link_put(to_lnk); |
c6f02ebe | 1146 | return to_lnk; |
95f7f3e7 KG |
1147 | |
1148 | err_out: | |
1149 | smcr_link_down_cond_sched(to_lnk); | |
1150 | smc_wr_tx_link_put(to_lnk); | |
1151 | return NULL; | |
c6f02ebe KG |
1152 | } |
1153 | ||
b8d19945 | 1154 | static void smcr_buf_unuse(struct smc_buf_desc *buf_desc, bool is_rmb, |
6d74c3a8 | 1155 | struct smc_link_group *lgr) |
b9247544 | 1156 | { |
aff7bfed | 1157 | struct rw_semaphore *lock; /* lock buffer list */ |
d5500667 KG |
1158 | int rc; |
1159 | ||
b8d19945 | 1160 | if (is_rmb && buf_desc->is_conf_rkey && !list_empty(&lgr->list)) { |
b9247544 | 1161 | /* unregister rmb with peer */ |
d5500667 KG |
1162 | rc = smc_llc_flow_initiate(lgr, SMC_LLC_FLOW_RKEY); |
1163 | if (!rc) { | |
1164 | /* protect against smc_llc_cli_rkey_exchange() */ | |
f6421014 | 1165 | down_read(&lgr->llc_conf_mutex); |
b8d19945 WG |
1166 | smc_llc_do_delete_rkey(lgr, buf_desc); |
1167 | buf_desc->is_conf_rkey = false; | |
f6421014 | 1168 | up_read(&lgr->llc_conf_mutex); |
d5500667 KG |
1169 | smc_llc_flow_stop(lgr, &lgr->llc_flow_lcl); |
1170 | } | |
b9247544 | 1171 | } |
d5500667 | 1172 | |
b8d19945 | 1173 | if (buf_desc->is_reg_err) { |
b9247544 | 1174 | /* buf registration failed, reuse not possible */ |
b8d19945 WG |
1175 | lock = is_rmb ? &lgr->rmbs_lock : |
1176 | &lgr->sndbufs_lock; | |
aff7bfed | 1177 | down_write(lock); |
d386d59b | 1178 | smc_lgr_buf_list_del(lgr, is_rmb, buf_desc); |
aff7bfed | 1179 | up_write(lock); |
b9247544 | 1180 | |
b8d19945 | 1181 | smc_buf_free(lgr, is_rmb, buf_desc); |
b9247544 | 1182 | } else { |
475f9ff6 W |
1183 | /* memzero_explicit provides potential memory barrier semantics */ |
1184 | memzero_explicit(buf_desc->cpu_addr, buf_desc->len); | |
1185 | WRITE_ONCE(buf_desc->used, 0); | |
b9247544 KG |
1186 | } |
1187 | } | |
1188 | ||
ae2be35c WG |
1189 | static void smcd_buf_detach(struct smc_connection *conn) |
1190 | { | |
1191 | struct smcd_dev *smcd = conn->lgr->smcd; | |
1192 | u64 peer_token = conn->peer_token; | |
1193 | ||
1194 | if (!conn->sndbuf_desc) | |
1195 | return; | |
1196 | ||
1197 | smc_ism_detach_dmb(smcd, peer_token); | |
1198 | ||
1199 | kfree(conn->sndbuf_desc); | |
1200 | conn->sndbuf_desc = NULL; | |
1201 | } | |
1202 | ||
fb692ec4 KG |
1203 | static void smc_buf_unuse(struct smc_connection *conn, |
1204 | struct smc_link_group *lgr) | |
cd6851f3 | 1205 | { |
e0d10354 WG |
1206 | struct smc_sock *smc = container_of(conn, struct smc_sock, conn); |
1207 | bool is_smcd = lgr->is_smcd; | |
1208 | int bufsize; | |
1209 | ||
1c552696 | 1210 | if (conn->sndbuf_desc) { |
e0d10354 WG |
1211 | bufsize = conn->sndbuf_desc->len; |
1212 | if (!is_smcd && conn->sndbuf_desc->is_vm) { | |
b8d19945 WG |
1213 | smcr_buf_unuse(conn->sndbuf_desc, false, lgr); |
1214 | } else { | |
e0d10354 | 1215 | memzero_explicit(conn->sndbuf_desc->cpu_addr, bufsize); |
475f9ff6 | 1216 | WRITE_ONCE(conn->sndbuf_desc->used, 0); |
b8d19945 | 1217 | } |
e0d10354 | 1218 | SMC_STAT_RMB_SIZE(smc, is_smcd, false, false, bufsize); |
1c552696 | 1219 | } |
b8d19945 | 1220 | if (conn->rmb_desc) { |
e0d10354 WG |
1221 | bufsize = conn->rmb_desc->len; |
1222 | if (!is_smcd) { | |
b8d19945 WG |
1223 | smcr_buf_unuse(conn->rmb_desc, true, lgr); |
1224 | } else { | |
e0d10354 WG |
1225 | bufsize += sizeof(struct smcd_cdc_msg); |
1226 | memzero_explicit(conn->rmb_desc->cpu_addr, bufsize); | |
475f9ff6 | 1227 | WRITE_ONCE(conn->rmb_desc->used, 0); |
b8d19945 | 1228 | } |
e0d10354 | 1229 | SMC_STAT_RMB_SIZE(smc, is_smcd, true, false, bufsize); |
1c552696 | 1230 | } |
cd6851f3 UB |
1231 | } |
1232 | ||
0cfdd8f9 UB |
1233 | /* remove a finished connection from its link group */ |
1234 | void smc_conn_free(struct smc_connection *conn) | |
1235 | { | |
fb692ec4 KG |
1236 | struct smc_link_group *lgr = conn->lgr; |
1237 | ||
61f434b0 WG |
1238 | if (!lgr || conn->freed) |
1239 | /* Connection has never been registered in a | |
1240 | * link group, or has already been freed. | |
1241 | */ | |
0cfdd8f9 | 1242 | return; |
61f434b0 WG |
1243 | |
1244 | conn->freed = 1; | |
ea89c6c0 | 1245 | if (!smc_conn_lgr_valid(conn)) |
61f434b0 WG |
1246 | /* Connection has already unregistered from |
1247 | * link group. | |
1248 | */ | |
1249 | goto lgr_put; | |
1250 | ||
fb692ec4 | 1251 | if (lgr->is_smcd) { |
42bfba9e UB |
1252 | if (!list_empty(&lgr->list)) |
1253 | smc_ism_unset_conn(conn); | |
ae2be35c WG |
1254 | if (smc_ism_support_dmb_nocopy(lgr->smcd)) |
1255 | smcd_buf_detach(conn); | |
be244f28 HW |
1256 | tasklet_kill(&conn->rx_tsklet); |
1257 | } else { | |
349d4312 | 1258 | smc_cdc_wait_pend_tx_wr(conn); |
b286a065 KG |
1259 | if (current_work() != &conn->abort_work) |
1260 | cancel_work_sync(&conn->abort_work); | |
be244f28 | 1261 | } |
2a0674ff | 1262 | if (!list_empty(&lgr->list)) { |
2a0674ff | 1263 | smc_buf_unuse(conn, lgr); /* allow buffer reuse */ |
0537f0a2 | 1264 | smc_lgr_unregister_conn(conn); |
2a0674ff | 1265 | } |
fb692ec4 KG |
1266 | |
1267 | if (!lgr->conns_num) | |
1268 | smc_lgr_schedule_free_work(lgr); | |
61f434b0 | 1269 | lgr_put: |
20c9398d WG |
1270 | if (!lgr->is_smcd) |
1271 | smcr_link_put(conn->lnk); /* link_hold in smc_conn_create() */ | |
61f434b0 | 1272 | smc_lgr_put(lgr); /* lgr_hold in smc_conn_create() */ |
0cfdd8f9 UB |
1273 | } |
1274 | ||
4a3641c1 KG |
1275 | /* unregister a link from a buf_desc */ |
1276 | static void smcr_buf_unmap_link(struct smc_buf_desc *buf_desc, bool is_rmb, | |
1277 | struct smc_link *lnk) | |
1278 | { | |
b8d19945 | 1279 | if (is_rmb || buf_desc->is_vm) |
4a3641c1 KG |
1280 | buf_desc->is_reg_mr[lnk->link_idx] = false; |
1281 | if (!buf_desc->is_map_ib[lnk->link_idx]) | |
1282 | return; | |
b8d19945 WG |
1283 | |
1284 | if ((is_rmb || buf_desc->is_vm) && | |
1285 | buf_desc->mr[lnk->link_idx]) { | |
1286 | smc_ib_put_memory_region(buf_desc->mr[lnk->link_idx]); | |
1287 | buf_desc->mr[lnk->link_idx] = NULL; | |
1288 | } | |
1289 | if (is_rmb) | |
4a3641c1 | 1290 | smc_ib_buf_unmap_sg(lnk, buf_desc, DMA_FROM_DEVICE); |
b8d19945 | 1291 | else |
4a3641c1 | 1292 | smc_ib_buf_unmap_sg(lnk, buf_desc, DMA_TO_DEVICE); |
b8d19945 | 1293 | |
4a3641c1 KG |
1294 | sg_free_table(&buf_desc->sgt[lnk->link_idx]); |
1295 | buf_desc->is_map_ib[lnk->link_idx] = false; | |
1296 | } | |
1297 | ||
1298 | /* unmap all buffers of lgr for a deleted link */ | |
1299 | static void smcr_buf_unmap_lgr(struct smc_link *lnk) | |
1300 | { | |
1301 | struct smc_link_group *lgr = lnk->lgr; | |
1302 | struct smc_buf_desc *buf_desc, *bf; | |
1303 | int i; | |
1304 | ||
1305 | for (i = 0; i < SMC_RMBE_SIZES; i++) { | |
aff7bfed | 1306 | down_write(&lgr->rmbs_lock); |
4a3641c1 KG |
1307 | list_for_each_entry_safe(buf_desc, bf, &lgr->rmbs[i], list) |
1308 | smcr_buf_unmap_link(buf_desc, true, lnk); | |
aff7bfed W |
1309 | up_write(&lgr->rmbs_lock); |
1310 | ||
1311 | down_write(&lgr->sndbufs_lock); | |
4a3641c1 KG |
1312 | list_for_each_entry_safe(buf_desc, bf, &lgr->sndbufs[i], |
1313 | list) | |
1314 | smcr_buf_unmap_link(buf_desc, false, lnk); | |
aff7bfed | 1315 | up_write(&lgr->sndbufs_lock); |
4a3641c1 KG |
1316 | } |
1317 | } | |
1318 | ||
1319 | static void smcr_rtoken_clear_link(struct smc_link *lnk) | |
1320 | { | |
1321 | struct smc_link_group *lgr = lnk->lgr; | |
1322 | int i; | |
1323 | ||
1324 | for (i = 0; i < SMC_RMBS_PER_LGR_MAX; i++) { | |
1325 | lgr->rtokens[i][lnk->link_idx].rkey = 0; | |
1326 | lgr->rtokens[i][lnk->link_idx].dma_addr = 0; | |
1327 | } | |
1328 | } | |
1329 | ||
20c9398d | 1330 | static void __smcr_link_clear(struct smc_link *lnk) |
0cfdd8f9 | 1331 | { |
61f434b0 | 1332 | struct smc_link_group *lgr = lnk->lgr; |
d854fcbf KG |
1333 | struct smc_ib_device *smcibdev; |
1334 | ||
f38ba179 | 1335 | smc_wr_free_link_mem(lnk); |
ddc99286 | 1336 | smc_ibdev_cnt_dec(lnk); |
f3c1dedd | 1337 | put_device(&lnk->smcibdev->ibdev->dev); |
d854fcbf KG |
1338 | smcibdev = lnk->smcibdev; |
1339 | memset(lnk, 0, sizeof(struct smc_link)); | |
1340 | lnk->state = SMC_LNK_UNUSED; | |
1341 | if (!atomic_dec_return(&smcibdev->lnk_cnt)) | |
1342 | wake_up(&smcibdev->lnks_deleted); | |
61f434b0 | 1343 | smc_lgr_put(lgr); /* lgr_hold in smcr_link_init() */ |
0cfdd8f9 UB |
1344 | } |
1345 | ||
20c9398d WG |
1346 | /* must be called under lgr->llc_conf_mutex lock */ |
1347 | void smcr_link_clear(struct smc_link *lnk, bool log) | |
1348 | { | |
1349 | if (!lnk->lgr || lnk->clearing || | |
1350 | lnk->state == SMC_LNK_UNUSED) | |
1351 | return; | |
1352 | lnk->clearing = 1; | |
1353 | lnk->peer_qpn = 0; | |
1354 | smc_llc_link_clear(lnk, log); | |
1355 | smcr_buf_unmap_lgr(lnk); | |
1356 | smcr_rtoken_clear_link(lnk); | |
1357 | smc_ib_modify_qp_error(lnk); | |
1358 | smc_wr_free_link(lnk); | |
1359 | smc_ib_destroy_queue_pair(lnk); | |
1360 | smc_ib_dealloc_protection_domain(lnk); | |
1361 | smcr_link_put(lnk); /* theoretically last link_put */ | |
1362 | } | |
1363 | ||
1364 | void smcr_link_hold(struct smc_link *lnk) | |
1365 | { | |
1366 | refcount_inc(&lnk->refcnt); | |
1367 | } | |
1368 | ||
1369 | void smcr_link_put(struct smc_link *lnk) | |
1370 | { | |
1371 | if (refcount_dec_and_test(&lnk->refcnt)) | |
1372 | __smcr_link_clear(lnk); | |
1373 | } | |
1374 | ||
c6ba7c9b HW |
1375 | static void smcr_buf_free(struct smc_link_group *lgr, bool is_rmb, |
1376 | struct smc_buf_desc *buf_desc) | |
cd6851f3 | 1377 | { |
b9247544 | 1378 | int i; |
6511aad3 | 1379 | |
4a3641c1 KG |
1380 | for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) |
1381 | smcr_buf_unmap_link(buf_desc, is_rmb, &lgr->lnk[i]); | |
387707fd | 1382 | |
b8d19945 | 1383 | if (!buf_desc->is_vm && buf_desc->pages) |
2ef4f27a | 1384 | __free_pages(buf_desc->pages, buf_desc->order); |
b8d19945 WG |
1385 | else if (buf_desc->is_vm && buf_desc->cpu_addr) |
1386 | vfree(buf_desc->cpu_addr); | |
3e034725 | 1387 | kfree(buf_desc); |
cd6851f3 UB |
1388 | } |
1389 | ||
c6ba7c9b HW |
1390 | static void smcd_buf_free(struct smc_link_group *lgr, bool is_dmb, |
1391 | struct smc_buf_desc *buf_desc) | |
1392 | { | |
be244f28 HW |
1393 | if (is_dmb) { |
1394 | /* restore original buf len */ | |
1395 | buf_desc->len += sizeof(struct smcd_cdc_msg); | |
c6ba7c9b | 1396 | smc_ism_unregister_dmb(lgr->smcd, buf_desc); |
be244f28 | 1397 | } else { |
c6ba7c9b | 1398 | kfree(buf_desc->cpu_addr); |
be244f28 | 1399 | } |
c6ba7c9b HW |
1400 | kfree(buf_desc); |
1401 | } | |
1402 | ||
1403 | static void smc_buf_free(struct smc_link_group *lgr, bool is_rmb, | |
1404 | struct smc_buf_desc *buf_desc) | |
1405 | { | |
1406 | if (lgr->is_smcd) | |
1407 | smcd_buf_free(lgr, is_rmb, buf_desc); | |
1408 | else | |
1409 | smcr_buf_free(lgr, is_rmb, buf_desc); | |
1410 | } | |
1411 | ||
3e034725 | 1412 | static void __smc_lgr_free_bufs(struct smc_link_group *lgr, bool is_rmb) |
cd6851f3 | 1413 | { |
3e034725 UB |
1414 | struct smc_buf_desc *buf_desc, *bf_desc; |
1415 | struct list_head *buf_list; | |
cd6851f3 UB |
1416 | int i; |
1417 | ||
1418 | for (i = 0; i < SMC_RMBE_SIZES; i++) { | |
3e034725 UB |
1419 | if (is_rmb) |
1420 | buf_list = &lgr->rmbs[i]; | |
1421 | else | |
1422 | buf_list = &lgr->sndbufs[i]; | |
1423 | list_for_each_entry_safe(buf_desc, bf_desc, buf_list, | |
cd6851f3 | 1424 | list) { |
d386d59b | 1425 | smc_lgr_buf_list_del(lgr, is_rmb, buf_desc); |
6511aad3 | 1426 | smc_buf_free(lgr, is_rmb, buf_desc); |
cd6851f3 UB |
1427 | } |
1428 | } | |
1429 | } | |
1430 | ||
3e034725 UB |
1431 | static void smc_lgr_free_bufs(struct smc_link_group *lgr) |
1432 | { | |
1433 | /* free send buffers */ | |
1434 | __smc_lgr_free_bufs(lgr, false); | |
1435 | /* free rmbs */ | |
1436 | __smc_lgr_free_bufs(lgr, true); | |
1437 | } | |
1438 | ||
61f434b0 WG |
1439 | /* won't be freed until no one accesses to lgr anymore */ |
1440 | static void __smc_lgr_free(struct smc_link_group *lgr) | |
1441 | { | |
1442 | smc_lgr_free_bufs(lgr); | |
1443 | if (lgr->is_smcd) { | |
1444 | if (!atomic_dec_return(&lgr->smcd->lgr_cnt)) | |
1445 | wake_up(&lgr->smcd->lgrs_deleted); | |
1446 | } else { | |
1447 | smc_wr_free_lgr_mem(lgr); | |
1448 | if (!atomic_dec_return(&lgr_cnt)) | |
1449 | wake_up(&lgrs_deleted); | |
1450 | } | |
1451 | kfree(lgr); | |
1452 | } | |
1453 | ||
0cfdd8f9 | 1454 | /* remove a link group */ |
3f3f0e36 | 1455 | static void smc_lgr_free(struct smc_link_group *lgr) |
0cfdd8f9 | 1456 | { |
b9247544 KG |
1457 | int i; |
1458 | ||
a52bcc91 | 1459 | if (!lgr->is_smcd) { |
b5dd4d69 | 1460 | down_write(&lgr->llc_conf_mutex); |
a52bcc91 KG |
1461 | for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { |
1462 | if (lgr->lnk[i].state != SMC_LNK_UNUSED) | |
0a99be43 | 1463 | smcr_link_clear(&lgr->lnk[i], false); |
a52bcc91 | 1464 | } |
b5dd4d69 | 1465 | up_write(&lgr->llc_conf_mutex); |
a52bcc91 KG |
1466 | smc_llc_lgr_clear(lgr); |
1467 | } | |
1468 | ||
22ef473d | 1469 | destroy_workqueue(lgr->tx_wq); |
b3cb53c0 | 1470 | if (lgr->is_smcd) { |
f9aab6f2 | 1471 | smc_ism_put_vlan(lgr->smcd, lgr->vlan_id); |
8c81ba20 | 1472 | put_device(lgr->smcd->ops->get_dev(lgr->smcd)); |
b3cb53c0 | 1473 | } |
61f434b0 WG |
1474 | smc_lgr_put(lgr); /* theoretically last lgr_put */ |
1475 | } | |
1476 | ||
1477 | void smc_lgr_hold(struct smc_link_group *lgr) | |
1478 | { | |
1479 | refcount_inc(&lgr->refcnt); | |
1480 | } | |
1481 | ||
1482 | void smc_lgr_put(struct smc_link_group *lgr) | |
1483 | { | |
1484 | if (refcount_dec_and_test(&lgr->refcnt)) | |
1485 | __smc_lgr_free(lgr); | |
0cfdd8f9 UB |
1486 | } |
1487 | ||
2a0674ff UB |
1488 | static void smc_sk_wake_ups(struct smc_sock *smc) |
1489 | { | |
1490 | smc->sk.sk_write_space(&smc->sk); | |
1491 | smc->sk.sk_data_ready(&smc->sk); | |
1492 | smc->sk.sk_state_change(&smc->sk); | |
1493 | } | |
1494 | ||
1495 | /* kill a connection */ | |
5421ec28 | 1496 | static void smc_conn_kill(struct smc_connection *conn, bool soft) |
2a0674ff UB |
1497 | { |
1498 | struct smc_sock *smc = container_of(conn, struct smc_sock, conn); | |
1499 | ||
50c6b20e UB |
1500 | if (conn->lgr->is_smcd && conn->lgr->peer_shutdown) |
1501 | conn->local_tx_ctrl.conn_state_flags.peer_conn_abort = 1; | |
1502 | else | |
1503 | smc_close_abort(conn); | |
2a0674ff | 1504 | conn->killed = 1; |
50c6b20e | 1505 | smc->sk.sk_err = ECONNABORTED; |
2a0674ff | 1506 | smc_sk_wake_ups(smc); |
42bfba9e UB |
1507 | if (conn->lgr->is_smcd) { |
1508 | smc_ism_unset_conn(conn); | |
ae2be35c WG |
1509 | if (smc_ism_support_dmb_nocopy(conn->lgr->smcd)) |
1510 | smcd_buf_detach(conn); | |
5421ec28 UB |
1511 | if (soft) |
1512 | tasklet_kill(&conn->rx_tsklet); | |
1513 | else | |
1514 | tasklet_unlock_wait(&conn->rx_tsklet); | |
6a37ad3d | 1515 | } else { |
349d4312 | 1516 | smc_cdc_wait_pend_tx_wr(conn); |
42bfba9e | 1517 | } |
2a0674ff | 1518 | smc_lgr_unregister_conn(conn); |
81cf4f47 | 1519 | smc_close_active_abort(smc); |
2a0674ff UB |
1520 | } |
1521 | ||
42bfba9e UB |
1522 | static void smc_lgr_cleanup(struct smc_link_group *lgr) |
1523 | { | |
1524 | if (lgr->is_smcd) { | |
1525 | smc_ism_signal_shutdown(lgr); | |
42bfba9e | 1526 | } else { |
3e0c40af KG |
1527 | u32 rsn = lgr->llc_termination_rsn; |
1528 | ||
1529 | if (!rsn) | |
1530 | rsn = SMC_LLC_DEL_PROG_INIT_TERM; | |
1531 | smc_llc_send_link_delete_all(lgr, false, rsn); | |
a52bcc91 | 1532 | smcr_lgr_link_deactivate_all(lgr); |
42bfba9e UB |
1533 | } |
1534 | } | |
1535 | ||
ba952060 KG |
1536 | /* terminate link group |
1537 | * @soft: true if link group shutdown can take its time | |
1538 | * false if immediate link group shutdown is required | |
1539 | */ | |
5421ec28 | 1540 | static void __smc_lgr_terminate(struct smc_link_group *lgr, bool soft) |
0cfdd8f9 UB |
1541 | { |
1542 | struct smc_connection *conn; | |
b38d7324 | 1543 | struct smc_sock *smc; |
0cfdd8f9 UB |
1544 | struct rb_node *node; |
1545 | ||
517c300e KG |
1546 | if (lgr->terminating) |
1547 | return; /* lgr already terminating */ | |
a52bcc91 | 1548 | /* cancel free_work sync, will terminate when lgr->freeing is set */ |
13085e1b | 1549 | cancel_delayed_work(&lgr->free_work); |
517c300e | 1550 | lgr->terminating = 1; |
0cfdd8f9 | 1551 | |
69318b52 UB |
1552 | /* kill remaining link group connections */ |
1553 | read_lock_bh(&lgr->conns_lock); | |
0cfdd8f9 UB |
1554 | node = rb_first(&lgr->conns_all); |
1555 | while (node) { | |
69318b52 | 1556 | read_unlock_bh(&lgr->conns_lock); |
0cfdd8f9 | 1557 | conn = rb_entry(node, struct smc_connection, alert_node); |
b38d7324 | 1558 | smc = container_of(conn, struct smc_sock, conn); |
81cf4f47 | 1559 | sock_hold(&smc->sk); /* sock_put below */ |
69318b52 | 1560 | lock_sock(&smc->sk); |
5421ec28 | 1561 | smc_conn_kill(conn, soft); |
69318b52 | 1562 | release_sock(&smc->sk); |
81cf4f47 | 1563 | sock_put(&smc->sk); /* sock_hold above */ |
69318b52 | 1564 | read_lock_bh(&lgr->conns_lock); |
0cfdd8f9 UB |
1565 | node = rb_first(&lgr->conns_all); |
1566 | } | |
69318b52 | 1567 | read_unlock_bh(&lgr->conns_lock); |
42bfba9e | 1568 | smc_lgr_cleanup(lgr); |
a52bcc91 | 1569 | smc_lgr_free(lgr); |
0cfdd8f9 UB |
1570 | } |
1571 | ||
5f78fe96 KG |
1572 | /* unlink link group and schedule termination */ |
1573 | void smc_lgr_terminate_sched(struct smc_link_group *lgr) | |
b9f227c3 | 1574 | { |
a0a62ee1 UB |
1575 | spinlock_t *lgr_lock; |
1576 | ||
1577 | smc_lgr_list_head(lgr, &lgr_lock); | |
1578 | spin_lock_bh(lgr_lock); | |
3739707c | 1579 | if (list_empty(&lgr->list) || lgr->terminating || lgr->freeing) { |
8caa6544 UB |
1580 | spin_unlock_bh(lgr_lock); |
1581 | return; /* lgr already terminating */ | |
1582 | } | |
1583 | list_del_init(&lgr->list); | |
a52bcc91 | 1584 | lgr->freeing = 1; |
a0a62ee1 | 1585 | spin_unlock_bh(lgr_lock); |
5f78fe96 | 1586 | schedule_work(&lgr->terminate_work); |
b9f227c3 HW |
1587 | } |
1588 | ||
5421ec28 | 1589 | /* Called when peer lgr shutdown (regularly or abnormally) is received */ |
b40584d1 WG |
1590 | void smc_smcd_terminate(struct smcd_dev *dev, struct smcd_gid *peer_gid, |
1591 | unsigned short vlan) | |
c6ba7c9b HW |
1592 | { |
1593 | struct smc_link_group *lgr, *l; | |
1594 | LIST_HEAD(lgr_free_list); | |
1595 | ||
1596 | /* run common cleanup function and build free list */ | |
a0a62ee1 | 1597 | spin_lock_bh(&dev->lgr_lock); |
a2351c5d | 1598 | list_for_each_entry_safe(lgr, l, &dev->lgr_list, list) { |
b40584d1 WG |
1599 | if ((!peer_gid->gid || |
1600 | (lgr->peer_gid.gid == peer_gid->gid && | |
b27696cd | 1601 | !smc_ism_is_emulated(dev) ? 1 : |
b40584d1 | 1602 | lgr->peer_gid.gid_ext == peer_gid->gid_ext)) && |
0512f69e | 1603 | (vlan == VLAN_VID_MASK || lgr->vlan_id == vlan)) { |
b40584d1 | 1604 | if (peer_gid->gid) /* peer triggered termination */ |
50c6b20e | 1605 | lgr->peer_shutdown = 1; |
c6ba7c9b | 1606 | list_move(&lgr->list, &lgr_free_list); |
a52bcc91 | 1607 | lgr->freeing = 1; |
c6ba7c9b HW |
1608 | } |
1609 | } | |
a0a62ee1 | 1610 | spin_unlock_bh(&dev->lgr_lock); |
c6ba7c9b HW |
1611 | |
1612 | /* cancel the regular free workers and actually free lgrs */ | |
1613 | list_for_each_entry_safe(lgr, l, &lgr_free_list, list) { | |
1614 | list_del_init(&lgr->list); | |
50c6b20e | 1615 | schedule_work(&lgr->terminate_work); |
c6ba7c9b HW |
1616 | } |
1617 | } | |
1618 | ||
5421ec28 UB |
1619 | /* Called when an SMCD device is removed or the smc module is unloaded */ |
1620 | void smc_smcd_terminate_all(struct smcd_dev *smcd) | |
1621 | { | |
1622 | struct smc_link_group *lgr, *lg; | |
1623 | LIST_HEAD(lgr_free_list); | |
1624 | ||
1625 | spin_lock_bh(&smcd->lgr_lock); | |
1626 | list_splice_init(&smcd->lgr_list, &lgr_free_list); | |
1627 | list_for_each_entry(lgr, &lgr_free_list, list) | |
1628 | lgr->freeing = 1; | |
1629 | spin_unlock_bh(&smcd->lgr_lock); | |
1630 | ||
1631 | list_for_each_entry_safe(lgr, lg, &lgr_free_list, list) { | |
1632 | list_del_init(&lgr->list); | |
1633 | __smc_lgr_terminate(lgr, false); | |
1634 | } | |
5edd6b9c UB |
1635 | |
1636 | if (atomic_read(&smcd->lgr_cnt)) | |
1637 | wait_event(smcd->lgrs_deleted, !atomic_read(&smcd->lgr_cnt)); | |
5421ec28 UB |
1638 | } |
1639 | ||
0b29ec64 UB |
1640 | /* Called when an SMCR device is removed or the smc module is unloaded. |
1641 | * If smcibdev is given, all SMCR link groups using this device are terminated. | |
1642 | * If smcibdev is NULL, all SMCR link groups are terminated. | |
1643 | */ | |
1644 | void smc_smcr_terminate_all(struct smc_ib_device *smcibdev) | |
1645 | { | |
1646 | struct smc_link_group *lgr, *lg; | |
1647 | LIST_HEAD(lgr_free_list); | |
b9247544 | 1648 | int i; |
0b29ec64 UB |
1649 | |
1650 | spin_lock_bh(&smc_lgr_list.lock); | |
1651 | if (!smcibdev) { | |
1652 | list_splice_init(&smc_lgr_list.list, &lgr_free_list); | |
1653 | list_for_each_entry(lgr, &lgr_free_list, list) | |
1654 | lgr->freeing = 1; | |
1655 | } else { | |
1656 | list_for_each_entry_safe(lgr, lg, &smc_lgr_list.list, list) { | |
b9247544 | 1657 | for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { |
87523930 | 1658 | if (lgr->lnk[i].smcibdev == smcibdev) |
56d99e81 | 1659 | smcr_link_down_cond_sched(&lgr->lnk[i]); |
0b29ec64 UB |
1660 | } |
1661 | } | |
1662 | } | |
1663 | spin_unlock_bh(&smc_lgr_list.lock); | |
1664 | ||
1665 | list_for_each_entry_safe(lgr, lg, &lgr_free_list, list) { | |
1666 | list_del_init(&lgr->list); | |
3e0c40af | 1667 | smc_llc_set_termination_rsn(lgr, SMC_LLC_DEL_OP_INIT_TERM); |
0b29ec64 | 1668 | __smc_lgr_terminate(lgr, false); |
349d4312 DL |
1669 | } |
1670 | ||
6dabd405 UB |
1671 | if (smcibdev) { |
1672 | if (atomic_read(&smcibdev->lnk_cnt)) | |
1673 | wait_event(smcibdev->lnks_deleted, | |
1674 | !atomic_read(&smcibdev->lnk_cnt)); | |
1675 | } else { | |
1676 | if (atomic_read(&lgr_cnt)) | |
1677 | wait_event(lgrs_deleted, !atomic_read(&lgr_cnt)); | |
1678 | } | |
0b29ec64 UB |
1679 | } |
1680 | ||
ad6c111b KG |
1681 | /* set new lgr type and clear all asymmetric link tagging */ |
1682 | void smcr_lgr_set_type(struct smc_link_group *lgr, enum smc_lgr_type new_type) | |
1683 | { | |
0a99be43 | 1684 | char *lgr_type = ""; |
ad6c111b KG |
1685 | int i; |
1686 | ||
1687 | for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) | |
1688 | if (smc_link_usable(&lgr->lnk[i])) | |
1689 | lgr->lnk[i].link_is_asym = false; | |
0a99be43 KG |
1690 | if (lgr->type == new_type) |
1691 | return; | |
ad6c111b | 1692 | lgr->type = new_type; |
0a99be43 KG |
1693 | |
1694 | switch (lgr->type) { | |
1695 | case SMC_LGR_NONE: | |
1696 | lgr_type = "NONE"; | |
1697 | break; | |
1698 | case SMC_LGR_SINGLE: | |
1699 | lgr_type = "SINGLE"; | |
1700 | break; | |
1701 | case SMC_LGR_SYMMETRIC: | |
1702 | lgr_type = "SYMMETRIC"; | |
1703 | break; | |
1704 | case SMC_LGR_ASYMMETRIC_PEER: | |
1705 | lgr_type = "ASYMMETRIC_PEER"; | |
1706 | break; | |
1707 | case SMC_LGR_ASYMMETRIC_LOCAL: | |
1708 | lgr_type = "ASYMMETRIC_LOCAL"; | |
1709 | break; | |
1710 | } | |
de2fea7b | 1711 | pr_warn_ratelimited("smc: SMC-R lg %*phN net %llu state changed: " |
0a99be43 | 1712 | "%s, pnetid %.16s\n", SMC_LGR_ID_SIZE, &lgr->id, |
de2fea7b | 1713 | lgr->net->net_cookie, lgr_type, lgr->pnet_id); |
ad6c111b KG |
1714 | } |
1715 | ||
1716 | /* set new lgr type and tag a link as asymmetric */ | |
1717 | void smcr_lgr_set_type_asym(struct smc_link_group *lgr, | |
1718 | enum smc_lgr_type new_type, int asym_lnk_idx) | |
1719 | { | |
1720 | smcr_lgr_set_type(lgr, new_type); | |
1721 | lgr->lnk[asym_lnk_idx].link_is_asym = true; | |
1722 | } | |
1723 | ||
b286a065 KG |
1724 | /* abort connection, abort_work scheduled from tasklet context */ |
1725 | static void smc_conn_abort_work(struct work_struct *work) | |
1726 | { | |
1727 | struct smc_connection *conn = container_of(work, | |
1728 | struct smc_connection, | |
1729 | abort_work); | |
1730 | struct smc_sock *smc = container_of(conn, struct smc_sock, conn); | |
1731 | ||
a18cee47 | 1732 | lock_sock(&smc->sk); |
b286a065 | 1733 | smc_conn_kill(conn, true); |
a18cee47 | 1734 | release_sock(&smc->sk); |
b286a065 KG |
1735 | sock_put(&smc->sk); /* sock_hold done by schedulers of abort_work */ |
1736 | } | |
1737 | ||
1f90a05d KG |
1738 | void smcr_port_add(struct smc_ib_device *smcibdev, u8 ibport) |
1739 | { | |
1f90a05d KG |
1740 | struct smc_link_group *lgr, *n; |
1741 | ||
f5146e3e | 1742 | spin_lock_bh(&smc_lgr_list.lock); |
1f90a05d | 1743 | list_for_each_entry_safe(lgr, n, &smc_lgr_list.list, list) { |
c48254fa KG |
1744 | struct smc_link *link; |
1745 | ||
1f90a05d KG |
1746 | if (strncmp(smcibdev->pnetid[ibport - 1], lgr->pnet_id, |
1747 | SMC_MAX_PNETID_LEN) || | |
1748 | lgr->type == SMC_LGR_SYMMETRIC || | |
0237a3a6 TL |
1749 | lgr->type == SMC_LGR_ASYMMETRIC_PEER || |
1750 | !rdma_dev_access_netns(smcibdev->ibdev, lgr->net)) | |
1f90a05d | 1751 | continue; |
c48254fa | 1752 | |
69b888e3 GW |
1753 | if (lgr->type == SMC_LGR_SINGLE && lgr->max_links <= 1) |
1754 | continue; | |
1755 | ||
c48254fa KG |
1756 | /* trigger local add link processing */ |
1757 | link = smc_llc_usable_link(lgr); | |
1758 | if (link) | |
1759 | smc_llc_add_link_local(link); | |
1f90a05d | 1760 | } |
f5146e3e | 1761 | spin_unlock_bh(&smc_lgr_list.lock); |
1f90a05d KG |
1762 | } |
1763 | ||
541afa10 KG |
1764 | /* link is down - switch connections to alternate link, |
1765 | * must be called under lgr->llc_conf_mutex lock | |
1766 | */ | |
1767 | static void smcr_link_down(struct smc_link *lnk) | |
1768 | { | |
1769 | struct smc_link_group *lgr = lnk->lgr; | |
1770 | struct smc_link *to_lnk; | |
1771 | int del_link_id; | |
1772 | ||
1773 | if (!lgr || lnk->state == SMC_LNK_UNUSED || list_empty(&lgr->list)) | |
1774 | return; | |
1775 | ||
c6f02ebe | 1776 | to_lnk = smc_switch_conns(lgr, lnk, true); |
541afa10 | 1777 | if (!to_lnk) { /* no backup link available */ |
0a99be43 | 1778 | smcr_link_clear(lnk, true); |
541afa10 KG |
1779 | return; |
1780 | } | |
ad6c111b | 1781 | smcr_lgr_set_type(lgr, SMC_LGR_SINGLE); |
541afa10 KG |
1782 | del_link_id = lnk->link_id; |
1783 | ||
1784 | if (lgr->role == SMC_SERV) { | |
1785 | /* trigger local delete link processing */ | |
4dadd151 | 1786 | smc_llc_srv_delete_link_local(to_lnk, del_link_id); |
541afa10 KG |
1787 | } else { |
1788 | if (lgr->llc_flow_lcl.type != SMC_LLC_FLOW_NONE) { | |
1789 | /* another llc task is ongoing */ | |
b5dd4d69 | 1790 | up_write(&lgr->llc_conf_mutex); |
6778a6be KG |
1791 | wait_event_timeout(lgr->llc_flow_waiter, |
1792 | (list_empty(&lgr->list) || | |
1793 | lgr->llc_flow_lcl.type == SMC_LLC_FLOW_NONE), | |
541afa10 | 1794 | SMC_LLC_WAIT_TIME); |
b5dd4d69 | 1795 | down_write(&lgr->llc_conf_mutex); |
541afa10 | 1796 | } |
68fd8942 | 1797 | if (!list_empty(&lgr->list)) { |
6778a6be KG |
1798 | smc_llc_send_delete_link(to_lnk, del_link_id, |
1799 | SMC_LLC_REQ, true, | |
1800 | SMC_LLC_DEL_LOST_PATH); | |
68fd8942 KG |
1801 | smcr_link_clear(lnk, true); |
1802 | } | |
6778a6be | 1803 | wake_up(&lgr->llc_flow_waiter); /* wake up next waiter */ |
541afa10 KG |
1804 | } |
1805 | } | |
1806 | ||
1807 | /* must be called under lgr->llc_conf_mutex lock */ | |
1808 | void smcr_link_down_cond(struct smc_link *lnk) | |
1809 | { | |
a3a0e81b TL |
1810 | if (smc_link_downing(&lnk->state)) { |
1811 | trace_smcr_link_down(lnk, __builtin_return_address(0)); | |
541afa10 | 1812 | smcr_link_down(lnk); |
a3a0e81b | 1813 | } |
541afa10 KG |
1814 | } |
1815 | ||
1816 | /* will get the lgr->llc_conf_mutex lock */ | |
1817 | void smcr_link_down_cond_sched(struct smc_link *lnk) | |
1818 | { | |
a3a0e81b TL |
1819 | if (smc_link_downing(&lnk->state)) { |
1820 | trace_smcr_link_down(lnk, __builtin_return_address(0)); | |
541afa10 | 1821 | schedule_work(&lnk->link_down_wrk); |
a3a0e81b | 1822 | } |
541afa10 KG |
1823 | } |
1824 | ||
1825 | void smcr_port_err(struct smc_ib_device *smcibdev, u8 ibport) | |
1826 | { | |
1827 | struct smc_link_group *lgr, *n; | |
1828 | int i; | |
1829 | ||
1830 | list_for_each_entry_safe(lgr, n, &smc_lgr_list.list, list) { | |
1831 | if (strncmp(smcibdev->pnetid[ibport - 1], lgr->pnet_id, | |
1832 | SMC_MAX_PNETID_LEN)) | |
1833 | continue; /* lgr is not affected */ | |
1834 | if (list_empty(&lgr->list)) | |
1835 | continue; | |
1836 | for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { | |
1837 | struct smc_link *lnk = &lgr->lnk[i]; | |
1838 | ||
1839 | if (smc_link_usable(lnk) && | |
1840 | lnk->smcibdev == smcibdev && lnk->ibport == ibport) | |
1841 | smcr_link_down_cond_sched(lnk); | |
1842 | } | |
1843 | } | |
1844 | } | |
1845 | ||
541afa10 KG |
1846 | static void smc_link_down_work(struct work_struct *work) |
1847 | { | |
1848 | struct smc_link *link = container_of(work, struct smc_link, | |
1849 | link_down_wrk); | |
1850 | struct smc_link_group *lgr = link->lgr; | |
1851 | ||
1852 | if (list_empty(&lgr->list)) | |
1853 | return; | |
6778a6be | 1854 | wake_up_all(&lgr->llc_msg_waiter); |
b5dd4d69 | 1855 | down_write(&lgr->llc_conf_mutex); |
541afa10 | 1856 | smcr_link_down(link); |
b5dd4d69 | 1857 | up_write(&lgr->llc_conf_mutex); |
541afa10 KG |
1858 | } |
1859 | ||
587acad4 KG |
1860 | static int smc_vlan_by_tcpsk_walk(struct net_device *lower_dev, |
1861 | struct netdev_nested_priv *priv) | |
1862 | { | |
1863 | unsigned short *vlan_id = (unsigned short *)priv->data; | |
1864 | ||
1865 | if (is_vlan_dev(lower_dev)) { | |
1866 | *vlan_id = vlan_dev_vlan_id(lower_dev); | |
1867 | return 1; | |
1868 | } | |
1869 | ||
1870 | return 0; | |
1871 | } | |
1872 | ||
1873 | /* Determine vlan of internal TCP socket. */ | |
bc36d2fc | 1874 | int smc_vlan_by_tcpsk(struct socket *clcsock, struct smc_init_info *ini) |
0cfdd8f9 UB |
1875 | { |
1876 | struct dst_entry *dst = sk_dst_get(clcsock->sk); | |
587acad4 | 1877 | struct netdev_nested_priv priv; |
cb9d43f6 | 1878 | struct net_device *ndev; |
587acad4 | 1879 | int rc = 0; |
0cfdd8f9 | 1880 | |
bc36d2fc | 1881 | ini->vlan_id = 0; |
0cfdd8f9 UB |
1882 | if (!dst) { |
1883 | rc = -ENOTCONN; | |
1884 | goto out; | |
1885 | } | |
1886 | if (!dst->dev) { | |
1887 | rc = -ENODEV; | |
1888 | goto out_rel; | |
1889 | } | |
1890 | ||
cb9d43f6 UB |
1891 | ndev = dst->dev; |
1892 | if (is_vlan_dev(ndev)) { | |
bc36d2fc | 1893 | ini->vlan_id = vlan_dev_vlan_id(ndev); |
cb9d43f6 UB |
1894 | goto out_rel; |
1895 | } | |
1896 | ||
587acad4 | 1897 | priv.data = (void *)&ini->vlan_id; |
cb9d43f6 | 1898 | rtnl_lock(); |
587acad4 | 1899 | netdev_walk_all_lower_dev(ndev, smc_vlan_by_tcpsk_walk, &priv); |
cb9d43f6 | 1900 | rtnl_unlock(); |
0cfdd8f9 UB |
1901 | |
1902 | out_rel: | |
1903 | dst_release(dst); | |
1904 | out: | |
1905 | return rc; | |
1906 | } | |
1907 | ||
e49300a6 KG |
1908 | static bool smcr_lgr_match(struct smc_link_group *lgr, u8 smcr_version, |
1909 | u8 peer_systemid[], | |
1910 | u8 peer_gid[], | |
1911 | u8 peer_mac_v1[], | |
0237a3a6 TL |
1912 | enum smc_lgr_role role, u32 clcqpn, |
1913 | struct net *net) | |
0cfdd8f9 | 1914 | { |
0237a3a6 | 1915 | struct smc_link *lnk; |
b9247544 KG |
1916 | int i; |
1917 | ||
e49300a6 | 1918 | if (memcmp(lgr->peer_systemid, peer_systemid, SMC_SYSTEMID_LEN) || |
b9247544 KG |
1919 | lgr->role != role) |
1920 | return false; | |
1921 | ||
1922 | for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { | |
0237a3a6 TL |
1923 | lnk = &lgr->lnk[i]; |
1924 | ||
1925 | if (!smc_link_active(lnk)) | |
b9247544 | 1926 | continue; |
0237a3a6 TL |
1927 | /* use verbs API to check netns, instead of lgr->net */ |
1928 | if (!rdma_dev_access_netns(lnk->smcibdev->ibdev, net)) | |
1929 | return false; | |
1930 | if ((lgr->role == SMC_SERV || lnk->peer_qpn == clcqpn) && | |
1931 | !memcmp(lnk->peer_gid, peer_gid, SMC_GID_SIZE) && | |
e49300a6 | 1932 | (smcr_version == SMC_V2 || |
0237a3a6 | 1933 | !memcmp(lnk->peer_mac, peer_mac_v1, ETH_ALEN))) |
b9247544 KG |
1934 | return true; |
1935 | } | |
1936 | return false; | |
c6ba7c9b | 1937 | } |
0cfdd8f9 | 1938 | |
c6ba7c9b | 1939 | static bool smcd_lgr_match(struct smc_link_group *lgr, |
b40584d1 WG |
1940 | struct smcd_dev *smcismdev, |
1941 | struct smcd_gid *peer_gid) | |
c6ba7c9b | 1942 | { |
c3dfcdb6 WG |
1943 | if (lgr->peer_gid.gid != peer_gid->gid || |
1944 | lgr->smcd != smcismdev) | |
1945 | return false; | |
1946 | ||
b27696cd | 1947 | if (smc_ism_is_emulated(smcismdev) && |
c3dfcdb6 WG |
1948 | lgr->peer_gid.gid_ext != peer_gid->gid_ext) |
1949 | return false; | |
1950 | ||
1951 | return true; | |
0cfdd8f9 UB |
1952 | } |
1953 | ||
1954 | /* create a new SMC connection (and a new link group if necessary) */ | |
bc36d2fc | 1955 | int smc_conn_create(struct smc_sock *smc, struct smc_init_info *ini) |
0cfdd8f9 UB |
1956 | { |
1957 | struct smc_connection *conn = &smc->conn; | |
0237a3a6 | 1958 | struct net *net = sock_net(&smc->sk); |
a2351c5d | 1959 | struct list_head *lgr_list; |
0cfdd8f9 | 1960 | struct smc_link_group *lgr; |
0cfdd8f9 | 1961 | enum smc_lgr_role role; |
a0a62ee1 | 1962 | spinlock_t *lgr_lock; |
0cfdd8f9 UB |
1963 | int rc = 0; |
1964 | ||
5c21c4cc | 1965 | lgr_list = ini->is_smcd ? &ini->ism_dev[ini->ism_selected]->lgr_list : |
3fc64937 | 1966 | &smc_lgr_list.list; |
5c21c4cc | 1967 | lgr_lock = ini->is_smcd ? &ini->ism_dev[ini->ism_selected]->lgr_lock : |
3fc64937 | 1968 | &smc_lgr_list.lock; |
5ac54d87 | 1969 | ini->first_contact_local = 1; |
0cfdd8f9 | 1970 | role = smc->listen_smc ? SMC_SERV : SMC_CLNT; |
5ac54d87 | 1971 | if (role == SMC_CLNT && ini->first_contact_peer) |
0cfdd8f9 UB |
1972 | /* create new link group as well */ |
1973 | goto create; | |
1974 | ||
1975 | /* determine if an existing link group can be reused */ | |
a0a62ee1 | 1976 | spin_lock_bh(lgr_lock); |
a2351c5d | 1977 | list_for_each_entry(lgr, lgr_list, list) { |
0cfdd8f9 | 1978 | write_lock_bh(&lgr->conns_lock); |
bc36d2fc | 1979 | if ((ini->is_smcd ? |
5c21c4cc | 1980 | smcd_lgr_match(lgr, ini->ism_dev[ini->ism_selected], |
b40584d1 | 1981 | &ini->ism_peer_gid[ini->ism_selected]) : |
e49300a6 KG |
1982 | smcr_lgr_match(lgr, ini->smcr_version, |
1983 | ini->peer_systemid, | |
1984 | ini->peer_gid, ini->peer_mac, role, | |
0237a3a6 | 1985 | ini->ib_clcqpn, net)) && |
0cfdd8f9 | 1986 | !lgr->sync_err && |
0530bd6e KG |
1987 | (ini->smcd_version == SMC_V2 || |
1988 | lgr->vlan_id == ini->vlan_id) && | |
a9e44502 | 1989 | (role == SMC_CLNT || ini->is_smcd || |
7f0620b9 | 1990 | (lgr->conns_num < lgr->max_conns && |
4940a1fd | 1991 | !bitmap_full(lgr->rtokens_used_mask, SMC_RMBS_PER_LGR_MAX)))) { |
0cfdd8f9 | 1992 | /* link group found */ |
5ac54d87 | 1993 | ini->first_contact_local = 0; |
0cfdd8f9 | 1994 | conn->lgr = lgr; |
56bc3b20 | 1995 | rc = smc_lgr_register_conn(conn, false); |
0cfdd8f9 | 1996 | write_unlock_bh(&lgr->conns_lock); |
b9247544 KG |
1997 | if (!rc && delayed_work_pending(&lgr->free_work)) |
1998 | cancel_delayed_work(&lgr->free_work); | |
0cfdd8f9 UB |
1999 | break; |
2000 | } | |
2001 | write_unlock_bh(&lgr->conns_lock); | |
2002 | } | |
a0a62ee1 | 2003 | spin_unlock_bh(lgr_lock); |
b9247544 KG |
2004 | if (rc) |
2005 | return rc; | |
0cfdd8f9 | 2006 | |
5ac54d87 UB |
2007 | if (role == SMC_CLNT && !ini->first_contact_peer && |
2008 | ini->first_contact_local) { | |
0cfdd8f9 UB |
2009 | /* Server reuses a link group, but Client wants to start |
2010 | * a new one | |
2011 | * send out_of_sync decline, reason synchr. error | |
2012 | */ | |
7a62725a | 2013 | return SMC_CLC_DECL_SYNCERR; |
0cfdd8f9 UB |
2014 | } |
2015 | ||
2016 | create: | |
5ac54d87 | 2017 | if (ini->first_contact_local) { |
bc36d2fc | 2018 | rc = smc_lgr_create(smc, ini); |
0cfdd8f9 UB |
2019 | if (rc) |
2020 | goto out; | |
44808792 HZ |
2021 | lgr = conn->lgr; |
2022 | write_lock_bh(&lgr->conns_lock); | |
56bc3b20 | 2023 | rc = smc_lgr_register_conn(conn, true); |
44808792 | 2024 | write_unlock_bh(&lgr->conns_lock); |
36595d8a WG |
2025 | if (rc) { |
2026 | smc_lgr_cleanup_early(lgr); | |
b9247544 | 2027 | goto out; |
36595d8a | 2028 | } |
0cfdd8f9 | 2029 | } |
61f434b0 | 2030 | smc_lgr_hold(conn->lgr); /* lgr_put in smc_conn_free() */ |
20c9398d WG |
2031 | if (!conn->lgr->is_smcd) |
2032 | smcr_link_hold(conn->lnk); /* link_put in smc_conn_free() */ | |
61f434b0 | 2033 | conn->freed = 0; |
5f08318f | 2034 | conn->local_tx_ctrl.common.type = SMC_CDC_MSG_TYPE; |
cbba07a7 | 2035 | conn->local_tx_ctrl.len = SMC_WR_TX_SIZE; |
de8474eb | 2036 | conn->urg_state = SMC_URG_READ; |
349d4312 | 2037 | init_waitqueue_head(&conn->cdc_pend_tx_wq); |
b286a065 | 2038 | INIT_WORK(&smc->conn.abort_work, smc_conn_abort_work); |
bc36d2fc | 2039 | if (ini->is_smcd) { |
be244f28 HW |
2040 | conn->rx_off = sizeof(struct smcd_cdc_msg); |
2041 | smcd_cdc_rx_init(conn); /* init tasklet for this conn */ | |
2d2bfeb8 UB |
2042 | } else { |
2043 | conn->rx_off = 0; | |
be244f28 | 2044 | } |
5f08318f UB |
2045 | #ifndef KERNEL_HAS_ATOMIC64 |
2046 | spin_lock_init(&conn->acurs_lock); | |
2047 | #endif | |
0cfdd8f9 UB |
2048 | |
2049 | out: | |
7a62725a | 2050 | return rc; |
0cfdd8f9 | 2051 | } |
cd6851f3 | 2052 | |
67161779 | 2053 | #define SMCD_DMBE_SIZES 6 /* 0 -> 16KB, 1 -> 32KB, .. 6 -> 1MB */ |
2f4b101c | 2054 | #define SMCR_RMBE_SIZES 15 /* 0 -> 16KB, 1 -> 32KB, .. 15 -> 512MB */ |
67161779 SR |
2055 | |
2056 | /* convert the RMB size into the compressed notation (minimum 16K, see | |
2057 | * SMCD/R_DMBE_SIZES. | |
2f6becaf HW |
2058 | * In contrast to plain ilog2, this rounds towards the next power of 2, |
2059 | * so the socket application gets at least its desired sndbuf / rcvbuf size. | |
2060 | */ | |
67161779 | 2061 | static u8 smc_compress_bufsize(int size, bool is_smcd, bool is_rmb) |
2f6becaf HW |
2062 | { |
2063 | u8 compressed; | |
2064 | ||
2065 | if (size <= SMC_BUF_MIN_SIZE) | |
2066 | return 0; | |
2067 | ||
67161779 SR |
2068 | size = (size - 1) >> 14; /* convert to 16K multiple */ |
2069 | compressed = min_t(u8, ilog2(size) + 1, | |
2070 | is_smcd ? SMCD_DMBE_SIZES : SMCR_RMBE_SIZES); | |
2071 | ||
3ac14b9d | 2072 | #ifdef CONFIG_ARCH_NO_SG_CHAIN |
67161779 SR |
2073 | if (!is_smcd && is_rmb) |
2074 | /* RMBs are backed by & limited to max size of scatterlists */ | |
3ac14b9d GW |
2075 | compressed = min_t(u8, compressed, ilog2((SG_MAX_SINGLE_ALLOC * PAGE_SIZE) >> 14)); |
2076 | #endif | |
67161779 | 2077 | |
2f6becaf HW |
2078 | return compressed; |
2079 | } | |
2080 | ||
2081 | /* convert the RMB size from compressed notation into integer */ | |
2082 | int smc_uncompress_bufsize(u8 compressed) | |
2083 | { | |
2084 | u32 size; | |
2085 | ||
2086 | size = 0x00000001 << (((int)compressed) + 14); | |
2087 | return (int)size; | |
2088 | } | |
2089 | ||
3e034725 UB |
2090 | /* try to reuse a sndbuf or rmb description slot for a certain |
2091 | * buffer size; if not available, return NULL | |
cd6851f3 | 2092 | */ |
8437bda0 | 2093 | static struct smc_buf_desc *smc_buf_get_slot(int compressed_bufsize, |
aff7bfed | 2094 | struct rw_semaphore *lock, |
8437bda0 | 2095 | struct list_head *buf_list) |
cd6851f3 | 2096 | { |
3e034725 | 2097 | struct smc_buf_desc *buf_slot; |
cd6851f3 | 2098 | |
aff7bfed | 2099 | down_read(lock); |
3e034725 UB |
2100 | list_for_each_entry(buf_slot, buf_list, list) { |
2101 | if (cmpxchg(&buf_slot->used, 0, 1) == 0) { | |
aff7bfed | 2102 | up_read(lock); |
3e034725 | 2103 | return buf_slot; |
cd6851f3 UB |
2104 | } |
2105 | } | |
aff7bfed | 2106 | up_read(lock); |
cd6851f3 UB |
2107 | return NULL; |
2108 | } | |
2109 | ||
952310cc UB |
2110 | /* one of the conditions for announcing a receiver's current window size is |
2111 | * that it "results in a minimum increase in the window size of 10% of the | |
2112 | * receive buffer space" [RFC7609] | |
2113 | */ | |
2114 | static inline int smc_rmb_wnd_update_limit(int rmbe_size) | |
2115 | { | |
6bf536eb | 2116 | return max_t(int, rmbe_size / 10, SOCK_MIN_SNDBUF / 2); |
952310cc UB |
2117 | } |
2118 | ||
b8d19945 | 2119 | /* map an buf to a link */ |
b9247544 KG |
2120 | static int smcr_buf_map_link(struct smc_buf_desc *buf_desc, bool is_rmb, |
2121 | struct smc_link *lnk) | |
2122 | { | |
b8d19945 WG |
2123 | int rc, i, nents, offset, buf_size, size, access_flags; |
2124 | struct scatterlist *sg; | |
2125 | void *buf; | |
b9247544 KG |
2126 | |
2127 | if (buf_desc->is_map_ib[lnk->link_idx]) | |
2128 | return 0; | |
2129 | ||
b8d19945 WG |
2130 | if (buf_desc->is_vm) { |
2131 | buf = buf_desc->cpu_addr; | |
2132 | buf_size = buf_desc->len; | |
2133 | offset = offset_in_page(buf_desc->cpu_addr); | |
2134 | nents = PAGE_ALIGN(buf_size + offset) / PAGE_SIZE; | |
2135 | } else { | |
2136 | nents = 1; | |
2137 | } | |
2138 | ||
2139 | rc = sg_alloc_table(&buf_desc->sgt[lnk->link_idx], nents, GFP_KERNEL); | |
b9247544 KG |
2140 | if (rc) |
2141 | return rc; | |
b8d19945 WG |
2142 | |
2143 | if (buf_desc->is_vm) { | |
2144 | /* virtually contiguous buffer */ | |
2145 | for_each_sg(buf_desc->sgt[lnk->link_idx].sgl, sg, nents, i) { | |
2146 | size = min_t(int, PAGE_SIZE - offset, buf_size); | |
2147 | sg_set_page(sg, vmalloc_to_page(buf), size, offset); | |
2148 | buf += size / sizeof(*buf); | |
2149 | buf_size -= size; | |
2150 | offset = 0; | |
2151 | } | |
2152 | } else { | |
2153 | /* physically contiguous buffer */ | |
2154 | sg_set_buf(buf_desc->sgt[lnk->link_idx].sgl, | |
2155 | buf_desc->cpu_addr, buf_desc->len); | |
2156 | } | |
b9247544 KG |
2157 | |
2158 | /* map sg table to DMA address */ | |
2159 | rc = smc_ib_buf_map_sg(lnk, buf_desc, | |
2160 | is_rmb ? DMA_FROM_DEVICE : DMA_TO_DEVICE); | |
2161 | /* SMC protocol depends on mapping to one DMA address only */ | |
b8d19945 | 2162 | if (rc != nents) { |
b9247544 KG |
2163 | rc = -EAGAIN; |
2164 | goto free_table; | |
2165 | } | |
2166 | ||
0ef69e78 GW |
2167 | buf_desc->is_dma_need_sync |= |
2168 | smc_ib_is_sg_need_sync(lnk, buf_desc) << lnk->link_idx; | |
2169 | ||
b8d19945 WG |
2170 | if (is_rmb || buf_desc->is_vm) { |
2171 | /* create a new memory region for the RMB or vzalloced sndbuf */ | |
2172 | access_flags = is_rmb ? | |
2173 | IB_ACCESS_REMOTE_WRITE | IB_ACCESS_LOCAL_WRITE : | |
2174 | IB_ACCESS_LOCAL_WRITE; | |
2175 | ||
2176 | rc = smc_ib_get_memory_region(lnk->roce_pd, access_flags, | |
b9247544 KG |
2177 | buf_desc, lnk->link_idx); |
2178 | if (rc) | |
2179 | goto buf_unmap; | |
b8d19945 WG |
2180 | smc_ib_sync_sg_for_device(lnk, buf_desc, |
2181 | is_rmb ? DMA_FROM_DEVICE : DMA_TO_DEVICE); | |
b9247544 KG |
2182 | } |
2183 | buf_desc->is_map_ib[lnk->link_idx] = true; | |
2184 | return 0; | |
2185 | ||
2186 | buf_unmap: | |
2187 | smc_ib_buf_unmap_sg(lnk, buf_desc, | |
2188 | is_rmb ? DMA_FROM_DEVICE : DMA_TO_DEVICE); | |
2189 | free_table: | |
2190 | sg_free_table(&buf_desc->sgt[lnk->link_idx]); | |
2191 | return rc; | |
2192 | } | |
2193 | ||
b8d19945 | 2194 | /* register a new buf on IB device, rmb or vzalloced sndbuf |
d5500667 KG |
2195 | * must be called under lgr->llc_conf_mutex lock |
2196 | */ | |
b8d19945 | 2197 | int smcr_link_reg_buf(struct smc_link *link, struct smc_buf_desc *buf_desc) |
7562a13d KG |
2198 | { |
2199 | if (list_empty(&link->lgr->list)) | |
2200 | return -ENOLINK; | |
b8d19945 WG |
2201 | if (!buf_desc->is_reg_mr[link->link_idx]) { |
2202 | /* register memory region for new buf */ | |
2203 | if (buf_desc->is_vm) | |
2204 | buf_desc->mr[link->link_idx]->iova = | |
2205 | (uintptr_t)buf_desc->cpu_addr; | |
2206 | if (smc_wr_reg_send(link, buf_desc->mr[link->link_idx])) { | |
2207 | buf_desc->is_reg_err = true; | |
7562a13d KG |
2208 | return -EFAULT; |
2209 | } | |
b8d19945 | 2210 | buf_desc->is_reg_mr[link->link_idx] = true; |
7562a13d KG |
2211 | } |
2212 | return 0; | |
2213 | } | |
2214 | ||
aff7bfed | 2215 | static int _smcr_buf_map_lgr(struct smc_link *lnk, struct rw_semaphore *lock, |
fb33d277 KG |
2216 | struct list_head *lst, bool is_rmb) |
2217 | { | |
2218 | struct smc_buf_desc *buf_desc, *bf; | |
2219 | int rc = 0; | |
2220 | ||
aff7bfed | 2221 | down_write(lock); |
fb33d277 KG |
2222 | list_for_each_entry_safe(buf_desc, bf, lst, list) { |
2223 | if (!buf_desc->used) | |
2224 | continue; | |
2225 | rc = smcr_buf_map_link(buf_desc, is_rmb, lnk); | |
2226 | if (rc) | |
2227 | goto out; | |
2228 | } | |
2229 | out: | |
aff7bfed | 2230 | up_write(lock); |
fb33d277 KG |
2231 | return rc; |
2232 | } | |
2233 | ||
2234 | /* map all used buffers of lgr for a new link */ | |
2235 | int smcr_buf_map_lgr(struct smc_link *lnk) | |
2236 | { | |
2237 | struct smc_link_group *lgr = lnk->lgr; | |
2238 | int i, rc = 0; | |
2239 | ||
2240 | for (i = 0; i < SMC_RMBE_SIZES; i++) { | |
2241 | rc = _smcr_buf_map_lgr(lnk, &lgr->rmbs_lock, | |
2242 | &lgr->rmbs[i], true); | |
2243 | if (rc) | |
2244 | return rc; | |
2245 | rc = _smcr_buf_map_lgr(lnk, &lgr->sndbufs_lock, | |
2246 | &lgr->sndbufs[i], false); | |
2247 | if (rc) | |
2248 | return rc; | |
2249 | } | |
2250 | return 0; | |
2251 | } | |
2252 | ||
d5500667 KG |
2253 | /* register all used buffers of lgr for a new link, |
2254 | * must be called under lgr->llc_conf_mutex lock | |
2255 | */ | |
fb33d277 KG |
2256 | int smcr_buf_reg_lgr(struct smc_link *lnk) |
2257 | { | |
2258 | struct smc_link_group *lgr = lnk->lgr; | |
2259 | struct smc_buf_desc *buf_desc, *bf; | |
2260 | int i, rc = 0; | |
2261 | ||
b8d19945 | 2262 | /* reg all RMBs for a new link */ |
aff7bfed | 2263 | down_write(&lgr->rmbs_lock); |
fb33d277 KG |
2264 | for (i = 0; i < SMC_RMBE_SIZES; i++) { |
2265 | list_for_each_entry_safe(buf_desc, bf, &lgr->rmbs[i], list) { | |
2266 | if (!buf_desc->used) | |
2267 | continue; | |
b8d19945 WG |
2268 | rc = smcr_link_reg_buf(lnk, buf_desc); |
2269 | if (rc) { | |
aff7bfed | 2270 | up_write(&lgr->rmbs_lock); |
b8d19945 WG |
2271 | return rc; |
2272 | } | |
fb33d277 KG |
2273 | } |
2274 | } | |
aff7bfed | 2275 | up_write(&lgr->rmbs_lock); |
b8d19945 WG |
2276 | |
2277 | if (lgr->buf_type == SMCR_PHYS_CONT_BUFS) | |
2278 | return rc; | |
2279 | ||
2280 | /* reg all vzalloced sndbufs for a new link */ | |
aff7bfed | 2281 | down_write(&lgr->sndbufs_lock); |
b8d19945 WG |
2282 | for (i = 0; i < SMC_RMBE_SIZES; i++) { |
2283 | list_for_each_entry_safe(buf_desc, bf, &lgr->sndbufs[i], list) { | |
2284 | if (!buf_desc->used || !buf_desc->is_vm) | |
2285 | continue; | |
2286 | rc = smcr_link_reg_buf(lnk, buf_desc); | |
2287 | if (rc) { | |
aff7bfed | 2288 | up_write(&lgr->sndbufs_lock); |
b8d19945 WG |
2289 | return rc; |
2290 | } | |
2291 | } | |
2292 | } | |
aff7bfed | 2293 | up_write(&lgr->sndbufs_lock); |
fb33d277 KG |
2294 | return rc; |
2295 | } | |
2296 | ||
c6ba7c9b | 2297 | static struct smc_buf_desc *smcr_new_buf_create(struct smc_link_group *lgr, |
0908503a | 2298 | int bufsize) |
b33982c3 UB |
2299 | { |
2300 | struct smc_buf_desc *buf_desc; | |
b33982c3 UB |
2301 | |
2302 | /* try to alloc a new buffer */ | |
2303 | buf_desc = kzalloc(sizeof(*buf_desc), GFP_KERNEL); | |
2304 | if (!buf_desc) | |
2305 | return ERR_PTR(-ENOMEM); | |
2306 | ||
b8d19945 WG |
2307 | switch (lgr->buf_type) { |
2308 | case SMCR_PHYS_CONT_BUFS: | |
2309 | case SMCR_MIXED_BUFS: | |
2310 | buf_desc->order = get_order(bufsize); | |
2311 | buf_desc->pages = alloc_pages(GFP_KERNEL | __GFP_NOWARN | | |
2312 | __GFP_NOMEMALLOC | __GFP_COMP | | |
2313 | __GFP_NORETRY | __GFP_ZERO, | |
2314 | buf_desc->order); | |
2315 | if (buf_desc->pages) { | |
2316 | buf_desc->cpu_addr = | |
2317 | (void *)page_address(buf_desc->pages); | |
2318 | buf_desc->len = bufsize; | |
2319 | buf_desc->is_vm = false; | |
2320 | break; | |
2321 | } | |
2322 | if (lgr->buf_type == SMCR_PHYS_CONT_BUFS) | |
2323 | goto out; | |
2324 | fallthrough; // try virtually continguous buf | |
2325 | case SMCR_VIRT_CONT_BUFS: | |
2326 | buf_desc->order = get_order(bufsize); | |
2327 | buf_desc->cpu_addr = vzalloc(PAGE_SIZE << buf_desc->order); | |
2328 | if (!buf_desc->cpu_addr) | |
2329 | goto out; | |
2330 | buf_desc->pages = NULL; | |
2331 | buf_desc->len = bufsize; | |
2332 | buf_desc->is_vm = true; | |
2333 | break; | |
2334 | } | |
b9247544 | 2335 | return buf_desc; |
b8d19945 WG |
2336 | |
2337 | out: | |
2338 | kfree(buf_desc); | |
2339 | return ERR_PTR(-EAGAIN); | |
b9247544 | 2340 | } |
b33982c3 | 2341 | |
b9247544 KG |
2342 | /* map buf_desc on all usable links, |
2343 | * unused buffers stay mapped as long as the link is up | |
2344 | */ | |
2345 | static int smcr_buf_map_usable_links(struct smc_link_group *lgr, | |
2346 | struct smc_buf_desc *buf_desc, bool is_rmb) | |
2347 | { | |
e738455b | 2348 | int i, rc = 0, cnt = 0; |
b33982c3 | 2349 | |
d5500667 | 2350 | /* protect against parallel link reconfiguration */ |
f6421014 | 2351 | down_read(&lgr->llc_conf_mutex); |
b9247544 KG |
2352 | for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { |
2353 | struct smc_link *lnk = &lgr->lnk[i]; | |
b33982c3 | 2354 | |
d854fcbf | 2355 | if (!smc_link_usable(lnk)) |
b9247544 KG |
2356 | continue; |
2357 | if (smcr_buf_map_link(buf_desc, is_rmb, lnk)) { | |
b9247544 KG |
2358 | rc = -ENOMEM; |
2359 | goto out; | |
b33982c3 | 2360 | } |
e738455b | 2361 | cnt++; |
b33982c3 | 2362 | } |
b9247544 | 2363 | out: |
f6421014 | 2364 | up_read(&lgr->llc_conf_mutex); |
e738455b WG |
2365 | if (!rc && !cnt) |
2366 | rc = -EINVAL; | |
b9247544 | 2367 | return rc; |
b33982c3 UB |
2368 | } |
2369 | ||
c6ba7c9b HW |
2370 | static struct smc_buf_desc *smcd_new_buf_create(struct smc_link_group *lgr, |
2371 | bool is_dmb, int bufsize) | |
2372 | { | |
2373 | struct smc_buf_desc *buf_desc; | |
2374 | int rc; | |
2375 | ||
c6ba7c9b HW |
2376 | /* try to alloc a new DMB */ |
2377 | buf_desc = kzalloc(sizeof(*buf_desc), GFP_KERNEL); | |
2378 | if (!buf_desc) | |
2379 | return ERR_PTR(-ENOMEM); | |
2380 | if (is_dmb) { | |
2381 | rc = smc_ism_register_dmb(lgr, bufsize, buf_desc); | |
2382 | if (rc) { | |
2383 | kfree(buf_desc); | |
96d6fded KG |
2384 | if (rc == -ENOMEM) |
2385 | return ERR_PTR(-EAGAIN); | |
2386 | if (rc == -ENOSPC) | |
2387 | return ERR_PTR(-ENOSPC); | |
2388 | return ERR_PTR(-EIO); | |
c6ba7c9b | 2389 | } |
be244f28 HW |
2390 | buf_desc->pages = virt_to_page(buf_desc->cpu_addr); |
2391 | /* CDC header stored in buf. So, pretend it was smaller */ | |
2392 | buf_desc->len = bufsize - sizeof(struct smcd_cdc_msg); | |
c6ba7c9b HW |
2393 | } else { |
2394 | buf_desc->cpu_addr = kzalloc(bufsize, GFP_KERNEL | | |
2395 | __GFP_NOWARN | __GFP_NORETRY | | |
2396 | __GFP_NOMEMALLOC); | |
2397 | if (!buf_desc->cpu_addr) { | |
2398 | kfree(buf_desc); | |
2399 | return ERR_PTR(-EAGAIN); | |
2400 | } | |
2401 | buf_desc->len = bufsize; | |
2402 | } | |
2403 | return buf_desc; | |
2404 | } | |
2405 | ||
2406 | static int __smc_buf_create(struct smc_sock *smc, bool is_smcd, bool is_rmb) | |
cd6851f3 | 2407 | { |
8437bda0 | 2408 | struct smc_buf_desc *buf_desc = ERR_PTR(-ENOMEM); |
cd6851f3 UB |
2409 | struct smc_connection *conn = &smc->conn; |
2410 | struct smc_link_group *lgr = conn->lgr; | |
3e034725 | 2411 | struct list_head *buf_list; |
833bac7e | 2412 | int bufsize, bufsize_comp; |
aff7bfed | 2413 | struct rw_semaphore *lock; /* lock buffer list */ |
e0e4b8fa | 2414 | bool is_dgraded = false; |
cd6851f3 | 2415 | |
3e034725 UB |
2416 | if (is_rmb) |
2417 | /* use socket recv buffer size (w/o overhead) as start value */ | |
833bac7e | 2418 | bufsize = smc->sk.sk_rcvbuf / 2; |
3e034725 UB |
2419 | else |
2420 | /* use socket send buffer size (w/o overhead) as start value */ | |
833bac7e | 2421 | bufsize = smc->sk.sk_sndbuf / 2; |
3e034725 | 2422 | |
833bac7e GB |
2423 | for (bufsize_comp = smc_compress_bufsize(bufsize, is_smcd, is_rmb); |
2424 | bufsize_comp >= 0; bufsize_comp--) { | |
3e034725 UB |
2425 | if (is_rmb) { |
2426 | lock = &lgr->rmbs_lock; | |
833bac7e | 2427 | buf_list = &lgr->rmbs[bufsize_comp]; |
3e034725 UB |
2428 | } else { |
2429 | lock = &lgr->sndbufs_lock; | |
833bac7e | 2430 | buf_list = &lgr->sndbufs[bufsize_comp]; |
9d8fb617 | 2431 | } |
833bac7e | 2432 | bufsize = smc_uncompress_bufsize(bufsize_comp); |
a3fe3d01 | 2433 | |
3e034725 | 2434 | /* check for reusable slot in the link group */ |
833bac7e | 2435 | buf_desc = smc_buf_get_slot(bufsize_comp, lock, buf_list); |
3e034725 | 2436 | if (buf_desc) { |
0ef69e78 | 2437 | buf_desc->is_dma_need_sync = 0; |
e0d10354 | 2438 | SMC_STAT_RMB_SIZE(smc, is_smcd, is_rmb, true, bufsize); |
194730a9 | 2439 | SMC_STAT_BUF_REUSE(smc, is_smcd, is_rmb); |
cd6851f3 UB |
2440 | break; /* found reusable slot */ |
2441 | } | |
a3fe3d01 | 2442 | |
c6ba7c9b HW |
2443 | if (is_smcd) |
2444 | buf_desc = smcd_new_buf_create(lgr, is_rmb, bufsize); | |
2445 | else | |
0908503a | 2446 | buf_desc = smcr_new_buf_create(lgr, bufsize); |
c6ba7c9b | 2447 | |
b33982c3 UB |
2448 | if (PTR_ERR(buf_desc) == -ENOMEM) |
2449 | break; | |
e0e4b8fa GG |
2450 | if (IS_ERR(buf_desc)) { |
2451 | if (!is_dgraded) { | |
2452 | is_dgraded = true; | |
194730a9 | 2453 | SMC_STAT_RMB_DOWNGRADED(smc, is_smcd, is_rmb); |
e0e4b8fa | 2454 | } |
a3fe3d01 | 2455 | continue; |
e0e4b8fa | 2456 | } |
897e1c24 | 2457 | |
194730a9 | 2458 | SMC_STAT_RMB_ALLOC(smc, is_smcd, is_rmb); |
e0d10354 | 2459 | SMC_STAT_RMB_SIZE(smc, is_smcd, is_rmb, true, bufsize); |
3e034725 | 2460 | buf_desc->used = 1; |
aff7bfed | 2461 | down_write(lock); |
d386d59b | 2462 | smc_lgr_buf_list_add(lgr, is_rmb, buf_list, buf_desc); |
aff7bfed | 2463 | up_write(lock); |
3e034725 | 2464 | break; /* found */ |
cd6851f3 | 2465 | } |
3e034725 | 2466 | |
b33982c3 | 2467 | if (IS_ERR(buf_desc)) |
72b7f6c4 | 2468 | return PTR_ERR(buf_desc); |
3e034725 | 2469 | |
b9247544 KG |
2470 | if (!is_smcd) { |
2471 | if (smcr_buf_map_usable_links(lgr, buf_desc, is_rmb)) { | |
b8d19945 | 2472 | smcr_buf_unuse(buf_desc, is_rmb, lgr); |
b9247544 KG |
2473 | return -ENOMEM; |
2474 | } | |
2475 | } | |
2476 | ||
3e034725 UB |
2477 | if (is_rmb) { |
2478 | conn->rmb_desc = buf_desc; | |
833bac7e GB |
2479 | conn->rmbe_size_comp = bufsize_comp; |
2480 | smc->sk.sk_rcvbuf = bufsize * 2; | |
5f08318f | 2481 | atomic_set(&conn->bytes_to_rcv, 0); |
be244f28 HW |
2482 | conn->rmbe_update_limit = |
2483 | smc_rmb_wnd_update_limit(buf_desc->len); | |
c6ba7c9b HW |
2484 | if (is_smcd) |
2485 | smc_ism_set_conn(conn); /* map RMB/smcd_dev to conn */ | |
cd6851f3 | 2486 | } else { |
3e034725 | 2487 | conn->sndbuf_desc = buf_desc; |
833bac7e | 2488 | smc->sk.sk_sndbuf = bufsize * 2; |
3e034725 | 2489 | atomic_set(&conn->sndbuf_space, bufsize); |
cd6851f3 | 2490 | } |
3e034725 UB |
2491 | return 0; |
2492 | } | |
2493 | ||
10428dd8 UB |
2494 | void smc_sndbuf_sync_sg_for_device(struct smc_connection *conn) |
2495 | { | |
0ef69e78 GW |
2496 | if (!conn->sndbuf_desc->is_dma_need_sync) |
2497 | return; | |
ea89c6c0 WG |
2498 | if (!smc_conn_lgr_valid(conn) || conn->lgr->is_smcd || |
2499 | !smc_link_active(conn->lnk)) | |
c6ba7c9b | 2500 | return; |
387707fd | 2501 | smc_ib_sync_sg_for_device(conn->lnk, conn->sndbuf_desc, DMA_TO_DEVICE); |
10428dd8 UB |
2502 | } |
2503 | ||
2504 | void smc_rmb_sync_sg_for_cpu(struct smc_connection *conn) | |
2505 | { | |
b9247544 | 2506 | int i; |
10428dd8 | 2507 | |
0ef69e78 GW |
2508 | if (!conn->rmb_desc->is_dma_need_sync) |
2509 | return; | |
ea89c6c0 | 2510 | if (!smc_conn_lgr_valid(conn) || conn->lgr->is_smcd) |
c6ba7c9b | 2511 | return; |
b9247544 | 2512 | for (i = 0; i < SMC_LINKS_PER_LGR_MAX; i++) { |
741a49a4 | 2513 | if (!smc_link_active(&conn->lgr->lnk[i])) |
b9247544 KG |
2514 | continue; |
2515 | smc_ib_sync_sg_for_cpu(&conn->lgr->lnk[i], conn->rmb_desc, | |
2516 | DMA_FROM_DEVICE); | |
2517 | } | |
10428dd8 UB |
2518 | } |
2519 | ||
3e034725 UB |
2520 | /* create the send and receive buffer for an SMC socket; |
2521 | * receive buffers are called RMBs; | |
2522 | * (even though the SMC protocol allows more than one RMB-element per RMB, | |
2523 | * the Linux implementation uses just one RMB-element per RMB, i.e. uses an | |
2524 | * extra RMB for every connection in a link group | |
2525 | */ | |
c6ba7c9b | 2526 | int smc_buf_create(struct smc_sock *smc, bool is_smcd) |
3e034725 UB |
2527 | { |
2528 | int rc; | |
2529 | ||
2530 | /* create send buffer */ | |
ae2be35c WG |
2531 | if (is_smcd && |
2532 | smc_ism_support_dmb_nocopy(smc->conn.lgr->smcd)) | |
2533 | goto create_rmb; | |
2534 | ||
c6ba7c9b | 2535 | rc = __smc_buf_create(smc, is_smcd, false); |
3e034725 UB |
2536 | if (rc) |
2537 | return rc; | |
ae2be35c WG |
2538 | |
2539 | create_rmb: | |
3e034725 | 2540 | /* create rmb */ |
c6ba7c9b | 2541 | rc = __smc_buf_create(smc, is_smcd, true); |
ae2be35c | 2542 | if (rc && smc->conn.sndbuf_desc) { |
aff7bfed | 2543 | down_write(&smc->conn.lgr->sndbufs_lock); |
d386d59b WG |
2544 | smc_lgr_buf_list_del(smc->conn.lgr, false, |
2545 | smc->conn.sndbuf_desc); | |
aff7bfed | 2546 | up_write(&smc->conn.lgr->sndbufs_lock); |
6511aad3 | 2547 | smc_buf_free(smc->conn.lgr, false, smc->conn.sndbuf_desc); |
1d8df41d | 2548 | smc->conn.sndbuf_desc = NULL; |
fd7f3a74 | 2549 | } |
3e034725 | 2550 | return rc; |
cd6851f3 | 2551 | } |
bd4ad577 | 2552 | |
ae2be35c WG |
2553 | int smcd_buf_attach(struct smc_sock *smc) |
2554 | { | |
2555 | struct smc_connection *conn = &smc->conn; | |
2556 | struct smcd_dev *smcd = conn->lgr->smcd; | |
2557 | u64 peer_token = conn->peer_token; | |
2558 | struct smc_buf_desc *buf_desc; | |
2559 | int rc; | |
2560 | ||
2561 | buf_desc = kzalloc(sizeof(*buf_desc), GFP_KERNEL); | |
2562 | if (!buf_desc) | |
2563 | return -ENOMEM; | |
2564 | ||
2565 | /* The ghost sndbuf_desc describes the same memory region as | |
2566 | * peer RMB. Its lifecycle is consistent with the connection's | |
2567 | * and it will be freed with the connections instead of the | |
2568 | * link group. | |
2569 | */ | |
2570 | rc = smc_ism_attach_dmb(smcd, peer_token, buf_desc); | |
2571 | if (rc) | |
2572 | goto free; | |
2573 | ||
2574 | smc->sk.sk_sndbuf = buf_desc->len; | |
2575 | buf_desc->cpu_addr = | |
2576 | (u8 *)buf_desc->cpu_addr + sizeof(struct smcd_cdc_msg); | |
2577 | buf_desc->len -= sizeof(struct smcd_cdc_msg); | |
2578 | conn->sndbuf_desc = buf_desc; | |
2579 | conn->sndbuf_desc->used = 1; | |
2580 | atomic_set(&conn->sndbuf_space, conn->sndbuf_desc->len); | |
2581 | return 0; | |
2582 | ||
2583 | free: | |
2584 | kfree(buf_desc); | |
2585 | return rc; | |
2586 | } | |
2587 | ||
bd4ad577 UB |
2588 | static inline int smc_rmb_reserve_rtoken_idx(struct smc_link_group *lgr) |
2589 | { | |
2590 | int i; | |
2591 | ||
2592 | for_each_clear_bit(i, lgr->rtokens_used_mask, SMC_RMBS_PER_LGR_MAX) { | |
2593 | if (!test_and_set_bit(i, lgr->rtokens_used_mask)) | |
2594 | return i; | |
2595 | } | |
2596 | return -ENOSPC; | |
2597 | } | |
2598 | ||
ba21abd2 KG |
2599 | static int smc_rtoken_find_by_link(struct smc_link_group *lgr, int lnk_idx, |
2600 | u32 rkey) | |
2601 | { | |
2602 | int i; | |
2603 | ||
2604 | for (i = 0; i < SMC_RMBS_PER_LGR_MAX; i++) { | |
2605 | if (test_bit(i, lgr->rtokens_used_mask) && | |
2606 | lgr->rtokens[i][lnk_idx].rkey == rkey) | |
2607 | return i; | |
2608 | } | |
2609 | return -ENOENT; | |
2610 | } | |
2611 | ||
2612 | /* set rtoken for a new link to an existing rmb */ | |
2613 | void smc_rtoken_set(struct smc_link_group *lgr, int link_idx, int link_idx_new, | |
2614 | __be32 nw_rkey_known, __be64 nw_vaddr, __be32 nw_rkey) | |
2615 | { | |
2616 | int rtok_idx; | |
2617 | ||
2618 | rtok_idx = smc_rtoken_find_by_link(lgr, link_idx, ntohl(nw_rkey_known)); | |
2619 | if (rtok_idx == -ENOENT) | |
2620 | return; | |
2621 | lgr->rtokens[rtok_idx][link_idx_new].rkey = ntohl(nw_rkey); | |
2622 | lgr->rtokens[rtok_idx][link_idx_new].dma_addr = be64_to_cpu(nw_vaddr); | |
2623 | } | |
2624 | ||
2625 | /* set rtoken for a new link whose link_id is given */ | |
2626 | void smc_rtoken_set2(struct smc_link_group *lgr, int rtok_idx, int link_id, | |
2627 | __be64 nw_vaddr, __be32 nw_rkey) | |
2628 | { | |
2629 | u64 dma_addr = be64_to_cpu(nw_vaddr); | |
2630 | u32 rkey = ntohl(nw_rkey); | |
2631 | bool found = false; | |
2632 | int link_idx; | |
2633 | ||
2634 | for (link_idx = 0; link_idx < SMC_LINKS_PER_LGR_MAX; link_idx++) { | |
2635 | if (lgr->lnk[link_idx].link_id == link_id) { | |
2636 | found = true; | |
2637 | break; | |
2638 | } | |
2639 | } | |
2640 | if (!found) | |
2641 | return; | |
2642 | lgr->rtokens[rtok_idx][link_idx].rkey = rkey; | |
2643 | lgr->rtokens[rtok_idx][link_idx].dma_addr = dma_addr; | |
2644 | } | |
2645 | ||
4ed75de5 | 2646 | /* add a new rtoken from peer */ |
387707fd | 2647 | int smc_rtoken_add(struct smc_link *lnk, __be64 nw_vaddr, __be32 nw_rkey) |
bd4ad577 | 2648 | { |
387707fd | 2649 | struct smc_link_group *lgr = smc_get_lgr(lnk); |
4ed75de5 KG |
2650 | u64 dma_addr = be64_to_cpu(nw_vaddr); |
2651 | u32 rkey = ntohl(nw_rkey); | |
bd4ad577 UB |
2652 | int i; |
2653 | ||
2654 | for (i = 0; i < SMC_RMBS_PER_LGR_MAX; i++) { | |
387707fd KG |
2655 | if (lgr->rtokens[i][lnk->link_idx].rkey == rkey && |
2656 | lgr->rtokens[i][lnk->link_idx].dma_addr == dma_addr && | |
bd4ad577 | 2657 | test_bit(i, lgr->rtokens_used_mask)) { |
4ed75de5 KG |
2658 | /* already in list */ |
2659 | return i; | |
2660 | } | |
2661 | } | |
2662 | i = smc_rmb_reserve_rtoken_idx(lgr); | |
2663 | if (i < 0) | |
2664 | return i; | |
387707fd KG |
2665 | lgr->rtokens[i][lnk->link_idx].rkey = rkey; |
2666 | lgr->rtokens[i][lnk->link_idx].dma_addr = dma_addr; | |
4ed75de5 KG |
2667 | return i; |
2668 | } | |
2669 | ||
e07d31dc | 2670 | /* delete an rtoken from all links */ |
387707fd | 2671 | int smc_rtoken_delete(struct smc_link *lnk, __be32 nw_rkey) |
4ed75de5 | 2672 | { |
387707fd | 2673 | struct smc_link_group *lgr = smc_get_lgr(lnk); |
4ed75de5 | 2674 | u32 rkey = ntohl(nw_rkey); |
e07d31dc | 2675 | int i, j; |
4ed75de5 KG |
2676 | |
2677 | for (i = 0; i < SMC_RMBS_PER_LGR_MAX; i++) { | |
387707fd | 2678 | if (lgr->rtokens[i][lnk->link_idx].rkey == rkey && |
4ed75de5 | 2679 | test_bit(i, lgr->rtokens_used_mask)) { |
e07d31dc KG |
2680 | for (j = 0; j < SMC_LINKS_PER_LGR_MAX; j++) { |
2681 | lgr->rtokens[i][j].rkey = 0; | |
2682 | lgr->rtokens[i][j].dma_addr = 0; | |
2683 | } | |
4ed75de5 | 2684 | clear_bit(i, lgr->rtokens_used_mask); |
bd4ad577 UB |
2685 | return 0; |
2686 | } | |
2687 | } | |
4ed75de5 KG |
2688 | return -ENOENT; |
2689 | } | |
2690 | ||
2691 | /* save rkey and dma_addr received from peer during clc handshake */ | |
2692 | int smc_rmb_rtoken_handling(struct smc_connection *conn, | |
e07d31dc | 2693 | struct smc_link *lnk, |
4ed75de5 KG |
2694 | struct smc_clc_msg_accept_confirm *clc) |
2695 | { | |
3d9725a6 UB |
2696 | conn->rtoken_idx = smc_rtoken_add(lnk, clc->r0.rmb_dma_addr, |
2697 | clc->r0.rmb_rkey); | |
bd4ad577 UB |
2698 | if (conn->rtoken_idx < 0) |
2699 | return conn->rtoken_idx; | |
bd4ad577 UB |
2700 | return 0; |
2701 | } | |
9fda3510 | 2702 | |
c3d9494e UB |
2703 | static void smc_core_going_away(void) |
2704 | { | |
2705 | struct smc_ib_device *smcibdev; | |
2706 | struct smcd_dev *smcd; | |
2707 | ||
92f3cb0e | 2708 | mutex_lock(&smc_ib_devices.mutex); |
c3d9494e UB |
2709 | list_for_each_entry(smcibdev, &smc_ib_devices.list, list) { |
2710 | int i; | |
2711 | ||
2712 | for (i = 0; i < SMC_MAX_PORTS; i++) | |
2713 | set_bit(i, smcibdev->ports_going_away); | |
2714 | } | |
92f3cb0e | 2715 | mutex_unlock(&smc_ib_devices.mutex); |
c3d9494e | 2716 | |
82087c03 | 2717 | mutex_lock(&smcd_dev_list.mutex); |
c3d9494e UB |
2718 | list_for_each_entry(smcd, &smcd_dev_list.list, list) { |
2719 | smcd->going_away = 1; | |
2720 | } | |
82087c03 | 2721 | mutex_unlock(&smcd_dev_list.mutex); |
c3d9494e UB |
2722 | } |
2723 | ||
5421ec28 UB |
2724 | /* Clean up all SMC link groups */ |
2725 | static void smc_lgrs_shutdown(void) | |
9fda3510 | 2726 | { |
a2351c5d | 2727 | struct smcd_dev *smcd; |
9fda3510 | 2728 | |
c3d9494e UB |
2729 | smc_core_going_away(); |
2730 | ||
0b29ec64 | 2731 | smc_smcr_terminate_all(NULL); |
a2351c5d | 2732 | |
82087c03 | 2733 | mutex_lock(&smcd_dev_list.mutex); |
a2351c5d | 2734 | list_for_each_entry(smcd, &smcd_dev_list.list, list) |
5421ec28 | 2735 | smc_smcd_terminate_all(smcd); |
82087c03 | 2736 | mutex_unlock(&smcd_dev_list.mutex); |
9fda3510 | 2737 | } |
5421ec28 | 2738 | |
a33a803c UB |
2739 | static int smc_core_reboot_event(struct notifier_block *this, |
2740 | unsigned long event, void *ptr) | |
2741 | { | |
2742 | smc_lgrs_shutdown(); | |
28a3b840 | 2743 | smc_ib_unregister_client(); |
8747716f | 2744 | smc_ism_exit(); |
a33a803c UB |
2745 | return 0; |
2746 | } | |
2747 | ||
2748 | static struct notifier_block smc_reboot_notifier = { | |
2749 | .notifier_call = smc_core_reboot_event, | |
2750 | }; | |
2751 | ||
6dabd405 UB |
2752 | int __init smc_core_init(void) |
2753 | { | |
a33a803c | 2754 | return register_reboot_notifier(&smc_reboot_notifier); |
6dabd405 UB |
2755 | } |
2756 | ||
5421ec28 UB |
2757 | /* Called (from smc_exit) when module is removed */ |
2758 | void smc_core_exit(void) | |
2759 | { | |
a33a803c | 2760 | unregister_reboot_notifier(&smc_reboot_notifier); |
5421ec28 UB |
2761 | smc_lgrs_shutdown(); |
2762 | } |