projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
ravb: do not write 1 to reserved bits
[linux-2.6-block.git] / net / smc / af_smc.c
CommitLineData
ac713874
UB		 1/*
2 * Shared Memory Communications over RDMA (SMC-R) and RoCE
3 *
4 * AF_SMC protocol family socket handler keeping the AF_INET sock address type
5 * applies to SOCK_STREAM sockets only
6 * offers an alternative communication option for TCP-protocol sockets
7 * applicable with RoCE-cards only
8 *
a046d57d 9 * Initial restrictions:
a046d57d 10 * - support for alternate links postponed
a046d57d 11 *
aaa4d33f 12 * Copyright IBM Corp. 2016, 2018
ac713874
UB		 13 *
14 * Author(s): Ursula Braun <ubraun@linux.vnet.ibm.com>
15 * based on prototype from Frank Blaschka
16 */
17
18#define KMSG_COMPONENT "smc"
19#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
20
21#include <linux/module.h>
22#include <linux/socket.h>
a046d57d 23#include <linux/workqueue.h>
5f08318f 24#include <linux/in.h>
c3edc401 25#include <linux/sched/signal.h>
41349844 26#include <linux/if_vlan.h>
c3edc401 27
ac713874 28#include <net/sock.h>
a046d57d 29#include <net/tcp.h>
f16a7dd5 30#include <net/smc.h>
9b67e26f 31#include <asm/ioctls.h>
ac713874
UB		 32
33#include "smc.h"
a046d57d 34#include "smc_clc.h"
9bf9abea 35#include "smc_llc.h"
5f08318f 36#include "smc_cdc.h"
0cfdd8f9 37#include "smc_core.h"
a4cf0443 38#include "smc_ib.h"
41349844 39#include "smc_ism.h"
6812baab 40#include "smc_pnet.h"
e6727f39 41#include "smc_tx.h"
952310cc 42#include "smc_rx.h"
b38d7324 43#include "smc_close.h"
ac713874 44
0cfdd8f9
UB		 45static DEFINE_MUTEX(smc_create_lgr_pending); /* serialize link group
46 * creation
47 */
48
a046d57d 49static void smc_tcp_listen_work(struct work_struct *);
24ac3a08 50static void smc_connect_work(struct work_struct *);
a046d57d 51
ac713874
UB		 52static void smc_set_keepalive(struct sock *sk, int val)
53{
54 struct smc_sock *smc = smc_sk(sk);
55
56 smc->clcsock->sk->sk_prot->keepalive(smc->clcsock->sk, val);
57}
58
f16a7dd5
UB		 59static struct smc_hashinfo smc_v4_hashinfo = {
60 .lock = __RW_LOCK_UNLOCKED(smc_v4_hashinfo.lock),
61};
62
aaa4d33f
KG		 63static struct smc_hashinfo smc_v6_hashinfo = {
64 .lock = __RW_LOCK_UNLOCKED(smc_v6_hashinfo.lock),
65};
66
f16a7dd5
UB		 67int smc_hash_sk(struct sock *sk)
68{
69 struct smc_hashinfo *h = sk->sk_prot->h.smc_hash;
70 struct hlist_head *head;
71
72 head = &h->ht;
73
74 write_lock_bh(&h->lock);
75 sk_add_node(sk, head);
76 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
77 write_unlock_bh(&h->lock);
78
79 return 0;
80}
81EXPORT_SYMBOL_GPL(smc_hash_sk);
82
83void smc_unhash_sk(struct sock *sk)
84{
85 struct smc_hashinfo *h = sk->sk_prot->h.smc_hash;
86
87 write_lock_bh(&h->lock);
88 if (sk_del_node_init(sk))
89 sock_prot_inuse_add(sock_net(sk), sk->sk_prot, -1);
90 write_unlock_bh(&h->lock);
91}
92EXPORT_SYMBOL_GPL(smc_unhash_sk);
93
94struct proto smc_proto = {
ac713874
UB		 95 .name = "SMC",
96 .owner = THIS_MODULE,
97 .keepalive = smc_set_keepalive,
f16a7dd5
UB		 98 .hash = smc_hash_sk,
99 .unhash = smc_unhash_sk,
ac713874 100 .obj_size = sizeof(struct smc_sock),
f16a7dd5 101 .h.smc_hash = &smc_v4_hashinfo,
5f0d5a3a 102 .slab_flags = SLAB_TYPESAFE_BY_RCU,
ac713874 103};
f16a7dd5 104EXPORT_SYMBOL_GPL(smc_proto);
ac713874 105
aaa4d33f
KG		 106struct proto smc_proto6 = {
107 .name = "SMC6",
108 .owner = THIS_MODULE,
109 .keepalive = smc_set_keepalive,
110 .hash = smc_hash_sk,
111 .unhash = smc_unhash_sk,
112 .obj_size = sizeof(struct smc_sock),
113 .h.smc_hash = &smc_v6_hashinfo,
114 .slab_flags = SLAB_TYPESAFE_BY_RCU,
115};
116EXPORT_SYMBOL_GPL(smc_proto6);
117
ac713874
UB		 118static int smc_release(struct socket *sock)
119{
120 struct sock *sk = sock->sk;
121 struct smc_sock *smc;
b38d7324 122 int rc = 0;
ac713874
UB		 123
124 if (!sk)
125 goto out;
126
127 smc = smc_sk(sk);
24ac3a08
UB		 128
129 /* cleanup for a dangling non-blocking connect */
130 flush_work(&smc->connect_work);
131 kfree(smc->connect_info);
132 smc->connect_info = NULL;
133
b38d7324
UB		 134 if (sk->sk_state == SMC_LISTEN)
135 /* smc_close_non_accepted() is called and acquires
136 * sock lock for child sockets again
137 */
138 lock_sock_nested(sk, SINGLE_DEPTH_NESTING);
139 else
140 lock_sock(sk);
ac713874 141
51f1de79 142 if (!smc->use_fallback) {
b38d7324
UB		 143 rc = smc_close_active(smc);
144 sock_set_flag(sk, SOCK_DEAD);
145 sk->sk_shutdown |= SHUTDOWN_MASK;
146 }
ac713874
UB		 147 if (smc->clcsock) {
148 sock_release(smc->clcsock);
149 smc->clcsock = NULL;
150 }
51f1de79 151 if (smc->use_fallback) {
e1bbdd57
UB		 152 if (sk->sk_state != SMC_LISTEN && sk->sk_state != SMC_INIT)
153 sock_put(sk); /* passive closing */
51f1de79
UB		 154 sk->sk_state = SMC_CLOSED;
155 sk->sk_state_change(sk);
156 }
ac713874
UB		 157
158 /* detach socket */
159 sock_orphan(sk);
160 sock->sk = NULL;
51f1de79 161 if (!smc->use_fallback && sk->sk_state == SMC_CLOSED)
b38d7324 162 smc_conn_free(&smc->conn);
ac713874
UB		 163 release_sock(sk);
164
51f1de79
UB		 165 sk->sk_prot->unhash(sk);
166 sock_put(sk); /* final sock_put */
ac713874 167out:
b38d7324 168 return rc;
ac713874
UB		 169}
170
171static void smc_destruct(struct sock *sk)
172{
173 if (sk->sk_state != SMC_CLOSED)
174 return;
175 if (!sock_flag(sk, SOCK_DEAD))
176 return;
177
178 sk_refcnt_debug_dec(sk);
179}
180
aaa4d33f
KG		 181static struct sock *smc_sock_alloc(struct net *net, struct socket *sock,
182 int protocol)
ac713874
UB		 183{
184 struct smc_sock *smc;
aaa4d33f 185 struct proto *prot;
ac713874
UB		 186 struct sock *sk;
187
aaa4d33f
KG		 188 prot = (protocol == SMCPROTO_SMC6) ? &smc_proto6 : &smc_proto;
189 sk = sk_alloc(net, PF_SMC, GFP_KERNEL, prot, 0);
ac713874
UB		 190 if (!sk)
191 return NULL;
192
193 sock_init_data(sock, sk); /* sets sk_refcnt to 1 */
194 sk->sk_state = SMC_INIT;
195 sk->sk_destruct = smc_destruct;
aaa4d33f 196 sk->sk_protocol = protocol;
ac713874 197 smc = smc_sk(sk);
a046d57d 198 INIT_WORK(&smc->tcp_listen_work, smc_tcp_listen_work);
24ac3a08 199 INIT_WORK(&smc->connect_work, smc_connect_work);
be7f3e59 200 INIT_DELAYED_WORK(&smc->conn.tx_work, smc_tx_work);
a046d57d
UB		 201 INIT_LIST_HEAD(&smc->accept_q);
202 spin_lock_init(&smc->accept_q_lock);
be7f3e59 203 spin_lock_init(&smc->conn.send_lock);
f16a7dd5 204 sk->sk_prot->hash(sk);
a046d57d 205 sk_refcnt_debug_inc(sk);
ac713874
UB		 206
207 return sk;
208}
209
210static int smc_bind(struct socket *sock, struct sockaddr *uaddr,
211 int addr_len)
212{
213 struct sockaddr_in *addr = (struct sockaddr_in *)uaddr;
214 struct sock *sk = sock->sk;
215 struct smc_sock *smc;
216 int rc;
217
218 smc = smc_sk(sk);
219
220 /* replicate tests from inet_bind(), to be safe wrt. future changes */
221 rc = -EINVAL;
222 if (addr_len < sizeof(struct sockaddr_in))
223 goto out;
224
225 rc = -EAFNOSUPPORT;
aaa4d33f
KG		 226 if (addr->sin_family != AF_INET &&
227 addr->sin_family != AF_INET6 &&
228 addr->sin_family != AF_UNSPEC)
229 goto out;
ac713874 230 /* accept AF_UNSPEC (mapped to AF_INET) only if s_addr is INADDR_ANY */
aaa4d33f
KG		 231 if (addr->sin_family == AF_UNSPEC &&
232 addr->sin_addr.s_addr != htonl(INADDR_ANY))
ac713874
UB		 233 goto out;
234
235 lock_sock(sk);
236
237 /* Check if socket is already active */
238 rc = -EINVAL;
239 if (sk->sk_state != SMC_INIT)
240 goto out_rel;
241
242 smc->clcsock->sk->sk_reuse = sk->sk_reuse;
243 rc = kernel_bind(smc->clcsock, uaddr, addr_len);
244
245out_rel:
246 release_sock(sk);
247out:
248 return rc;
249}
250
251static void smc_copy_sock_settings(struct sock *nsk, struct sock *osk,
252 unsigned long mask)
253{
254 /* options we don't get control via setsockopt for */
255 nsk->sk_type = osk->sk_type;
256 nsk->sk_sndbuf = osk->sk_sndbuf;
257 nsk->sk_rcvbuf = osk->sk_rcvbuf;
258 nsk->sk_sndtimeo = osk->sk_sndtimeo;
259 nsk->sk_rcvtimeo = osk->sk_rcvtimeo;
260 nsk->sk_mark = osk->sk_mark;
261 nsk->sk_priority = osk->sk_priority;
262 nsk->sk_rcvlowat = osk->sk_rcvlowat;
263 nsk->sk_bound_dev_if = osk->sk_bound_dev_if;
264 nsk->sk_err = osk->sk_err;
265
266 nsk->sk_flags &= ~mask;
267 nsk->sk_flags |= osk->sk_flags & mask;
268}
269
270#define SK_FLAGS_SMC_TO_CLC ((1UL << SOCK_URGINLINE) | \
271 (1UL << SOCK_KEEPOPEN) | \
272 (1UL << SOCK_LINGER) | \
273 (1UL << SOCK_BROADCAST) | \
274 (1UL << SOCK_TIMESTAMP) | \
275 (1UL << SOCK_DBG) | \
276 (1UL << SOCK_RCVTSTAMP) | \
277 (1UL << SOCK_RCVTSTAMPNS) | \
278 (1UL << SOCK_LOCALROUTE) | \
279 (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE) | \
280 (1UL << SOCK_RXQ_OVFL) | \
281 (1UL << SOCK_WIFI_STATUS) | \
282 (1UL << SOCK_NOFCS) | \
283 (1UL << SOCK_FILTER_LOCKED))
284/* copy only relevant settings and flags of SOL_SOCKET level from smc to
285 * clc socket (since smc is not called for these options from net/core)
286 */
287static void smc_copy_sock_settings_to_clc(struct smc_sock *smc)
288{
289 smc_copy_sock_settings(smc->clcsock->sk, &smc->sk, SK_FLAGS_SMC_TO_CLC);
290}
291
292#define SK_FLAGS_CLC_TO_SMC ((1UL << SOCK_URGINLINE) | \
293 (1UL << SOCK_KEEPOPEN) | \
294 (1UL << SOCK_LINGER) | \
295 (1UL << SOCK_DBG))
296/* copy only settings and flags relevant for smc from clc to smc socket */
297static void smc_copy_sock_settings_to_smc(struct smc_sock *smc)
298{
299 smc_copy_sock_settings(&smc->sk, smc->clcsock->sk, SK_FLAGS_CLC_TO_SMC);
300}
301
44aa81ce
KG		 302/* register a new rmb, optionally send confirm_rkey msg to register with peer */
303static int smc_reg_rmb(struct smc_link *link, struct smc_buf_desc *rmb_desc,
304 bool conf_rkey)
e63a5f8c
KG		 305{
306 /* register memory region for new rmb */
a6920d1d
KG		 307 if (smc_wr_reg_send(link, rmb_desc->mr_rx[SMC_SINGLE_LINK])) {
308 rmb_desc->regerr = 1;
e63a5f8c 309 return -EFAULT;
a6920d1d 310 }
44aa81ce
KG		 311 if (!conf_rkey)
312 return 0;
313 /* exchange confirm_rkey msg with peer */
314 if (smc_llc_do_confirm_rkey(link, rmb_desc)) {
315 rmb_desc->regerr = 1;
316 return -EFAULT;
317 }
e63a5f8c
KG		 318 return 0;
319}
320
0f627126 321static int smc_clnt_conf_first_link(struct smc_sock *smc)
9bf9abea 322{
877ae5be 323 struct net *net = sock_net(smc->clcsock->sk);
9bf9abea
UB		 324 struct smc_link_group *lgr = smc->conn.lgr;
325 struct smc_link *link;
326 int rest;
327 int rc;
328
329 link = &lgr->lnk[SMC_SINGLE_LINK];
330 /* receive CONFIRM LINK request from server over RoCE fabric */
331 rest = wait_for_completion_interruptible_timeout(
332 &link->llc_confirm,
333 SMC_LLC_WAIT_FIRST_TIME);
334 if (rest <= 0) {
335 struct smc_clc_msg_decline dclc;
336
337 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
338 SMC_CLC_DECLINE);
339 return rc;
340 }
341
75d320d6
KG		 342 if (link->llc_confirm_rc)
343 return SMC_CLC_DECL_RMBE_EC;
344
9bf9abea
UB		 345 rc = smc_ib_modify_qp_rts(link);
346 if (rc)
603cc149 347 return SMC_CLC_DECL_ERR_RDYLNK;
9bf9abea
UB		 348
349 smc_wr_remember_qp_attr(link);
652a1e41 350
44aa81ce 351 if (smc_reg_rmb(link, smc->conn.rmb_desc, false))
603cc149 352 return SMC_CLC_DECL_ERR_REGRMB;
652a1e41 353
9bf9abea 354 /* send CONFIRM LINK response over RoCE fabric */
947541f3 355 rc = smc_llc_send_confirm_link(link, SMC_LLC_RESP);
9bf9abea 356 if (rc < 0)
603cc149 357 return SMC_CLC_DECL_TIMEOUT_CL;
9bf9abea 358
52bedf37
KG		 359 /* receive ADD LINK request from server over RoCE fabric */
360 rest = wait_for_completion_interruptible_timeout(&link->llc_add,
361 SMC_LLC_WAIT_TIME);
362 if (rest <= 0) {
363 struct smc_clc_msg_decline dclc;
364
365 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
366 SMC_CLC_DECLINE);
367 return rc;
368 }
369
370 /* send add link reject message, only one link supported for now */
371 rc = smc_llc_send_add_link(link,
372 link->smcibdev->mac[link->ibport - 1],
7005ada6 373 link->gid, SMC_LLC_RESP);
52bedf37 374 if (rc < 0)
603cc149 375 return SMC_CLC_DECL_TIMEOUT_AL;
52bedf37 376
877ae5be 377 smc_llc_link_active(link, net->ipv4.sysctl_tcp_keepalive_time);
52bedf37 378
75d320d6 379 return 0;
9bf9abea
UB		 380}
381
41349844
HW		 382static void smcr_conn_save_peer_info(struct smc_sock *smc,
383 struct smc_clc_msg_accept_confirm *clc)
0cfdd8f9 384{
95d8d263
HW		 385 int bufsize = smc_uncompress_bufsize(clc->rmbe_size);
386
92a138e3 387 smc->conn.peer_rmbe_idx = clc->rmbe_idx;
5f08318f 388 smc->conn.local_tx_ctrl.token = ntohl(clc->rmbe_alert_token);
95d8d263 389 smc->conn.peer_rmbe_size = bufsize;
cd6851f3 390 atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size);
95d8d263 391 smc->conn.tx_off = bufsize * (smc->conn.peer_rmbe_idx - 1);
0cfdd8f9
UB		 392}
393
41349844
HW		 394static void smcd_conn_save_peer_info(struct smc_sock *smc,
395 struct smc_clc_msg_accept_confirm *clc)
396{
397 int bufsize = smc_uncompress_bufsize(clc->dmbe_size);
398
399 smc->conn.peer_rmbe_idx = clc->dmbe_idx;
400 smc->conn.peer_token = clc->token;
401 /* msg header takes up space in the buffer */
402 smc->conn.peer_rmbe_size = bufsize - sizeof(struct smcd_cdc_msg);
403 atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size);
404 smc->conn.tx_off = bufsize * smc->conn.peer_rmbe_idx;
405}
406
407static void smc_conn_save_peer_info(struct smc_sock *smc,
408 struct smc_clc_msg_accept_confirm *clc)
409{
410 if (smc->conn.lgr->is_smcd)
411 smcd_conn_save_peer_info(smc, clc);
412 else
413 smcr_conn_save_peer_info(smc, clc);
414}
415
0cfdd8f9
UB		 416static void smc_link_save_peer_info(struct smc_link *link,
417 struct smc_clc_msg_accept_confirm *clc)
418{
419 link->peer_qpn = ntoh24(clc->qpn);
420 memcpy(link->peer_gid, clc->lcl.gid, SMC_GID_SIZE);
421 memcpy(link->peer_mac, clc->lcl.mac, sizeof(link->peer_mac));
422 link->peer_psn = ntoh24(clc->psn);
423 link->peer_mtu = clc->qp_mtu;
424}
425
3b2dec26 426/* fall back during connect */
603cc149 427static int smc_connect_fallback(struct smc_sock *smc, int reason_code)
a046d57d 428{
3b2dec26 429 smc->use_fallback = true;
603cc149 430 smc->fallback_rsn = reason_code;
3b2dec26
HW		 431 smc_copy_sock_settings_to_clc(smc);
432 if (smc->sk.sk_state == SMC_INIT)
433 smc->sk.sk_state = SMC_ACTIVE;
434 return 0;
435}
51f1de79 436
3b2dec26
HW		 437/* decline and fall back during connect */
438static int smc_connect_decline_fallback(struct smc_sock *smc, int reason_code)
439{
440 int rc;
ee9dfbef 441
e1bbdd57
UB		 442 if (reason_code < 0) { /* error, fallback is not possible */
443 if (smc->sk.sk_state == SMC_INIT)
444 sock_put(&smc->sk); /* passive closing */
3b2dec26 445 return reason_code;
e1bbdd57 446 }
603cc149 447 if (reason_code != SMC_CLC_DECL_PEERDECL) {
3b2dec26 448 rc = smc_clc_send_decline(smc, reason_code);
e1bbdd57
UB		 449 if (rc < 0) {
450 if (smc->sk.sk_state == SMC_INIT)
451 sock_put(&smc->sk); /* passive closing */
3b2dec26 452 return rc;
e1bbdd57 453 }
c5c1cc9c 454 }
603cc149 455 return smc_connect_fallback(smc, reason_code);
3b2dec26 456}
c5c1cc9c 457
3b2dec26
HW		 458/* abort connecting */
459static int smc_connect_abort(struct smc_sock *smc, int reason_code,
460 int local_contact)
461{
462 if (local_contact == SMC_FIRST_CONTACT)
463 smc_lgr_forget(smc->conn.lgr);
464 mutex_unlock(&smc_create_lgr_pending);
465 smc_conn_free(&smc->conn);
3b2dec26
HW		 466 return reason_code;
467}
468
469/* check if there is a rdma device available for this connection. */
470/* called for connect and listen */
471static int smc_check_rdma(struct smc_sock *smc, struct smc_ib_device **ibdev,
7005ada6 472 u8 *ibport, unsigned short vlan_id, u8 gid[])
3b2dec26
HW		 473{
474 int reason_code = 0;
a046d57d
UB		 475
476 /* PNET table look up: search active ib_device and port
477 * within same PNETID that also contains the ethernet device
478 * used for the internal TCP socket
479 */
7005ada6
UB		 480 smc_pnet_find_roce_resource(smc->clcsock->sk, ibdev, ibport, vlan_id,
481 gid);
3b2dec26 482 if (!(*ibdev))
a046d57d 483 reason_code = SMC_CLC_DECL_CNFERR; /* configuration error */
3b2dec26
HW		 484
485 return reason_code;
486}
487
41349844
HW		 488/* check if there is an ISM device available for this connection. */
489/* called for connect and listen */
490static int smc_check_ism(struct smc_sock *smc, struct smcd_dev **ismdev)
491{
492 /* Find ISM device with same PNETID as connecting interface */
493 smc_pnet_find_ism_resource(smc->clcsock->sk, ismdev);
494 if (!(*ismdev))
495 return SMC_CLC_DECL_CNFERR; /* configuration error */
496 return 0;
497}
498
499/* Check for VLAN ID and register it on ISM device just for CLC handshake */
500static int smc_connect_ism_vlan_setup(struct smc_sock *smc,
501 struct smcd_dev *ismdev,
502 unsigned short vlan_id)
503{
504 if (vlan_id && smc_ism_get_vlan(ismdev, vlan_id))
505 return SMC_CLC_DECL_CNFERR;
506 return 0;
507}
508
509/* cleanup temporary VLAN ID registration used for CLC handshake. If ISM is
510 * used, the VLAN ID will be registered again during the connection setup.
511 */
512static int smc_connect_ism_vlan_cleanup(struct smc_sock *smc, bool is_smcd,
513 struct smcd_dev *ismdev,
514 unsigned short vlan_id)
515{
516 if (!is_smcd)
517 return 0;
518 if (vlan_id && smc_ism_put_vlan(ismdev, vlan_id))
519 return SMC_CLC_DECL_CNFERR;
520 return 0;
521}
522
3b2dec26 523/* CLC handshake during connect */
c758dfdd 524static int smc_connect_clc(struct smc_sock *smc, int smc_type,
3b2dec26 525 struct smc_clc_msg_accept_confirm *aclc,
41349844 526 struct smc_ib_device *ibdev, u8 ibport,
7005ada6 527 u8 gid[], struct smcd_dev *ismdev)
3b2dec26
HW		 528{
529 int rc = 0;
a046d57d
UB		 530
531 /* do inband token exchange */
7005ada6 532 rc = smc_clc_send_proposal(smc, smc_type, ibdev, ibport, gid, ismdev);
3b2dec26
HW		 533 if (rc)
534 return rc;
a046d57d 535 /* receive SMC Accept CLC message */
3b2dec26
HW		 536 return smc_clc_wait_msg(smc, aclc, sizeof(*aclc), SMC_CLC_ACCEPT);
537}
538
539/* setup for RDMA connection of client */
540static int smc_connect_rdma(struct smc_sock *smc,
541 struct smc_clc_msg_accept_confirm *aclc,
542 struct smc_ib_device *ibdev, u8 ibport)
543{
544 int local_contact = SMC_FIRST_CONTACT;
545 struct smc_link *link;
546 int reason_code = 0;
a046d57d 547
0cfdd8f9 548 mutex_lock(&smc_create_lgr_pending);
c6ba7c9b
HW		 549 local_contact = smc_conn_create(smc, false, aclc->hdr.flag, ibdev,
550 ibport, &aclc->lcl, NULL, 0);
0cfdd8f9 551 if (local_contact < 0) {
3b2dec26 552 if (local_contact == -ENOMEM)
0cfdd8f9 553 reason_code = SMC_CLC_DECL_MEM;/* insufficient memory*/
3b2dec26 554 else if (local_contact == -ENOLINK)
0cfdd8f9 555 reason_code = SMC_CLC_DECL_SYNCERR; /* synchr. error */
1401ea04
KG		 556 else
557 reason_code = SMC_CLC_DECL_INTERR; /* other error */
3b2dec26 558 return smc_connect_abort(smc, reason_code, 0);
0cfdd8f9
UB		 559 }
560 link = &smc->conn.lgr->lnk[SMC_SINGLE_LINK];
a046d57d 561
3b2dec26 562 smc_conn_save_peer_info(smc, aclc);
cd6851f3 563
3e034725 564 /* create send buffer and rmb */
c6ba7c9b 565 if (smc_buf_create(smc, false))
3b2dec26 566 return smc_connect_abort(smc, SMC_CLC_DECL_MEM, local_contact);
cd6851f3 567
0cfdd8f9 568 if (local_contact == SMC_FIRST_CONTACT)
3b2dec26 569 smc_link_save_peer_info(link, aclc);
bd4ad577 570
3b2dec26 571 if (smc_rmb_rtoken_handling(&smc->conn, aclc))
603cc149 572 return smc_connect_abort(smc, SMC_CLC_DECL_ERR_RTOK,
3b2dec26 573 local_contact);
bd4ad577 574
46c28dbd
UB		 575 smc_close_init(smc);
576 smc_rx_init(smc);
577
bd4ad577 578 if (local_contact == SMC_FIRST_CONTACT) {
3b2dec26 579 if (smc_ib_ready_link(link))
603cc149 580 return smc_connect_abort(smc, SMC_CLC_DECL_ERR_RDYLNK,
3b2dec26 581 local_contact);
652a1e41 582 } else {
3b2dec26
HW		 583 if (!smc->conn.rmb_desc->reused &&
584 smc_reg_rmb(link, smc->conn.rmb_desc, true))
603cc149 585 return smc_connect_abort(smc, SMC_CLC_DECL_ERR_REGRMB,
3b2dec26 586 local_contact);
bd4ad577 587 }
10428dd8 588 smc_rmb_sync_sg_for_device(&smc->conn);
a046d57d 589
3b2dec26
HW		 590 reason_code = smc_clc_send_confirm(smc);
591 if (reason_code)
592 return smc_connect_abort(smc, reason_code, local_contact);
593
594 smc_tx_init(smc);
a046d57d 595
9bf9abea
UB		 596 if (local_contact == SMC_FIRST_CONTACT) {
597 /* QP confirmation over RoCE fabric */
0f627126 598 reason_code = smc_clnt_conf_first_link(smc);
3b2dec26
HW		 599 if (reason_code)
600 return smc_connect_abort(smc, reason_code,
601 local_contact);
9bf9abea 602 }
0cfdd8f9 603 mutex_unlock(&smc_create_lgr_pending);
e6727f39 604
a046d57d 605 smc_copy_sock_settings_to_clc(smc);
b38d7324
UB		 606 if (smc->sk.sk_state == SMC_INIT)
607 smc->sk.sk_state = SMC_ACTIVE;
a046d57d 608
3b2dec26
HW		 609 return 0;
610}
a046d57d 611
41349844
HW		 612/* setup for ISM connection of client */
613static int smc_connect_ism(struct smc_sock *smc,
614 struct smc_clc_msg_accept_confirm *aclc,
615 struct smcd_dev *ismdev)
616{
617 int local_contact = SMC_FIRST_CONTACT;
618 int rc = 0;
619
620 mutex_lock(&smc_create_lgr_pending);
621 local_contact = smc_conn_create(smc, true, aclc->hdr.flag, NULL, 0,
622 NULL, ismdev, aclc->gid);
623 if (local_contact < 0)
624 return smc_connect_abort(smc, SMC_CLC_DECL_MEM, 0);
625
626 /* Create send and receive buffers */
627 if (smc_buf_create(smc, true))
628 return smc_connect_abort(smc, SMC_CLC_DECL_MEM, local_contact);
629
630 smc_conn_save_peer_info(smc, aclc);
631 smc_close_init(smc);
632 smc_rx_init(smc);
633 smc_tx_init(smc);
634
635 rc = smc_clc_send_confirm(smc);
636 if (rc)
637 return smc_connect_abort(smc, rc, local_contact);
638 mutex_unlock(&smc_create_lgr_pending);
639
640 smc_copy_sock_settings_to_clc(smc);
641 if (smc->sk.sk_state == SMC_INIT)
642 smc->sk.sk_state = SMC_ACTIVE;
643
644 return 0;
645}
646
3b2dec26
HW		 647/* perform steps before actually connecting */
648static int __smc_connect(struct smc_sock *smc)
649{
41349844 650 bool ism_supported = false, rdma_supported = false;
3b2dec26
HW		 651 struct smc_clc_msg_accept_confirm aclc;
652 struct smc_ib_device *ibdev;
41349844 653 struct smcd_dev *ismdev;
7005ada6 654 u8 gid[SMC_GID_SIZE];
41349844
HW		 655 unsigned short vlan;
656 int smc_type;
3b2dec26
HW		 657 int rc = 0;
658 u8 ibport;
a046d57d 659
3b2dec26
HW		 660 sock_hold(&smc->sk); /* sock put in passive closing */
661
662 if (smc->use_fallback)
603cc149 663 return smc_connect_fallback(smc, smc->fallback_rsn);
3b2dec26
HW		 664
665 /* if peer has not signalled SMC-capability, fall back */
666 if (!tcp_sk(smc->clcsock->sk)->syn_smc)
603cc149 667 return smc_connect_fallback(smc, SMC_CLC_DECL_PEERNOSMC);
3b2dec26
HW		 668
669 /* IPSec connections opt out of SMC-R optimizations */
670 if (using_ipsec(smc))
671 return smc_connect_decline_fallback(smc, SMC_CLC_DECL_IPSEC);
672
41349844
HW		 673 /* check for VLAN ID */
674 if (smc_vlan_by_tcpsk(smc->clcsock, &vlan))
675 return smc_connect_decline_fallback(smc, SMC_CLC_DECL_CNFERR);
676
677 /* check if there is an ism device available */
678 if (!smc_check_ism(smc, &ismdev) &&
679 !smc_connect_ism_vlan_setup(smc, ismdev, vlan)) {
680 /* ISM is supported for this connection */
681 ism_supported = true;
682 smc_type = SMC_TYPE_D;
683 }
684
685 /* check if there is a rdma device available */
7005ada6 686 if (!smc_check_rdma(smc, &ibdev, &ibport, vlan, gid)) {
41349844
HW		 687 /* RDMA is supported for this connection */
688 rdma_supported = true;
689 if (ism_supported)
690 smc_type = SMC_TYPE_B; /* both */
691 else
692 smc_type = SMC_TYPE_R; /* only RDMA */
693 }
694
695 /* if neither ISM nor RDMA are supported, fallback */
696 if (!rdma_supported && !ism_supported)
603cc149 697 return smc_connect_decline_fallback(smc, SMC_CLC_DECL_NOSMCDEV);
3b2dec26
HW		 698
699 /* perform CLC handshake */
7005ada6 700 rc = smc_connect_clc(smc, smc_type, &aclc, ibdev, ibport, gid, ismdev);
41349844
HW		 701 if (rc) {
702 smc_connect_ism_vlan_cleanup(smc, ism_supported, ismdev, vlan);
3b2dec26 703 return smc_connect_decline_fallback(smc, rc);
41349844 704 }
3b2dec26 705
41349844
HW		 706 /* depending on previous steps, connect using rdma or ism */
707 if (rdma_supported && aclc.hdr.path == SMC_TYPE_R)
708 rc = smc_connect_rdma(smc, &aclc, ibdev, ibport);
709 else if (ism_supported && aclc.hdr.path == SMC_TYPE_D)
710 rc = smc_connect_ism(smc, &aclc, ismdev);
711 else
603cc149 712 rc = SMC_CLC_DECL_MODEUNSUPP;
41349844
HW		 713 if (rc) {
714 smc_connect_ism_vlan_cleanup(smc, ism_supported, ismdev, vlan);
3b2dec26 715 return smc_connect_decline_fallback(smc, rc);
41349844 716 }
3b2dec26 717
41349844 718 smc_connect_ism_vlan_cleanup(smc, ism_supported, ismdev, vlan);
3b2dec26 719 return 0;
a046d57d
UB		 720}
721
24ac3a08
UB		 722static void smc_connect_work(struct work_struct *work)
723{
724 struct smc_sock *smc = container_of(work, struct smc_sock,
725 connect_work);
726 int rc;
727
728 lock_sock(&smc->sk);
729 rc = kernel_connect(smc->clcsock, &smc->connect_info->addr,
730 smc->connect_info->alen, smc->connect_info->flags);
731 if (smc->clcsock->sk->sk_err) {
732 smc->sk.sk_err = smc->clcsock->sk->sk_err;
733 goto out;
734 }
735 if (rc < 0) {
736 smc->sk.sk_err = -rc;
737 goto out;
738 }
739
740 rc = __smc_connect(smc);
741 if (rc < 0)
742 smc->sk.sk_err = -rc;
743
744out:
745 smc->sk.sk_state_change(&smc->sk);
746 kfree(smc->connect_info);
747 smc->connect_info = NULL;
748 release_sock(&smc->sk);
749}
750
ac713874
UB		 751static int smc_connect(struct socket *sock, struct sockaddr *addr,
752 int alen, int flags)
753{
754 struct sock *sk = sock->sk;
755 struct smc_sock *smc;
756 int rc = -EINVAL;
757
758 smc = smc_sk(sk);
759
760 /* separate smc parameter checking to be safe */
761 if (alen < sizeof(addr->sa_family))
762 goto out_err;
aaa4d33f 763 if (addr->sa_family != AF_INET && addr->sa_family != AF_INET6)
ac713874
UB		 764 goto out_err;
765
766 lock_sock(sk);
767 switch (sk->sk_state) {
768 default:
769 goto out;
770 case SMC_ACTIVE:
771 rc = -EISCONN;
772 goto out;
773 case SMC_INIT:
774 rc = 0;
775 break;
776 }
777
778 smc_copy_sock_settings_to_clc(smc);
c5c1cc9c 779 tcp_sk(smc->clcsock->sk)->syn_smc = 1;
24ac3a08
UB		 780 if (flags & O_NONBLOCK) {
781 if (smc->connect_info) {
782 rc = -EALREADY;
783 goto out;
784 }
785 smc->connect_info = kzalloc(alen + 2 * sizeof(int), GFP_KERNEL);
786 if (!smc->connect_info) {
787 rc = -ENOMEM;
788 goto out;
789 }
790 smc->connect_info->alen = alen;
791 smc->connect_info->flags = flags ^ O_NONBLOCK;
792 memcpy(&smc->connect_info->addr, addr, alen);
793 schedule_work(&smc->connect_work);
794 rc = -EINPROGRESS;
795 } else {
796 rc = kernel_connect(smc->clcsock, addr, alen, flags);
797 if (rc)
798 goto out;
ac713874 799
24ac3a08
UB		 800 rc = __smc_connect(smc);
801 if (rc < 0)
802 goto out;
803 else
804 rc = 0; /* success cases including fallback */
805 }
ac713874
UB		 806
807out:
808 release_sock(sk);
809out_err:
810 return rc;
811}
812
813static int smc_clcsock_accept(struct smc_sock *lsmc, struct smc_sock **new_smc)
814{
3163c507
UB		 815 struct socket *new_clcsock = NULL;
816 struct sock *lsk = &lsmc->sk;
ac713874
UB		 817 struct sock *new_sk;
818 int rc;
819
3163c507 820 release_sock(lsk);
aaa4d33f 821 new_sk = smc_sock_alloc(sock_net(lsk), NULL, lsk->sk_protocol);
ac713874
UB		 822 if (!new_sk) {
823 rc = -ENOMEM;
3163c507 824 lsk->sk_err = ENOMEM;
ac713874 825 *new_smc = NULL;
3163c507 826 lock_sock(lsk);
ac713874
UB		 827 goto out;
828 }
829 *new_smc = smc_sk(new_sk);
830
831 rc = kernel_accept(lsmc->clcsock, &new_clcsock, 0);
3163c507 832 lock_sock(lsk);
35a6b178 833 if (rc < 0)
3163c507 834 lsk->sk_err = -rc;
35a6b178 835 if (rc < 0 || lsk->sk_state == SMC_CLOSED) {
a046d57d
UB		 836 if (new_clcsock)
837 sock_release(new_clcsock);
838 new_sk->sk_state = SMC_CLOSED;
839 sock_set_flag(new_sk, SOCK_DEAD);
3163c507 840 new_sk->sk_prot->unhash(new_sk);
51f1de79 841 sock_put(new_sk); /* final */
ac713874
UB		 842 *new_smc = NULL;
843 goto out;
844 }
845
846 (*new_smc)->clcsock = new_clcsock;
847out:
848 return rc;
849}
850
a046d57d
UB		 851/* add a just created sock to the accept queue of the listen sock as
852 * candidate for a following socket accept call from user space
853 */
854static void smc_accept_enqueue(struct sock *parent, struct sock *sk)
855{
856 struct smc_sock *par = smc_sk(parent);
857
51f1de79 858 sock_hold(sk); /* sock_put in smc_accept_unlink () */
a046d57d
UB		 859 spin_lock(&par->accept_q_lock);
860 list_add_tail(&smc_sk(sk)->accept_q, &par->accept_q);
861 spin_unlock(&par->accept_q_lock);
862 sk_acceptq_added(parent);
863}
864
865/* remove a socket from the accept queue of its parental listening socket */
866static void smc_accept_unlink(struct sock *sk)
867{
868 struct smc_sock *par = smc_sk(sk)->listen_smc;
869
870 spin_lock(&par->accept_q_lock);
871 list_del_init(&smc_sk(sk)->accept_q);
872 spin_unlock(&par->accept_q_lock);
873 sk_acceptq_removed(&smc_sk(sk)->listen_smc->sk);
51f1de79 874 sock_put(sk); /* sock_hold in smc_accept_enqueue */
a046d57d
UB		 875}
876
877/* remove a sock from the accept queue to bind it to a new socket created
878 * for a socket accept call from user space
879 */
b38d7324
UB		 880struct sock *smc_accept_dequeue(struct sock *parent,
881 struct socket *new_sock)
a046d57d
UB		 882{
883 struct smc_sock *isk, *n;
884 struct sock *new_sk;
885
886 list_for_each_entry_safe(isk, n, &smc_sk(parent)->accept_q, accept_q) {
887 new_sk = (struct sock *)isk;
888
889 smc_accept_unlink(new_sk);
890 if (new_sk->sk_state == SMC_CLOSED) {
127f4970
UB		 891 if (isk->clcsock) {
892 sock_release(isk->clcsock);
893 isk->clcsock = NULL;
894 }
288c8390 895 new_sk->sk_prot->unhash(new_sk);
51f1de79 896 sock_put(new_sk); /* final */
a046d57d
UB		 897 continue;
898 }
899 if (new_sock)
900 sock_graft(new_sk, new_sock);
901 return new_sk;
902 }
903 return NULL;
904}
905
906/* clean up for a created but never accepted sock */
b38d7324 907void smc_close_non_accepted(struct sock *sk)
a046d57d
UB		 908{
909 struct smc_sock *smc = smc_sk(sk);
910
b38d7324
UB		 911 lock_sock(sk);
912 if (!sk->sk_lingertime)
913 /* wait for peer closing */
914 sk->sk_lingertime = SMC_MAX_STREAM_WAIT_TIMEOUT;
51f1de79 915 if (!smc->use_fallback) {
b38d7324 916 smc_close_active(smc);
288c8390
UB		 917 sock_set_flag(sk, SOCK_DEAD);
918 sk->sk_shutdown |= SHUTDOWN_MASK;
919 }
a046d57d
UB		 920 if (smc->clcsock) {
921 struct socket *tcp;
922
923 tcp = smc->clcsock;
924 smc->clcsock = NULL;
925 sock_release(tcp);
926 }
b38d7324 927 if (smc->use_fallback) {
51f1de79
UB		 928 sock_put(sk); /* passive closing */
929 sk->sk_state = SMC_CLOSED;
930 } else {
931 if (sk->sk_state == SMC_CLOSED)
932 smc_conn_free(&smc->conn);
b38d7324
UB		 933 }
934 release_sock(sk);
51f1de79
UB		 935 sk->sk_prot->unhash(sk);
936 sock_put(sk); /* final sock_put */
a046d57d
UB		 937}
938
9bf9abea
UB		 939static int smc_serv_conf_first_link(struct smc_sock *smc)
940{
877ae5be 941 struct net *net = sock_net(smc->clcsock->sk);
9bf9abea
UB		 942 struct smc_link_group *lgr = smc->conn.lgr;
943 struct smc_link *link;
944 int rest;
945 int rc;
946
947 link = &lgr->lnk[SMC_SINGLE_LINK];
652a1e41 948
44aa81ce 949 if (smc_reg_rmb(link, smc->conn.rmb_desc, false))
603cc149 950 return SMC_CLC_DECL_ERR_REGRMB;
652a1e41 951
9bf9abea 952 /* send CONFIRM LINK request to client over the RoCE fabric */
947541f3 953 rc = smc_llc_send_confirm_link(link, SMC_LLC_REQ);
9bf9abea 954 if (rc < 0)
603cc149 955 return SMC_CLC_DECL_TIMEOUT_CL;
9bf9abea
UB		 956
957 /* receive CONFIRM LINK response from client over the RoCE fabric */
958 rest = wait_for_completion_interruptible_timeout(
959 &link->llc_confirm_resp,
960 SMC_LLC_WAIT_FIRST_TIME);
961 if (rest <= 0) {
962 struct smc_clc_msg_decline dclc;
963
964 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
965 SMC_CLC_DECLINE);
75d320d6 966 return rc;
9bf9abea
UB		 967 }
968
75d320d6
KG		 969 if (link->llc_confirm_resp_rc)
970 return SMC_CLC_DECL_RMBE_EC;
971
52bedf37
KG		 972 /* send ADD LINK request to client over the RoCE fabric */
973 rc = smc_llc_send_add_link(link,
974 link->smcibdev->mac[link->ibport - 1],
7005ada6 975 link->gid, SMC_LLC_REQ);
52bedf37 976 if (rc < 0)
603cc149 977 return SMC_CLC_DECL_TIMEOUT_AL;
52bedf37
KG		 978
979 /* receive ADD LINK response from client over the RoCE fabric */
980 rest = wait_for_completion_interruptible_timeout(&link->llc_add_resp,
981 SMC_LLC_WAIT_TIME);
982 if (rest <= 0) {
983 struct smc_clc_msg_decline dclc;
984
985 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
986 SMC_CLC_DECLINE);
987 return rc;
988 }
989
877ae5be 990 smc_llc_link_active(link, net->ipv4.sysctl_tcp_keepalive_time);
52bedf37 991
75d320d6 992 return 0;
9bf9abea
UB		 993}
994
3b2dec26
HW		 995/* listen worker: finish */
996static void smc_listen_out(struct smc_sock *new_smc)
a046d57d 997{
a046d57d 998 struct smc_sock *lsmc = new_smc->listen_smc;
a046d57d 999 struct sock *newsmcsk = &new_smc->sk;
a046d57d 1000
3b2dec26
HW		 1001 lock_sock_nested(&lsmc->sk, SINGLE_DEPTH_NESTING);
1002 if (lsmc->sk.sk_state == SMC_LISTEN) {
1003 smc_accept_enqueue(&lsmc->sk, newsmcsk);
1004 } else { /* no longer listening */
1005 smc_close_non_accepted(newsmcsk);
c5c1cc9c 1006 }
3b2dec26 1007 release_sock(&lsmc->sk);
c5c1cc9c 1008
3b2dec26
HW		 1009 /* Wake up accept */
1010 lsmc->sk.sk_data_ready(&lsmc->sk);
1011 sock_put(&lsmc->sk); /* sock_hold in smc_tcp_listen_work */
1012}
a046d57d 1013
3b2dec26
HW		 1014/* listen worker: finish in state connected */
1015static void smc_listen_out_connected(struct smc_sock *new_smc)
1016{
1017 struct sock *newsmcsk = &new_smc->sk;
a046d57d 1018
3b2dec26
HW		 1019 sk_refcnt_debug_inc(newsmcsk);
1020 if (newsmcsk->sk_state == SMC_INIT)
1021 newsmcsk->sk_state = SMC_ACTIVE;
1022
1023 smc_listen_out(new_smc);
1024}
1025
1026/* listen worker: finish in error state */
1027static void smc_listen_out_err(struct smc_sock *new_smc)
1028{
1029 struct sock *newsmcsk = &new_smc->sk;
1030
1031 if (newsmcsk->sk_state == SMC_INIT)
1032 sock_put(&new_smc->sk); /* passive closing */
1033 newsmcsk->sk_state = SMC_CLOSED;
1034 smc_conn_free(&new_smc->conn);
1035
1036 smc_listen_out(new_smc);
1037}
1038
1039/* listen worker: decline and fall back if possible */
1040static void smc_listen_decline(struct smc_sock *new_smc, int reason_code,
1041 int local_contact)
1042{
1043 /* RDMA setup failed, switch back to TCP */
1044 if (local_contact == SMC_FIRST_CONTACT)
1045 smc_lgr_forget(new_smc->conn.lgr);
1046 if (reason_code < 0) { /* error, no fallback possible */
1047 smc_listen_out_err(new_smc);
1048 return;
1049 }
1050 smc_conn_free(&new_smc->conn);
1051 new_smc->use_fallback = true;
603cc149
KG		 1052 new_smc->fallback_rsn = reason_code;
1053 if (reason_code && reason_code != SMC_CLC_DECL_PEERDECL) {
3b2dec26
HW		 1054 if (smc_clc_send_decline(new_smc, reason_code) < 0) {
1055 smc_listen_out_err(new_smc);
1056 return;
1057 }
a046d57d 1058 }
3b2dec26
HW		 1059 smc_listen_out_connected(new_smc);
1060}
1061
1062/* listen worker: check prefixes */
1063static int smc_listen_rdma_check(struct smc_sock *new_smc,
1064 struct smc_clc_msg_proposal *pclc)
1065{
1066 struct smc_clc_msg_proposal_prefix *pclc_prfx;
1067 struct socket *newclcsock = new_smc->clcsock;
a046d57d 1068
e7b7a64a 1069 pclc_prfx = smc_clc_proposal_get_prefix(pclc);
3b2dec26
HW		 1070 if (smc_clc_prfx_match(newclcsock, pclc_prfx))
1071 return SMC_CLC_DECL_CNFERR;
c246d942 1072
3b2dec26
HW		 1073 return 0;
1074}
a046d57d 1075
3b2dec26
HW		 1076/* listen worker: initialize connection and buffers */
1077static int smc_listen_rdma_init(struct smc_sock *new_smc,
1078 struct smc_clc_msg_proposal *pclc,
1079 struct smc_ib_device *ibdev, u8 ibport,
1080 int *local_contact)
1081{
0cfdd8f9 1082 /* allocate connection / link group */
c6ba7c9b
HW		 1083 *local_contact = smc_conn_create(new_smc, false, 0, ibdev, ibport,
1084 &pclc->lcl, NULL, 0);
3b2dec26
HW		 1085 if (*local_contact < 0) {
1086 if (*local_contact == -ENOMEM)
1087 return SMC_CLC_DECL_MEM;/* insufficient memory*/
1088 return SMC_CLC_DECL_INTERR; /* other error */
0cfdd8f9 1089 }
a046d57d 1090
3e034725 1091 /* create send buffer and rmb */
c6ba7c9b 1092 if (smc_buf_create(new_smc, false))
3b2dec26 1093 return SMC_CLC_DECL_MEM;
a046d57d 1094
3b2dec26
HW		 1095 return 0;
1096}
1097
41349844
HW		 1098/* listen worker: initialize connection and buffers for SMC-D */
1099static int smc_listen_ism_init(struct smc_sock *new_smc,
1100 struct smc_clc_msg_proposal *pclc,
1101 struct smcd_dev *ismdev,
1102 int *local_contact)
1103{
1104 struct smc_clc_msg_smcd *pclc_smcd;
1105
1106 pclc_smcd = smc_get_clc_msg_smcd(pclc);
1107 *local_contact = smc_conn_create(new_smc, true, 0, NULL, 0, NULL,
1108 ismdev, pclc_smcd->gid);
1109 if (*local_contact < 0) {
1110 if (*local_contact == -ENOMEM)
1111 return SMC_CLC_DECL_MEM;/* insufficient memory*/
1112 return SMC_CLC_DECL_INTERR; /* other error */
1113 }
1114
1115 /* Check if peer can be reached via ISM device */
1116 if (smc_ism_cantalk(new_smc->conn.lgr->peer_gid,
1117 new_smc->conn.lgr->vlan_id,
1118 new_smc->conn.lgr->smcd)) {
1119 if (*local_contact == SMC_FIRST_CONTACT)
1120 smc_lgr_forget(new_smc->conn.lgr);
1121 smc_conn_free(&new_smc->conn);
1122 return SMC_CLC_DECL_CNFERR;
1123 }
1124
1125 /* Create send and receive buffers */
1126 if (smc_buf_create(new_smc, true)) {
1127 if (*local_contact == SMC_FIRST_CONTACT)
1128 smc_lgr_forget(new_smc->conn.lgr);
1129 smc_conn_free(&new_smc->conn);
1130 return SMC_CLC_DECL_MEM;
1131 }
1132
1133 return 0;
1134}
1135
3b2dec26
HW		 1136/* listen worker: register buffers */
1137static int smc_listen_rdma_reg(struct smc_sock *new_smc, int local_contact)
1138{
1139 struct smc_link *link = &new_smc->conn.lgr->lnk[SMC_SINGLE_LINK];
46c28dbd 1140
652a1e41 1141 if (local_contact != SMC_FIRST_CONTACT) {
e63a5f8c 1142 if (!new_smc->conn.rmb_desc->reused) {
3b2dec26 1143 if (smc_reg_rmb(link, new_smc->conn.rmb_desc, true))
603cc149 1144 return SMC_CLC_DECL_ERR_REGRMB;
652a1e41
UB		 1145 }
1146 }
10428dd8 1147 smc_rmb_sync_sg_for_device(&new_smc->conn);
652a1e41 1148
3b2dec26
HW		 1149 return 0;
1150}
1151
1152/* listen worker: finish RDMA setup */
1153static void smc_listen_rdma_finish(struct smc_sock *new_smc,
1154 struct smc_clc_msg_accept_confirm *cclc,
1155 int local_contact)
1156{
1157 struct smc_link *link = &new_smc->conn.lgr->lnk[SMC_SINGLE_LINK];
1158 int reason_code = 0;
a046d57d 1159
0cfdd8f9 1160 if (local_contact == SMC_FIRST_CONTACT)
3b2dec26 1161 smc_link_save_peer_info(link, cclc);
a046d57d 1162
3b2dec26 1163 if (smc_rmb_rtoken_handling(&new_smc->conn, cclc)) {
603cc149 1164 reason_code = SMC_CLC_DECL_ERR_RTOK;
3b2dec26 1165 goto decline;
bd4ad577
UB		 1166 }
1167
bd4ad577 1168 if (local_contact == SMC_FIRST_CONTACT) {
3b2dec26 1169 if (smc_ib_ready_link(link)) {
603cc149 1170 reason_code = SMC_CLC_DECL_ERR_RDYLNK;
3b2dec26 1171 goto decline;
bd4ad577 1172 }
9bf9abea
UB		 1173 /* QP confirmation over RoCE fabric */
1174 reason_code = smc_serv_conf_first_link(new_smc);
3b2dec26
HW		 1175 if (reason_code)
1176 goto decline;
bd4ad577 1177 }
3b2dec26 1178 return;
a046d57d 1179
3b2dec26 1180decline:
145686ba 1181 mutex_unlock(&smc_create_lgr_pending);
3b2dec26
HW		 1182 smc_listen_decline(new_smc, reason_code, local_contact);
1183}
e6727f39 1184
3b2dec26
HW		 1185/* setup for RDMA connection of server */
1186static void smc_listen_work(struct work_struct *work)
1187{
1188 struct smc_sock *new_smc = container_of(work, struct smc_sock,
1189 smc_listen_work);
1190 struct socket *newclcsock = new_smc->clcsock;
1191 struct smc_clc_msg_accept_confirm cclc;
1192 struct smc_clc_msg_proposal *pclc;
1193 struct smc_ib_device *ibdev;
41349844
HW		 1194 bool ism_supported = false;
1195 struct smcd_dev *ismdev;
3b2dec26
HW		 1196 u8 buf[SMC_CLC_MAX_LEN];
1197 int local_contact = 0;
7005ada6 1198 unsigned short vlan;
3b2dec26
HW		 1199 int reason_code = 0;
1200 int rc = 0;
1201 u8 ibport;
1202
1203 if (new_smc->use_fallback) {
1204 smc_listen_out_connected(new_smc);
1205 return;
a046d57d 1206 }
a046d57d 1207
3b2dec26
HW		 1208 /* check if peer is smc capable */
1209 if (!tcp_sk(newclcsock->sk)->syn_smc) {
1210 new_smc->use_fallback = true;
603cc149 1211 new_smc->fallback_rsn = SMC_CLC_DECL_PEERNOSMC;
3b2dec26
HW		 1212 smc_listen_out_connected(new_smc);
1213 return;
1214 }
a046d57d 1215
3b2dec26
HW		 1216 /* do inband token exchange -
1217 * wait for and receive SMC Proposal CLC message
1218 */
1219 pclc = (struct smc_clc_msg_proposal *)&buf;
1220 reason_code = smc_clc_wait_msg(new_smc, pclc, SMC_CLC_MAX_LEN,
1221 SMC_CLC_PROPOSAL);
1222 if (reason_code) {
1223 smc_listen_decline(new_smc, reason_code, 0);
1224 return;
a046d57d 1225 }
a046d57d 1226
3b2dec26
HW		 1227 /* IPSec connections opt out of SMC-R optimizations */
1228 if (using_ipsec(new_smc)) {
1229 smc_listen_decline(new_smc, SMC_CLC_DECL_IPSEC, 0);
1230 return;
1231 }
1232
1233 mutex_lock(&smc_create_lgr_pending);
1234 smc_close_init(new_smc);
1235 smc_rx_init(new_smc);
1236 smc_tx_init(new_smc);
1237
41349844
HW		 1238 /* check if ISM is available */
1239 if ((pclc->hdr.path == SMC_TYPE_D || pclc->hdr.path == SMC_TYPE_B) &&
1240 !smc_check_ism(new_smc, &ismdev) &&
1241 !smc_listen_ism_init(new_smc, pclc, ismdev, &local_contact)) {
1242 ism_supported = true;
1243 }
1244
3b2dec26 1245 /* check if RDMA is available */
41349844
HW		 1246 if (!ism_supported &&
1247 ((pclc->hdr.path != SMC_TYPE_R && pclc->hdr.path != SMC_TYPE_B) ||
7005ada6
UB		 1248 smc_vlan_by_tcpsk(new_smc->clcsock, &vlan) ||
1249 smc_check_rdma(new_smc, &ibdev, &ibport, vlan, NULL) ||
41349844
HW		 1250 smc_listen_rdma_check(new_smc, pclc) ||
1251 smc_listen_rdma_init(new_smc, pclc, ibdev, ibport,
1252 &local_contact) ||
1253 smc_listen_rdma_reg(new_smc, local_contact))) {
3b2dec26
HW		 1254 /* SMC not supported, decline */
1255 mutex_unlock(&smc_create_lgr_pending);
603cc149
KG		 1256 smc_listen_decline(new_smc, SMC_CLC_DECL_MODEUNSUPP,
1257 local_contact);
3b2dec26
HW		 1258 return;
1259 }
1260
1261 /* send SMC Accept CLC message */
1262 rc = smc_clc_send_accept(new_smc, local_contact);
1263 if (rc) {
1264 mutex_unlock(&smc_create_lgr_pending);
1265 smc_listen_decline(new_smc, rc, local_contact);
1266 return;
1267 }
1268
1269 /* receive SMC Confirm CLC message */
1270 reason_code = smc_clc_wait_msg(new_smc, &cclc, sizeof(cclc),
1271 SMC_CLC_CONFIRM);
1272 if (reason_code) {
1273 mutex_unlock(&smc_create_lgr_pending);
1274 smc_listen_decline(new_smc, reason_code, local_contact);
1275 return;
1276 }
1277
1278 /* finish worker */
41349844
HW		 1279 if (!ism_supported)
1280 smc_listen_rdma_finish(new_smc, &cclc, local_contact);
3b2dec26 1281 smc_conn_save_peer_info(new_smc, &cclc);
145686ba 1282 mutex_unlock(&smc_create_lgr_pending);
3b2dec26 1283 smc_listen_out_connected(new_smc);
a046d57d
UB		 1284}
1285
1286static void smc_tcp_listen_work(struct work_struct *work)
1287{
1288 struct smc_sock *lsmc = container_of(work, struct smc_sock,
1289 tcp_listen_work);
3163c507 1290 struct sock *lsk = &lsmc->sk;
a046d57d
UB		 1291 struct smc_sock *new_smc;
1292 int rc = 0;
1293
3163c507
UB		 1294 lock_sock(lsk);
1295 while (lsk->sk_state == SMC_LISTEN) {
a046d57d
UB		 1296 rc = smc_clcsock_accept(lsmc, &new_smc);
1297 if (rc)
1298 goto out;
1299 if (!new_smc)
1300 continue;
1301
1302 new_smc->listen_smc = lsmc;
ee9dfbef 1303 new_smc->use_fallback = lsmc->use_fallback;
603cc149 1304 new_smc->fallback_rsn = lsmc->fallback_rsn;
3163c507 1305 sock_hold(lsk); /* sock_put in smc_listen_work */
a046d57d
UB		 1306 INIT_WORK(&new_smc->smc_listen_work, smc_listen_work);
1307 smc_copy_sock_settings_to_smc(new_smc);
bd58c7e0
UB		 1308 new_smc->sk.sk_sndbuf = lsmc->sk.sk_sndbuf;
1309 new_smc->sk.sk_rcvbuf = lsmc->sk.sk_rcvbuf;
51f1de79
UB		 1310 sock_hold(&new_smc->sk); /* sock_put in passive closing */
1311 if (!schedule_work(&new_smc->smc_listen_work))
1312 sock_put(&new_smc->sk);
a046d57d
UB		 1313 }
1314
1315out:
3163c507 1316 release_sock(lsk);
51f1de79 1317 sock_put(&lsmc->sk); /* sock_hold in smc_listen */
a046d57d
UB		 1318}
1319
ac713874
UB		 1320static int smc_listen(struct socket *sock, int backlog)
1321{
1322 struct sock *sk = sock->sk;
1323 struct smc_sock *smc;
1324 int rc;
1325
1326 smc = smc_sk(sk);
1327 lock_sock(sk);
1328
1329 rc = -EINVAL;
1330 if ((sk->sk_state != SMC_INIT) && (sk->sk_state != SMC_LISTEN))
1331 goto out;
1332
1333 rc = 0;
1334 if (sk->sk_state == SMC_LISTEN) {
1335 sk->sk_max_ack_backlog = backlog;
1336 goto out;
1337 }
1338 /* some socket options are handled in core, so we could not apply
1339 * them to the clc socket -- copy smc socket options to clc socket
1340 */
1341 smc_copy_sock_settings_to_clc(smc);
ee9dfbef
UB		 1342 if (!smc->use_fallback)
1343 tcp_sk(smc->clcsock->sk)->syn_smc = 1;
ac713874
UB		 1344
1345 rc = kernel_listen(smc->clcsock, backlog);
1346 if (rc)
1347 goto out;
1348 sk->sk_max_ack_backlog = backlog;
1349 sk->sk_ack_backlog = 0;
1350 sk->sk_state = SMC_LISTEN;
a046d57d 1351 INIT_WORK(&smc->tcp_listen_work, smc_tcp_listen_work);
51f1de79
UB		 1352 sock_hold(sk); /* sock_hold in tcp_listen_worker */
1353 if (!schedule_work(&smc->tcp_listen_work))
1354 sock_put(sk);
ac713874
UB		 1355
1356out:
1357 release_sock(sk);
1358 return rc;
1359}
1360
1361static int smc_accept(struct socket *sock, struct socket *new_sock,
cdfbabfb 1362 int flags, bool kern)
ac713874 1363{
a046d57d
UB		 1364 struct sock *sk = sock->sk, *nsk;
1365 DECLARE_WAITQUEUE(wait, current);
ac713874 1366 struct smc_sock *lsmc;
a046d57d
UB		 1367 long timeo;
1368 int rc = 0;
ac713874
UB		 1369
1370 lsmc = smc_sk(sk);
51f1de79 1371 sock_hold(sk); /* sock_put below */
ac713874
UB		 1372 lock_sock(sk);
1373
1374 if (lsmc->sk.sk_state != SMC_LISTEN) {
1375 rc = -EINVAL;
abb190f1 1376 release_sock(sk);
ac713874
UB		 1377 goto out;
1378 }
1379
a046d57d
UB		 1380 /* Wait for an incoming connection */
1381 timeo = sock_rcvtimeo(sk, flags & O_NONBLOCK);
1382 add_wait_queue_exclusive(sk_sleep(sk), &wait);
1383 while (!(nsk = smc_accept_dequeue(sk, new_sock))) {
1384 set_current_state(TASK_INTERRUPTIBLE);
1385 if (!timeo) {
1386 rc = -EAGAIN;
1387 break;
1388 }
1389 release_sock(sk);
1390 timeo = schedule_timeout(timeo);
1391 /* wakeup by sk_data_ready in smc_listen_work() */
1392 sched_annotate_sleep();
1393 lock_sock(sk);
1394 if (signal_pending(current)) {
1395 rc = sock_intr_errno(timeo);
1396 break;
1397 }
1398 }
1399 set_current_state(TASK_RUNNING);
1400 remove_wait_queue(sk_sleep(sk), &wait);
ac713874 1401
a046d57d
UB		 1402 if (!rc)
1403 rc = sock_error(nsk);
abb190f1
UB		 1404 release_sock(sk);
1405 if (rc)
1406 goto out;
1407
1408 if (lsmc->sockopt_defer_accept && !(flags & O_NONBLOCK)) {
1409 /* wait till data arrives on the socket */
1410 timeo = msecs_to_jiffies(lsmc->sockopt_defer_accept *
1411 MSEC_PER_SEC);
1412 if (smc_sk(nsk)->use_fallback) {
1413 struct sock *clcsk = smc_sk(nsk)->clcsock->sk;
1414
1415 lock_sock(clcsk);
1416 if (skb_queue_empty(&clcsk->sk_receive_queue))
1417 sk_wait_data(clcsk, &timeo, NULL);
1418 release_sock(clcsk);
1419 } else if (!atomic_read(&smc_sk(nsk)->conn.bytes_to_rcv)) {
1420 lock_sock(nsk);
b51fa1b1 1421 smc_rx_wait(smc_sk(nsk), &timeo, smc_rx_data_available);
abb190f1
UB		 1422 release_sock(nsk);
1423 }
1424 }
ac713874
UB		 1425
1426out:
51f1de79 1427 sock_put(sk); /* sock_hold above */
ac713874
UB		 1428 return rc;
1429}
1430
1431static int smc_getname(struct socket *sock, struct sockaddr *addr,
9b2c45d4 1432 int peer)
ac713874
UB		 1433{
1434 struct smc_sock *smc;
1435
b38d7324
UB		 1436 if (peer && (sock->sk->sk_state != SMC_ACTIVE) &&
1437 (sock->sk->sk_state != SMC_APPCLOSEWAIT1))
ac713874
UB		 1438 return -ENOTCONN;
1439
1440 smc = smc_sk(sock->sk);
1441
9b2c45d4 1442 return smc->clcsock->ops->getname(smc->clcsock, addr, peer);
ac713874
UB		 1443}
1444
1445static int smc_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
1446{
1447 struct sock *sk = sock->sk;
1448 struct smc_sock *smc;
1449 int rc = -EPIPE;
1450
1451 smc = smc_sk(sk);
1452 lock_sock(sk);
b38d7324
UB		 1453 if ((sk->sk_state != SMC_ACTIVE) &&
1454 (sk->sk_state != SMC_APPCLOSEWAIT1) &&
1455 (sk->sk_state != SMC_INIT))
ac713874 1456 goto out;
ee9dfbef
UB		 1457
1458 if (msg->msg_flags & MSG_FASTOPEN) {
1459 if (sk->sk_state == SMC_INIT) {
1460 smc->use_fallback = true;
603cc149 1461 smc->fallback_rsn = SMC_CLC_DECL_OPTUNSUPP;
ee9dfbef
UB		 1462 } else {
1463 rc = -EINVAL;
1464 goto out;
1465 }
1466 }
1467
ac713874
UB		 1468 if (smc->use_fallback)
1469 rc = smc->clcsock->ops->sendmsg(smc->clcsock, msg, len);
1470 else
e6727f39 1471 rc = smc_tx_sendmsg(smc, msg, len);
ac713874
UB		 1472out:
1473 release_sock(sk);
1474 return rc;
1475}
1476
1477static int smc_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
1478 int flags)
1479{
1480 struct sock *sk = sock->sk;
1481 struct smc_sock *smc;
1482 int rc = -ENOTCONN;
1483
1484 smc = smc_sk(sk);
1485 lock_sock(sk);
b38d7324
UB		 1486 if ((sk->sk_state == SMC_INIT) ||
1487 (sk->sk_state == SMC_LISTEN) ||
1488 (sk->sk_state == SMC_CLOSED))
ac713874
UB		 1489 goto out;
1490
b38d7324
UB		 1491 if (sk->sk_state == SMC_PEERFINCLOSEWAIT) {
1492 rc = 0;
1493 goto out;
1494 }
1495
9014db20 1496 if (smc->use_fallback) {
ac713874 1497 rc = smc->clcsock->ops->recvmsg(smc->clcsock, msg, len, flags);
9014db20
SR		 1498 } else {
1499 msg->msg_namelen = 0;
1500 rc = smc_rx_recvmsg(smc, msg, NULL, len, flags);
1501 }
b38d7324 1502
ac713874
UB		 1503out:
1504 release_sock(sk);
1505 return rc;
1506}
1507
ade994f4 1508static __poll_t smc_accept_poll(struct sock *parent)
a046d57d 1509{
8dce2786 1510 struct smc_sock *isk = smc_sk(parent);
63e2480c 1511 __poll_t mask = 0;
a046d57d 1512
8dce2786
UB		 1513 spin_lock(&isk->accept_q_lock);
1514 if (!list_empty(&isk->accept_q))
a9a08845 1515 mask = EPOLLIN | EPOLLRDNORM;
8dce2786 1516 spin_unlock(&isk->accept_q_lock);
a046d57d 1517
8dce2786 1518 return mask;
a046d57d
UB		 1519}
1520
a11e1d43
LT		 1521static __poll_t smc_poll(struct file *file, struct socket *sock,
1522 poll_table *wait)
ac713874
UB		 1523{
1524 struct sock *sk = sock->sk;
e6c8adca 1525 __poll_t mask = 0;
ac713874
UB		 1526 struct smc_sock *smc;
1527
8dce2786 1528 if (!sk)
a9a08845 1529 return EPOLLNVAL;
8dce2786 1530
ac713874 1531 smc = smc_sk(sock->sk);
a046d57d
UB		 1532 if ((sk->sk_state == SMC_INIT) || smc->use_fallback) {
1533 /* delegate to CLC child sock */
a11e1d43 1534 mask = smc->clcsock->ops->poll(file, smc->clcsock, wait);
784813ae 1535 sk->sk_err = smc->clcsock->sk->sk_err;
24ac3a08 1536 if (sk->sk_err)
784813ae 1537 mask |= EPOLLERR;
ac713874 1538 } else {
410da1e1 1539 if (sk->sk_state != SMC_CLOSED)
dd979b4d 1540 sock_poll_wait(file, wait);
a046d57d 1541 if (sk->sk_err)
a9a08845 1542 mask |= EPOLLERR;
b38d7324
UB		 1543 if ((sk->sk_shutdown == SHUTDOWN_MASK) ||
1544 (sk->sk_state == SMC_CLOSED))
a9a08845 1545 mask |= EPOLLHUP;
8dce2786
UB		 1546 if (sk->sk_state == SMC_LISTEN) {
1547 /* woken up by sk_data_ready in smc_listen_work() */
1548 mask = smc_accept_poll(sk);
1549 } else {
1550 if (atomic_read(&smc->conn.sndbuf_space) ||
1551 sk->sk_shutdown & SEND_SHUTDOWN) {
a9a08845 1552 mask |= EPOLLOUT | EPOLLWRNORM;
8dce2786
UB		 1553 } else {
1554 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
1555 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
1556 }
1557 if (atomic_read(&smc->conn.bytes_to_rcv))
a9a08845 1558 mask |= EPOLLIN | EPOLLRDNORM;
8dce2786 1559 if (sk->sk_shutdown & RCV_SHUTDOWN)
a9a08845 1560 mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
8dce2786 1561 if (sk->sk_state == SMC_APPCLOSEWAIT1)
a9a08845 1562 mask |= EPOLLIN;
8dce2786 1563 }
de8474eb
SR		 1564 if (smc->conn.urg_state == SMC_URG_VALID)
1565 mask |= EPOLLPRI;
ac713874
UB		 1566 }
1567
1568 return mask;
1569}
1570
1571static int smc_shutdown(struct socket *sock, int how)
1572{
1573 struct sock *sk = sock->sk;
1574 struct smc_sock *smc;
1575 int rc = -EINVAL;
b38d7324 1576 int rc1 = 0;
ac713874
UB		 1577
1578 smc = smc_sk(sk);
1579
1580 if ((how < SHUT_RD) || (how > SHUT_RDWR))
b38d7324 1581 return rc;
ac713874
UB		 1582
1583 lock_sock(sk);
1584
1585 rc = -ENOTCONN;
caa21e19 1586 if ((sk->sk_state != SMC_ACTIVE) &&
b38d7324
UB		 1587 (sk->sk_state != SMC_PEERCLOSEWAIT1) &&
1588 (sk->sk_state != SMC_PEERCLOSEWAIT2) &&
1589 (sk->sk_state != SMC_APPCLOSEWAIT1) &&
1590 (sk->sk_state != SMC_APPCLOSEWAIT2) &&
1591 (sk->sk_state != SMC_APPFINCLOSEWAIT))
ac713874
UB		 1592 goto out;
1593 if (smc->use_fallback) {
1594 rc = kernel_sock_shutdown(smc->clcsock, how);
1595 sk->sk_shutdown = smc->clcsock->sk->sk_shutdown;
1596 if (sk->sk_shutdown == SHUTDOWN_MASK)
1597 sk->sk_state = SMC_CLOSED;
b38d7324 1598 goto out;
ac713874 1599 }
b38d7324
UB		 1600 switch (how) {
1601 case SHUT_RDWR: /* shutdown in both directions */
1602 rc = smc_close_active(smc);
1603 break;
1604 case SHUT_WR:
1605 rc = smc_close_shutdown_write(smc);
1606 break;
1607 case SHUT_RD:
1255fcb2
UB		 1608 rc = 0;
1609 /* nothing more to do because peer is not involved */
b38d7324
UB		 1610 break;
1611 }
1255fcb2
UB		 1612 if (smc->clcsock)
1613 rc1 = kernel_sock_shutdown(smc->clcsock, how);
b38d7324
UB		 1614 /* map sock_shutdown_cmd constants to sk_shutdown value range */
1615 sk->sk_shutdown |= how + 1;
ac713874
UB		 1616
1617out:
1618 release_sock(sk);
b38d7324 1619 return rc ? rc : rc1;
ac713874
UB		 1620}
1621
1622static int smc_setsockopt(struct socket *sock, int level, int optname,
1623 char __user *optval, unsigned int optlen)
1624{
1625 struct sock *sk = sock->sk;
1626 struct smc_sock *smc;
01d2f7e2 1627 int val, rc;
ac713874
UB		 1628
1629 smc = smc_sk(sk);
1630
1631 /* generic setsockopts reaching us here always apply to the
1632 * CLC socket
1633 */
ee9dfbef
UB		 1634 rc = smc->clcsock->ops->setsockopt(smc->clcsock, level, optname,
1635 optval, optlen);
1636 if (smc->clcsock->sk->sk_err) {
1637 sk->sk_err = smc->clcsock->sk->sk_err;
1638 sk->sk_error_report(sk);
1639 }
1640 if (rc)
1641 return rc;
1642
01d2f7e2 1643 if (optlen < sizeof(int))
3dc9f558 1644 return -EINVAL;
ac0107ed
UB		 1645 if (get_user(val, (int __user *)optval))
1646 return -EFAULT;
01d2f7e2 1647
ee9dfbef
UB		 1648 lock_sock(sk);
1649 switch (optname) {
1650 case TCP_ULP:
1651 case TCP_FASTOPEN:
1652 case TCP_FASTOPEN_CONNECT:
1653 case TCP_FASTOPEN_KEY:
1654 case TCP_FASTOPEN_NO_COOKIE:
1655 /* option not supported by SMC */
1656 if (sk->sk_state == SMC_INIT) {
1657 smc->use_fallback = true;
603cc149 1658 smc->fallback_rsn = SMC_CLC_DECL_OPTUNSUPP;
ee9dfbef
UB		 1659 } else {
1660 if (!smc->use_fallback)
1661 rc = -EINVAL;
1662 }
1663 break;
01d2f7e2
UB		 1664 case TCP_NODELAY:
1665 if (sk->sk_state != SMC_INIT && sk->sk_state != SMC_LISTEN) {
569bc643 1666 if (val && !smc->use_fallback)
01d2f7e2
UB		 1667 mod_delayed_work(system_wq, &smc->conn.tx_work,
1668 0);
1669 }
1670 break;
1671 case TCP_CORK:
1672 if (sk->sk_state != SMC_INIT && sk->sk_state != SMC_LISTEN) {
569bc643 1673 if (!val && !smc->use_fallback)
01d2f7e2
UB		 1674 mod_delayed_work(system_wq, &smc->conn.tx_work,
1675 0);
1676 }
1677 break;
abb190f1
UB		 1678 case TCP_DEFER_ACCEPT:
1679 smc->sockopt_defer_accept = val;
1680 break;
ee9dfbef
UB		 1681 default:
1682 break;
1683 }
1684 release_sock(sk);
1685
1686 return rc;
ac713874
UB		 1687}
1688
1689static int smc_getsockopt(struct socket *sock, int level, int optname,
1690 char __user *optval, int __user *optlen)
1691{
1692 struct smc_sock *smc;
1693
1694 smc = smc_sk(sock->sk);
1695 /* socket options apply to the CLC socket */
1696 return smc->clcsock->ops->getsockopt(smc->clcsock, level, optname,
1697 optval, optlen);
1698}
1699
1700static int smc_ioctl(struct socket *sock, unsigned int cmd,
1701 unsigned long arg)
1702{
de8474eb
SR		 1703 union smc_host_cursor cons, urg;
1704 struct smc_connection *conn;
ac713874 1705 struct smc_sock *smc;
9b67e26f 1706 int answ;
ac713874
UB		 1707
1708 smc = smc_sk(sock->sk);
de8474eb 1709 conn = &smc->conn;
7311d665 1710 lock_sock(&smc->sk);
9b67e26f 1711 if (smc->use_fallback) {
7311d665
UB		 1712 if (!smc->clcsock) {
1713 release_sock(&smc->sk);
9b67e26f 1714 return -EBADF;
7311d665
UB		 1715 }
1716 answ = smc->clcsock->ops->ioctl(smc->clcsock, cmd, arg);
1717 release_sock(&smc->sk);
1718 return answ;
9b67e26f
UB		 1719 }
1720 switch (cmd) {
1721 case SIOCINQ: /* same as FIONREAD */
1992d998
UB		 1722 if (smc->sk.sk_state == SMC_LISTEN) {
1723 release_sock(&smc->sk);
9b67e26f 1724 return -EINVAL;
1992d998 1725 }
2351abe6
UB		 1726 if (smc->sk.sk_state == SMC_INIT ||
1727 smc->sk.sk_state == SMC_CLOSED)
1728 answ = 0;
1729 else
1730 answ = atomic_read(&smc->conn.bytes_to_rcv);
9b67e26f
UB		 1731 break;
1732 case SIOCOUTQ:
1733 /* output queue size (not send + not acked) */
1992d998
UB		 1734 if (smc->sk.sk_state == SMC_LISTEN) {
1735 release_sock(&smc->sk);
9b67e26f 1736 return -EINVAL;
1992d998 1737 }
2351abe6
UB		 1738 if (smc->sk.sk_state == SMC_INIT ||
1739 smc->sk.sk_state == SMC_CLOSED)
1740 answ = 0;
1741 else
1742 answ = smc->conn.sndbuf_desc->len -
9b67e26f
UB		 1743 atomic_read(&smc->conn.sndbuf_space);
1744 break;
1745 case SIOCOUTQNSD:
1746 /* output queue size (not send only) */
1992d998
UB		 1747 if (smc->sk.sk_state == SMC_LISTEN) {
1748 release_sock(&smc->sk);
9b67e26f 1749 return -EINVAL;
1992d998 1750 }
2351abe6
UB		 1751 if (smc->sk.sk_state == SMC_INIT ||
1752 smc->sk.sk_state == SMC_CLOSED)
1753 answ = 0;
1754 else
1755 answ = smc_tx_prepared_sends(&smc->conn);
9b67e26f 1756 break;
de8474eb 1757 case SIOCATMARK:
1992d998
UB		 1758 if (smc->sk.sk_state == SMC_LISTEN) {
1759 release_sock(&smc->sk);
de8474eb 1760 return -EINVAL;
1992d998 1761 }
de8474eb
SR		 1762 if (smc->sk.sk_state == SMC_INIT ||
1763 smc->sk.sk_state == SMC_CLOSED) {
1764 answ = 0;
1765 } else {
bac6de7b
SR		 1766 smc_curs_copy(&cons, &conn->local_tx_ctrl.cons, conn);
1767 smc_curs_copy(&urg, &conn->urg_curs, conn);
de8474eb
SR		 1768 answ = smc_curs_diff(conn->rmb_desc->len,
1769 &cons, &urg) == 1;
1770 }
1771 break;
9b67e26f 1772 default:
1992d998 1773 release_sock(&smc->sk);
9b67e26f
UB		 1774 return -ENOIOCTLCMD;
1775 }
1992d998 1776 release_sock(&smc->sk);
9b67e26f
UB		 1777
1778 return put_user(answ, (int __user *)arg);
ac713874
UB		 1779}
1780
1781static ssize_t smc_sendpage(struct socket *sock, struct page *page,
1782 int offset, size_t size, int flags)
1783{
1784 struct sock *sk = sock->sk;
1785 struct smc_sock *smc;
1786 int rc = -EPIPE;
1787
1788 smc = smc_sk(sk);
1789 lock_sock(sk);
bda27ff5
SR		 1790 if (sk->sk_state != SMC_ACTIVE) {
1791 release_sock(sk);
ac713874 1792 goto out;
bda27ff5
SR		 1793 }
1794 release_sock(sk);
ac713874
UB		 1795 if (smc->use_fallback)
1796 rc = kernel_sendpage(smc->clcsock, page, offset,
1797 size, flags);
1798 else
1799 rc = sock_no_sendpage(sock, page, offset, size, flags);
1800
1801out:
ac713874
UB		 1802 return rc;
1803}
1804
9014db20
SR		 1805/* Map the affected portions of the rmbe into an spd, note the number of bytes
1806 * to splice in conn->splice_pending, and press 'go'. Delays consumer cursor
1807 * updates till whenever a respective page has been fully processed.
1808 * Note that subsequent recv() calls have to wait till all splice() processing
1809 * completed.
1810 */
ac713874
UB		 1811static ssize_t smc_splice_read(struct socket *sock, loff_t *ppos,
1812 struct pipe_inode_info *pipe, size_t len,
9014db20 1813 unsigned int flags)
ac713874
UB		 1814{
1815 struct sock *sk = sock->sk;
1816 struct smc_sock *smc;
1817 int rc = -ENOTCONN;
1818
1819 smc = smc_sk(sk);
1820 lock_sock(sk);
9014db20
SR		 1821
1822 if (sk->sk_state == SMC_INIT ||
1823 sk->sk_state == SMC_LISTEN ||
1824 sk->sk_state == SMC_CLOSED)
1825 goto out;
1826
1827 if (sk->sk_state == SMC_PEERFINCLOSEWAIT) {
1828 rc = 0;
ac713874 1829 goto out;
9014db20
SR		 1830 }
1831
ac713874
UB		 1832 if (smc->use_fallback) {
1833 rc = smc->clcsock->ops->splice_read(smc->clcsock, ppos,
1834 pipe, len, flags);
1835 } else {
9014db20
SR		 1836 if (*ppos) {
1837 rc = -ESPIPE;
1838 goto out;
1839 }
1840 if (flags & SPLICE_F_NONBLOCK)
1841 flags = MSG_DONTWAIT;
1842 else
1843 flags = 0;
1844 rc = smc_rx_recvmsg(smc, NULL, pipe, len, flags);
ac713874
UB		 1845 }
1846out:
1847 release_sock(sk);
9014db20 1848
ac713874
UB		 1849 return rc;
1850}
1851
1852/* must look like tcp */
1853static const struct proto_ops smc_sock_ops = {
1854 .family = PF_SMC,
1855 .owner = THIS_MODULE,
1856 .release = smc_release,
1857 .bind = smc_bind,
1858 .connect = smc_connect,
1859 .socketpair = sock_no_socketpair,
1860 .accept = smc_accept,
1861 .getname = smc_getname,
a11e1d43 1862 .poll = smc_poll,
ac713874
UB		 1863 .ioctl = smc_ioctl,
1864 .listen = smc_listen,
1865 .shutdown = smc_shutdown,
1866 .setsockopt = smc_setsockopt,
1867 .getsockopt = smc_getsockopt,
1868 .sendmsg = smc_sendmsg,
1869 .recvmsg = smc_recvmsg,
1870 .mmap = sock_no_mmap,
1871 .sendpage = smc_sendpage,
1872 .splice_read = smc_splice_read,
1873};
1874
1875static int smc_create(struct net *net, struct socket *sock, int protocol,
1876 int kern)
1877{
aaa4d33f 1878 int family = (protocol == SMCPROTO_SMC6) ? PF_INET6 : PF_INET;
ac713874
UB		 1879 struct smc_sock *smc;
1880 struct sock *sk;
1881 int rc;
1882
1883 rc = -ESOCKTNOSUPPORT;
1884 if (sock->type != SOCK_STREAM)
1885 goto out;
1886
1887 rc = -EPROTONOSUPPORT;
aaa4d33f 1888 if (protocol != SMCPROTO_SMC && protocol != SMCPROTO_SMC6)
ac713874
UB		 1889 goto out;
1890
1891 rc = -ENOBUFS;
1892 sock->ops = &smc_sock_ops;
aaa4d33f 1893 sk = smc_sock_alloc(net, sock, protocol);
ac713874
UB		 1894 if (!sk)
1895 goto out;
1896
1897 /* create internal TCP socket for CLC handshake and fallback */
1898 smc = smc_sk(sk);
a046d57d 1899 smc->use_fallback = false; /* assume rdma capability first */
603cc149 1900 smc->fallback_rsn = 0;
aaa4d33f
KG		 1901 rc = sock_create_kern(net, family, SOCK_STREAM, IPPROTO_TCP,
1902 &smc->clcsock);
a5dcb73b 1903 if (rc) {
ac713874 1904 sk_common_release(sk);
a5dcb73b
DC		 1905 goto out;
1906 }
cd6851f3
UB		 1907 smc->sk.sk_sndbuf = max(smc->clcsock->sk->sk_sndbuf, SMC_BUF_MIN_SIZE);
1908 smc->sk.sk_rcvbuf = max(smc->clcsock->sk->sk_rcvbuf, SMC_BUF_MIN_SIZE);
ac713874
UB		 1909
1910out:
1911 return rc;
1912}
1913
1914static const struct net_proto_family smc_sock_family_ops = {
1915 .family = PF_SMC,
1916 .owner = THIS_MODULE,
1917 .create = smc_create,
1918};
1919
1920static int __init smc_init(void)
1921{
1922 int rc;
1923
6812baab
TR		 1924 rc = smc_pnet_init();
1925 if (rc)
1926 return rc;
1927
9bf9abea
UB		 1928 rc = smc_llc_init();
1929 if (rc) {
1930 pr_err("%s: smc_llc_init fails with %d\n", __func__, rc);
1931 goto out_pnet;
1932 }
1933
5f08318f
UB		 1934 rc = smc_cdc_init();
1935 if (rc) {
1936 pr_err("%s: smc_cdc_init fails with %d\n", __func__, rc);
1937 goto out_pnet;
1938 }
1939
ac713874
UB		 1940 rc = proto_register(&smc_proto, 1);
1941 if (rc) {
aaa4d33f 1942 pr_err("%s: proto_register(v4) fails with %d\n", __func__, rc);
6812baab 1943 goto out_pnet;
ac713874
UB		 1944 }
1945
aaa4d33f
KG		 1946 rc = proto_register(&smc_proto6, 1);
1947 if (rc) {
1948 pr_err("%s: proto_register(v6) fails with %d\n", __func__, rc);
1949 goto out_proto;
1950 }
1951
ac713874
UB		 1952 rc = sock_register(&smc_sock_family_ops);
1953 if (rc) {
1954 pr_err("%s: sock_register fails with %d\n", __func__, rc);
aaa4d33f 1955 goto out_proto6;
ac713874 1956 }
f16a7dd5 1957 INIT_HLIST_HEAD(&smc_v4_hashinfo.ht);
aaa4d33f 1958 INIT_HLIST_HEAD(&smc_v6_hashinfo.ht);
ac713874 1959
a4cf0443
UB		 1960 rc = smc_ib_register_client();
1961 if (rc) {
1962 pr_err("%s: ib_register fails with %d\n", __func__, rc);
1963 goto out_sock;
1964 }
1965
c5c1cc9c 1966 static_branch_enable(&tcp_have_smc);
ac713874
UB		 1967 return 0;
1968
a4cf0443
UB		 1969out_sock:
1970 sock_unregister(PF_SMC);
aaa4d33f
KG		 1971out_proto6:
1972 proto_unregister(&smc_proto6);
ac713874
UB		 1973out_proto:
1974 proto_unregister(&smc_proto);
6812baab
TR		 1975out_pnet:
1976 smc_pnet_exit();
ac713874
UB		 1977 return rc;
1978}
1979
1980static void __exit smc_exit(void)
1981{
9fda3510 1982 smc_core_exit();
c5c1cc9c 1983 static_branch_disable(&tcp_have_smc);
a4cf0443 1984 smc_ib_unregister_client();
ac713874 1985 sock_unregister(PF_SMC);
aaa4d33f 1986 proto_unregister(&smc_proto6);
ac713874 1987 proto_unregister(&smc_proto);
6812baab 1988 smc_pnet_exit();
ac713874
UB		 1989}
1990
1991module_init(smc_init);
1992module_exit(smc_exit);
1993
1994MODULE_AUTHOR("Ursula Braun <ubraun@linux.vnet.ibm.com>");
1995MODULE_DESCRIPTION("smc socket address family");
1996MODULE_LICENSE("GPL");
1997MODULE_ALIAS_NETPROTO(PF_SMC);
Linux 6.x block layer and io_uring tree(s)