projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
smc: receive data from RMBE
[linux-2.6-block.git] / net / smc / af_smc.c
CommitLineData
ac713874
UB		 1/*
2 * Shared Memory Communications over RDMA (SMC-R) and RoCE
3 *
4 * AF_SMC protocol family socket handler keeping the AF_INET sock address type
5 * applies to SOCK_STREAM sockets only
6 * offers an alternative communication option for TCP-protocol sockets
7 * applicable with RoCE-cards only
8 *
a046d57d
UB		 9 * Initial restrictions:
10 * - non-blocking connect postponed
11 * - IPv6 support postponed
12 * - support for alternate links postponed
13 * - partial support for non-blocking sockets only
14 * - support for urgent data postponed
15 *
ac713874
UB		 16 * Copyright IBM Corp. 2016
17 *
18 * Author(s): Ursula Braun <ubraun@linux.vnet.ibm.com>
19 * based on prototype from Frank Blaschka
20 */
21
22#define KMSG_COMPONENT "smc"
23#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
24
25#include <linux/module.h>
26#include <linux/socket.h>
a046d57d
UB		 27#include <linux/inetdevice.h>
28#include <linux/workqueue.h>
5f08318f 29#include <linux/in.h>
ac713874 30#include <net/sock.h>
a046d57d 31#include <net/tcp.h>
ac713874
UB		 32
33#include "smc.h"
a046d57d 34#include "smc_clc.h"
9bf9abea 35#include "smc_llc.h"
5f08318f 36#include "smc_cdc.h"
0cfdd8f9 37#include "smc_core.h"
a4cf0443 38#include "smc_ib.h"
6812baab 39#include "smc_pnet.h"
e6727f39 40#include "smc_tx.h"
952310cc 41#include "smc_rx.h"
ac713874 42
0cfdd8f9
UB		 43static DEFINE_MUTEX(smc_create_lgr_pending); /* serialize link group
44 * creation
45 */
46
47struct smc_lgr_list smc_lgr_list = { /* established link groups */
48 .lock = __SPIN_LOCK_UNLOCKED(smc_lgr_list.lock),
49 .list = LIST_HEAD_INIT(smc_lgr_list.list),
50};
51
a046d57d
UB		 52static void smc_tcp_listen_work(struct work_struct *);
53
ac713874
UB		 54static void smc_set_keepalive(struct sock *sk, int val)
55{
56 struct smc_sock *smc = smc_sk(sk);
57
58 smc->clcsock->sk->sk_prot->keepalive(smc->clcsock->sk, val);
59}
60
61static struct proto smc_proto = {
62 .name = "SMC",
63 .owner = THIS_MODULE,
64 .keepalive = smc_set_keepalive,
65 .obj_size = sizeof(struct smc_sock),
66 .slab_flags = SLAB_DESTROY_BY_RCU,
67};
68
69static int smc_release(struct socket *sock)
70{
71 struct sock *sk = sock->sk;
72 struct smc_sock *smc;
73
74 if (!sk)
75 goto out;
76
77 smc = smc_sk(sk);
78 lock_sock(sk);
79
80 sk->sk_state = SMC_CLOSED;
81 if (smc->clcsock) {
82 sock_release(smc->clcsock);
83 smc->clcsock = NULL;
84 }
85
86 /* detach socket */
87 sock_orphan(sk);
88 sock->sk = NULL;
89 release_sock(sk);
90
91 sock_put(sk);
92out:
93 return 0;
94}
95
96static void smc_destruct(struct sock *sk)
97{
98 if (sk->sk_state != SMC_CLOSED)
99 return;
100 if (!sock_flag(sk, SOCK_DEAD))
101 return;
102
103 sk_refcnt_debug_dec(sk);
104}
105
106static struct sock *smc_sock_alloc(struct net *net, struct socket *sock)
107{
108 struct smc_sock *smc;
109 struct sock *sk;
110
111 sk = sk_alloc(net, PF_SMC, GFP_KERNEL, &smc_proto, 0);
112 if (!sk)
113 return NULL;
114
115 sock_init_data(sock, sk); /* sets sk_refcnt to 1 */
116 sk->sk_state = SMC_INIT;
117 sk->sk_destruct = smc_destruct;
118 sk->sk_protocol = SMCPROTO_SMC;
ac713874 119 smc = smc_sk(sk);
a046d57d
UB		 120 INIT_WORK(&smc->tcp_listen_work, smc_tcp_listen_work);
121 INIT_LIST_HEAD(&smc->accept_q);
122 spin_lock_init(&smc->accept_q_lock);
123 sk_refcnt_debug_inc(sk);
ac713874
UB		 124
125 return sk;
126}
127
128static int smc_bind(struct socket *sock, struct sockaddr *uaddr,
129 int addr_len)
130{
131 struct sockaddr_in *addr = (struct sockaddr_in *)uaddr;
132 struct sock *sk = sock->sk;
133 struct smc_sock *smc;
134 int rc;
135
136 smc = smc_sk(sk);
137
138 /* replicate tests from inet_bind(), to be safe wrt. future changes */
139 rc = -EINVAL;
140 if (addr_len < sizeof(struct sockaddr_in))
141 goto out;
142
143 rc = -EAFNOSUPPORT;
144 /* accept AF_UNSPEC (mapped to AF_INET) only if s_addr is INADDR_ANY */
145 if ((addr->sin_family != AF_INET) &&
146 ((addr->sin_family != AF_UNSPEC) ||
147 (addr->sin_addr.s_addr != htonl(INADDR_ANY))))
148 goto out;
149
150 lock_sock(sk);
151
152 /* Check if socket is already active */
153 rc = -EINVAL;
154 if (sk->sk_state != SMC_INIT)
155 goto out_rel;
156
157 smc->clcsock->sk->sk_reuse = sk->sk_reuse;
158 rc = kernel_bind(smc->clcsock, uaddr, addr_len);
159
160out_rel:
161 release_sock(sk);
162out:
163 return rc;
164}
165
166static void smc_copy_sock_settings(struct sock *nsk, struct sock *osk,
167 unsigned long mask)
168{
169 /* options we don't get control via setsockopt for */
170 nsk->sk_type = osk->sk_type;
171 nsk->sk_sndbuf = osk->sk_sndbuf;
172 nsk->sk_rcvbuf = osk->sk_rcvbuf;
173 nsk->sk_sndtimeo = osk->sk_sndtimeo;
174 nsk->sk_rcvtimeo = osk->sk_rcvtimeo;
175 nsk->sk_mark = osk->sk_mark;
176 nsk->sk_priority = osk->sk_priority;
177 nsk->sk_rcvlowat = osk->sk_rcvlowat;
178 nsk->sk_bound_dev_if = osk->sk_bound_dev_if;
179 nsk->sk_err = osk->sk_err;
180
181 nsk->sk_flags &= ~mask;
182 nsk->sk_flags |= osk->sk_flags & mask;
183}
184
185#define SK_FLAGS_SMC_TO_CLC ((1UL << SOCK_URGINLINE) | \
186 (1UL << SOCK_KEEPOPEN) | \
187 (1UL << SOCK_LINGER) | \
188 (1UL << SOCK_BROADCAST) | \
189 (1UL << SOCK_TIMESTAMP) | \
190 (1UL << SOCK_DBG) | \
191 (1UL << SOCK_RCVTSTAMP) | \
192 (1UL << SOCK_RCVTSTAMPNS) | \
193 (1UL << SOCK_LOCALROUTE) | \
194 (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE) | \
195 (1UL << SOCK_RXQ_OVFL) | \
196 (1UL << SOCK_WIFI_STATUS) | \
197 (1UL << SOCK_NOFCS) | \
198 (1UL << SOCK_FILTER_LOCKED))
199/* copy only relevant settings and flags of SOL_SOCKET level from smc to
200 * clc socket (since smc is not called for these options from net/core)
201 */
202static void smc_copy_sock_settings_to_clc(struct smc_sock *smc)
203{
204 smc_copy_sock_settings(smc->clcsock->sk, &smc->sk, SK_FLAGS_SMC_TO_CLC);
205}
206
207#define SK_FLAGS_CLC_TO_SMC ((1UL << SOCK_URGINLINE) | \
208 (1UL << SOCK_KEEPOPEN) | \
209 (1UL << SOCK_LINGER) | \
210 (1UL << SOCK_DBG))
211/* copy only settings and flags relevant for smc from clc to smc socket */
212static void smc_copy_sock_settings_to_smc(struct smc_sock *smc)
213{
214 smc_copy_sock_settings(&smc->sk, smc->clcsock->sk, SK_FLAGS_CLC_TO_SMC);
215}
216
a046d57d
UB		 217/* determine subnet and mask of internal TCP socket */
218int smc_netinfo_by_tcpsk(struct socket *clcsock,
219 __be32 *subnet, u8 *prefix_len)
220{
221 struct dst_entry *dst = sk_dst_get(clcsock->sk);
222 struct sockaddr_in addr;
223 int rc = -ENOENT;
224 int len;
225
226 if (!dst) {
227 rc = -ENOTCONN;
228 goto out;
229 }
230 if (!dst->dev) {
231 rc = -ENODEV;
232 goto out_rel;
233 }
234
235 /* get address to which the internal TCP socket is bound */
236 kernel_getsockname(clcsock, (struct sockaddr *)&addr, &len);
237 /* analyze IPv4 specific data of net_device belonging to TCP socket */
238 for_ifa(dst->dev->ip_ptr) {
239 if (ifa->ifa_address != addr.sin_addr.s_addr)
240 continue;
241 *prefix_len = inet_mask_len(ifa->ifa_mask);
242 *subnet = ifa->ifa_address & ifa->ifa_mask;
243 rc = 0;
244 break;
245 } endfor_ifa(dst->dev->ip_ptr);
246
247out_rel:
248 dst_release(dst);
249out:
250 return rc;
251}
252
9bf9abea
UB		 253static int smc_clnt_conf_first_link(struct smc_sock *smc, union ib_gid *gid)
254{
255 struct smc_link_group *lgr = smc->conn.lgr;
256 struct smc_link *link;
257 int rest;
258 int rc;
259
260 link = &lgr->lnk[SMC_SINGLE_LINK];
261 /* receive CONFIRM LINK request from server over RoCE fabric */
262 rest = wait_for_completion_interruptible_timeout(
263 &link->llc_confirm,
264 SMC_LLC_WAIT_FIRST_TIME);
265 if (rest <= 0) {
266 struct smc_clc_msg_decline dclc;
267
268 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
269 SMC_CLC_DECLINE);
270 return rc;
271 }
272
273 rc = smc_ib_modify_qp_rts(link);
274 if (rc)
275 return SMC_CLC_DECL_INTERR;
276
277 smc_wr_remember_qp_attr(link);
278 /* send CONFIRM LINK response over RoCE fabric */
279 rc = smc_llc_send_confirm_link(link,
280 link->smcibdev->mac[link->ibport - 1],
281 gid, SMC_LLC_RESP);
282 if (rc < 0)
283 return SMC_CLC_DECL_TCL;
284
285 return rc;
286}
287
0cfdd8f9
UB		 288static void smc_conn_save_peer_info(struct smc_sock *smc,
289 struct smc_clc_msg_accept_confirm *clc)
290{
291 smc->conn.peer_conn_idx = clc->conn_idx;
5f08318f 292 smc->conn.local_tx_ctrl.token = ntohl(clc->rmbe_alert_token);
cd6851f3
UB		 293 smc->conn.peer_rmbe_size = smc_uncompress_bufsize(clc->rmbe_size);
294 atomic_set(&smc->conn.peer_rmbe_space, smc->conn.peer_rmbe_size);
0cfdd8f9
UB		 295}
296
297static void smc_link_save_peer_info(struct smc_link *link,
298 struct smc_clc_msg_accept_confirm *clc)
299{
300 link->peer_qpn = ntoh24(clc->qpn);
301 memcpy(link->peer_gid, clc->lcl.gid, SMC_GID_SIZE);
302 memcpy(link->peer_mac, clc->lcl.mac, sizeof(link->peer_mac));
303 link->peer_psn = ntoh24(clc->psn);
304 link->peer_mtu = clc->qp_mtu;
305}
306
a046d57d
UB		 307/* setup for RDMA connection of client */
308static int smc_connect_rdma(struct smc_sock *smc)
309{
0cfdd8f9 310 struct sockaddr_in *inaddr = (struct sockaddr_in *)smc->addr;
a046d57d 311 struct smc_clc_msg_accept_confirm aclc;
0cfdd8f9 312 int local_contact = SMC_FIRST_CONTACT;
a046d57d 313 struct smc_ib_device *smcibdev;
0cfdd8f9
UB		 314 struct smc_link *link;
315 u8 srv_first_contact;
a046d57d
UB		 316 int reason_code = 0;
317 int rc = 0;
318 u8 ibport;
319
320 /* IPSec connections opt out of SMC-R optimizations */
321 if (using_ipsec(smc)) {
322 reason_code = SMC_CLC_DECL_IPSEC;
323 goto decline_rdma;
324 }
325
326 /* PNET table look up: search active ib_device and port
327 * within same PNETID that also contains the ethernet device
328 * used for the internal TCP socket
329 */
330 smc_pnet_find_roce_resource(smc->clcsock->sk, &smcibdev, &ibport);
331 if (!smcibdev) {
332 reason_code = SMC_CLC_DECL_CNFERR; /* configuration error */
333 goto decline_rdma;
334 }
335
336 /* do inband token exchange */
337 reason_code = smc_clc_send_proposal(smc, smcibdev, ibport);
338 if (reason_code < 0) {
339 rc = reason_code;
340 goto out_err;
341 }
342 if (reason_code > 0) /* configuration error */
343 goto decline_rdma;
344 /* receive SMC Accept CLC message */
345 reason_code = smc_clc_wait_msg(smc, &aclc, sizeof(aclc),
346 SMC_CLC_ACCEPT);
347 if (reason_code < 0) {
348 rc = reason_code;
349 goto out_err;
350 }
351 if (reason_code > 0)
352 goto decline_rdma;
353
0cfdd8f9
UB		 354 srv_first_contact = aclc.hdr.flag;
355 mutex_lock(&smc_create_lgr_pending);
356 local_contact = smc_conn_create(smc, inaddr->sin_addr.s_addr, smcibdev,
357 ibport, &aclc.lcl, srv_first_contact);
358 if (local_contact < 0) {
359 rc = local_contact;
360 if (rc == -ENOMEM)
361 reason_code = SMC_CLC_DECL_MEM;/* insufficient memory*/
362 else if (rc == -ENOLINK)
363 reason_code = SMC_CLC_DECL_SYNCERR; /* synchr. error */
364 goto decline_rdma_unlock;
365 }
366 link = &smc->conn.lgr->lnk[SMC_SINGLE_LINK];
a046d57d 367
0cfdd8f9 368 smc_conn_save_peer_info(smc, &aclc);
cd6851f3
UB		 369
370 rc = smc_sndbuf_create(smc);
371 if (rc) {
372 reason_code = SMC_CLC_DECL_MEM;
373 goto decline_rdma_unlock;
374 }
375 rc = smc_rmb_create(smc);
376 if (rc) {
377 reason_code = SMC_CLC_DECL_MEM;
378 goto decline_rdma_unlock;
379 }
380
0cfdd8f9
UB		 381 if (local_contact == SMC_FIRST_CONTACT)
382 smc_link_save_peer_info(link, &aclc);
bd4ad577
UB		 383
384 rc = smc_rmb_rtoken_handling(&smc->conn, &aclc);
385 if (rc) {
386 reason_code = SMC_CLC_DECL_INTERR;
387 goto decline_rdma_unlock;
388 }
389
390 if (local_contact == SMC_FIRST_CONTACT) {
391 rc = smc_ib_ready_link(link);
392 if (rc) {
393 reason_code = SMC_CLC_DECL_INTERR;
394 goto decline_rdma_unlock;
395 }
396 }
a046d57d
UB		 397
398 rc = smc_clc_send_confirm(smc);
399 if (rc)
0cfdd8f9 400 goto out_err_unlock;
a046d57d 401
9bf9abea
UB		 402 if (local_contact == SMC_FIRST_CONTACT) {
403 /* QP confirmation over RoCE fabric */
404 reason_code = smc_clnt_conf_first_link(
405 smc, &smcibdev->gid[ibport - 1]);
406 if (reason_code < 0) {
407 rc = reason_code;
408 goto out_err_unlock;
409 }
410 if (reason_code > 0)
411 goto decline_rdma_unlock;
412 }
a046d57d 413
0cfdd8f9 414 mutex_unlock(&smc_create_lgr_pending);
e6727f39 415 smc_tx_init(smc);
952310cc 416 smc_rx_init(smc);
e6727f39 417
a046d57d
UB		 418out_connected:
419 smc_copy_sock_settings_to_clc(smc);
420 smc->sk.sk_state = SMC_ACTIVE;
421
0cfdd8f9 422 return rc ? rc : local_contact;
a046d57d 423
0cfdd8f9
UB		 424decline_rdma_unlock:
425 mutex_unlock(&smc_create_lgr_pending);
426 smc_conn_free(&smc->conn);
a046d57d
UB		 427decline_rdma:
428 /* RDMA setup failed, switch back to TCP */
429 smc->use_fallback = true;
430 if (reason_code && (reason_code != SMC_CLC_DECL_REPLY)) {
431 rc = smc_clc_send_decline(smc, reason_code, 0);
432 if (rc < sizeof(struct smc_clc_msg_decline))
433 goto out_err;
434 }
435 goto out_connected;
436
0cfdd8f9
UB		 437out_err_unlock:
438 mutex_unlock(&smc_create_lgr_pending);
439 smc_conn_free(&smc->conn);
a046d57d
UB		 440out_err:
441 return rc;
442}
443
ac713874
UB		 444static int smc_connect(struct socket *sock, struct sockaddr *addr,
445 int alen, int flags)
446{
447 struct sock *sk = sock->sk;
448 struct smc_sock *smc;
449 int rc = -EINVAL;
450
451 smc = smc_sk(sk);
452
453 /* separate smc parameter checking to be safe */
454 if (alen < sizeof(addr->sa_family))
455 goto out_err;
456 if (addr->sa_family != AF_INET)
457 goto out_err;
a046d57d 458 smc->addr = addr; /* needed for nonblocking connect */
ac713874
UB		 459
460 lock_sock(sk);
461 switch (sk->sk_state) {
462 default:
463 goto out;
464 case SMC_ACTIVE:
465 rc = -EISCONN;
466 goto out;
467 case SMC_INIT:
468 rc = 0;
469 break;
470 }
471
472 smc_copy_sock_settings_to_clc(smc);
473 rc = kernel_connect(smc->clcsock, addr, alen, flags);
474 if (rc)
475 goto out;
476
a046d57d
UB		 477 /* setup RDMA connection */
478 rc = smc_connect_rdma(smc);
479 if (rc < 0)
480 goto out;
481 else
482 rc = 0; /* success cases including fallback */
ac713874
UB		 483
484out:
485 release_sock(sk);
486out_err:
487 return rc;
488}
489
490static int smc_clcsock_accept(struct smc_sock *lsmc, struct smc_sock **new_smc)
491{
492 struct sock *sk = &lsmc->sk;
493 struct socket *new_clcsock;
494 struct sock *new_sk;
495 int rc;
496
a046d57d 497 release_sock(&lsmc->sk);
ac713874
UB		 498 new_sk = smc_sock_alloc(sock_net(sk), NULL);
499 if (!new_sk) {
500 rc = -ENOMEM;
501 lsmc->sk.sk_err = ENOMEM;
502 *new_smc = NULL;
a046d57d 503 lock_sock(&lsmc->sk);
ac713874
UB		 504 goto out;
505 }
506 *new_smc = smc_sk(new_sk);
507
508 rc = kernel_accept(lsmc->clcsock, &new_clcsock, 0);
a046d57d
UB		 509 lock_sock(&lsmc->sk);
510 if (rc < 0) {
511 lsmc->sk.sk_err = -rc;
512 new_sk->sk_state = SMC_CLOSED;
513 sock_set_flag(new_sk, SOCK_DEAD);
514 sock_put(new_sk);
515 *new_smc = NULL;
516 goto out;
517 }
518 if (lsmc->sk.sk_state == SMC_CLOSED) {
519 if (new_clcsock)
520 sock_release(new_clcsock);
521 new_sk->sk_state = SMC_CLOSED;
522 sock_set_flag(new_sk, SOCK_DEAD);
ac713874
UB		 523 sock_put(new_sk);
524 *new_smc = NULL;
525 goto out;
526 }
527
528 (*new_smc)->clcsock = new_clcsock;
529out:
530 return rc;
531}
532
a046d57d
UB		 533/* add a just created sock to the accept queue of the listen sock as
534 * candidate for a following socket accept call from user space
535 */
536static void smc_accept_enqueue(struct sock *parent, struct sock *sk)
537{
538 struct smc_sock *par = smc_sk(parent);
539
540 sock_hold(sk);
541 spin_lock(&par->accept_q_lock);
542 list_add_tail(&smc_sk(sk)->accept_q, &par->accept_q);
543 spin_unlock(&par->accept_q_lock);
544 sk_acceptq_added(parent);
545}
546
547/* remove a socket from the accept queue of its parental listening socket */
548static void smc_accept_unlink(struct sock *sk)
549{
550 struct smc_sock *par = smc_sk(sk)->listen_smc;
551
552 spin_lock(&par->accept_q_lock);
553 list_del_init(&smc_sk(sk)->accept_q);
554 spin_unlock(&par->accept_q_lock);
555 sk_acceptq_removed(&smc_sk(sk)->listen_smc->sk);
556 sock_put(sk);
557}
558
559/* remove a sock from the accept queue to bind it to a new socket created
560 * for a socket accept call from user space
561 */
562static struct sock *smc_accept_dequeue(struct sock *parent,
563 struct socket *new_sock)
564{
565 struct smc_sock *isk, *n;
566 struct sock *new_sk;
567
568 list_for_each_entry_safe(isk, n, &smc_sk(parent)->accept_q, accept_q) {
569 new_sk = (struct sock *)isk;
570
571 smc_accept_unlink(new_sk);
572 if (new_sk->sk_state == SMC_CLOSED) {
573 /* tbd in follow-on patch: close this sock */
574 continue;
575 }
576 if (new_sock)
577 sock_graft(new_sk, new_sock);
578 return new_sk;
579 }
580 return NULL;
581}
582
583/* clean up for a created but never accepted sock */
584static void smc_close_non_accepted(struct sock *sk)
585{
586 struct smc_sock *smc = smc_sk(sk);
587
588 sock_hold(sk);
589 if (smc->clcsock) {
590 struct socket *tcp;
591
592 tcp = smc->clcsock;
593 smc->clcsock = NULL;
594 sock_release(tcp);
595 }
596 /* more closing stuff to be added with socket closing patch */
597 sock_put(sk);
598}
599
9bf9abea
UB		 600static int smc_serv_conf_first_link(struct smc_sock *smc)
601{
602 struct smc_link_group *lgr = smc->conn.lgr;
603 struct smc_link *link;
604 int rest;
605 int rc;
606
607 link = &lgr->lnk[SMC_SINGLE_LINK];
608 /* send CONFIRM LINK request to client over the RoCE fabric */
609 rc = smc_llc_send_confirm_link(link,
610 link->smcibdev->mac[link->ibport - 1],
611 &link->smcibdev->gid[link->ibport - 1],
612 SMC_LLC_REQ);
613 if (rc < 0)
614 return SMC_CLC_DECL_TCL;
615
616 /* receive CONFIRM LINK response from client over the RoCE fabric */
617 rest = wait_for_completion_interruptible_timeout(
618 &link->llc_confirm_resp,
619 SMC_LLC_WAIT_FIRST_TIME);
620 if (rest <= 0) {
621 struct smc_clc_msg_decline dclc;
622
623 rc = smc_clc_wait_msg(smc, &dclc, sizeof(dclc),
624 SMC_CLC_DECLINE);
625 }
626
627 return rc;
628}
629
a046d57d
UB		 630/* setup for RDMA connection of server */
631static void smc_listen_work(struct work_struct *work)
632{
633 struct smc_sock *new_smc = container_of(work, struct smc_sock,
634 smc_listen_work);
635 struct socket *newclcsock = new_smc->clcsock;
636 struct smc_sock *lsmc = new_smc->listen_smc;
637 struct smc_clc_msg_accept_confirm cclc;
0cfdd8f9 638 int local_contact = SMC_REUSE_CONTACT;
a046d57d
UB		 639 struct sock *newsmcsk = &new_smc->sk;
640 struct smc_clc_msg_proposal pclc;
641 struct smc_ib_device *smcibdev;
642 struct sockaddr_in peeraddr;
0cfdd8f9 643 struct smc_link *link;
a046d57d
UB		 644 int reason_code = 0;
645 int rc = 0, len;
646 __be32 subnet;
647 u8 prefix_len;
648 u8 ibport;
649
650 /* do inband token exchange -
651 *wait for and receive SMC Proposal CLC message
652 */
653 reason_code = smc_clc_wait_msg(new_smc, &pclc, sizeof(pclc),
654 SMC_CLC_PROPOSAL);
655 if (reason_code < 0)
656 goto out_err;
657 if (reason_code > 0)
658 goto decline_rdma;
659
660 /* IPSec connections opt out of SMC-R optimizations */
661 if (using_ipsec(new_smc)) {
662 reason_code = SMC_CLC_DECL_IPSEC;
663 goto decline_rdma;
664 }
665
666 /* PNET table look up: search active ib_device and port
667 * within same PNETID that also contains the ethernet device
668 * used for the internal TCP socket
669 */
670 smc_pnet_find_roce_resource(newclcsock->sk, &smcibdev, &ibport);
671 if (!smcibdev) {
672 reason_code = SMC_CLC_DECL_CNFERR; /* configuration error */
673 goto decline_rdma;
674 }
675
676 /* determine subnet and mask from internal TCP socket */
677 rc = smc_netinfo_by_tcpsk(newclcsock, &subnet, &prefix_len);
678 if (rc) {
679 reason_code = SMC_CLC_DECL_CNFERR; /* configuration error */
680 goto decline_rdma;
681 }
682 if ((pclc.outgoing_subnet != subnet) ||
683 (pclc.prefix_len != prefix_len)) {
684 reason_code = SMC_CLC_DECL_CNFERR; /* configuration error */
685 goto decline_rdma;
686 }
687
688 /* get address of the peer connected to the internal TCP socket */
689 kernel_getpeername(newclcsock, (struct sockaddr *)&peeraddr, &len);
690
0cfdd8f9
UB		 691 /* allocate connection / link group */
692 mutex_lock(&smc_create_lgr_pending);
693 local_contact = smc_conn_create(new_smc, peeraddr.sin_addr.s_addr,
694 smcibdev, ibport, &pclc.lcl, 0);
695 if (local_contact == SMC_REUSE_CONTACT)
696 /* lock no longer needed, free it due to following
697 * smc_clc_wait_msg() call
698 */
699 mutex_unlock(&smc_create_lgr_pending);
700 if (local_contact < 0) {
701 rc = local_contact;
702 if (rc == -ENOMEM)
703 reason_code = SMC_CLC_DECL_MEM;/* insufficient memory*/
704 else if (rc == -ENOLINK)
705 reason_code = SMC_CLC_DECL_SYNCERR; /* synchr. error */
706 goto decline_rdma;
707 }
708 link = &new_smc->conn.lgr->lnk[SMC_SINGLE_LINK];
a046d57d 709
cd6851f3
UB		 710 rc = smc_sndbuf_create(new_smc);
711 if (rc) {
712 reason_code = SMC_CLC_DECL_MEM;
713 goto decline_rdma;
714 }
715 rc = smc_rmb_create(new_smc);
716 if (rc) {
717 reason_code = SMC_CLC_DECL_MEM;
718 goto decline_rdma;
719 }
a046d57d 720
0cfdd8f9 721 rc = smc_clc_send_accept(new_smc, local_contact);
a046d57d
UB		 722 if (rc)
723 goto out_err;
724
725 /* receive SMC Confirm CLC message */
726 reason_code = smc_clc_wait_msg(new_smc, &cclc, sizeof(cclc),
727 SMC_CLC_CONFIRM);
728 if (reason_code < 0)
729 goto out_err;
730 if (reason_code > 0)
731 goto decline_rdma;
0cfdd8f9
UB		 732 smc_conn_save_peer_info(new_smc, &cclc);
733 if (local_contact == SMC_FIRST_CONTACT)
734 smc_link_save_peer_info(link, &cclc);
a046d57d 735
bd4ad577
UB		 736 rc = smc_rmb_rtoken_handling(&new_smc->conn, &cclc);
737 if (rc) {
738 reason_code = SMC_CLC_DECL_INTERR;
739 goto decline_rdma;
740 }
741
bd4ad577
UB		 742 if (local_contact == SMC_FIRST_CONTACT) {
743 rc = smc_ib_ready_link(link);
744 if (rc) {
745 reason_code = SMC_CLC_DECL_INTERR;
746 goto decline_rdma;
747 }
9bf9abea
UB		 748 /* QP confirmation over RoCE fabric */
749 reason_code = smc_serv_conf_first_link(new_smc);
750 if (reason_code < 0) {
751 /* peer is not aware of a problem */
752 rc = reason_code;
753 goto out_err;
754 }
755 if (reason_code > 0)
756 goto decline_rdma;
bd4ad577 757 }
a046d57d 758
e6727f39 759 smc_tx_init(new_smc);
952310cc 760 smc_rx_init(new_smc);
e6727f39 761
a046d57d
UB		 762out_connected:
763 sk_refcnt_debug_inc(newsmcsk);
764 newsmcsk->sk_state = SMC_ACTIVE;
765enqueue:
0cfdd8f9
UB		 766 if (local_contact == SMC_FIRST_CONTACT)
767 mutex_unlock(&smc_create_lgr_pending);
a046d57d
UB		 768 lock_sock(&lsmc->sk);
769 if (lsmc->sk.sk_state == SMC_LISTEN) {
770 smc_accept_enqueue(&lsmc->sk, newsmcsk);
771 } else { /* no longer listening */
772 smc_close_non_accepted(newsmcsk);
773 }
774 release_sock(&lsmc->sk);
775
776 /* Wake up accept */
777 lsmc->sk.sk_data_ready(&lsmc->sk);
778 sock_put(&lsmc->sk); /* sock_hold in smc_tcp_listen_work */
779 return;
780
781decline_rdma:
782 /* RDMA setup failed, switch back to TCP */
0cfdd8f9 783 smc_conn_free(&new_smc->conn);
a046d57d
UB		 784 new_smc->use_fallback = true;
785 if (reason_code && (reason_code != SMC_CLC_DECL_REPLY)) {
786 rc = smc_clc_send_decline(new_smc, reason_code, 0);
787 if (rc < sizeof(struct smc_clc_msg_decline))
788 goto out_err;
789 }
790 goto out_connected;
791
792out_err:
793 newsmcsk->sk_state = SMC_CLOSED;
794 goto enqueue; /* queue new sock with sk_err set */
795}
796
797static void smc_tcp_listen_work(struct work_struct *work)
798{
799 struct smc_sock *lsmc = container_of(work, struct smc_sock,
800 tcp_listen_work);
801 struct smc_sock *new_smc;
802 int rc = 0;
803
804 lock_sock(&lsmc->sk);
805 while (lsmc->sk.sk_state == SMC_LISTEN) {
806 rc = smc_clcsock_accept(lsmc, &new_smc);
807 if (rc)
808 goto out;
809 if (!new_smc)
810 continue;
811
812 new_smc->listen_smc = lsmc;
813 new_smc->use_fallback = false; /* assume rdma capability first*/
814 sock_hold(&lsmc->sk); /* sock_put in smc_listen_work */
815 INIT_WORK(&new_smc->smc_listen_work, smc_listen_work);
816 smc_copy_sock_settings_to_smc(new_smc);
817 schedule_work(&new_smc->smc_listen_work);
818 }
819
820out:
821 release_sock(&lsmc->sk);
822 lsmc->sk.sk_data_ready(&lsmc->sk); /* no more listening, wake accept */
823}
824
ac713874
UB		 825static int smc_listen(struct socket *sock, int backlog)
826{
827 struct sock *sk = sock->sk;
828 struct smc_sock *smc;
829 int rc;
830
831 smc = smc_sk(sk);
832 lock_sock(sk);
833
834 rc = -EINVAL;
835 if ((sk->sk_state != SMC_INIT) && (sk->sk_state != SMC_LISTEN))
836 goto out;
837
838 rc = 0;
839 if (sk->sk_state == SMC_LISTEN) {
840 sk->sk_max_ack_backlog = backlog;
841 goto out;
842 }
843 /* some socket options are handled in core, so we could not apply
844 * them to the clc socket -- copy smc socket options to clc socket
845 */
846 smc_copy_sock_settings_to_clc(smc);
847
848 rc = kernel_listen(smc->clcsock, backlog);
849 if (rc)
850 goto out;
851 sk->sk_max_ack_backlog = backlog;
852 sk->sk_ack_backlog = 0;
853 sk->sk_state = SMC_LISTEN;
a046d57d
UB		 854 INIT_WORK(&smc->tcp_listen_work, smc_tcp_listen_work);
855 schedule_work(&smc->tcp_listen_work);
ac713874
UB		 856
857out:
858 release_sock(sk);
859 return rc;
860}
861
862static int smc_accept(struct socket *sock, struct socket *new_sock,
863 int flags)
864{
a046d57d
UB		 865 struct sock *sk = sock->sk, *nsk;
866 DECLARE_WAITQUEUE(wait, current);
ac713874 867 struct smc_sock *lsmc;
a046d57d
UB		 868 long timeo;
869 int rc = 0;
ac713874
UB		 870
871 lsmc = smc_sk(sk);
872 lock_sock(sk);
873
874 if (lsmc->sk.sk_state != SMC_LISTEN) {
875 rc = -EINVAL;
876 goto out;
877 }
878
a046d57d
UB		 879 /* Wait for an incoming connection */
880 timeo = sock_rcvtimeo(sk, flags & O_NONBLOCK);
881 add_wait_queue_exclusive(sk_sleep(sk), &wait);
882 while (!(nsk = smc_accept_dequeue(sk, new_sock))) {
883 set_current_state(TASK_INTERRUPTIBLE);
884 if (!timeo) {
885 rc = -EAGAIN;
886 break;
887 }
888 release_sock(sk);
889 timeo = schedule_timeout(timeo);
890 /* wakeup by sk_data_ready in smc_listen_work() */
891 sched_annotate_sleep();
892 lock_sock(sk);
893 if (signal_pending(current)) {
894 rc = sock_intr_errno(timeo);
895 break;
896 }
897 }
898 set_current_state(TASK_RUNNING);
899 remove_wait_queue(sk_sleep(sk), &wait);
ac713874 900
a046d57d
UB		 901 if (!rc)
902 rc = sock_error(nsk);
ac713874
UB		 903
904out:
905 release_sock(sk);
906 return rc;
907}
908
909static int smc_getname(struct socket *sock, struct sockaddr *addr,
910 int *len, int peer)
911{
912 struct smc_sock *smc;
913
914 if (peer && (sock->sk->sk_state != SMC_ACTIVE))
915 return -ENOTCONN;
916
917 smc = smc_sk(sock->sk);
918
919 return smc->clcsock->ops->getname(smc->clcsock, addr, len, peer);
920}
921
922static int smc_sendmsg(struct socket *sock, struct msghdr *msg, size_t len)
923{
924 struct sock *sk = sock->sk;
925 struct smc_sock *smc;
926 int rc = -EPIPE;
927
928 smc = smc_sk(sk);
929 lock_sock(sk);
930 if (sk->sk_state != SMC_ACTIVE)
931 goto out;
932 if (smc->use_fallback)
933 rc = smc->clcsock->ops->sendmsg(smc->clcsock, msg, len);
934 else
e6727f39 935 rc = smc_tx_sendmsg(smc, msg, len);
ac713874
UB		 936out:
937 release_sock(sk);
938 return rc;
939}
940
941static int smc_recvmsg(struct socket *sock, struct msghdr *msg, size_t len,
942 int flags)
943{
944 struct sock *sk = sock->sk;
945 struct smc_sock *smc;
946 int rc = -ENOTCONN;
947
948 smc = smc_sk(sk);
949 lock_sock(sk);
950 if ((sk->sk_state != SMC_ACTIVE) && (sk->sk_state != SMC_CLOSED))
951 goto out;
952
953 if (smc->use_fallback)
954 rc = smc->clcsock->ops->recvmsg(smc->clcsock, msg, len, flags);
955 else
952310cc 956 rc = smc_rx_recvmsg(smc, msg, len, flags);
ac713874
UB		 957out:
958 release_sock(sk);
959 return rc;
960}
961
a046d57d
UB		 962static unsigned int smc_accept_poll(struct sock *parent)
963{
964 struct smc_sock *isk;
965 struct sock *sk;
966
967 lock_sock(parent);
968 list_for_each_entry(isk, &smc_sk(parent)->accept_q, accept_q) {
969 sk = (struct sock *)isk;
970
971 if (sk->sk_state == SMC_ACTIVE) {
972 release_sock(parent);
973 return POLLIN | POLLRDNORM;
974 }
975 }
976 release_sock(parent);
977
978 return 0;
979}
980
ac713874
UB		 981static unsigned int smc_poll(struct file *file, struct socket *sock,
982 poll_table *wait)
983{
984 struct sock *sk = sock->sk;
985 unsigned int mask = 0;
986 struct smc_sock *smc;
a046d57d 987 int rc;
ac713874
UB		 988
989 smc = smc_sk(sock->sk);
a046d57d
UB		 990 if ((sk->sk_state == SMC_INIT) || smc->use_fallback) {
991 /* delegate to CLC child sock */
ac713874
UB		 992 mask = smc->clcsock->ops->poll(file, smc->clcsock, wait);
993 /* if non-blocking connect finished ... */
994 lock_sock(sk);
995 if ((sk->sk_state == SMC_INIT) && (mask & POLLOUT)) {
a046d57d
UB		 996 sk->sk_err = smc->clcsock->sk->sk_err;
997 if (sk->sk_err) {
998 mask |= POLLERR;
999 } else {
1000 rc = smc_connect_rdma(smc);
1001 if (rc < 0)
1002 mask |= POLLERR;
1003 else
1004 /* success cases including fallback */
1005 mask |= POLLOUT | POLLWRNORM;
1006 }
ac713874
UB		 1007 }
1008 release_sock(sk);
1009 } else {
a046d57d
UB		 1010 sock_poll_wait(file, sk_sleep(sk), wait);
1011 if (sk->sk_state == SMC_LISTEN)
1012 /* woken up by sk_data_ready in smc_listen_work() */
1013 mask |= smc_accept_poll(sk);
1014 if (sk->sk_err)
1015 mask |= POLLERR;
e6727f39
UB		 1016 if (atomic_read(&smc->conn.sndbuf_space)) {
1017 mask |= POLLOUT | POLLWRNORM;
1018 } else {
1019 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
1020 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
1021 }
952310cc
UB		 1022 if (atomic_read(&smc->conn.bytes_to_rcv))
1023 mask |= POLLIN | POLLRDNORM;
a046d57d 1024 /* for now - to be enhanced in follow-on patch */
ac713874
UB		 1025 }
1026
1027 return mask;
1028}
1029
1030static int smc_shutdown(struct socket *sock, int how)
1031{
1032 struct sock *sk = sock->sk;
1033 struct smc_sock *smc;
1034 int rc = -EINVAL;
1035
1036 smc = smc_sk(sk);
1037
1038 if ((how < SHUT_RD) || (how > SHUT_RDWR))
1039 goto out_err;
1040
1041 lock_sock(sk);
1042
1043 rc = -ENOTCONN;
1044 if (sk->sk_state == SMC_CLOSED)
1045 goto out;
1046 if (smc->use_fallback) {
1047 rc = kernel_sock_shutdown(smc->clcsock, how);
1048 sk->sk_shutdown = smc->clcsock->sk->sk_shutdown;
1049 if (sk->sk_shutdown == SHUTDOWN_MASK)
1050 sk->sk_state = SMC_CLOSED;
1051 } else {
1052 rc = sock_no_shutdown(sock, how);
1053 }
1054
1055out:
1056 release_sock(sk);
1057
1058out_err:
1059 return rc;
1060}
1061
1062static int smc_setsockopt(struct socket *sock, int level, int optname,
1063 char __user *optval, unsigned int optlen)
1064{
1065 struct sock *sk = sock->sk;
1066 struct smc_sock *smc;
1067
1068 smc = smc_sk(sk);
1069
1070 /* generic setsockopts reaching us here always apply to the
1071 * CLC socket
1072 */
1073 return smc->clcsock->ops->setsockopt(smc->clcsock, level, optname,
1074 optval, optlen);
1075}
1076
1077static int smc_getsockopt(struct socket *sock, int level, int optname,
1078 char __user *optval, int __user *optlen)
1079{
1080 struct smc_sock *smc;
1081
1082 smc = smc_sk(sock->sk);
1083 /* socket options apply to the CLC socket */
1084 return smc->clcsock->ops->getsockopt(smc->clcsock, level, optname,
1085 optval, optlen);
1086}
1087
1088static int smc_ioctl(struct socket *sock, unsigned int cmd,
1089 unsigned long arg)
1090{
1091 struct smc_sock *smc;
1092
1093 smc = smc_sk(sock->sk);
1094 if (smc->use_fallback)
1095 return smc->clcsock->ops->ioctl(smc->clcsock, cmd, arg);
1096 else
1097 return sock_no_ioctl(sock, cmd, arg);
1098}
1099
1100static ssize_t smc_sendpage(struct socket *sock, struct page *page,
1101 int offset, size_t size, int flags)
1102{
1103 struct sock *sk = sock->sk;
1104 struct smc_sock *smc;
1105 int rc = -EPIPE;
1106
1107 smc = smc_sk(sk);
1108 lock_sock(sk);
1109 if (sk->sk_state != SMC_ACTIVE)
1110 goto out;
1111 if (smc->use_fallback)
1112 rc = kernel_sendpage(smc->clcsock, page, offset,
1113 size, flags);
1114 else
1115 rc = sock_no_sendpage(sock, page, offset, size, flags);
1116
1117out:
1118 release_sock(sk);
1119 return rc;
1120}
1121
1122static ssize_t smc_splice_read(struct socket *sock, loff_t *ppos,
1123 struct pipe_inode_info *pipe, size_t len,
1124 unsigned int flags)
1125{
1126 struct sock *sk = sock->sk;
1127 struct smc_sock *smc;
1128 int rc = -ENOTCONN;
1129
1130 smc = smc_sk(sk);
1131 lock_sock(sk);
1132 if ((sk->sk_state != SMC_ACTIVE) && (sk->sk_state != SMC_CLOSED))
1133 goto out;
1134 if (smc->use_fallback) {
1135 rc = smc->clcsock->ops->splice_read(smc->clcsock, ppos,
1136 pipe, len, flags);
1137 } else {
1138 rc = -EOPNOTSUPP;
1139 }
1140out:
1141 release_sock(sk);
1142 return rc;
1143}
1144
1145/* must look like tcp */
1146static const struct proto_ops smc_sock_ops = {
1147 .family = PF_SMC,
1148 .owner = THIS_MODULE,
1149 .release = smc_release,
1150 .bind = smc_bind,
1151 .connect = smc_connect,
1152 .socketpair = sock_no_socketpair,
1153 .accept = smc_accept,
1154 .getname = smc_getname,
1155 .poll = smc_poll,
1156 .ioctl = smc_ioctl,
1157 .listen = smc_listen,
1158 .shutdown = smc_shutdown,
1159 .setsockopt = smc_setsockopt,
1160 .getsockopt = smc_getsockopt,
1161 .sendmsg = smc_sendmsg,
1162 .recvmsg = smc_recvmsg,
1163 .mmap = sock_no_mmap,
1164 .sendpage = smc_sendpage,
1165 .splice_read = smc_splice_read,
1166};
1167
1168static int smc_create(struct net *net, struct socket *sock, int protocol,
1169 int kern)
1170{
1171 struct smc_sock *smc;
1172 struct sock *sk;
1173 int rc;
1174
1175 rc = -ESOCKTNOSUPPORT;
1176 if (sock->type != SOCK_STREAM)
1177 goto out;
1178
1179 rc = -EPROTONOSUPPORT;
1180 if ((protocol != IPPROTO_IP) && (protocol != IPPROTO_TCP))
1181 goto out;
1182
1183 rc = -ENOBUFS;
1184 sock->ops = &smc_sock_ops;
1185 sk = smc_sock_alloc(net, sock);
1186 if (!sk)
1187 goto out;
1188
1189 /* create internal TCP socket for CLC handshake and fallback */
1190 smc = smc_sk(sk);
a046d57d 1191 smc->use_fallback = false; /* assume rdma capability first */
ac713874
UB		 1192 rc = sock_create_kern(net, PF_INET, SOCK_STREAM,
1193 IPPROTO_TCP, &smc->clcsock);
1194 if (rc)
1195 sk_common_release(sk);
cd6851f3
UB		 1196 smc->sk.sk_sndbuf = max(smc->clcsock->sk->sk_sndbuf, SMC_BUF_MIN_SIZE);
1197 smc->sk.sk_rcvbuf = max(smc->clcsock->sk->sk_rcvbuf, SMC_BUF_MIN_SIZE);
ac713874
UB		 1198
1199out:
1200 return rc;
1201}
1202
1203static const struct net_proto_family smc_sock_family_ops = {
1204 .family = PF_SMC,
1205 .owner = THIS_MODULE,
1206 .create = smc_create,
1207};
1208
1209static int __init smc_init(void)
1210{
1211 int rc;
1212
6812baab
TR		 1213 rc = smc_pnet_init();
1214 if (rc)
1215 return rc;
1216
9bf9abea
UB		 1217 rc = smc_llc_init();
1218 if (rc) {
1219 pr_err("%s: smc_llc_init fails with %d\n", __func__, rc);
1220 goto out_pnet;
1221 }
1222
5f08318f
UB		 1223 rc = smc_cdc_init();
1224 if (rc) {
1225 pr_err("%s: smc_cdc_init fails with %d\n", __func__, rc);
1226 goto out_pnet;
1227 }
1228
ac713874
UB		 1229 rc = proto_register(&smc_proto, 1);
1230 if (rc) {
1231 pr_err("%s: proto_register fails with %d\n", __func__, rc);
6812baab 1232 goto out_pnet;
ac713874
UB		 1233 }
1234
1235 rc = sock_register(&smc_sock_family_ops);
1236 if (rc) {
1237 pr_err("%s: sock_register fails with %d\n", __func__, rc);
1238 goto out_proto;
1239 }
1240
a4cf0443
UB		 1241 rc = smc_ib_register_client();
1242 if (rc) {
1243 pr_err("%s: ib_register fails with %d\n", __func__, rc);
1244 goto out_sock;
1245 }
1246
ac713874
UB		 1247 return 0;
1248
a4cf0443
UB		 1249out_sock:
1250 sock_unregister(PF_SMC);
ac713874
UB		 1251out_proto:
1252 proto_unregister(&smc_proto);
6812baab
TR		 1253out_pnet:
1254 smc_pnet_exit();
ac713874
UB		 1255 return rc;
1256}
1257
1258static void __exit smc_exit(void)
1259{
0cfdd8f9
UB		 1260 struct smc_link_group *lgr, *lg;
1261 LIST_HEAD(lgr_freeing_list);
1262
1263 spin_lock_bh(&smc_lgr_list.lock);
1264 if (!list_empty(&smc_lgr_list.list))
1265 list_splice_init(&smc_lgr_list.list, &lgr_freeing_list);
1266 spin_unlock_bh(&smc_lgr_list.lock);
1267 list_for_each_entry_safe(lgr, lg, &lgr_freeing_list, list) {
1268 list_del_init(&lgr->list);
1269 smc_lgr_free(lgr); /* free link group */
1270 }
a4cf0443 1271 smc_ib_unregister_client();
ac713874
UB		 1272 sock_unregister(PF_SMC);
1273 proto_unregister(&smc_proto);
6812baab 1274 smc_pnet_exit();
ac713874
UB		 1275}
1276
1277module_init(smc_init);
1278module_exit(smc_exit);
1279
1280MODULE_AUTHOR("Ursula Braun <ubraun@linux.vnet.ibm.com>");
1281MODULE_DESCRIPTION("smc socket address family");
1282MODULE_LICENSE("GPL");
1283MODULE_ALIAS_NETPROTO(PF_SMC);
Linux 6.x block layer and io_uring tree(s)