projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
treewide: Replace GPLv2 boilerplate/reference with SPDX - rule 499
[linux-2.6-block.git] / drivers / crypto / chelsio / chtls / chtls_cm.c
CommitLineData
cc35c88a
AG		 1/*
2 * Copyright (c) 2018 Chelsio Communications, Inc.
3 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of the GNU General Public License version 2 as
6 * published by the Free Software Foundation.
7 *
8 * Written by: Atul Gupta (atul.gupta@chelsio.com)
9 */
10
11#include <linux/module.h>
12#include <linux/list.h>
13#include <linux/workqueue.h>
14#include <linux/skbuff.h>
15#include <linux/timer.h>
16#include <linux/notifier.h>
17#include <linux/inetdevice.h>
18#include <linux/ip.h>
19#include <linux/tcp.h>
20#include <linux/sched/signal.h>
21#include <linux/kallsyms.h>
22#include <linux/kprobes.h>
23#include <linux/if_vlan.h>
0c3a16be 24#include <net/inet_common.h>
cc35c88a
AG		 25#include <net/tcp.h>
26#include <net/dst.h>
76f7164d 27#include <net/tls.h>
cc35c88a
AG		 28
29#include "chtls.h"
30#include "chtls_cm.h"
31
32/*
33 * State transitions and actions for close. Note that if we are in SYN_SENT
34 * we remain in that state as we cannot control a connection while it's in
35 * SYN_SENT; such connections are allowed to establish and are then aborted.
36 */
37static unsigned char new_state[16] = {
38 /* current state: new state: action: */
39 /* (Invalid) */ TCP_CLOSE,
40 /* TCP_ESTABLISHED */ TCP_FIN_WAIT1 | TCP_ACTION_FIN,
41 /* TCP_SYN_SENT */ TCP_SYN_SENT,
42 /* TCP_SYN_RECV */ TCP_FIN_WAIT1 | TCP_ACTION_FIN,
43 /* TCP_FIN_WAIT1 */ TCP_FIN_WAIT1,
44 /* TCP_FIN_WAIT2 */ TCP_FIN_WAIT2,
45 /* TCP_TIME_WAIT */ TCP_CLOSE,
46 /* TCP_CLOSE */ TCP_CLOSE,
47 /* TCP_CLOSE_WAIT */ TCP_LAST_ACK | TCP_ACTION_FIN,
48 /* TCP_LAST_ACK */ TCP_LAST_ACK,
49 /* TCP_LISTEN */ TCP_CLOSE,
50 /* TCP_CLOSING */ TCP_CLOSING,
51};
52
53static struct chtls_sock *chtls_sock_create(struct chtls_dev *cdev)
54{
55 struct chtls_sock *csk = kzalloc(sizeof(*csk), GFP_ATOMIC);
56
57 if (!csk)
58 return NULL;
59
60 csk->txdata_skb_cache = alloc_skb(TXDATA_SKB_LEN, GFP_ATOMIC);
61 if (!csk->txdata_skb_cache) {
62 kfree(csk);
63 return NULL;
64 }
65
66 kref_init(&csk->kref);
67 csk->cdev = cdev;
68 skb_queue_head_init(&csk->txq);
69 csk->wr_skb_head = NULL;
70 csk->wr_skb_tail = NULL;
71 csk->mss = MAX_MSS;
72 csk->tlshws.ofld = 1;
73 csk->tlshws.txkey = -1;
74 csk->tlshws.rxkey = -1;
75 csk->tlshws.mfs = TLS_MFS;
76 skb_queue_head_init(&csk->tlshws.sk_recv_queue);
77 return csk;
78}
79
80static void chtls_sock_release(struct kref *ref)
81{
82 struct chtls_sock *csk =
83 container_of(ref, struct chtls_sock, kref);
84
85 kfree(csk);
86}
87
88static struct net_device *chtls_ipv4_netdev(struct chtls_dev *cdev,
89 struct sock *sk)
90{
91 struct net_device *ndev = cdev->ports[0];
92
93 if (likely(!inet_sk(sk)->inet_rcv_saddr))
94 return ndev;
95
96 ndev = ip_dev_find(&init_net, inet_sk(sk)->inet_rcv_saddr);
97 if (!ndev)
98 return NULL;
99
100 if (is_vlan_dev(ndev))
101 return vlan_dev_real_dev(ndev);
102 return ndev;
103}
104
105static void assign_rxopt(struct sock *sk, unsigned int opt)
106{
107 const struct chtls_dev *cdev;
108 struct chtls_sock *csk;
109 struct tcp_sock *tp;
110
111 csk = rcu_dereference_sk_user_data(sk);
112 tp = tcp_sk(sk);
113
114 cdev = csk->cdev;
115 tp->tcp_header_len = sizeof(struct tcphdr);
116 tp->rx_opt.mss_clamp = cdev->mtus[TCPOPT_MSS_G(opt)] - 40;
117 tp->mss_cache = tp->rx_opt.mss_clamp;
118 tp->rx_opt.tstamp_ok = TCPOPT_TSTAMP_G(opt);
119 tp->rx_opt.snd_wscale = TCPOPT_SACK_G(opt);
120 tp->rx_opt.wscale_ok = TCPOPT_WSCALE_OK_G(opt);
121 SND_WSCALE(tp) = TCPOPT_SND_WSCALE_G(opt);
122 if (!tp->rx_opt.wscale_ok)
123 tp->rx_opt.rcv_wscale = 0;
124 if (tp->rx_opt.tstamp_ok) {
125 tp->tcp_header_len += TCPOLEN_TSTAMP_ALIGNED;
126 tp->rx_opt.mss_clamp -= TCPOLEN_TSTAMP_ALIGNED;
127 } else if (csk->opt2 & TSTAMPS_EN_F) {
128 csk->opt2 &= ~TSTAMPS_EN_F;
129 csk->mtu_idx = TCPOPT_MSS_G(opt);
130 }
131}
132
133static void chtls_purge_receive_queue(struct sock *sk)
134{
135 struct sk_buff *skb;
136
137 while ((skb = __skb_dequeue(&sk->sk_receive_queue)) != NULL) {
138 skb_dst_set(skb, (void *)NULL);
139 kfree_skb(skb);
140 }
141}
142
143static void chtls_purge_write_queue(struct sock *sk)
144{
145 struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
146 struct sk_buff *skb;
147
148 while ((skb = __skb_dequeue(&csk->txq))) {
149 sk->sk_wmem_queued -= skb->truesize;
150 __kfree_skb(skb);
151 }
152}
153
154static void chtls_purge_recv_queue(struct sock *sk)
155{
156 struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
157 struct chtls_hws *tlsk = &csk->tlshws;
158 struct sk_buff *skb;
159
160 while ((skb = __skb_dequeue(&tlsk->sk_recv_queue)) != NULL) {
161 skb_dst_set(skb, NULL);
162 kfree_skb(skb);
163 }
164}
165
166static void abort_arp_failure(void *handle, struct sk_buff *skb)
167{
168 struct cpl_abort_req *req = cplhdr(skb);
169 struct chtls_dev *cdev;
170
171 cdev = (struct chtls_dev *)handle;
172 req->cmd = CPL_ABORT_NO_RST;
173 cxgb4_ofld_send(cdev->lldi->ports[0], skb);
174}
175
176static struct sk_buff *alloc_ctrl_skb(struct sk_buff *skb, int len)
177{
178 if (likely(skb && !skb_shared(skb) && !skb_cloned(skb))) {
179 __skb_trim(skb, 0);
180 refcount_add(2, &skb->users);
181 } else {
182 skb = alloc_skb(len, GFP_KERNEL | __GFP_NOFAIL);
183 }
184 return skb;
185}
186
187static void chtls_send_abort(struct sock *sk, int mode, struct sk_buff *skb)
188{
189 struct cpl_abort_req *req;
190 struct chtls_sock *csk;
191 struct tcp_sock *tp;
192
193 csk = rcu_dereference_sk_user_data(sk);
194 tp = tcp_sk(sk);
195
196 if (!skb)
197 skb = alloc_ctrl_skb(csk->txdata_skb_cache, sizeof(*req));
198
199 req = (struct cpl_abort_req *)skb_put(skb, sizeof(*req));
200 INIT_TP_WR_CPL(req, CPL_ABORT_REQ, csk->tid);
201 skb_set_queue_mapping(skb, (csk->txq_idx << 1) | CPL_PRIORITY_DATA);
202 req->rsvd0 = htonl(tp->snd_nxt);
203 req->rsvd1 = !csk_flag_nochk(csk, CSK_TX_DATA_SENT);
204 req->cmd = mode;
205 t4_set_arp_err_handler(skb, csk->cdev, abort_arp_failure);
206 send_or_defer(sk, tp, skb, mode == CPL_ABORT_SEND_RST);
207}
208
209static void chtls_send_reset(struct sock *sk, int mode, struct sk_buff *skb)
210{
211 struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
212
213 if (unlikely(csk_flag_nochk(csk, CSK_ABORT_SHUTDOWN) ||
214 !csk->cdev)) {
215 if (sk->sk_state == TCP_SYN_RECV)
216 csk_set_flag(csk, CSK_RST_ABORTED);
217 goto out;
218 }
219
220 if (!csk_flag_nochk(csk, CSK_TX_DATA_SENT)) {
221 struct tcp_sock *tp = tcp_sk(sk);
222
223 if (send_tx_flowc_wr(sk, 0, tp->snd_nxt, tp->rcv_nxt) < 0)
224 WARN_ONCE(1, "send tx flowc error");
225 csk_set_flag(csk, CSK_TX_DATA_SENT);
226 }
227
228 csk_set_flag(csk, CSK_ABORT_RPL_PENDING);
229 chtls_purge_write_queue(sk);
230
231 csk_set_flag(csk, CSK_ABORT_SHUTDOWN);
232 if (sk->sk_state != TCP_SYN_RECV)
233 chtls_send_abort(sk, mode, skb);
234 else
235 goto out;
236
237 return;
238out:
ce1294d9 239 kfree_skb(skb);
cc35c88a
AG		 240}
241
242static void release_tcp_port(struct sock *sk)
243{
244 if (inet_csk(sk)->icsk_bind_hash)
245 inet_put_port(sk);
246}
247
248static void tcp_uncork(struct sock *sk)
249{
250 struct tcp_sock *tp = tcp_sk(sk);
251
252 if (tp->nonagle & TCP_NAGLE_CORK) {
253 tp->nonagle &= ~TCP_NAGLE_CORK;
254 chtls_tcp_push(sk, 0);
255 }
256}
257
258static void chtls_close_conn(struct sock *sk)
259{
260 struct cpl_close_con_req *req;
261 struct chtls_sock *csk;
262 struct sk_buff *skb;
263 unsigned int tid;
264 unsigned int len;
265
266 len = roundup(sizeof(struct cpl_close_con_req), 16);
267 csk = rcu_dereference_sk_user_data(sk);
268 tid = csk->tid;
269
270 skb = alloc_skb(len, GFP_KERNEL | __GFP_NOFAIL);
271 req = (struct cpl_close_con_req *)__skb_put(skb, len);
272 memset(req, 0, len);
273 req->wr.wr_hi = htonl(FW_WR_OP_V(FW_TP_WR) |
274 FW_WR_IMMDLEN_V(sizeof(*req) -
275 sizeof(req->wr)));
276 req->wr.wr_mid = htonl(FW_WR_LEN16_V(DIV_ROUND_UP(sizeof(*req), 16)) |
277 FW_WR_FLOWID_V(tid));
278
279 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid));
280
281 tcp_uncork(sk);
282 skb_entail(sk, skb, ULPCB_FLAG_NO_HDR | ULPCB_FLAG_NO_APPEND);
283 if (sk->sk_state != TCP_SYN_SENT)
284 chtls_push_frames(csk, 1);
285}
286
287/*
288 * Perform a state transition during close and return the actions indicated
289 * for the transition. Do not make this function inline, the main reason
290 * it exists at all is to avoid multiple inlining of tcp_set_state.
291 */
292static int make_close_transition(struct sock *sk)
293{
294 int next = (int)new_state[sk->sk_state];
295
296 tcp_set_state(sk, next & TCP_STATE_MASK);
297 return next & TCP_ACTION_FIN;
298}
299
300void chtls_close(struct sock *sk, long timeout)
301{
302 int data_lost, prev_state;
303 struct chtls_sock *csk;
304
305 csk = rcu_dereference_sk_user_data(sk);
306
307 lock_sock(sk);
308 sk->sk_shutdown |= SHUTDOWN_MASK;
309
310 data_lost = skb_queue_len(&sk->sk_receive_queue);
311 data_lost |= skb_queue_len(&csk->tlshws.sk_recv_queue);
312 chtls_purge_recv_queue(sk);
313 chtls_purge_receive_queue(sk);
314
315 if (sk->sk_state == TCP_CLOSE) {
316 goto wait;
317 } else if (data_lost || sk->sk_state == TCP_SYN_SENT) {
318 chtls_send_reset(sk, CPL_ABORT_SEND_RST, NULL);
319 release_tcp_port(sk);
320 goto unlock;
321 } else if (sock_flag(sk, SOCK_LINGER) && !sk->sk_lingertime) {
322 sk->sk_prot->disconnect(sk, 0);
323 } else if (make_close_transition(sk)) {
324 chtls_close_conn(sk);
325 }
326wait:
327 if (timeout)
328 sk_stream_wait_close(sk, timeout);
329
330unlock:
331 prev_state = sk->sk_state;
332 sock_hold(sk);
333 sock_orphan(sk);
334
335 release_sock(sk);
336
337 local_bh_disable();
338 bh_lock_sock(sk);
339
340 if (prev_state != TCP_CLOSE && sk->sk_state == TCP_CLOSE)
341 goto out;
342
343 if (sk->sk_state == TCP_FIN_WAIT2 && tcp_sk(sk)->linger2 < 0 &&
344 !csk_flag(sk, CSK_ABORT_SHUTDOWN)) {
345 struct sk_buff *skb;
346
347 skb = alloc_skb(sizeof(struct cpl_abort_req), GFP_ATOMIC);
348 if (skb)
349 chtls_send_reset(sk, CPL_ABORT_SEND_RST, skb);
350 }
351
352 if (sk->sk_state == TCP_CLOSE)
353 inet_csk_destroy_sock(sk);
354
355out:
356 bh_unlock_sock(sk);
357 local_bh_enable();
358 sock_put(sk);
359}
360
361/*
362 * Wait until a socket enters on of the given states.
363 */
364static int wait_for_states(struct sock *sk, unsigned int states)
365{
366 DECLARE_WAITQUEUE(wait, current);
367 struct socket_wq _sk_wq;
368 long current_timeo;
369 int err = 0;
370
371 current_timeo = 200;
372
373 /*
374 * We want this to work even when there's no associated struct socket.
375 * In that case we provide a temporary wait_queue_head_t.
376 */
377 if (!sk->sk_wq) {
378 init_waitqueue_head(&_sk_wq.wait);
379 _sk_wq.fasync_list = NULL;
380 init_rcu_head_on_stack(&_sk_wq.rcu);
381 RCU_INIT_POINTER(sk->sk_wq, &_sk_wq);
382 }
383
384 add_wait_queue(sk_sleep(sk), &wait);
385 while (!sk_in_state(sk, states)) {
386 if (!current_timeo) {
387 err = -EBUSY;
388 break;
389 }
390 if (signal_pending(current)) {
391 err = sock_intr_errno(current_timeo);
392 break;
393 }
394 set_current_state(TASK_UNINTERRUPTIBLE);
395 release_sock(sk);
396 if (!sk_in_state(sk, states))
397 current_timeo = schedule_timeout(current_timeo);
398 __set_current_state(TASK_RUNNING);
399 lock_sock(sk);
400 }
401 remove_wait_queue(sk_sleep(sk), &wait);
402
403 if (rcu_dereference(sk->sk_wq) == &_sk_wq)
404 sk->sk_wq = NULL;
405 return err;
406}
407
408int chtls_disconnect(struct sock *sk, int flags)
409{
cc35c88a
AG		 410 struct tcp_sock *tp;
411 int err;
412
413 tp = tcp_sk(sk);
cc35c88a
AG		 414 chtls_purge_recv_queue(sk);
415 chtls_purge_receive_queue(sk);
416 chtls_purge_write_queue(sk);
417
418 if (sk->sk_state != TCP_CLOSE) {
419 sk->sk_err = ECONNRESET;
420 chtls_send_reset(sk, CPL_ABORT_SEND_RST, NULL);
421 err = wait_for_states(sk, TCPF_CLOSE);
422 if (err)
423 return err;
424 }
425 chtls_purge_recv_queue(sk);
426 chtls_purge_receive_queue(sk);
427 tp->max_window = 0xFFFF << (tp->rx_opt.snd_wscale);
428 return tcp_disconnect(sk, flags);
429}
430
431#define SHUTDOWN_ELIGIBLE_STATE (TCPF_ESTABLISHED | \
432 TCPF_SYN_RECV | TCPF_CLOSE_WAIT)
433void chtls_shutdown(struct sock *sk, int how)
434{
435 if ((how & SEND_SHUTDOWN) &&
436 sk_in_state(sk, SHUTDOWN_ELIGIBLE_STATE) &&
437 make_close_transition(sk))
438 chtls_close_conn(sk);
439}
440
441void chtls_destroy_sock(struct sock *sk)
442{
443 struct chtls_sock *csk;
444
445 csk = rcu_dereference_sk_user_data(sk);
446 chtls_purge_recv_queue(sk);
447 csk->ulp_mode = ULP_MODE_NONE;
448 chtls_purge_write_queue(sk);
449 free_tls_keyid(sk);
450 kref_put(&csk->kref, chtls_sock_release);
451 sk->sk_prot = &tcp_prot;
452 sk->sk_prot->destroy(sk);
453}
454
455static void reset_listen_child(struct sock *child)
456{
457 struct chtls_sock *csk = rcu_dereference_sk_user_data(child);
458 struct sk_buff *skb;
459
460 skb = alloc_ctrl_skb(csk->txdata_skb_cache,
461 sizeof(struct cpl_abort_req));
462
463 chtls_send_reset(child, CPL_ABORT_SEND_RST, skb);
464 sock_orphan(child);
465 INC_ORPHAN_COUNT(child);
466 if (child->sk_state == TCP_CLOSE)
467 inet_csk_destroy_sock(child);
468}
469
470static void chtls_disconnect_acceptq(struct sock *listen_sk)
471{
472 struct request_sock **pprev;
473
474 pprev = ACCEPT_QUEUE(listen_sk);
475 while (*pprev) {
476 struct request_sock *req = *pprev;
477
478 if (req->rsk_ops == &chtls_rsk_ops) {
479 struct sock *child = req->sk;
480
481 *pprev = req->dl_next;
482 sk_acceptq_removed(listen_sk);
483 reqsk_put(req);
484 sock_hold(child);
485 local_bh_disable();
486 bh_lock_sock(child);
487 release_tcp_port(child);
488 reset_listen_child(child);
489 bh_unlock_sock(child);
490 local_bh_enable();
491 sock_put(child);
492 } else {
493 pprev = &req->dl_next;
494 }
495 }
496}
497
498static int listen_hashfn(const struct sock *sk)
499{
500 return ((unsigned long)sk >> 10) & (LISTEN_INFO_HASH_SIZE - 1);
501}
502
503static struct listen_info *listen_hash_add(struct chtls_dev *cdev,
504 struct sock *sk,
505 unsigned int stid)
506{
507 struct listen_info *p = kmalloc(sizeof(*p), GFP_KERNEL);
508
509 if (p) {
510 int key = listen_hashfn(sk);
511
512 p->sk = sk;
513 p->stid = stid;
514 spin_lock(&cdev->listen_lock);
515 p->next = cdev->listen_hash_tab[key];
516 cdev->listen_hash_tab[key] = p;
517 spin_unlock(&cdev->listen_lock);
518 }
519 return p;
520}
521
522static int listen_hash_find(struct chtls_dev *cdev,
523 struct sock *sk)
524{
525 struct listen_info *p;
526 int stid = -1;
527 int key;
528
529 key = listen_hashfn(sk);
530
531 spin_lock(&cdev->listen_lock);
532 for (p = cdev->listen_hash_tab[key]; p; p = p->next)
533 if (p->sk == sk) {
534 stid = p->stid;
535 break;
536 }
537 spin_unlock(&cdev->listen_lock);
538 return stid;
539}
540
541static int listen_hash_del(struct chtls_dev *cdev,
542 struct sock *sk)
543{
544 struct listen_info *p, **prev;
545 int stid = -1;
546 int key;
547
548 key = listen_hashfn(sk);
549 prev = &cdev->listen_hash_tab[key];
550
551 spin_lock(&cdev->listen_lock);
552 for (p = *prev; p; prev = &p->next, p = p->next)
553 if (p->sk == sk) {
554 stid = p->stid;
555 *prev = p->next;
556 kfree(p);
557 break;
558 }
559 spin_unlock(&cdev->listen_lock);
560 return stid;
561}
562
563static void cleanup_syn_rcv_conn(struct sock *child, struct sock *parent)
564{
565 struct request_sock *req;
566 struct chtls_sock *csk;
567
568 csk = rcu_dereference_sk_user_data(child);
569 req = csk->passive_reap_next;
570
571 reqsk_queue_removed(&inet_csk(parent)->icsk_accept_queue, req);
572 __skb_unlink((struct sk_buff *)&csk->synq, &csk->listen_ctx->synq);
573 chtls_reqsk_free(req);
574 csk->passive_reap_next = NULL;
575}
576
577static void chtls_reset_synq(struct listen_ctx *listen_ctx)
578{
579 struct sock *listen_sk = listen_ctx->lsk;
580
581 while (!skb_queue_empty(&listen_ctx->synq)) {
582 struct chtls_sock *csk =
583 container_of((struct synq *)__skb_dequeue
584 (&listen_ctx->synq), struct chtls_sock, synq);
585 struct sock *child = csk->sk;
586
587 cleanup_syn_rcv_conn(child, listen_sk);
588 sock_hold(child);
589 local_bh_disable();
590 bh_lock_sock(child);
591 release_tcp_port(child);
592 reset_listen_child(child);
593 bh_unlock_sock(child);
594 local_bh_enable();
595 sock_put(child);
596 }
597}
598
599int chtls_listen_start(struct chtls_dev *cdev, struct sock *sk)
600{
601 struct net_device *ndev;
602 struct listen_ctx *ctx;
603 struct adapter *adap;
604 struct port_info *pi;
605 int stid;
606 int ret;
607
608 if (sk->sk_family != PF_INET)
609 return -EAGAIN;
610
611 rcu_read_lock();
612 ndev = chtls_ipv4_netdev(cdev, sk);
613 rcu_read_unlock();
614 if (!ndev)
615 return -EBADF;
616
617 pi = netdev_priv(ndev);
618 adap = pi->adapter;
80f61f19 619 if (!(adap->flags & CXGB4_FULL_INIT_DONE))
cc35c88a
AG		 620 return -EBADF;
621
622 if (listen_hash_find(cdev, sk) >= 0) /* already have it */
623 return -EADDRINUSE;
624
625 ctx = kmalloc(sizeof(*ctx), GFP_KERNEL);
626 if (!ctx)
627 return -ENOMEM;
628
629 __module_get(THIS_MODULE);
630 ctx->lsk = sk;
631 ctx->cdev = cdev;
632 ctx->state = T4_LISTEN_START_PENDING;
633 skb_queue_head_init(&ctx->synq);
634
635 stid = cxgb4_alloc_stid(cdev->tids, sk->sk_family, ctx);
636 if (stid < 0)
637 goto free_ctx;
638
639 sock_hold(sk);
640 if (!listen_hash_add(cdev, sk, stid))
641 goto free_stid;
642
643 ret = cxgb4_create_server(ndev, stid,
644 inet_sk(sk)->inet_rcv_saddr,
645 inet_sk(sk)->inet_sport, 0,
646 cdev->lldi->rxq_ids[0]);
647 if (ret > 0)
648 ret = net_xmit_errno(ret);
649 if (ret)
650 goto del_hash;
651 return 0;
652del_hash:
653 listen_hash_del(cdev, sk);
654free_stid:
655 cxgb4_free_stid(cdev->tids, stid, sk->sk_family);
656 sock_put(sk);
657free_ctx:
658 kfree(ctx);
659 module_put(THIS_MODULE);
660 return -EBADF;
661}
662
663void chtls_listen_stop(struct chtls_dev *cdev, struct sock *sk)
664{
665 struct listen_ctx *listen_ctx;
666 int stid;
667
668 stid = listen_hash_del(cdev, sk);
669 if (stid < 0)
670 return;
671
672 listen_ctx = (struct listen_ctx *)lookup_stid(cdev->tids, stid);
673 chtls_reset_synq(listen_ctx);
674
675 cxgb4_remove_server(cdev->lldi->ports[0], stid,
676 cdev->lldi->rxq_ids[0], 0);
677 chtls_disconnect_acceptq(sk);
678}
679
680static int chtls_pass_open_rpl(struct chtls_dev *cdev, struct sk_buff *skb)
681{
682 struct cpl_pass_open_rpl *rpl = cplhdr(skb) + RSS_HDR;
683 unsigned int stid = GET_TID(rpl);
684 struct listen_ctx *listen_ctx;
685
686 listen_ctx = (struct listen_ctx *)lookup_stid(cdev->tids, stid);
687 if (!listen_ctx)
688 return CPL_RET_BUF_DONE;
689
690 if (listen_ctx->state == T4_LISTEN_START_PENDING) {
691 listen_ctx->state = T4_LISTEN_STARTED;
692 return CPL_RET_BUF_DONE;
693 }
694
695 if (rpl->status != CPL_ERR_NONE) {
696 pr_info("Unexpected PASS_OPEN_RPL status %u for STID %u\n",
697 rpl->status, stid);
698 return CPL_RET_BUF_DONE;
699 }
700 cxgb4_free_stid(cdev->tids, stid, listen_ctx->lsk->sk_family);
701 sock_put(listen_ctx->lsk);
702 kfree(listen_ctx);
703 module_put(THIS_MODULE);
704
705 return 0;
706}
707
708static int chtls_close_listsrv_rpl(struct chtls_dev *cdev, struct sk_buff *skb)
709{
710 struct cpl_close_listsvr_rpl *rpl = cplhdr(skb) + RSS_HDR;
711 struct listen_ctx *listen_ctx;
712 unsigned int stid;
713 void *data;
714
715 stid = GET_TID(rpl);
716 data = lookup_stid(cdev->tids, stid);
717 listen_ctx = (struct listen_ctx *)data;
718
719 if (rpl->status != CPL_ERR_NONE) {
720 pr_info("Unexpected CLOSE_LISTSRV_RPL status %u for STID %u\n",
721 rpl->status, stid);
722 return CPL_RET_BUF_DONE;
723 }
724
725 cxgb4_free_stid(cdev->tids, stid, listen_ctx->lsk->sk_family);
726 sock_put(listen_ctx->lsk);
727 kfree(listen_ctx);
728 module_put(THIS_MODULE);
729
730 return 0;
731}
732
733static void chtls_release_resources(struct sock *sk)
734{
735 struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
736 struct chtls_dev *cdev = csk->cdev;
737 unsigned int tid = csk->tid;
738 struct tid_info *tids;
739
740 if (!cdev)
741 return;
742
743 tids = cdev->tids;
744 kfree_skb(csk->txdata_skb_cache);
745 csk->txdata_skb_cache = NULL;
746
747 if (csk->l2t_entry) {
748 cxgb4_l2t_release(csk->l2t_entry);
749 csk->l2t_entry = NULL;
750 }
751
752 cxgb4_remove_tid(tids, csk->port_id, tid, sk->sk_family);
753 sock_put(sk);
754}
755
756static void chtls_conn_done(struct sock *sk)
757{
758 if (sock_flag(sk, SOCK_DEAD))
759 chtls_purge_receive_queue(sk);
760 sk_wakeup_sleepers(sk, 0);
761 tcp_done(sk);
762}
763
764static void do_abort_syn_rcv(struct sock *child, struct sock *parent)
765{
766 /*
767 * If the server is still open we clean up the child connection,
768 * otherwise the server already did the clean up as it was purging
769 * its SYN queue and the skb was just sitting in its backlog.
770 */
771 if (likely(parent->sk_state == TCP_LISTEN)) {
772 cleanup_syn_rcv_conn(child, parent);
773 /* Without the below call to sock_orphan,
774 * we leak the socket resource with syn_flood test
775 * as inet_csk_destroy_sock will not be called
776 * in tcp_done since SOCK_DEAD flag is not set.
777 * Kernel handles this differently where new socket is
778 * created only after 3 way handshake is done.
779 */
780 sock_orphan(child);
781 percpu_counter_inc((child)->sk_prot->orphan_count);
782 chtls_release_resources(child);
783 chtls_conn_done(child);
784 } else {
785 if (csk_flag(child, CSK_RST_ABORTED)) {
786 chtls_release_resources(child);
787 chtls_conn_done(child);
788 }
789 }
790}
791
792static void pass_open_abort(struct sock *child, struct sock *parent,
793 struct sk_buff *skb)
794{
795 do_abort_syn_rcv(child, parent);
796 kfree_skb(skb);
797}
798
799static void bl_pass_open_abort(struct sock *lsk, struct sk_buff *skb)
800{
801 pass_open_abort(skb->sk, lsk, skb);
802}
803
804static void chtls_pass_open_arp_failure(struct sock *sk,
805 struct sk_buff *skb)
806{
807 const struct request_sock *oreq;
808 struct chtls_sock *csk;
809 struct chtls_dev *cdev;
810 struct sock *parent;
811 void *data;
812
813 csk = rcu_dereference_sk_user_data(sk);
814 cdev = csk->cdev;
815
816 /*
817 * If the connection is being aborted due to the parent listening
818 * socket going away there's nothing to do, the ABORT_REQ will close
819 * the connection.
820 */
821 if (csk_flag(sk, CSK_ABORT_RPL_PENDING)) {
822 kfree_skb(skb);
823 return;
824 }
825
826 oreq = csk->passive_reap_next;
827 data = lookup_stid(cdev->tids, oreq->ts_recent);
828 parent = ((struct listen_ctx *)data)->lsk;
829
830 bh_lock_sock(parent);
831 if (!sock_owned_by_user(parent)) {
832 pass_open_abort(sk, parent, skb);
833 } else {
834 BLOG_SKB_CB(skb)->backlog_rcv = bl_pass_open_abort;
835 __sk_add_backlog(parent, skb);
836 }
837 bh_unlock_sock(parent);
838}
839
840static void chtls_accept_rpl_arp_failure(void *handle,
841 struct sk_buff *skb)
842{
843 struct sock *sk = (struct sock *)handle;
844
845 sock_hold(sk);
846 process_cpl_msg(chtls_pass_open_arp_failure, sk, skb);
847 sock_put(sk);
848}
849
850static unsigned int chtls_select_mss(const struct chtls_sock *csk,
851 unsigned int pmtu,
852 struct cpl_pass_accept_req *req)
853{
854 struct chtls_dev *cdev;
855 struct dst_entry *dst;
856 unsigned int tcpoptsz;
857 unsigned int iphdrsz;
858 unsigned int mtu_idx;
859 struct tcp_sock *tp;
860 unsigned int mss;
861 struct sock *sk;
862
863 mss = ntohs(req->tcpopt.mss);
864 sk = csk->sk;
865 dst = __sk_dst_get(sk);
866 cdev = csk->cdev;
867 tp = tcp_sk(sk);
868 tcpoptsz = 0;
869
870 iphdrsz = sizeof(struct iphdr) + sizeof(struct tcphdr);
871 if (req->tcpopt.tstamp)
872 tcpoptsz += round_up(TCPOLEN_TIMESTAMP, 4);
873
874 tp->advmss = dst_metric_advmss(dst);
875 if (USER_MSS(tp) && tp->advmss > USER_MSS(tp))
876 tp->advmss = USER_MSS(tp);
877 if (tp->advmss > pmtu - iphdrsz)
878 tp->advmss = pmtu - iphdrsz;
879 if (mss && tp->advmss > mss)
880 tp->advmss = mss;
881
882 tp->advmss = cxgb4_best_aligned_mtu(cdev->lldi->mtus,
883 iphdrsz + tcpoptsz,
884 tp->advmss - tcpoptsz,
885 8, &mtu_idx);
886 tp->advmss -= iphdrsz;
887
888 inet_csk(sk)->icsk_pmtu_cookie = pmtu;
889 return mtu_idx;
890}
891
cc35c88a
AG		 892static unsigned int select_rcv_wscale(int space, int wscale_ok, int win_clamp)
893{
894 int wscale = 0;
895
896 if (space > MAX_RCV_WND)
897 space = MAX_RCV_WND;
898 if (win_clamp && win_clamp < space)
899 space = win_clamp;
900
901 if (wscale_ok) {
902 while (wscale < 14 && (65535 << wscale) < space)
903 wscale++;
904 }
905 return wscale;
906}
907
908static void chtls_pass_accept_rpl(struct sk_buff *skb,
909 struct cpl_pass_accept_req *req,
910 unsigned int tid)
911
912{
913 struct cpl_t5_pass_accept_rpl *rpl5;
914 struct cxgb4_lld_info *lldi;
915 const struct tcphdr *tcph;
916 const struct tcp_sock *tp;
917 struct chtls_sock *csk;
918 unsigned int len;
919 struct sock *sk;
920 u32 opt2, hlen;
921 u64 opt0;
922
923 sk = skb->sk;
924 tp = tcp_sk(sk);
925 csk = sk->sk_user_data;
926 csk->tid = tid;
927 lldi = csk->cdev->lldi;
928 len = roundup(sizeof(*rpl5), 16);
929
930 rpl5 = __skb_put_zero(skb, len);
931 INIT_TP_WR(rpl5, tid);
932
933 OPCODE_TID(rpl5) = cpu_to_be32(MK_OPCODE_TID(CPL_PASS_ACCEPT_RPL,
934 csk->tid));
935 csk->mtu_idx = chtls_select_mss(csk, dst_mtu(__sk_dst_get(sk)),
936 req);
937 opt0 = TCAM_BYPASS_F |
0c3a16be 938 WND_SCALE_V(RCV_WSCALE(tp)) |
cc35c88a
AG		 939 MSS_IDX_V(csk->mtu_idx) |
940 L2T_IDX_V(csk->l2t_entry->idx) |
941 NAGLE_V(!(tp->nonagle & TCP_NAGLE_OFF)) |
942 TX_CHAN_V(csk->tx_chan) |
943 SMAC_SEL_V(csk->smac_idx) |
944 DSCP_V(csk->tos >> 2) |
945 ULP_MODE_V(ULP_MODE_TLS) |
946 RCV_BUFSIZ_V(min(tp->rcv_wnd >> 10, RCV_BUFSIZ_M));
947
948 opt2 = RX_CHANNEL_V(0) |
949 RSS_QUEUE_VALID_F | RSS_QUEUE_V(csk->rss_qid);
950
951 if (!is_t5(lldi->adapter_type))
952 opt2 |= RX_FC_DISABLE_F;
953 if (req->tcpopt.tstamp)
954 opt2 |= TSTAMPS_EN_F;
955 if (req->tcpopt.sack)
956 opt2 |= SACK_EN_F;
957 hlen = ntohl(req->hdr_len);
958
959 tcph = (struct tcphdr *)((u8 *)(req + 1) +
960 T6_ETH_HDR_LEN_G(hlen) + T6_IP_HDR_LEN_G(hlen));
961 if (tcph->ece && tcph->cwr)
962 opt2 |= CCTRL_ECN_V(1);
963 opt2 |= CONG_CNTRL_V(CONG_ALG_NEWRENO);
964 opt2 |= T5_ISS_F;
965 opt2 |= T5_OPT_2_VALID_F;
966 rpl5->opt0 = cpu_to_be64(opt0);
967 rpl5->opt2 = cpu_to_be32(opt2);
968 rpl5->iss = cpu_to_be32((prandom_u32() & ~7UL) - 1);
969 set_wr_txq(skb, CPL_PRIORITY_SETUP, csk->port_id);
970 t4_set_arp_err_handler(skb, sk, chtls_accept_rpl_arp_failure);
971 cxgb4_l2t_send(csk->egress_dev, skb, csk->l2t_entry);
972}
973
974static void inet_inherit_port(struct inet_hashinfo *hash_info,
975 struct sock *lsk, struct sock *newsk)
976{
977 local_bh_disable();
978 __inet_inherit_port(lsk, newsk);
979 local_bh_enable();
980}
981
982static int chtls_backlog_rcv(struct sock *sk, struct sk_buff *skb)
983{
984 if (skb->protocol) {
985 kfree_skb(skb);
986 return 0;
987 }
988 BLOG_SKB_CB(skb)->backlog_rcv(sk, skb);
989 return 0;
990}
991
0c3a16be
AG		 992static void chtls_set_tcp_window(struct chtls_sock *csk)
993{
994 struct net_device *ndev = csk->egress_dev;
995 struct port_info *pi = netdev_priv(ndev);
996 unsigned int linkspeed;
997 u8 scale;
998
999 linkspeed = pi->link_cfg.speed;
1000 scale = linkspeed / SPEED_10000;
1001#define CHTLS_10G_RCVWIN (256 * 1024)
1002 csk->rcv_win = CHTLS_10G_RCVWIN;
1003 if (scale)
1004 csk->rcv_win *= scale;
1005#define CHTLS_10G_SNDWIN (256 * 1024)
1006 csk->snd_win = CHTLS_10G_SNDWIN;
1007 if (scale)
1008 csk->snd_win *= scale;
1009}
1010
cc35c88a
AG		 1011static struct sock *chtls_recv_sock(struct sock *lsk,
1012 struct request_sock *oreq,
1013 void *network_hdr,
1014 const struct cpl_pass_accept_req *req,
1015 struct chtls_dev *cdev)
1016{
cc35c88a
AG		 1017 struct inet_sock *newinet;
1018 const struct iphdr *iph;
76f7164d 1019 struct tls_context *ctx;
cc35c88a
AG		 1020 struct net_device *ndev;
1021 struct chtls_sock *csk;
1022 struct dst_entry *dst;
1023 struct neighbour *n;
1024 struct tcp_sock *tp;
1025 struct sock *newsk;
1026 u16 port_id;
1027 int rxq_idx;
1028 int step;
1029
1030 iph = (const struct iphdr *)network_hdr;
1031 newsk = tcp_create_openreq_child(lsk, oreq, cdev->askb);
1032 if (!newsk)
1033 goto free_oreq;
1034
1035 dst = inet_csk_route_child_sock(lsk, newsk, oreq);
1036 if (!dst)
1037 goto free_sk;
1038
cc35c88a
AG		 1039 n = dst_neigh_lookup(dst, &iph->saddr);
1040 if (!n)
1041 goto free_sk;
1042
1043 ndev = n->dev;
1044 if (!ndev)
1045 goto free_dst;
1046 port_id = cxgb4_port_idx(ndev);
1047
1048 csk = chtls_sock_create(cdev);
1049 if (!csk)
1050 goto free_dst;
1051
1052 csk->l2t_entry = cxgb4_l2t_get(cdev->lldi->l2t, n, ndev, 0);
1053 if (!csk->l2t_entry)
1054 goto free_csk;
1055
1056 newsk->sk_user_data = csk;
1057 newsk->sk_backlog_rcv = chtls_backlog_rcv;
1058
1059 tp = tcp_sk(newsk);
1060 newinet = inet_sk(newsk);
1061
1062 newinet->inet_daddr = iph->saddr;
1063 newinet->inet_rcv_saddr = iph->daddr;
1064 newinet->inet_saddr = iph->daddr;
1065
1066 oreq->ts_recent = PASS_OPEN_TID_G(ntohl(req->tos_stid));
1067 sk_setup_caps(newsk, dst);
76f7164d
AG		 1068 ctx = tls_get_ctx(lsk);
1069 newsk->sk_destruct = ctx->sk_destruct;
cc35c88a
AG		 1070 csk->sk = newsk;
1071 csk->passive_reap_next = oreq;
1072 csk->tx_chan = cxgb4_port_chan(ndev);
1073 csk->port_id = port_id;
1074 csk->egress_dev = ndev;
1075 csk->tos = PASS_OPEN_TOS_G(ntohl(req->tos_stid));
0c3a16be
AG		 1076 chtls_set_tcp_window(csk);
1077 tp->rcv_wnd = csk->rcv_win;
1078 csk->sndbuf = csk->snd_win;
cc35c88a
AG		 1079 csk->ulp_mode = ULP_MODE_TLS;
1080 step = cdev->lldi->nrxq / cdev->lldi->nchan;
1081 csk->rss_qid = cdev->lldi->rxq_ids[port_id * step];
1082 rxq_idx = port_id * step;
1083 csk->txq_idx = (rxq_idx < cdev->lldi->ntxq) ? rxq_idx :
1084 port_id * step;
1085 csk->sndbuf = newsk->sk_sndbuf;
02d805dc 1086 csk->smac_idx = ((struct port_info *)netdev_priv(ndev))->smt_idx;
cc35c88a 1087 RCV_WSCALE(tp) = select_rcv_wscale(tcp_full_space(newsk),
0c3a16be
AG		 1088 sock_net(newsk)->
1089 ipv4.sysctl_tcp_window_scaling,
cc35c88a
AG		 1090 tp->window_clamp);
1091 neigh_release(n);
1092 inet_inherit_port(&tcp_hashinfo, lsk, newsk);
1093 csk_set_flag(csk, CSK_CONN_INLINE);
1094 bh_unlock_sock(newsk); /* tcp_create_openreq_child ->sk_clone_lock */
1095
1096 return newsk;
1097free_csk:
1098 chtls_sock_release(&csk->kref);
1099free_dst:
1100 dst_release(dst);
1101free_sk:
1102 inet_csk_prepare_forced_close(newsk);
1103 tcp_done(newsk);
1104free_oreq:
1105 chtls_reqsk_free(oreq);
1106 return NULL;
1107}
1108
1109/*
1110 * Populate a TID_RELEASE WR. The skb must be already propely sized.
1111 */
1112static void mk_tid_release(struct sk_buff *skb,
1113 unsigned int chan, unsigned int tid)
1114{
1115 struct cpl_tid_release *req;
1116 unsigned int len;
1117
1118 len = roundup(sizeof(struct cpl_tid_release), 16);
1119 req = (struct cpl_tid_release *)__skb_put(skb, len);
1120 memset(req, 0, len);
1121 set_wr_txq(skb, CPL_PRIORITY_SETUP, chan);
1122 INIT_TP_WR_CPL(req, CPL_TID_RELEASE, tid);
1123}
1124
1125static int chtls_get_module(struct sock *sk)
1126{
1127 struct inet_connection_sock *icsk = inet_csk(sk);
1128
1129 if (!try_module_get(icsk->icsk_ulp_ops->owner))
1130 return -1;
1131
1132 return 0;
1133}
1134
1135static void chtls_pass_accept_request(struct sock *sk,
1136 struct sk_buff *skb)
1137{
1138 struct cpl_t5_pass_accept_rpl *rpl;
1139 struct cpl_pass_accept_req *req;
1140 struct listen_ctx *listen_ctx;
0c3a16be 1141 struct vlan_ethhdr *vlan_eh;
cc35c88a
AG		 1142 struct request_sock *oreq;
1143 struct sk_buff *reply_skb;
1144 struct chtls_sock *csk;
1145 struct chtls_dev *cdev;
1146 struct tcphdr *tcph;
1147 struct sock *newsk;
1148 struct ethhdr *eh;
1149 struct iphdr *iph;
1150 void *network_hdr;
1151 unsigned int stid;
1152 unsigned int len;
1153 unsigned int tid;
0c3a16be
AG		 1154 bool th_ecn, ect;
1155 __u8 ip_dsfield; /* IPv4 tos or IPv6 dsfield */
1156 u16 eth_hdr_len;
1157 bool ecn_ok;
cc35c88a
AG		 1158
1159 req = cplhdr(skb) + RSS_HDR;
1160 tid = GET_TID(req);
1161 cdev = BLOG_SKB_CB(skb)->cdev;
1162 newsk = lookup_tid(cdev->tids, tid);
1163 stid = PASS_OPEN_TID_G(ntohl(req->tos_stid));
1164 if (newsk) {
1165 pr_info("tid (%d) already in use\n", tid);
1166 return;
1167 }
1168
1169 len = roundup(sizeof(*rpl), 16);
1170 reply_skb = alloc_skb(len, GFP_ATOMIC);
1171 if (!reply_skb) {
1172 cxgb4_remove_tid(cdev->tids, 0, tid, sk->sk_family);
1173 kfree_skb(skb);
1174 return;
1175 }
1176
1177 if (sk->sk_state != TCP_LISTEN)
1178 goto reject;
1179
1180 if (inet_csk_reqsk_queue_is_full(sk))
1181 goto reject;
1182
1183 if (sk_acceptq_is_full(sk))
1184 goto reject;
1185
1186 oreq = inet_reqsk_alloc(&chtls_rsk_ops, sk, true);
1187 if (!oreq)
1188 goto reject;
1189
1190 oreq->rsk_rcv_wnd = 0;
1191 oreq->rsk_window_clamp = 0;
1192 oreq->cookie_ts = 0;
1193 oreq->mss = 0;
1194 oreq->ts_recent = 0;
1195
0c3a16be
AG		 1196 eth_hdr_len = T6_ETH_HDR_LEN_G(ntohl(req->hdr_len));
1197 if (eth_hdr_len == ETH_HLEN) {
1198 eh = (struct ethhdr *)(req + 1);
1199 iph = (struct iphdr *)(eh + 1);
1200 network_hdr = (void *)(eh + 1);
1201 } else {
1202 vlan_eh = (struct vlan_ethhdr *)(req + 1);
1203 iph = (struct iphdr *)(vlan_eh + 1);
1204 network_hdr = (void *)(vlan_eh + 1);
1205 }
cc35c88a
AG		 1206 if (iph->version != 0x4)
1207 goto free_oreq;
1208
cc35c88a 1209 tcph = (struct tcphdr *)(iph + 1);
0c3a16be 1210 skb_set_network_header(skb, (void *)iph - (void *)req);
cc35c88a
AG		 1211
1212 tcp_rsk(oreq)->tfo_listener = false;
1213 tcp_rsk(oreq)->rcv_isn = ntohl(tcph->seq);
1214 chtls_set_req_port(oreq, tcph->source, tcph->dest);
cc35c88a 1215 chtls_set_req_addr(oreq, iph->daddr, iph->saddr);
0c3a16be
AG		 1216 ip_dsfield = ipv4_get_dsfield(iph);
1217 if (req->tcpopt.wsf <= 14 &&
1218 sock_net(sk)->ipv4.sysctl_tcp_window_scaling) {
cc35c88a
AG		 1219 inet_rsk(oreq)->wscale_ok = 1;
1220 inet_rsk(oreq)->snd_wscale = req->tcpopt.wsf;
1221 }
1222 inet_rsk(oreq)->ir_iif = sk->sk_bound_dev_if;
0c3a16be
AG		 1223 th_ecn = tcph->ece && tcph->cwr;
1224 if (th_ecn) {
1225 ect = !INET_ECN_is_not_ect(ip_dsfield);
1226 ecn_ok = sock_net(sk)->ipv4.sysctl_tcp_ecn;
1227 if ((!ect && ecn_ok) || tcp_ca_needs_ecn(sk))
1228 inet_rsk(oreq)->ecn_ok = 1;
1229 }
cc35c88a
AG		 1230
1231 newsk = chtls_recv_sock(sk, oreq, network_hdr, req, cdev);
1232 if (!newsk)
1233 goto reject;
1234
1235 if (chtls_get_module(newsk))
1236 goto reject;
1237 inet_csk_reqsk_queue_added(sk);
1238 reply_skb->sk = newsk;
1239 chtls_install_cpl_ops(newsk);
1240 cxgb4_insert_tid(cdev->tids, newsk, tid, newsk->sk_family);
1241 csk = rcu_dereference_sk_user_data(newsk);
1242 listen_ctx = (struct listen_ctx *)lookup_stid(cdev->tids, stid);
1243 csk->listen_ctx = listen_ctx;
1244 __skb_queue_tail(&listen_ctx->synq, (struct sk_buff *)&csk->synq);
1245 chtls_pass_accept_rpl(reply_skb, req, tid);
1246 kfree_skb(skb);
1247 return;
1248
1249free_oreq:
1250 chtls_reqsk_free(oreq);
1251reject:
1252 mk_tid_release(reply_skb, 0, tid);
1253 cxgb4_ofld_send(cdev->lldi->ports[0], reply_skb);
1254 kfree_skb(skb);
1255}
1256
1257/*
1258 * Handle a CPL_PASS_ACCEPT_REQ message.
1259 */
1260static int chtls_pass_accept_req(struct chtls_dev *cdev, struct sk_buff *skb)
1261{
1262 struct cpl_pass_accept_req *req = cplhdr(skb) + RSS_HDR;
1263 struct listen_ctx *ctx;
1264 unsigned int stid;
1265 unsigned int tid;
1266 struct sock *lsk;
1267 void *data;
1268
1269 stid = PASS_OPEN_TID_G(ntohl(req->tos_stid));
1270 tid = GET_TID(req);
1271
1272 data = lookup_stid(cdev->tids, stid);
1273 if (!data)
1274 return 1;
1275
1276 ctx = (struct listen_ctx *)data;
1277 lsk = ctx->lsk;
1278
1279 if (unlikely(tid >= cdev->tids->ntids)) {
1280 pr_info("passive open TID %u too large\n", tid);
1281 return 1;
1282 }
1283
1284 BLOG_SKB_CB(skb)->cdev = cdev;
1285 process_cpl_msg(chtls_pass_accept_request, lsk, skb);
1286 return 0;
1287}
1288
1289/*
1290 * Completes some final bits of initialization for just established connections
1291 * and changes their state to TCP_ESTABLISHED.
1292 *
1293 * snd_isn here is the ISN after the SYN, i.e., the true ISN + 1.
1294 */
1295static void make_established(struct sock *sk, u32 snd_isn, unsigned int opt)
1296{
1297 struct tcp_sock *tp = tcp_sk(sk);
1298
1299 tp->pushed_seq = snd_isn;
1300 tp->write_seq = snd_isn;
1301 tp->snd_nxt = snd_isn;
1302 tp->snd_una = snd_isn;
1303 inet_sk(sk)->inet_id = tp->write_seq ^ jiffies;
1304 assign_rxopt(sk, opt);
1305
1306 if (tp->rcv_wnd > (RCV_BUFSIZ_M << 10))
1307 tp->rcv_wup -= tp->rcv_wnd - (RCV_BUFSIZ_M << 10);
1308
1309 smp_mb();
1310 tcp_set_state(sk, TCP_ESTABLISHED);
1311}
1312
1313static void chtls_abort_conn(struct sock *sk, struct sk_buff *skb)
1314{
1315 struct sk_buff *abort_skb;
1316
1317 abort_skb = alloc_skb(sizeof(struct cpl_abort_req), GFP_ATOMIC);
1318 if (abort_skb)
1319 chtls_send_reset(sk, CPL_ABORT_SEND_RST, abort_skb);
1320}
1321
1322static struct sock *reap_list;
1323static DEFINE_SPINLOCK(reap_list_lock);
1324
1325/*
1326 * Process the reap list.
1327 */
1328DECLARE_TASK_FUNC(process_reap_list, task_param)
1329{
1330 spin_lock_bh(&reap_list_lock);
1331 while (reap_list) {
1332 struct sock *sk = reap_list;
1333 struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
1334
1335 reap_list = csk->passive_reap_next;
1336 csk->passive_reap_next = NULL;
1337 spin_unlock(&reap_list_lock);
1338 sock_hold(sk);
1339
1340 bh_lock_sock(sk);
1341 chtls_abort_conn(sk, NULL);
1342 sock_orphan(sk);
1343 if (sk->sk_state == TCP_CLOSE)
1344 inet_csk_destroy_sock(sk);
1345 bh_unlock_sock(sk);
1346 sock_put(sk);
1347 spin_lock(&reap_list_lock);
1348 }
1349 spin_unlock_bh(&reap_list_lock);
1350}
1351
1352static DECLARE_WORK(reap_task, process_reap_list);
1353
1354static void add_to_reap_list(struct sock *sk)
1355{
1356 struct chtls_sock *csk = sk->sk_user_data;
1357
1358 local_bh_disable();
1359 bh_lock_sock(sk);
1360 release_tcp_port(sk); /* release the port immediately */
1361
1362 spin_lock(&reap_list_lock);
1363 csk->passive_reap_next = reap_list;
1364 reap_list = sk;
1365 if (!csk->passive_reap_next)
1366 schedule_work(&reap_task);
1367 spin_unlock(&reap_list_lock);
1368 bh_unlock_sock(sk);
1369 local_bh_enable();
1370}
1371
1372static void add_pass_open_to_parent(struct sock *child, struct sock *lsk,
1373 struct chtls_dev *cdev)
1374{
1375 struct request_sock *oreq;
1376 struct chtls_sock *csk;
1377
1378 if (lsk->sk_state != TCP_LISTEN)
1379 return;
1380
1381 csk = child->sk_user_data;
1382 oreq = csk->passive_reap_next;
1383 csk->passive_reap_next = NULL;
1384
1385 reqsk_queue_removed(&inet_csk(lsk)->icsk_accept_queue, oreq);
1386 __skb_unlink((struct sk_buff *)&csk->synq, &csk->listen_ctx->synq);
1387
1388 if (sk_acceptq_is_full(lsk)) {
1389 chtls_reqsk_free(oreq);
1390 add_to_reap_list(child);
1391 } else {
1392 refcount_set(&oreq->rsk_refcnt, 1);
1393 inet_csk_reqsk_queue_add(lsk, oreq, child);
1394 lsk->sk_data_ready(lsk);
1395 }
1396}
1397
1398static void bl_add_pass_open_to_parent(struct sock *lsk, struct sk_buff *skb)
1399{
1400 struct sock *child = skb->sk;
1401
1402 skb->sk = NULL;
1403 add_pass_open_to_parent(child, lsk, BLOG_SKB_CB(skb)->cdev);
1404 kfree_skb(skb);
1405}
1406
1407static int chtls_pass_establish(struct chtls_dev *cdev, struct sk_buff *skb)
1408{
1409 struct cpl_pass_establish *req = cplhdr(skb) + RSS_HDR;
1410 struct chtls_sock *csk;
1411 struct sock *lsk, *sk;
1412 unsigned int hwtid;
1413
1414 hwtid = GET_TID(req);
1415 sk = lookup_tid(cdev->tids, hwtid);
1416 if (!sk)
1417 return (CPL_RET_UNKNOWN_TID | CPL_RET_BUF_DONE);
1418
1419 bh_lock_sock(sk);
1420 if (unlikely(sock_owned_by_user(sk))) {
1421 kfree_skb(skb);
1422 } else {
1423 unsigned int stid;
1424 void *data;
1425
1426 csk = sk->sk_user_data;
1427 csk->wr_max_credits = 64;
1428 csk->wr_credits = 64;
1429 csk->wr_unacked = 0;
1430 make_established(sk, ntohl(req->snd_isn), ntohs(req->tcp_opt));
1431 stid = PASS_OPEN_TID_G(ntohl(req->tos_stid));
1432 sk->sk_state_change(sk);
1433 if (unlikely(sk->sk_socket))
1434 sk_wake_async(sk, 0, POLL_OUT);
1435
1436 data = lookup_stid(cdev->tids, stid);
1437 lsk = ((struct listen_ctx *)data)->lsk;
1438
1439 bh_lock_sock(lsk);
1440 if (unlikely(skb_queue_empty(&csk->listen_ctx->synq))) {
1441 /* removed from synq */
1442 bh_unlock_sock(lsk);
1443 kfree_skb(skb);
1444 goto unlock;
1445 }
1446
1447 if (likely(!sock_owned_by_user(lsk))) {
1448 kfree_skb(skb);
1449 add_pass_open_to_parent(sk, lsk, cdev);
1450 } else {
1451 skb->sk = sk;
1452 BLOG_SKB_CB(skb)->cdev = cdev;
1453 BLOG_SKB_CB(skb)->backlog_rcv =
1454 bl_add_pass_open_to_parent;
1455 __sk_add_backlog(lsk, skb);
1456 }
1457 bh_unlock_sock(lsk);
1458 }
1459unlock:
1460 bh_unlock_sock(sk);
1461 return 0;
1462}
1463
1464/*
1465 * Handle receipt of an urgent pointer.
1466 */
1467static void handle_urg_ptr(struct sock *sk, u32 urg_seq)
1468{
1469 struct tcp_sock *tp = tcp_sk(sk);
1470
1471 urg_seq--;
1472 if (tp->urg_data && !after(urg_seq, tp->urg_seq))
1473 return; /* duplicate pointer */
1474
1475 sk_send_sigurg(sk);
1476 if (tp->urg_seq == tp->copied_seq && tp->urg_data &&
1477 !sock_flag(sk, SOCK_URGINLINE) &&
1478 tp->copied_seq != tp->rcv_nxt) {
1479 struct sk_buff *skb = skb_peek(&sk->sk_receive_queue);
1480
1481 tp->copied_seq++;
1482 if (skb && tp->copied_seq - ULP_SKB_CB(skb)->seq >= skb->len)
1483 chtls_free_skb(sk, skb);
1484 }
1485
1486 tp->urg_data = TCP_URG_NOTYET;
1487 tp->urg_seq = urg_seq;
1488}
1489
1490static void check_sk_callbacks(struct chtls_sock *csk)
1491{
1492 struct sock *sk = csk->sk;
1493
1494 if (unlikely(sk->sk_user_data &&
1495 !csk_flag_nochk(csk, CSK_CALLBACKS_CHKD)))
1496 csk_set_flag(csk, CSK_CALLBACKS_CHKD);
1497}
1498
1499/*
1500 * Handles Rx data that arrives in a state where the socket isn't accepting
1501 * new data.
1502 */
1503static void handle_excess_rx(struct sock *sk, struct sk_buff *skb)
1504{
1505 if (!csk_flag(sk, CSK_ABORT_SHUTDOWN))
1506 chtls_abort_conn(sk, skb);
1507
1508 kfree_skb(skb);
1509}
1510
1511static void chtls_recv_data(struct sock *sk, struct sk_buff *skb)
1512{
1513 struct cpl_rx_data *hdr = cplhdr(skb) + RSS_HDR;
1514 struct chtls_sock *csk;
1515 struct tcp_sock *tp;
1516
1517 csk = rcu_dereference_sk_user_data(sk);
1518 tp = tcp_sk(sk);
1519
1520 if (unlikely(sk->sk_shutdown & RCV_SHUTDOWN)) {
1521 handle_excess_rx(sk, skb);
1522 return;
1523 }
1524
1525 ULP_SKB_CB(skb)->seq = ntohl(hdr->seq);
1526 ULP_SKB_CB(skb)->psh = hdr->psh;
1527 skb_ulp_mode(skb) = ULP_MODE_NONE;
1528
1529 skb_reset_transport_header(skb);
1530 __skb_pull(skb, sizeof(*hdr) + RSS_HDR);
1531 if (!skb->data_len)
1532 __skb_trim(skb, ntohs(hdr->len));
1533
1534 if (unlikely(hdr->urg))
1535 handle_urg_ptr(sk, tp->rcv_nxt + ntohs(hdr->urg));
1536 if (unlikely(tp->urg_data == TCP_URG_NOTYET &&
1537 tp->urg_seq - tp->rcv_nxt < skb->len))
1538 tp->urg_data = TCP_URG_VALID |
1539 skb->data[tp->urg_seq - tp->rcv_nxt];
1540
1541 if (unlikely(hdr->dack_mode != csk->delack_mode)) {
1542 csk->delack_mode = hdr->dack_mode;
1543 csk->delack_seq = tp->rcv_nxt;
1544 }
1545
1546 tcp_hdr(skb)->fin = 0;
1547 tp->rcv_nxt += skb->len;
1548
1549 __skb_queue_tail(&sk->sk_receive_queue, skb);
1550
1551 if (!sock_flag(sk, SOCK_DEAD)) {
1552 check_sk_callbacks(csk);
1553 sk->sk_data_ready(sk);
1554 }
1555}
1556
1557static int chtls_rx_data(struct chtls_dev *cdev, struct sk_buff *skb)
1558{
1559 struct cpl_rx_data *req = cplhdr(skb) + RSS_HDR;
1560 unsigned int hwtid = GET_TID(req);
1561 struct sock *sk;
1562
1563 sk = lookup_tid(cdev->tids, hwtid);
3d8ccf9f
GS		 1564 if (unlikely(!sk)) {
1565 pr_err("can't find conn. for hwtid %u.\n", hwtid);
1566 return -EINVAL;
1567 }
cc35c88a
AG		 1568 skb_dst_set(skb, NULL);
1569 process_cpl_msg(chtls_recv_data, sk, skb);
1570 return 0;
1571}
1572
1573static void chtls_recv_pdu(struct sock *sk, struct sk_buff *skb)
1574{
1575 struct cpl_tls_data *hdr = cplhdr(skb);
1576 struct chtls_sock *csk;
1577 struct chtls_hws *tlsk;
1578 struct tcp_sock *tp;
1579
1580 csk = rcu_dereference_sk_user_data(sk);
1581 tlsk = &csk->tlshws;
1582 tp = tcp_sk(sk);
1583
1584 if (unlikely(sk->sk_shutdown & RCV_SHUTDOWN)) {
1585 handle_excess_rx(sk, skb);
1586 return;
1587 }
1588
1589 ULP_SKB_CB(skb)->seq = ntohl(hdr->seq);
1590 ULP_SKB_CB(skb)->flags = 0;
1591 skb_ulp_mode(skb) = ULP_MODE_TLS;
1592
1593 skb_reset_transport_header(skb);
1594 __skb_pull(skb, sizeof(*hdr));
1595 if (!skb->data_len)
1596 __skb_trim(skb,
1597 CPL_TLS_DATA_LENGTH_G(ntohl(hdr->length_pkd)));
1598
1599 if (unlikely(tp->urg_data == TCP_URG_NOTYET && tp->urg_seq -
1600 tp->rcv_nxt < skb->len))
1601 tp->urg_data = TCP_URG_VALID |
1602 skb->data[tp->urg_seq - tp->rcv_nxt];
1603
1604 tcp_hdr(skb)->fin = 0;
1605 tlsk->pldlen = CPL_TLS_DATA_LENGTH_G(ntohl(hdr->length_pkd));
1606 __skb_queue_tail(&tlsk->sk_recv_queue, skb);
1607}
1608
1609static int chtls_rx_pdu(struct chtls_dev *cdev, struct sk_buff *skb)
1610{
1611 struct cpl_tls_data *req = cplhdr(skb);
1612 unsigned int hwtid = GET_TID(req);
1613 struct sock *sk;
1614
1615 sk = lookup_tid(cdev->tids, hwtid);
3d8ccf9f
GS		 1616 if (unlikely(!sk)) {
1617 pr_err("can't find conn. for hwtid %u.\n", hwtid);
1618 return -EINVAL;
1619 }
cc35c88a
AG		 1620 skb_dst_set(skb, NULL);
1621 process_cpl_msg(chtls_recv_pdu, sk, skb);
1622 return 0;
1623}
1624
1625static void chtls_set_hdrlen(struct sk_buff *skb, unsigned int nlen)
1626{
1627 struct tlsrx_cmp_hdr *tls_cmp_hdr = cplhdr(skb);
1628
1629 skb->hdr_len = ntohs((__force __be16)tls_cmp_hdr->length);
1630 tls_cmp_hdr->length = ntohs((__force __be16)nlen);
1631}
1632
1633static void chtls_rx_hdr(struct sock *sk, struct sk_buff *skb)
1634{
17a7d24a
AG		 1635 struct tlsrx_cmp_hdr *tls_hdr_pkt;
1636 struct cpl_rx_tls_cmp *cmp_cpl;
cc35c88a
AG		 1637 struct sk_buff *skb_rec;
1638 struct chtls_sock *csk;
1639 struct chtls_hws *tlsk;
1640 struct tcp_sock *tp;
1641
17a7d24a 1642 cmp_cpl = cplhdr(skb);
cc35c88a
AG		 1643 csk = rcu_dereference_sk_user_data(sk);
1644 tlsk = &csk->tlshws;
1645 tp = tcp_sk(sk);
1646
1647 ULP_SKB_CB(skb)->seq = ntohl(cmp_cpl->seq);
1648 ULP_SKB_CB(skb)->flags = 0;
1649
1650 skb_reset_transport_header(skb);
1651 __skb_pull(skb, sizeof(*cmp_cpl));
17a7d24a
AG		 1652 tls_hdr_pkt = (struct tlsrx_cmp_hdr *)skb->data;
1653 if (tls_hdr_pkt->res_to_mac_error & TLSRX_HDR_PKT_ERROR_M)
1654 tls_hdr_pkt->type = CONTENT_TYPE_ERROR;
cc35c88a 1655 if (!skb->data_len)
17a7d24a 1656 __skb_trim(skb, TLS_HEADER_LENGTH);
cc35c88a
AG		 1657
1658 tp->rcv_nxt +=
1659 CPL_RX_TLS_CMP_PDULENGTH_G(ntohl(cmp_cpl->pdulength_length));
1660
17a7d24a 1661 ULP_SKB_CB(skb)->flags |= ULPCB_FLAG_TLS_HDR;
cc35c88a
AG		 1662 skb_rec = __skb_dequeue(&tlsk->sk_recv_queue);
1663 if (!skb_rec) {
cc35c88a
AG		 1664 __skb_queue_tail(&sk->sk_receive_queue, skb);
1665 } else {
1666 chtls_set_hdrlen(skb, tlsk->pldlen);
1667 tlsk->pldlen = 0;
1668 __skb_queue_tail(&sk->sk_receive_queue, skb);
1669 __skb_queue_tail(&sk->sk_receive_queue, skb_rec);
1670 }
1671
1672 if (!sock_flag(sk, SOCK_DEAD)) {
1673 check_sk_callbacks(csk);
1674 sk->sk_data_ready(sk);
1675 }
1676}
1677
1678static int chtls_rx_cmp(struct chtls_dev *cdev, struct sk_buff *skb)
1679{
1680 struct cpl_rx_tls_cmp *req = cplhdr(skb);
1681 unsigned int hwtid = GET_TID(req);
1682 struct sock *sk;
1683
1684 sk = lookup_tid(cdev->tids, hwtid);
3d8ccf9f
GS		 1685 if (unlikely(!sk)) {
1686 pr_err("can't find conn. for hwtid %u.\n", hwtid);
1687 return -EINVAL;
1688 }
cc35c88a
AG		 1689 skb_dst_set(skb, NULL);
1690 process_cpl_msg(chtls_rx_hdr, sk, skb);
1691
1692 return 0;
1693}
1694
1695static void chtls_timewait(struct sock *sk)
1696{
1697 struct tcp_sock *tp = tcp_sk(sk);
1698
1699 tp->rcv_nxt++;
cca9bab1 1700 tp->rx_opt.ts_recent_stamp = ktime_get_seconds();
cc35c88a
AG		 1701 tp->srtt_us = 0;
1702 tcp_time_wait(sk, TCP_TIME_WAIT, 0);
1703}
1704
1705static void chtls_peer_close(struct sock *sk, struct sk_buff *skb)
1706{
1707 struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
1708
1709 sk->sk_shutdown |= RCV_SHUTDOWN;
1710 sock_set_flag(sk, SOCK_DONE);
1711
1712 switch (sk->sk_state) {
1713 case TCP_SYN_RECV:
1714 case TCP_ESTABLISHED:
1715 tcp_set_state(sk, TCP_CLOSE_WAIT);
1716 break;
1717 case TCP_FIN_WAIT1:
1718 tcp_set_state(sk, TCP_CLOSING);
1719 break;
1720 case TCP_FIN_WAIT2:
1721 chtls_release_resources(sk);
1722 if (csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING))
1723 chtls_conn_done(sk);
1724 else
1725 chtls_timewait(sk);
1726 break;
1727 default:
1728 pr_info("cpl_peer_close in bad state %d\n", sk->sk_state);
1729 }
1730
1731 if (!sock_flag(sk, SOCK_DEAD)) {
1732 sk->sk_state_change(sk);
1733 /* Do not send POLL_HUP for half duplex close. */
1734
1735 if ((sk->sk_shutdown & SEND_SHUTDOWN) ||
1736 sk->sk_state == TCP_CLOSE)
1737 sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_HUP);
1738 else
1739 sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN);
1740 }
1741}
1742
1743static void chtls_close_con_rpl(struct sock *sk, struct sk_buff *skb)
1744{
1745 struct cpl_close_con_rpl *rpl = cplhdr(skb) + RSS_HDR;
1746 struct chtls_sock *csk;
1747 struct tcp_sock *tp;
1748
1749 csk = rcu_dereference_sk_user_data(sk);
1750 tp = tcp_sk(sk);
1751
1752 tp->snd_una = ntohl(rpl->snd_nxt) - 1; /* exclude FIN */
1753
1754 switch (sk->sk_state) {
1755 case TCP_CLOSING:
1756 chtls_release_resources(sk);
1757 if (csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING))
1758 chtls_conn_done(sk);
1759 else
1760 chtls_timewait(sk);
1761 break;
1762 case TCP_LAST_ACK:
1763 chtls_release_resources(sk);
1764 chtls_conn_done(sk);
1765 break;
1766 case TCP_FIN_WAIT1:
1767 tcp_set_state(sk, TCP_FIN_WAIT2);
1768 sk->sk_shutdown |= SEND_SHUTDOWN;
1769
1770 if (!sock_flag(sk, SOCK_DEAD))
1771 sk->sk_state_change(sk);
1772 else if (tcp_sk(sk)->linger2 < 0 &&
1773 !csk_flag_nochk(csk, CSK_ABORT_SHUTDOWN))
1774 chtls_abort_conn(sk, skb);
1775 break;
1776 default:
1777 pr_info("close_con_rpl in bad state %d\n", sk->sk_state);
1778 }
1779 kfree_skb(skb);
1780}
1781
1782static struct sk_buff *get_cpl_skb(struct sk_buff *skb,
1783 size_t len, gfp_t gfp)
1784{
1785 if (likely(!skb_is_nonlinear(skb) && !skb_cloned(skb))) {
1786 WARN_ONCE(skb->len < len, "skb alloc error");
1787 __skb_trim(skb, len);
1788 skb_get(skb);
1789 } else {
1790 skb = alloc_skb(len, gfp);
1791 if (skb)
1792 __skb_put(skb, len);
1793 }
1794 return skb;
1795}
1796
1797static void set_abort_rpl_wr(struct sk_buff *skb, unsigned int tid,
1798 int cmd)
1799{
1800 struct cpl_abort_rpl *rpl = cplhdr(skb);
1801
1802 INIT_TP_WR_CPL(rpl, CPL_ABORT_RPL, tid);
1803 rpl->cmd = cmd;
1804}
1805
1806static void send_defer_abort_rpl(struct chtls_dev *cdev, struct sk_buff *skb)
1807{
1808 struct cpl_abort_req_rss *req = cplhdr(skb);
1809 struct sk_buff *reply_skb;
1810
1811 reply_skb = alloc_skb(sizeof(struct cpl_abort_rpl),
1812 GFP_KERNEL | __GFP_NOFAIL);
1813 __skb_put(reply_skb, sizeof(struct cpl_abort_rpl));
1814 set_abort_rpl_wr(reply_skb, GET_TID(req),
1815 (req->status & CPL_ABORT_NO_RST));
1816 set_wr_txq(reply_skb, CPL_PRIORITY_DATA, req->status >> 1);
1817 cxgb4_ofld_send(cdev->lldi->ports[0], reply_skb);
1818 kfree_skb(skb);
1819}
1820
1821static void send_abort_rpl(struct sock *sk, struct sk_buff *skb,
1822 struct chtls_dev *cdev, int status, int queue)
1823{
1824 struct cpl_abort_req_rss *req = cplhdr(skb);
1825 struct sk_buff *reply_skb;
1826 struct chtls_sock *csk;
1827
1828 csk = rcu_dereference_sk_user_data(sk);
1829
1830 reply_skb = alloc_skb(sizeof(struct cpl_abort_rpl),
1831 GFP_KERNEL);
1832
1833 if (!reply_skb) {
1834 req->status = (queue << 1);
1835 send_defer_abort_rpl(cdev, skb);
1836 return;
1837 }
1838
1839 set_abort_rpl_wr(reply_skb, GET_TID(req), status);
1840 kfree_skb(skb);
1841
1842 set_wr_txq(reply_skb, CPL_PRIORITY_DATA, queue);
1843 if (csk_conn_inline(csk)) {
1844 struct l2t_entry *e = csk->l2t_entry;
1845
1846 if (e && sk->sk_state != TCP_SYN_RECV) {
1847 cxgb4_l2t_send(csk->egress_dev, reply_skb, e);
1848 return;
1849 }
1850 }
1851 cxgb4_ofld_send(cdev->lldi->ports[0], reply_skb);
1852}
1853
1854/*
1855 * Add an skb to the deferred skb queue for processing from process context.
1856 */
1857static void t4_defer_reply(struct sk_buff *skb, struct chtls_dev *cdev,
1858 defer_handler_t handler)
1859{
1860 DEFERRED_SKB_CB(skb)->handler = handler;
1861 spin_lock_bh(&cdev->deferq.lock);
1862 __skb_queue_tail(&cdev->deferq, skb);
1863 if (skb_queue_len(&cdev->deferq) == 1)
1864 schedule_work(&cdev->deferq_task);
1865 spin_unlock_bh(&cdev->deferq.lock);
1866}
1867
1868static void chtls_send_abort_rpl(struct sock *sk, struct sk_buff *skb,
1869 struct chtls_dev *cdev,
1870 int status, int queue)
1871{
1872 struct cpl_abort_req_rss *req = cplhdr(skb) + RSS_HDR;
1873 struct sk_buff *reply_skb;
1874 struct chtls_sock *csk;
1875 unsigned int tid;
1876
1877 csk = rcu_dereference_sk_user_data(sk);
1878 tid = GET_TID(req);
1879
1880 reply_skb = get_cpl_skb(skb, sizeof(struct cpl_abort_rpl), gfp_any());
1881 if (!reply_skb) {
1882 req->status = (queue << 1) | status;
1883 t4_defer_reply(skb, cdev, send_defer_abort_rpl);
1884 return;
1885 }
1886
1887 set_abort_rpl_wr(reply_skb, tid, status);
1888 set_wr_txq(reply_skb, CPL_PRIORITY_DATA, queue);
1889 if (csk_conn_inline(csk)) {
1890 struct l2t_entry *e = csk->l2t_entry;
1891
1892 if (e && sk->sk_state != TCP_SYN_RECV) {
1893 cxgb4_l2t_send(csk->egress_dev, reply_skb, e);
1894 return;
1895 }
1896 }
1897 cxgb4_ofld_send(cdev->lldi->ports[0], reply_skb);
1898 kfree_skb(skb);
1899}
1900
1901/*
1902 * This is run from a listener's backlog to abort a child connection in
1903 * SYN_RCV state (i.e., one on the listener's SYN queue).
1904 */
1905static void bl_abort_syn_rcv(struct sock *lsk, struct sk_buff *skb)
1906{
1907 struct chtls_sock *csk;
1908 struct sock *child;
1909 int queue;
1910
1911 child = skb->sk;
1912 csk = rcu_dereference_sk_user_data(child);
1913 queue = csk->txq_idx;
1914
1915 skb->sk = NULL;
1916 do_abort_syn_rcv(child, lsk);
1917 send_abort_rpl(child, skb, BLOG_SKB_CB(skb)->cdev,
1918 CPL_ABORT_NO_RST, queue);
1919}
1920
1921static int abort_syn_rcv(struct sock *sk, struct sk_buff *skb)
1922{
1923 const struct request_sock *oreq;
1924 struct listen_ctx *listen_ctx;
1925 struct chtls_sock *csk;
1926 struct chtls_dev *cdev;
1927 struct sock *psk;
1928 void *ctx;
1929
1930 csk = sk->sk_user_data;
1931 oreq = csk->passive_reap_next;
1932 cdev = csk->cdev;
1933
1934 if (!oreq)
1935 return -1;
1936
1937 ctx = lookup_stid(cdev->tids, oreq->ts_recent);
1938 if (!ctx)
1939 return -1;
1940
1941 listen_ctx = (struct listen_ctx *)ctx;
1942 psk = listen_ctx->lsk;
1943
1944 bh_lock_sock(psk);
1945 if (!sock_owned_by_user(psk)) {
1946 int queue = csk->txq_idx;
1947
1948 do_abort_syn_rcv(sk, psk);
1949 send_abort_rpl(sk, skb, cdev, CPL_ABORT_NO_RST, queue);
1950 } else {
1951 skb->sk = sk;
1952 BLOG_SKB_CB(skb)->backlog_rcv = bl_abort_syn_rcv;
1953 __sk_add_backlog(psk, skb);
1954 }
1955 bh_unlock_sock(psk);
1956 return 0;
1957}
1958
1959static void chtls_abort_req_rss(struct sock *sk, struct sk_buff *skb)
1960{
1961 const struct cpl_abort_req_rss *req = cplhdr(skb) + RSS_HDR;
1962 struct chtls_sock *csk = sk->sk_user_data;
1963 int rst_status = CPL_ABORT_NO_RST;
1964 int queue = csk->txq_idx;
1965
1966 if (is_neg_adv(req->status)) {
1967 if (sk->sk_state == TCP_SYN_RECV)
1968 chtls_set_tcb_tflag(sk, 0, 0);
1969
1970 kfree_skb(skb);
1971 return;
1972 }
1973
1974 csk_reset_flag(csk, CSK_ABORT_REQ_RCVD);
1975
1976 if (!csk_flag_nochk(csk, CSK_ABORT_SHUTDOWN) &&
1977 !csk_flag_nochk(csk, CSK_TX_DATA_SENT)) {
1978 struct tcp_sock *tp = tcp_sk(sk);
1979
1980 if (send_tx_flowc_wr(sk, 0, tp->snd_nxt, tp->rcv_nxt) < 0)
1981 WARN_ONCE(1, "send_tx_flowc error");
1982 csk_set_flag(csk, CSK_TX_DATA_SENT);
1983 }
1984
1985 csk_set_flag(csk, CSK_ABORT_SHUTDOWN);
1986
1987 if (!csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING)) {
1988 sk->sk_err = ETIMEDOUT;
1989
1990 if (!sock_flag(sk, SOCK_DEAD))
1991 sk->sk_error_report(sk);
1992
1993 if (sk->sk_state == TCP_SYN_RECV && !abort_syn_rcv(sk, skb))
1994 return;
1995
1996 chtls_release_resources(sk);
1997 chtls_conn_done(sk);
1998 }
1999
2000 chtls_send_abort_rpl(sk, skb, csk->cdev, rst_status, queue);
2001}
2002
2003static void chtls_abort_rpl_rss(struct sock *sk, struct sk_buff *skb)
2004{
2005 struct cpl_abort_rpl_rss *rpl = cplhdr(skb) + RSS_HDR;
2006 struct chtls_sock *csk;
2007 struct chtls_dev *cdev;
2008
2009 csk = rcu_dereference_sk_user_data(sk);
2010 cdev = csk->cdev;
2011
2012 if (csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING)) {
2013 csk_reset_flag(csk, CSK_ABORT_RPL_PENDING);
2014 if (!csk_flag_nochk(csk, CSK_ABORT_REQ_RCVD)) {
2015 if (sk->sk_state == TCP_SYN_SENT) {
2016 cxgb4_remove_tid(cdev->tids,
2017 csk->port_id,
2018 GET_TID(rpl),
2019 sk->sk_family);
2020 sock_put(sk);
2021 }
2022 chtls_release_resources(sk);
2023 chtls_conn_done(sk);
2024 }
2025 }
2026 kfree_skb(skb);
2027}
2028
2029static int chtls_conn_cpl(struct chtls_dev *cdev, struct sk_buff *skb)
2030{
2031 struct cpl_peer_close *req = cplhdr(skb) + RSS_HDR;
2032 void (*fn)(struct sock *sk, struct sk_buff *skb);
2033 unsigned int hwtid = GET_TID(req);
2034 struct sock *sk;
2035 u8 opcode;
2036
2037 opcode = ((const struct rss_header *)cplhdr(skb))->opcode;
2038
2039 sk = lookup_tid(cdev->tids, hwtid);
2040 if (!sk)
2041 goto rel_skb;
2042
2043 switch (opcode) {
2044 case CPL_PEER_CLOSE:
2045 fn = chtls_peer_close;
2046 break;
2047 case CPL_CLOSE_CON_RPL:
2048 fn = chtls_close_con_rpl;
2049 break;
2050 case CPL_ABORT_REQ_RSS:
2051 fn = chtls_abort_req_rss;
2052 break;
2053 case CPL_ABORT_RPL_RSS:
2054 fn = chtls_abort_rpl_rss;
2055 break;
2056 default:
2057 goto rel_skb;
2058 }
2059
2060 process_cpl_msg(fn, sk, skb);
2061 return 0;
2062
2063rel_skb:
2064 kfree_skb(skb);
2065 return 0;
2066}
2067
2068static struct sk_buff *dequeue_wr(struct sock *sk)
2069{
2070 struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
2071 struct sk_buff *skb = csk->wr_skb_head;
2072
2073 if (likely(skb)) {
2074 /* Don't bother clearing the tail */
2075 csk->wr_skb_head = WR_SKB_CB(skb)->next_wr;
2076 WR_SKB_CB(skb)->next_wr = NULL;
2077 }
2078 return skb;
2079}
2080
2081static void chtls_rx_ack(struct sock *sk, struct sk_buff *skb)
2082{
2083 struct cpl_fw4_ack *hdr = cplhdr(skb) + RSS_HDR;
2084 struct chtls_sock *csk = sk->sk_user_data;
2085 struct tcp_sock *tp = tcp_sk(sk);
2086 u32 credits = hdr->credits;
2087 u32 snd_una;
2088
2089 snd_una = ntohl(hdr->snd_una);
2090 csk->wr_credits += credits;
2091
2092 if (csk->wr_unacked > csk->wr_max_credits - csk->wr_credits)
2093 csk->wr_unacked = csk->wr_max_credits - csk->wr_credits;
2094
2095 while (credits) {
2096 struct sk_buff *pskb = csk->wr_skb_head;
2097 u32 csum;
2098
2099 if (unlikely(!pskb)) {
2100 if (csk->wr_nondata)
2101 csk->wr_nondata -= credits;
2102 break;
2103 }
2104 csum = (__force u32)pskb->csum;
2105 if (unlikely(credits < csum)) {
2106 pskb->csum = (__force __wsum)(csum - credits);
2107 break;
2108 }
2109 dequeue_wr(sk);
2110 credits -= csum;
2111 kfree_skb(pskb);
2112 }
2113 if (hdr->seq_vld & CPL_FW4_ACK_FLAGS_SEQVAL) {
2114 if (unlikely(before(snd_una, tp->snd_una))) {
2115 kfree_skb(skb);
2116 return;
2117 }
2118
2119 if (tp->snd_una != snd_una) {
2120 tp->snd_una = snd_una;
2121 tp->rcv_tstamp = tcp_time_stamp(tp);
2122 if (tp->snd_una == tp->snd_nxt &&
2123 !csk_flag_nochk(csk, CSK_TX_FAILOVER))
2124 csk_reset_flag(csk, CSK_TX_WAIT_IDLE);
2125 }
2126 }
2127
2128 if (hdr->seq_vld & CPL_FW4_ACK_FLAGS_CH) {
2129 unsigned int fclen16 = roundup(failover_flowc_wr_len, 16);
2130
2131 csk->wr_credits -= fclen16;
2132 csk_reset_flag(csk, CSK_TX_WAIT_IDLE);
2133 csk_reset_flag(csk, CSK_TX_FAILOVER);
2134 }
2135 if (skb_queue_len(&csk->txq) && chtls_push_frames(csk, 0))
2136 sk->sk_write_space(sk);
2137
2138 kfree_skb(skb);
2139}
2140
2141static int chtls_wr_ack(struct chtls_dev *cdev, struct sk_buff *skb)
2142{
2143 struct cpl_fw4_ack *rpl = cplhdr(skb) + RSS_HDR;
2144 unsigned int hwtid = GET_TID(rpl);
2145 struct sock *sk;
2146
2147 sk = lookup_tid(cdev->tids, hwtid);
3d8ccf9f
GS		 2148 if (unlikely(!sk)) {
2149 pr_err("can't find conn. for hwtid %u.\n", hwtid);
2150 return -EINVAL;
2151 }
cc35c88a
AG		 2152 process_cpl_msg(chtls_rx_ack, sk, skb);
2153
2154 return 0;
2155}
2156
2157chtls_handler_func chtls_handlers[NUM_CPL_CMDS] = {
2158 [CPL_PASS_OPEN_RPL] = chtls_pass_open_rpl,
2159 [CPL_CLOSE_LISTSRV_RPL] = chtls_close_listsrv_rpl,
2160 [CPL_PASS_ACCEPT_REQ] = chtls_pass_accept_req,
2161 [CPL_PASS_ESTABLISH] = chtls_pass_establish,
2162 [CPL_RX_DATA] = chtls_rx_data,
2163 [CPL_TLS_DATA] = chtls_rx_pdu,
2164 [CPL_RX_TLS_CMP] = chtls_rx_cmp,
2165 [CPL_PEER_CLOSE] = chtls_conn_cpl,
2166 [CPL_CLOSE_CON_RPL] = chtls_conn_cpl,
2167 [CPL_ABORT_REQ_RSS] = chtls_conn_cpl,
2168 [CPL_ABORT_RPL_RSS] = chtls_conn_cpl,
2169 [CPL_FW4_ACK] = chtls_wr_ack,
2170};
Linux 6.x block layer and io_uring tree(s)