projects / linux-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
[SCSI] cxgb3i: Add cxgb3i iSCSI driver.
[linux-block.git] / drivers / scsi / cxgb3i / cxgb3i_offload.c
CommitLineData
c3673464
KX		 1/*
2 * cxgb3i_offload.c: Chelsio S3xx iscsi offloaded tcp connection management
3 *
4 * Copyright (C) 2003-2008 Chelsio Communications. All rights reserved.
5 *
6 * This program is distributed in the hope that it will be useful, but WITHOUT
7 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
8 * FITNESS FOR A PARTICULAR PURPOSE. See the LICENSE file included in this
9 * release for licensing terms and conditions.
10 *
11 * Written by: Dimitris Michailidis (dm@chelsio.com)
12 * Karen Xie (kxie@chelsio.com)
13 */
14
15#include <linux/if_vlan.h>
16#include <linux/version.h>
17
18#include "cxgb3_defs.h"
19#include "cxgb3_ctl_defs.h"
20#include "firmware_exports.h"
21#include "cxgb3i_offload.h"
22#include "cxgb3i_pdu.h"
23#include "cxgb3i_ddp.h"
24
25#ifdef __DEBUG_C3CN_CONN__
26#define c3cn_conn_debug cxgb3i_log_info
27#else
28#define c3cn_conn_debug(fmt...)
29#endif
30
31#ifdef __DEBUG_C3CN_TX__
32#define c3cn_tx_debug cxgb3i_log_debug
33#else
34#define c3cn_tx_debug(fmt...)
35#endif
36
37#ifdef __DEBUG_C3CN_RX__
38#define c3cn_rx_debug cxgb3i_log_debug
39#else
40#define c3cn_rx_debug(fmt...)
41#endif
42
43/*
44 * module parameters releated to offloaded iscsi connection
45 */
46static int cxgb3_rcv_win = 256 * 1024;
47module_param(cxgb3_rcv_win, int, 0644);
48MODULE_PARM_DESC(cxgb3_rcv_win, "TCP receive window in bytes (default=256KB)");
49
50static int cxgb3_snd_win = 64 * 1024;
51module_param(cxgb3_snd_win, int, 0644);
52MODULE_PARM_DESC(cxgb3_snd_win, "TCP send window in bytes (default=64KB)");
53
54static int cxgb3_rx_credit_thres = 10 * 1024;
55module_param(cxgb3_rx_credit_thres, int, 0644);
56MODULE_PARM_DESC(rx_credit_thres,
57 "RX credits return threshold in bytes (default=10KB)");
58
59static unsigned int cxgb3_max_connect = 8 * 1024;
60module_param(cxgb3_max_connect, uint, 0644);
61MODULE_PARM_DESC(cxgb3_max_connect, "Max. # of connections (default=8092)");
62
63static unsigned int cxgb3_sport_base = 20000;
64module_param(cxgb3_sport_base, uint, 0644);
65MODULE_PARM_DESC(cxgb3_sport_base, "starting port number (default=20000)");
66
67/*
68 * cxgb3i tcp connection data(per adapter) list
69 */
70static LIST_HEAD(cdata_list);
71static DEFINE_RWLOCK(cdata_rwlock);
72
73static int c3cn_push_tx_frames(struct s3_conn *c3cn, int req_completion);
74static void c3cn_release_offload_resources(struct s3_conn *c3cn);
75
76/*
77 * iscsi source port management
78 *
79 * Find a free source port in the port allocation map. We use a very simple
80 * rotor scheme to look for the next free port.
81 *
82 * If a source port has been specified make sure that it doesn't collide with
83 * our normal source port allocation map. If it's outside the range of our
84 * allocation/deallocation scheme just let them use it.
85 *
86 * If the source port is outside our allocation range, the caller is
87 * responsible for keeping track of their port usage.
88 */
89static int c3cn_get_port(struct s3_conn *c3cn, struct cxgb3i_sdev_data *cdata)
90{
91 unsigned int start;
92 int idx;
93
94 if (!cdata)
95 goto error_out;
96
97 if (c3cn->saddr.sin_port != 0) {
98 idx = ntohs(c3cn->saddr.sin_port) - cxgb3_sport_base;
99 if (idx < 0 || idx >= cxgb3_max_connect)
100 return 0;
101 if (!test_and_set_bit(idx, cdata->sport_map))
102 return -EADDRINUSE;
103 }
104
105 /* the sport_map_next may not be accurate but that is okay, sport_map
106 should be */
107 start = idx = cdata->sport_map_next;
108 do {
109 if (++idx >= cxgb3_max_connect)
110 idx = 0;
111 if (!(test_and_set_bit(idx, cdata->sport_map))) {
112 c3cn->saddr.sin_port = htons(cxgb3_sport_base + idx);
113 cdata->sport_map_next = idx;
114 c3cn_conn_debug("%s reserve port %u.\n",
115 cdata->cdev->name,
116 cxgb3_sport_base + idx);
117 return 0;
118 }
119 } while (idx != start);
120
121error_out:
122 return -EADDRNOTAVAIL;
123}
124
125static void c3cn_put_port(struct s3_conn *c3cn)
126{
127 struct cxgb3i_sdev_data *cdata = CXGB3_SDEV_DATA(c3cn->cdev);
128
129 if (c3cn->saddr.sin_port) {
130 int idx = ntohs(c3cn->saddr.sin_port) - cxgb3_sport_base;
131
132 c3cn->saddr.sin_port = 0;
133 if (idx < 0 || idx >= cxgb3_max_connect)
134 return;
135 clear_bit(idx, cdata->sport_map);
136 c3cn_conn_debug("%s, release port %u.\n",
137 cdata->cdev->name, cxgb3_sport_base + idx);
138 }
139}
140
141static inline void c3cn_set_flag(struct s3_conn *c3cn, enum c3cn_flags flag)
142{
143 __set_bit(flag, &c3cn->flags);
144 c3cn_conn_debug("c3cn 0x%p, set %d, s %u, f 0x%lx.\n",
145 c3cn, flag, c3cn->state, c3cn->flags);
146}
147
148static inline void c3cn_clear_flag(struct s3_conn *c3cn, enum c3cn_flags flag)
149{
150 __clear_bit(flag, &c3cn->flags);
151 c3cn_conn_debug("c3cn 0x%p, clear %d, s %u, f 0x%lx.\n",
152 c3cn, flag, c3cn->state, c3cn->flags);
153}
154
155static inline int c3cn_flag(struct s3_conn *c3cn, enum c3cn_flags flag)
156{
157 if (c3cn == NULL)
158 return 0;
159 return test_bit(flag, &c3cn->flags);
160}
161
162static void c3cn_set_state(struct s3_conn *c3cn, int state)
163{
164 c3cn_conn_debug("c3cn 0x%p state -> %u.\n", c3cn, state);
165 c3cn->state = state;
166}
167
168static inline void c3cn_hold(struct s3_conn *c3cn)
169{
170 atomic_inc(&c3cn->refcnt);
171}
172
173static inline void c3cn_put(struct s3_conn *c3cn)
174{
175 if (atomic_dec_and_test(&c3cn->refcnt)) {
176 c3cn_conn_debug("free c3cn 0x%p, s %u, f 0x%lx.\n",
177 c3cn, c3cn->state, c3cn->flags);
178 kfree(c3cn);
179 }
180}
181
182static void c3cn_closed(struct s3_conn *c3cn)
183{
184 c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
185 c3cn, c3cn->state, c3cn->flags);
186
187 c3cn_put_port(c3cn);
188 c3cn_release_offload_resources(c3cn);
189 c3cn_set_state(c3cn, C3CN_STATE_CLOSED);
190 cxgb3i_conn_closing(c3cn);
191}
192
193/*
194 * CPL (Chelsio Protocol Language) defines a message passing interface between
195 * the host driver and T3 asic.
196 * The section below implments CPLs that related to iscsi tcp connection
197 * open/close/abort and data send/receive.
198 */
199
200/*
201 * CPL connection active open request: host ->
202 */
203static unsigned int find_best_mtu(const struct t3c_data *d, unsigned short mtu)
204{
205 int i = 0;
206
207 while (i < d->nmtus - 1 && d->mtus[i + 1] <= mtu)
208 ++i;
209 return i;
210}
211
212static unsigned int select_mss(struct s3_conn *c3cn, unsigned int pmtu)
213{
214 unsigned int idx;
215 struct dst_entry *dst = c3cn->dst_cache;
216 struct t3cdev *cdev = c3cn->cdev;
217 const struct t3c_data *td = T3C_DATA(cdev);
218 u16 advmss = dst_metric(dst, RTAX_ADVMSS);
219
220 if (advmss > pmtu - 40)
221 advmss = pmtu - 40;
222 if (advmss < td->mtus[0] - 40)
223 advmss = td->mtus[0] - 40;
224 idx = find_best_mtu(td, advmss + 40);
225 return idx;
226}
227
228static inline int compute_wscale(int win)
229{
230 int wscale = 0;
231 while (wscale < 14 && (65535<<wscale) < win)
232 wscale++;
233 return wscale;
234}
235
236static inline unsigned int calc_opt0h(struct s3_conn *c3cn)
237{
238 int wscale = compute_wscale(cxgb3_rcv_win);
239 return V_KEEP_ALIVE(1) |
240 F_TCAM_BYPASS |
241 V_WND_SCALE(wscale) |
242 V_MSS_IDX(c3cn->mss_idx);
243}
244
245static inline unsigned int calc_opt0l(struct s3_conn *c3cn)
246{
247 return V_ULP_MODE(ULP_MODE_ISCSI) |
248 V_RCV_BUFSIZ(cxgb3_rcv_win>>10);
249}
250
251static void make_act_open_req(struct s3_conn *c3cn, struct sk_buff *skb,
252 unsigned int atid, const struct l2t_entry *e)
253{
254 struct cpl_act_open_req *req;
255
256 c3cn_conn_debug("c3cn 0x%p, atid 0x%x.\n", c3cn, atid);
257
258 skb->priority = CPL_PRIORITY_SETUP;
259 req = (struct cpl_act_open_req *)__skb_put(skb, sizeof(*req));
260 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
261 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, atid));
262 req->local_port = c3cn->saddr.sin_port;
263 req->peer_port = c3cn->daddr.sin_port;
264 req->local_ip = c3cn->saddr.sin_addr.s_addr;
265 req->peer_ip = c3cn->daddr.sin_addr.s_addr;
266 req->opt0h = htonl(calc_opt0h(c3cn) | V_L2T_IDX(e->idx) |
267 V_TX_CHANNEL(e->smt_idx));
268 req->opt0l = htonl(calc_opt0l(c3cn));
269 req->params = 0;
270}
271
272static void fail_act_open(struct s3_conn *c3cn, int errno)
273{
274 c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
275 c3cn, c3cn->state, c3cn->flags);
276 c3cn->err = errno;
277 c3cn_closed(c3cn);
278}
279
280static void act_open_req_arp_failure(struct t3cdev *dev, struct sk_buff *skb)
281{
282 struct s3_conn *c3cn = (struct s3_conn *)skb->sk;
283
284 c3cn_conn_debug("c3cn 0x%p, state %u.\n", c3cn, c3cn->state);
285
286 c3cn_hold(c3cn);
287 spin_lock_bh(&c3cn->lock);
288 if (c3cn->state == C3CN_STATE_CONNECTING)
289 fail_act_open(c3cn, EHOSTUNREACH);
290 spin_unlock_bh(&c3cn->lock);
291 c3cn_put(c3cn);
292 __kfree_skb(skb);
293}
294
295/*
296 * CPL connection close request: host ->
297 *
298 * Close a connection by sending a CPL_CLOSE_CON_REQ message and queue it to
299 * the write queue (i.e., after any unsent txt data).
300 */
301static void skb_entail(struct s3_conn *c3cn, struct sk_buff *skb,
302 int flags)
303{
304 CXGB3_SKB_CB(skb)->seq = c3cn->write_seq;
305 CXGB3_SKB_CB(skb)->flags = flags;
306 __skb_queue_tail(&c3cn->write_queue, skb);
307}
308
309static void send_close_req(struct s3_conn *c3cn)
310{
311 struct sk_buff *skb = c3cn->cpl_close;
312 struct cpl_close_con_req *req = (struct cpl_close_con_req *)skb->head;
313 unsigned int tid = c3cn->tid;
314
315 c3cn_conn_debug("c3cn 0x%p, state 0x%x, flag 0x%lx.\n",
316 c3cn, c3cn->state, c3cn->flags);
317
318 c3cn->cpl_close = NULL;
319
320 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_CLOSE_CON));
321 req->wr.wr_lo = htonl(V_WR_TID(tid));
322 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid));
323 req->rsvd = htonl(c3cn->write_seq);
324
325 skb_entail(c3cn, skb, C3CB_FLAG_NO_APPEND);
326 if (c3cn->state != C3CN_STATE_CONNECTING)
327 c3cn_push_tx_frames(c3cn, 1);
328}
329
330/*
331 * CPL connection abort request: host ->
332 *
333 * Send an ABORT_REQ message. Makes sure we do not send multiple ABORT_REQs
334 * for the same connection and also that we do not try to send a message
335 * after the connection has closed.
336 */
337static void abort_arp_failure(struct t3cdev *cdev, struct sk_buff *skb)
338{
339 struct cpl_abort_req *req = cplhdr(skb);
340
341 c3cn_conn_debug("tdev 0x%p.\n", cdev);
342
343 req->cmd = CPL_ABORT_NO_RST;
344 cxgb3_ofld_send(cdev, skb);
345}
346
347static inline void c3cn_purge_write_queue(struct s3_conn *c3cn)
348{
349 struct sk_buff *skb;
350
351 while ((skb = __skb_dequeue(&c3cn->write_queue)))
352 __kfree_skb(skb);
353}
354
355static void send_abort_req(struct s3_conn *c3cn)
356{
357 struct sk_buff *skb = c3cn->cpl_abort_req;
358 struct cpl_abort_req *req;
359 unsigned int tid = c3cn->tid;
360
361 if (unlikely(c3cn->state == C3CN_STATE_ABORTING) || !skb ||
362 !c3cn->cdev)
363 return;
364
365 c3cn_set_state(c3cn, C3CN_STATE_ABORTING);
366
367 c3cn_conn_debug("c3cn 0x%p, flag ABORT_RPL + ABORT_SHUT.\n", c3cn);
368
369 c3cn_set_flag(c3cn, C3CN_ABORT_RPL_PENDING);
370
371 /* Purge the send queue so we don't send anything after an abort. */
372 c3cn_purge_write_queue(c3cn);
373
374 c3cn->cpl_abort_req = NULL;
375 req = (struct cpl_abort_req *)skb->head;
376
377 skb->priority = CPL_PRIORITY_DATA;
378 set_arp_failure_handler(skb, abort_arp_failure);
379
380 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_REQ));
381 req->wr.wr_lo = htonl(V_WR_TID(tid));
382 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_ABORT_REQ, tid));
383 req->rsvd0 = htonl(c3cn->snd_nxt);
384 req->rsvd1 = !c3cn_flag(c3cn, C3CN_TX_DATA_SENT);
385 req->cmd = CPL_ABORT_SEND_RST;
386
387 l2t_send(c3cn->cdev, skb, c3cn->l2t);
388}
389
390/*
391 * CPL connection abort reply: host ->
392 *
393 * Send an ABORT_RPL message in response of the ABORT_REQ received.
394 */
395static void send_abort_rpl(struct s3_conn *c3cn, int rst_status)
396{
397 struct sk_buff *skb = c3cn->cpl_abort_rpl;
398 struct cpl_abort_rpl *rpl = (struct cpl_abort_rpl *)skb->head;
399
400 c3cn->cpl_abort_rpl = NULL;
401
402 skb->priority = CPL_PRIORITY_DATA;
403 rpl->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_RPL));
404 rpl->wr.wr_lo = htonl(V_WR_TID(c3cn->tid));
405 OPCODE_TID(rpl) = htonl(MK_OPCODE_TID(CPL_ABORT_RPL, c3cn->tid));
406 rpl->cmd = rst_status;
407
408 cxgb3_ofld_send(c3cn->cdev, skb);
409}
410
411/*
412 * CPL connection rx data ack: host ->
413 * Send RX credits through an RX_DATA_ACK CPL message. Returns the number of
414 * credits sent.
415 */
416static u32 send_rx_credits(struct s3_conn *c3cn, u32 credits, u32 dack)
417{
418 struct sk_buff *skb;
419 struct cpl_rx_data_ack *req;
420
421 skb = alloc_skb(sizeof(*req), GFP_ATOMIC);
422 if (!skb)
423 return 0;
424
425 req = (struct cpl_rx_data_ack *)__skb_put(skb, sizeof(*req));
426 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
427 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RX_DATA_ACK, c3cn->tid));
428 req->credit_dack = htonl(dack | V_RX_CREDITS(credits));
429 skb->priority = CPL_PRIORITY_ACK;
430 cxgb3_ofld_send(c3cn->cdev, skb);
431 return credits;
432}
433
434/*
435 * CPL connection tx data: host ->
436 *
437 * Send iscsi PDU via TX_DATA CPL message. Returns the number of
438 * credits sent.
439 * Each TX_DATA consumes work request credit (wrs), so we need to keep track of
440 * how many we've used so far and how many are pending (i.e., yet ack'ed by T3).
441 */
442
443/*
444 * For ULP connections HW may inserts digest bytes into the pdu. Those digest
445 * bytes are not sent by the host but are part of the TCP payload and therefore
446 * consume TCP sequence space.
447 */
448static const unsigned int cxgb3_ulp_extra_len[] = { 0, 4, 4, 8 };
449static inline unsigned int ulp_extra_len(const struct sk_buff *skb)
450{
451 return cxgb3_ulp_extra_len[skb_ulp_mode(skb) & 3];
452}
453
454static unsigned int wrlen __read_mostly;
455
456/*
457 * The number of WRs needed for an skb depends on the number of fragments
458 * in the skb and whether it has any payload in its main body. This maps the
459 * length of the gather list represented by an skb into the # of necessary WRs.
460 *
461 * The max. length of an skb is controlled by the max pdu size which is ~16K.
462 * Also, assume the min. fragment length is the sector size (512), then add
463 * extra fragment counts for iscsi bhs and payload padding.
464 */
465#define SKB_WR_LIST_SIZE (16384/512 + 3)
466static unsigned int skb_wrs[SKB_WR_LIST_SIZE] __read_mostly;
467
468static void s3_init_wr_tab(unsigned int wr_len)
469{
470 int i;
471
472 if (skb_wrs[1]) /* already initialized */
473 return;
474
475 for (i = 1; i < SKB_WR_LIST_SIZE; i++) {
476 int sgl_len = (3 * i) / 2 + (i & 1);
477
478 sgl_len += 3;
479 skb_wrs[i] = (sgl_len <= wr_len
480 ? 1 : 1 + (sgl_len - 2) / (wr_len - 1));
481 }
482
483 wrlen = wr_len * 8;
484}
485
486static inline void reset_wr_list(struct s3_conn *c3cn)
487{
488 c3cn->wr_pending_head = NULL;
489}
490
491/*
492 * Add a WR to a connections's list of pending WRs. This is a singly-linked
493 * list of sk_buffs operating as a FIFO. The head is kept in wr_pending_head
494 * and the tail in wr_pending_tail.
495 */
496static inline void enqueue_wr(struct s3_conn *c3cn,
497 struct sk_buff *skb)
498{
499 skb->sp = NULL;
500
501 /*
502 * We want to take an extra reference since both us and the driver
503 * need to free the packet before it's really freed. We know there's
504 * just one user currently so we use atomic_set rather than skb_get
505 * to avoid the atomic op.
506 */
507 atomic_set(&skb->users, 2);
508
509 if (!c3cn->wr_pending_head)
510 c3cn->wr_pending_head = skb;
511 else
512 c3cn->wr_pending_tail->sp = (void *)skb;
513 c3cn->wr_pending_tail = skb;
514}
515
516static inline struct sk_buff *peek_wr(const struct s3_conn *c3cn)
517{
518 return c3cn->wr_pending_head;
519}
520
521static inline void free_wr_skb(struct sk_buff *skb)
522{
523 kfree_skb(skb);
524}
525
526static inline struct sk_buff *dequeue_wr(struct s3_conn *c3cn)
527{
528 struct sk_buff *skb = c3cn->wr_pending_head;
529
530 if (likely(skb)) {
531 /* Don't bother clearing the tail */
532 c3cn->wr_pending_head = (struct sk_buff *)skb->sp;
533 skb->sp = NULL;
534 }
535 return skb;
536}
537
538static void purge_wr_queue(struct s3_conn *c3cn)
539{
540 struct sk_buff *skb;
541 while ((skb = dequeue_wr(c3cn)) != NULL)
542 free_wr_skb(skb);
543}
544
545static inline void make_tx_data_wr(struct s3_conn *c3cn, struct sk_buff *skb,
546 int len)
547{
548 struct tx_data_wr *req;
549
550 skb_reset_transport_header(skb);
551 req = (struct tx_data_wr *)__skb_push(skb, sizeof(*req));
552 req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_OFLD_TX_DATA));
553 req->wr_lo = htonl(V_WR_TID(c3cn->tid));
554 req->sndseq = htonl(c3cn->snd_nxt);
555 /* len includes the length of any HW ULP additions */
556 req->len = htonl(len);
557 req->param = htonl(V_TX_PORT(c3cn->l2t->smt_idx));
558 /* V_TX_ULP_SUBMODE sets both the mode and submode */
559 req->flags = htonl(V_TX_ULP_SUBMODE(skb_ulp_mode(skb)) |
560 V_TX_SHOVE((skb_peek(&c3cn->write_queue) ? 0 : 1)));
561
562 if (!c3cn_flag(c3cn, C3CN_TX_DATA_SENT)) {
563 req->flags |= htonl(V_TX_ACK_PAGES(2) | F_TX_INIT |
564 V_TX_CPU_IDX(c3cn->qset));
565 /* Sendbuffer is in units of 32KB. */
566 req->param |= htonl(V_TX_SNDBUF(cxgb3_snd_win >> 15));
567 c3cn_set_flag(c3cn, C3CN_TX_DATA_SENT);
568 }
569}
570
571/**
572 * c3cn_push_tx_frames -- start transmit
573 * @c3cn: the offloaded connection
574 * @req_completion: request wr_ack or not
575 *
576 * Prepends TX_DATA_WR or CPL_CLOSE_CON_REQ headers to buffers waiting in a
577 * connection's send queue and sends them on to T3. Must be called with the
578 * connection's lock held. Returns the amount of send buffer space that was
579 * freed as a result of sending queued data to T3.
580 */
581static void arp_failure_discard(struct t3cdev *cdev, struct sk_buff *skb)
582{
583 kfree_skb(skb);
584}
585
586static int c3cn_push_tx_frames(struct s3_conn *c3cn, int req_completion)
587{
588 int total_size = 0;
589 struct sk_buff *skb;
590 struct t3cdev *cdev;
591 struct cxgb3i_sdev_data *cdata;
592
593 if (unlikely(c3cn->state == C3CN_STATE_CONNECTING ||
594 c3cn->state == C3CN_STATE_CLOSE_WAIT_1 ||
595 c3cn->state == C3CN_STATE_ABORTING)) {
596 c3cn_tx_debug("c3cn 0x%p, in closing state %u.\n",
597 c3cn, c3cn->state);
598 return 0;
599 }
600
601 cdev = c3cn->cdev;
602 cdata = CXGB3_SDEV_DATA(cdev);
603
604 while (c3cn->wr_avail
605 && (skb = skb_peek(&c3cn->write_queue)) != NULL) {
606 int len = skb->len; /* length before skb_push */
607 int frags = skb_shinfo(skb)->nr_frags + (len != skb->data_len);
608 int wrs_needed = skb_wrs[frags];
609
610 if (wrs_needed > 1 && len + sizeof(struct tx_data_wr) <= wrlen)
611 wrs_needed = 1;
612
613 WARN_ON(frags >= SKB_WR_LIST_SIZE || wrs_needed < 1);
614
615 if (c3cn->wr_avail < wrs_needed) {
616 c3cn_tx_debug("c3cn 0x%p, skb len %u/%u, frag %u, "
617 "wr %d < %u.\n",
618 c3cn, skb->len, skb->datalen, frags,
619 wrs_needed, c3cn->wr_avail);
620 break;
621 }
622
623 __skb_unlink(skb, &c3cn->write_queue);
624 skb->priority = CPL_PRIORITY_DATA;
625 skb->csum = wrs_needed; /* remember this until the WR_ACK */
626 c3cn->wr_avail -= wrs_needed;
627 c3cn->wr_unacked += wrs_needed;
628 enqueue_wr(c3cn, skb);
629
630 if (likely(CXGB3_SKB_CB(skb)->flags & C3CB_FLAG_NEED_HDR)) {
631 len += ulp_extra_len(skb);
632 make_tx_data_wr(c3cn, skb, len);
633 c3cn->snd_nxt += len;
634 if ((req_completion
635 && c3cn->wr_unacked == wrs_needed)
636 || (CXGB3_SKB_CB(skb)->flags & C3CB_FLAG_COMPL)
637 || c3cn->wr_unacked >= c3cn->wr_max / 2) {
638 struct work_request_hdr *wr = cplhdr(skb);
639
640 wr->wr_hi |= htonl(F_WR_COMPL);
641 c3cn->wr_unacked = 0;
642 }
643 CXGB3_SKB_CB(skb)->flags &= ~C3CB_FLAG_NEED_HDR;
644 }
645
646 total_size += skb->truesize;
647 set_arp_failure_handler(skb, arp_failure_discard);
648 l2t_send(cdev, skb, c3cn->l2t);
649 }
650 return total_size;
651}
652
653/*
654 * process_cpl_msg: -> host
655 * Top-level CPL message processing used by most CPL messages that
656 * pertain to connections.
657 */
658static inline void process_cpl_msg(void (*fn)(struct s3_conn *,
659 struct sk_buff *),
660 struct s3_conn *c3cn,
661 struct sk_buff *skb)
662{
663 spin_lock_bh(&c3cn->lock);
664 fn(c3cn, skb);
665 spin_unlock_bh(&c3cn->lock);
666}
667
668/*
669 * process_cpl_msg_ref: -> host
670 * Similar to process_cpl_msg() but takes an extra connection reference around
671 * the call to the handler. Should be used if the handler may drop a
672 * connection reference.
673 */
674static inline void process_cpl_msg_ref(void (*fn) (struct s3_conn *,
675 struct sk_buff *),
676 struct s3_conn *c3cn,
677 struct sk_buff *skb)
678{
679 c3cn_hold(c3cn);
680 process_cpl_msg(fn, c3cn, skb);
681 c3cn_put(c3cn);
682}
683
684/*
685 * Process a CPL_ACT_ESTABLISH message: -> host
686 * Updates connection state from an active establish CPL message. Runs with
687 * the connection lock held.
688 */
689
690static inline void s3_free_atid(struct t3cdev *cdev, unsigned int tid)
691{
692 struct s3_conn *c3cn = cxgb3_free_atid(cdev, tid);
693 if (c3cn)
694 c3cn_put(c3cn);
695}
696
697static void c3cn_established(struct s3_conn *c3cn, u32 snd_isn,
698 unsigned int opt)
699{
700 c3cn_conn_debug("c3cn 0x%p, state %u.\n", c3cn, c3cn->state);
701
702 c3cn->write_seq = c3cn->snd_nxt = c3cn->snd_una = snd_isn;
703
704 /*
705 * Causes the first RX_DATA_ACK to supply any Rx credits we couldn't
706 * pass through opt0.
707 */
708 if (cxgb3_rcv_win > (M_RCV_BUFSIZ << 10))
709 c3cn->rcv_wup -= cxgb3_rcv_win - (M_RCV_BUFSIZ << 10);
710
711 dst_confirm(c3cn->dst_cache);
712
713 smp_mb();
714
715 c3cn_set_state(c3cn, C3CN_STATE_ESTABLISHED);
716}
717
718static void process_act_establish(struct s3_conn *c3cn, struct sk_buff *skb)
719{
720 struct cpl_act_establish *req = cplhdr(skb);
721 u32 rcv_isn = ntohl(req->rcv_isn); /* real RCV_ISN + 1 */
722
723 c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
724 c3cn, c3cn->state, c3cn->flags);
725
726 if (unlikely(c3cn->state != C3CN_STATE_CONNECTING))
727 cxgb3i_log_error("TID %u expected SYN_SENT, got EST., s %u\n",
728 c3cn->tid, c3cn->state);
729
730 c3cn->copied_seq = c3cn->rcv_wup = c3cn->rcv_nxt = rcv_isn;
731 c3cn_established(c3cn, ntohl(req->snd_isn), ntohs(req->tcp_opt));
732
733 __kfree_skb(skb);
734
735 if (unlikely(c3cn_flag(c3cn, C3CN_ACTIVE_CLOSE_NEEDED)))
736 /* upper layer has requested closing */
737 send_abort_req(c3cn);
738 else if (c3cn_push_tx_frames(c3cn, 1))
739 cxgb3i_conn_tx_open(c3cn);
740}
741
742static int do_act_establish(struct t3cdev *cdev, struct sk_buff *skb,
743 void *ctx)
744{
745 struct cpl_act_establish *req = cplhdr(skb);
746 unsigned int tid = GET_TID(req);
747 unsigned int atid = G_PASS_OPEN_TID(ntohl(req->tos_tid));
748 struct s3_conn *c3cn = ctx;
749 struct cxgb3i_sdev_data *cdata = CXGB3_SDEV_DATA(cdev);
750
751 c3cn_conn_debug("rcv, tid 0x%x, c3cn 0x%p, s %u, f 0x%lx.\n",
752 tid, c3cn, c3cn->state, c3cn->flags);
753
754 c3cn->tid = tid;
755 c3cn_hold(c3cn);
756 cxgb3_insert_tid(cdata->cdev, cdata->client, c3cn, tid);
757 s3_free_atid(cdev, atid);
758
759 c3cn->qset = G_QNUM(ntohl(skb->csum));
760
761 process_cpl_msg(process_act_establish, c3cn, skb);
762 return 0;
763}
764
765/*
766 * Process a CPL_ACT_OPEN_RPL message: -> host
767 * Handle active open failures.
768 */
769static int act_open_rpl_status_to_errno(int status)
770{
771 switch (status) {
772 case CPL_ERR_CONN_RESET:
773 return ECONNREFUSED;
774 case CPL_ERR_ARP_MISS:
775 return EHOSTUNREACH;
776 case CPL_ERR_CONN_TIMEDOUT:
777 return ETIMEDOUT;
778 case CPL_ERR_TCAM_FULL:
779 return ENOMEM;
780 case CPL_ERR_CONN_EXIST:
781 cxgb3i_log_error("ACTIVE_OPEN_RPL: 4-tuple in use\n");
782 return EADDRINUSE;
783 default:
784 return EIO;
785 }
786}
787
788static void act_open_retry_timer(unsigned long data)
789{
790 struct sk_buff *skb;
791 struct s3_conn *c3cn = (struct s3_conn *)data;
792
793 c3cn_conn_debug("c3cn 0x%p, state %u.\n", c3cn, c3cn->state);
794
795 spin_lock_bh(&c3cn->lock);
796 skb = alloc_skb(sizeof(struct cpl_act_open_req), GFP_ATOMIC);
797 if (!skb)
798 fail_act_open(c3cn, ENOMEM);
799 else {
800 skb->sk = (struct sock *)c3cn;
801 set_arp_failure_handler(skb, act_open_req_arp_failure);
802 make_act_open_req(c3cn, skb, c3cn->tid, c3cn->l2t);
803 l2t_send(c3cn->cdev, skb, c3cn->l2t);
804 }
805 spin_unlock_bh(&c3cn->lock);
806 c3cn_put(c3cn);
807}
808
809static void process_act_open_rpl(struct s3_conn *c3cn, struct sk_buff *skb)
810{
811 struct cpl_act_open_rpl *rpl = cplhdr(skb);
812
813 c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
814 c3cn, c3cn->state, c3cn->flags);
815
816 if (rpl->status == CPL_ERR_CONN_EXIST &&
817 c3cn->retry_timer.function != act_open_retry_timer) {
818 c3cn->retry_timer.function = act_open_retry_timer;
819 if (!mod_timer(&c3cn->retry_timer, jiffies + HZ / 2))
820 c3cn_hold(c3cn);
821 } else
822 fail_act_open(c3cn, act_open_rpl_status_to_errno(rpl->status));
823 __kfree_skb(skb);
824}
825
826static int do_act_open_rpl(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
827{
828 struct s3_conn *c3cn = ctx;
829 struct cpl_act_open_rpl *rpl = cplhdr(skb);
830
831 c3cn_conn_debug("rcv, status 0x%x, c3cn 0x%p, s %u, f 0x%lx.\n",
832 rpl->status, c3cn, c3cn->state, c3cn->flags);
833
834 if (rpl->status != CPL_ERR_TCAM_FULL &&
835 rpl->status != CPL_ERR_CONN_EXIST &&
836 rpl->status != CPL_ERR_ARP_MISS)
837 cxgb3_queue_tid_release(cdev, GET_TID(rpl));
838
839 process_cpl_msg_ref(process_act_open_rpl, c3cn, skb);
840 return 0;
841}
842
843/*
844 * Process PEER_CLOSE CPL messages: -> host
845 * Handle peer FIN.
846 */
847static void process_peer_close(struct s3_conn *c3cn, struct sk_buff *skb)
848{
849 c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
850 c3cn, c3cn->state, c3cn->flags);
851
852 if (c3cn_flag(c3cn, C3CN_ABORT_RPL_PENDING))
853 goto out;
854
855 switch (c3cn->state) {
856 case C3CN_STATE_ESTABLISHED:
857 c3cn_set_state(c3cn, C3CN_STATE_PASSIVE_CLOSE);
858 break;
859 case C3CN_STATE_ACTIVE_CLOSE:
860 c3cn_set_state(c3cn, C3CN_STATE_CLOSE_WAIT_2);
861 break;
862 case C3CN_STATE_CLOSE_WAIT_1:
863 c3cn_closed(c3cn);
864 break;
865 case C3CN_STATE_ABORTING:
866 break;
867 default:
868 cxgb3i_log_error("%s: peer close, TID %u in bad state %u\n",
869 c3cn->cdev->name, c3cn->tid, c3cn->state);
870 }
871
872 cxgb3i_conn_closing(c3cn);
873out:
874 __kfree_skb(skb);
875}
876
877static int do_peer_close(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
878{
879 struct s3_conn *c3cn = ctx;
880
881 c3cn_conn_debug("rcv, c3cn 0x%p, s %u, f 0x%lx.\n",
882 c3cn, c3cn->state, c3cn->flags);
883 process_cpl_msg_ref(process_peer_close, c3cn, skb);
884 return 0;
885}
886
887/*
888 * Process CLOSE_CONN_RPL CPL message: -> host
889 * Process a peer ACK to our FIN.
890 */
891static void process_close_con_rpl(struct s3_conn *c3cn, struct sk_buff *skb)
892{
893 struct cpl_close_con_rpl *rpl = cplhdr(skb);
894
895 c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
896 c3cn, c3cn->state, c3cn->flags);
897
898 c3cn->snd_una = ntohl(rpl->snd_nxt) - 1; /* exclude FIN */
899
900 if (c3cn_flag(c3cn, C3CN_ABORT_RPL_PENDING))
901 goto out;
902
903 switch (c3cn->state) {
904 case C3CN_STATE_ACTIVE_CLOSE:
905 c3cn_set_state(c3cn, C3CN_STATE_CLOSE_WAIT_1);
906 break;
907 case C3CN_STATE_CLOSE_WAIT_1:
908 case C3CN_STATE_CLOSE_WAIT_2:
909 c3cn_closed(c3cn);
910 break;
911 case C3CN_STATE_ABORTING:
912 break;
913 default:
914 cxgb3i_log_error("%s: close_rpl, TID %u in bad state %u\n",
915 c3cn->cdev->name, c3cn->tid, c3cn->state);
916 }
917
918out:
919 kfree_skb(skb);
920}
921
922static int do_close_con_rpl(struct t3cdev *cdev, struct sk_buff *skb,
923 void *ctx)
924{
925 struct s3_conn *c3cn = ctx;
926
927 c3cn_conn_debug("rcv, c3cn 0x%p, s %u, f 0x%lx.\n",
928 c3cn, c3cn->state, c3cn->flags);
929
930 process_cpl_msg_ref(process_close_con_rpl, c3cn, skb);
931 return 0;
932}
933
934/*
935 * Process ABORT_REQ_RSS CPL message: -> host
936 * Process abort requests. If we are waiting for an ABORT_RPL we ignore this
937 * request except that we need to reply to it.
938 */
939
940static int abort_status_to_errno(struct s3_conn *c3cn, int abort_reason,
941 int *need_rst)
942{
943 switch (abort_reason) {
944 case CPL_ERR_BAD_SYN: /* fall through */
945 case CPL_ERR_CONN_RESET:
946 return c3cn->state > C3CN_STATE_ESTABLISHED ?
947 EPIPE : ECONNRESET;
948 case CPL_ERR_XMIT_TIMEDOUT:
949 case CPL_ERR_PERSIST_TIMEDOUT:
950 case CPL_ERR_FINWAIT2_TIMEDOUT:
951 case CPL_ERR_KEEPALIVE_TIMEDOUT:
952 return ETIMEDOUT;
953 default:
954 return EIO;
955 }
956}
957
958static void process_abort_req(struct s3_conn *c3cn, struct sk_buff *skb)
959{
960 int rst_status = CPL_ABORT_NO_RST;
961 const struct cpl_abort_req_rss *req = cplhdr(skb);
962
963 c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
964 c3cn, c3cn->state, c3cn->flags);
965
966 if (!c3cn_flag(c3cn, C3CN_ABORT_REQ_RCVD)) {
967 c3cn_set_flag(c3cn, C3CN_ABORT_REQ_RCVD);
968 c3cn_set_state(c3cn, C3CN_STATE_ABORTING);
969 __kfree_skb(skb);
970 return;
971 }
972
973 c3cn_clear_flag(c3cn, C3CN_ABORT_REQ_RCVD);
974 send_abort_rpl(c3cn, rst_status);
975
976 if (!c3cn_flag(c3cn, C3CN_ABORT_RPL_PENDING)) {
977 c3cn->err =
978 abort_status_to_errno(c3cn, req->status, &rst_status);
979 c3cn_closed(c3cn);
980 }
981}
982
983static int do_abort_req(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
984{
985 const struct cpl_abort_req_rss *req = cplhdr(skb);
986 struct s3_conn *c3cn = ctx;
987
988 c3cn_conn_debug("rcv, c3cn 0x%p, s 0x%x, f 0x%lx.\n",
989 c3cn, c3cn->state, c3cn->flags);
990
991 if (req->status == CPL_ERR_RTX_NEG_ADVICE ||
992 req->status == CPL_ERR_PERSIST_NEG_ADVICE) {
993 __kfree_skb(skb);
994 return 0;
995 }
996
997 process_cpl_msg_ref(process_abort_req, c3cn, skb);
998 return 0;
999}
1000
1001/*
1002 * Process ABORT_RPL_RSS CPL message: -> host
1003 * Process abort replies. We only process these messages if we anticipate
1004 * them as the coordination between SW and HW in this area is somewhat lacking
1005 * and sometimes we get ABORT_RPLs after we are done with the connection that
1006 * originated the ABORT_REQ.
1007 */
1008static void process_abort_rpl(struct s3_conn *c3cn, struct sk_buff *skb)
1009{
1010 c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
1011 c3cn, c3cn->state, c3cn->flags);
1012
1013 if (c3cn_flag(c3cn, C3CN_ABORT_RPL_PENDING)) {
1014 if (!c3cn_flag(c3cn, C3CN_ABORT_RPL_RCVD))
1015 c3cn_set_flag(c3cn, C3CN_ABORT_RPL_RCVD);
1016 else {
1017 c3cn_clear_flag(c3cn, C3CN_ABORT_RPL_RCVD);
1018 c3cn_clear_flag(c3cn, C3CN_ABORT_RPL_PENDING);
1019 if (c3cn_flag(c3cn, C3CN_ABORT_REQ_RCVD))
1020 cxgb3i_log_error("%s tid %u, ABORT_RPL_RSS\n",
1021 c3cn->cdev->name, c3cn->tid);
1022 c3cn_closed(c3cn);
1023 }
1024 }
1025 __kfree_skb(skb);
1026}
1027
1028static int do_abort_rpl(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
1029{
1030 struct cpl_abort_rpl_rss *rpl = cplhdr(skb);
1031 struct s3_conn *c3cn = ctx;
1032
1033 c3cn_conn_debug("rcv, status 0x%x, c3cn 0x%p, s %u, 0x%lx.\n",
1034 rpl->status, c3cn, c3cn ? c3cn->state : 0,
1035 c3cn ? c3cn->flags : 0UL);
1036
1037 /*
1038 * Ignore replies to post-close aborts indicating that the abort was
1039 * requested too late. These connections are terminated when we get
1040 * PEER_CLOSE or CLOSE_CON_RPL and by the time the abort_rpl_rss
1041 * arrives the TID is either no longer used or it has been recycled.
1042 */
1043 if (rpl->status == CPL_ERR_ABORT_FAILED)
1044 goto discard;
1045
1046 /*
1047 * Sometimes we've already closed the connection, e.g., a post-close
1048 * abort races with ABORT_REQ_RSS, the latter frees the connection
1049 * expecting the ABORT_REQ will fail with CPL_ERR_ABORT_FAILED,
1050 * but FW turns the ABORT_REQ into a regular one and so we get
1051 * ABORT_RPL_RSS with status 0 and no connection.
1052 */
1053 if (!c3cn)
1054 goto discard;
1055
1056 process_cpl_msg_ref(process_abort_rpl, c3cn, skb);
1057 return 0;
1058
1059discard:
1060 __kfree_skb(skb);
1061 return 0;
1062}
1063
1064/*
1065 * Process RX_ISCSI_HDR CPL message: -> host
1066 * Handle received PDUs, the payload could be DDP'ed. If not, the payload
1067 * follow after the bhs.
1068 */
1069static void process_rx_iscsi_hdr(struct s3_conn *c3cn, struct sk_buff *skb)
1070{
1071 struct cpl_iscsi_hdr *hdr_cpl = cplhdr(skb);
1072 struct cpl_iscsi_hdr_norss data_cpl;
1073 struct cpl_rx_data_ddp_norss ddp_cpl;
1074 unsigned int hdr_len, data_len, status;
1075 unsigned int len;
1076 int err;
1077
1078 if (unlikely(c3cn->state >= C3CN_STATE_PASSIVE_CLOSE)) {
1079 if (c3cn->state != C3CN_STATE_ABORTING)
1080 send_abort_req(c3cn);
1081 __kfree_skb(skb);
1082 return;
1083 }
1084
1085 CXGB3_SKB_CB(skb)->seq = ntohl(hdr_cpl->seq);
1086 CXGB3_SKB_CB(skb)->flags = 0;
1087
1088 skb_reset_transport_header(skb);
1089 __skb_pull(skb, sizeof(struct cpl_iscsi_hdr));
1090
1091 len = hdr_len = ntohs(hdr_cpl->len);
1092 /* msg coalesce is off or not enough data received */
1093 if (skb->len <= hdr_len) {
1094 cxgb3i_log_error("%s: TID %u, ISCSI_HDR, skb len %u < %u.\n",
1095 c3cn->cdev->name, c3cn->tid,
1096 skb->len, hdr_len);
1097 goto abort_conn;
1098 }
1099
1100 err = skb_copy_bits(skb, skb->len - sizeof(ddp_cpl), &ddp_cpl,
1101 sizeof(ddp_cpl));
1102 if (err < 0)
1103 goto abort_conn;
1104
1105 skb_ulp_mode(skb) = ULP2_FLAG_DATA_READY;
1106 skb_ulp_pdulen(skb) = ntohs(ddp_cpl.len);
1107 skb_ulp_ddigest(skb) = ntohl(ddp_cpl.ulp_crc);
1108 status = ntohl(ddp_cpl.ddp_status);
1109
1110 c3cn_rx_debug("rx skb 0x%p, len %u, pdulen %u, ddp status 0x%x.\n",
1111 skb, skb->len, skb_ulp_pdulen(skb), status);
1112
1113 if (status & (1 << RX_DDP_STATUS_HCRC_SHIFT))
1114 skb_ulp_mode(skb) |= ULP2_FLAG_HCRC_ERROR;
1115 if (status & (1 << RX_DDP_STATUS_DCRC_SHIFT))
1116 skb_ulp_mode(skb) |= ULP2_FLAG_DCRC_ERROR;
1117 if (status & (1 << RX_DDP_STATUS_PAD_SHIFT))
1118 skb_ulp_mode(skb) |= ULP2_FLAG_PAD_ERROR;
1119
1120 if (skb->len > (hdr_len + sizeof(ddp_cpl))) {
1121 err = skb_copy_bits(skb, hdr_len, &data_cpl, sizeof(data_cpl));
1122 if (err < 0)
1123 goto abort_conn;
1124 data_len = ntohs(data_cpl.len);
1125 len += sizeof(data_cpl) + data_len;
1126 } else if (status & (1 << RX_DDP_STATUS_DDP_SHIFT))
1127 skb_ulp_mode(skb) |= ULP2_FLAG_DATA_DDPED;
1128
1129 c3cn->rcv_nxt = ntohl(ddp_cpl.seq) + skb_ulp_pdulen(skb);
1130 __pskb_trim(skb, len);
1131 __skb_queue_tail(&c3cn->receive_queue, skb);
1132 cxgb3i_conn_pdu_ready(c3cn);
1133
1134 return;
1135
1136abort_conn:
1137 send_abort_req(c3cn);
1138 __kfree_skb(skb);
1139}
1140
1141static int do_iscsi_hdr(struct t3cdev *t3dev, struct sk_buff *skb, void *ctx)
1142{
1143 struct s3_conn *c3cn = ctx;
1144
1145 process_cpl_msg(process_rx_iscsi_hdr, c3cn, skb);
1146 return 0;
1147}
1148
1149/*
1150 * Process TX_DATA_ACK CPL messages: -> host
1151 * Process an acknowledgment of WR completion. Advance snd_una and send the
1152 * next batch of work requests from the write queue.
1153 */
1154static void process_wr_ack(struct s3_conn *c3cn, struct sk_buff *skb)
1155{
1156 struct cpl_wr_ack *hdr = cplhdr(skb);
1157 unsigned int credits = ntohs(hdr->credits);
1158 u32 snd_una = ntohl(hdr->snd_una);
1159
1160 c3cn->wr_avail += credits;
1161 if (c3cn->wr_unacked > c3cn->wr_max - c3cn->wr_avail)
1162 c3cn->wr_unacked = c3cn->wr_max - c3cn->wr_avail;
1163
1164 while (credits) {
1165 struct sk_buff *p = peek_wr(c3cn);
1166
1167 if (unlikely(!p)) {
1168 cxgb3i_log_error("%u WR_ACK credits for TID %u with "
1169 "nothing pending, state %u\n",
1170 credits, c3cn->tid, c3cn->state);
1171 break;
1172 }
1173 if (unlikely(credits < p->csum)) {
1174 p->csum -= credits;
1175 break;
1176 } else {
1177 dequeue_wr(c3cn);
1178 credits -= p->csum;
1179 free_wr_skb(p);
1180 }
1181 }
1182
1183 if (unlikely(before(snd_una, c3cn->snd_una)))
1184 goto out_free;
1185
1186 if (c3cn->snd_una != snd_una) {
1187 c3cn->snd_una = snd_una;
1188 dst_confirm(c3cn->dst_cache);
1189 }
1190
1191 if (skb_queue_len(&c3cn->write_queue) && c3cn_push_tx_frames(c3cn, 0))
1192 cxgb3i_conn_tx_open(c3cn);
1193out_free:
1194 __kfree_skb(skb);
1195}
1196
1197static int do_wr_ack(struct t3cdev *cdev, struct sk_buff *skb, void *ctx)
1198{
1199 struct s3_conn *c3cn = ctx;
1200
1201 process_cpl_msg(process_wr_ack, c3cn, skb);
1202 return 0;
1203}
1204
1205/*
1206 * for each connection, pre-allocate skbs needed for close/abort requests. So
1207 * that we can service the request right away.
1208 */
1209static void c3cn_free_cpl_skbs(struct s3_conn *c3cn)
1210{
1211 if (c3cn->cpl_close)
1212 kfree_skb(c3cn->cpl_close);
1213 if (c3cn->cpl_abort_req)
1214 kfree_skb(c3cn->cpl_abort_req);
1215 if (c3cn->cpl_abort_rpl)
1216 kfree_skb(c3cn->cpl_abort_rpl);
1217}
1218
1219static int c3cn_alloc_cpl_skbs(struct s3_conn *c3cn)
1220{
1221 c3cn->cpl_close = alloc_skb(sizeof(struct cpl_close_con_req),
1222 GFP_KERNEL);
1223 if (!c3cn->cpl_close)
1224 return -ENOMEM;
1225 skb_put(c3cn->cpl_close, sizeof(struct cpl_close_con_req));
1226
1227 c3cn->cpl_abort_req = alloc_skb(sizeof(struct cpl_abort_req),
1228 GFP_KERNEL);
1229 if (!c3cn->cpl_abort_req)
1230 goto free_cpl_skbs;
1231 skb_put(c3cn->cpl_abort_req, sizeof(struct cpl_abort_req));
1232
1233 c3cn->cpl_abort_rpl = alloc_skb(sizeof(struct cpl_abort_rpl),
1234 GFP_KERNEL);
1235 if (!c3cn->cpl_abort_rpl)
1236 goto free_cpl_skbs;
1237 skb_put(c3cn->cpl_abort_rpl, sizeof(struct cpl_abort_rpl));
1238
1239 return 0;
1240
1241free_cpl_skbs:
1242 c3cn_free_cpl_skbs(c3cn);
1243 return -ENOMEM;
1244}
1245
1246/**
1247 * c3cn_release_offload_resources - release offload resource
1248 * @c3cn: the offloaded iscsi tcp connection.
1249 * Release resources held by an offload connection (TID, L2T entry, etc.)
1250 */
1251static void c3cn_release_offload_resources(struct s3_conn *c3cn)
1252{
1253 struct t3cdev *cdev = c3cn->cdev;
1254 unsigned int tid = c3cn->tid;
1255
1256 if (!cdev)
1257 return;
1258
1259 c3cn->qset = 0;
1260
1261 c3cn_free_cpl_skbs(c3cn);
1262
1263 if (c3cn->wr_avail != c3cn->wr_max) {
1264 purge_wr_queue(c3cn);
1265 reset_wr_list(c3cn);
1266 }
1267
1268 if (c3cn->l2t) {
1269 l2t_release(L2DATA(cdev), c3cn->l2t);
1270 c3cn->l2t = NULL;
1271 }
1272
1273 if (c3cn->state == C3CN_STATE_CONNECTING) /* we have ATID */
1274 s3_free_atid(cdev, tid);
1275 else { /* we have TID */
1276 cxgb3_remove_tid(cdev, (void *)c3cn, tid);
1277 c3cn_put(c3cn);
1278 }
1279
1280 c3cn->cdev = NULL;
1281}
1282
1283/**
1284 * cxgb3i_c3cn_create - allocate and initialize an s3_conn structure
1285 * returns the s3_conn structure allocated.
1286 */
1287struct s3_conn *cxgb3i_c3cn_create(void)
1288{
1289 struct s3_conn *c3cn;
1290
1291 c3cn = kzalloc(sizeof(*c3cn), GFP_KERNEL);
1292 if (!c3cn)
1293 return NULL;
1294
1295 /* pre-allocate close/abort cpl, so we don't need to wait for memory
1296 when close/abort is requested. */
1297 if (c3cn_alloc_cpl_skbs(c3cn) < 0)
1298 goto free_c3cn;
1299
1300 c3cn_conn_debug("alloc c3cn 0x%p.\n", c3cn);
1301
1302 c3cn->flags = 0;
1303 spin_lock_init(&c3cn->lock);
1304 atomic_set(&c3cn->refcnt, 1);
1305 skb_queue_head_init(&c3cn->receive_queue);
1306 skb_queue_head_init(&c3cn->write_queue);
1307 setup_timer(&c3cn->retry_timer, NULL, (unsigned long)c3cn);
1308 rwlock_init(&c3cn->callback_lock);
1309
1310 return c3cn;
1311
1312free_c3cn:
1313 kfree(c3cn);
1314 return NULL;
1315}
1316
1317static void c3cn_active_close(struct s3_conn *c3cn)
1318{
1319 int data_lost;
1320 int close_req = 0;
1321
1322 c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
1323 c3cn, c3cn->state, c3cn->flags);
1324
1325 dst_confirm(c3cn->dst_cache);
1326
1327 c3cn_hold(c3cn);
1328 spin_lock_bh(&c3cn->lock);
1329
1330 data_lost = skb_queue_len(&c3cn->receive_queue);
1331 __skb_queue_purge(&c3cn->receive_queue);
1332
1333 switch (c3cn->state) {
1334 case C3CN_STATE_CLOSED:
1335 case C3CN_STATE_ACTIVE_CLOSE:
1336 case C3CN_STATE_CLOSE_WAIT_1:
1337 case C3CN_STATE_CLOSE_WAIT_2:
1338 case C3CN_STATE_ABORTING:
1339 /* nothing need to be done */
1340 break;
1341 case C3CN_STATE_CONNECTING:
1342 /* defer until cpl_act_open_rpl or cpl_act_establish */
1343 c3cn_set_flag(c3cn, C3CN_ACTIVE_CLOSE_NEEDED);
1344 break;
1345 case C3CN_STATE_ESTABLISHED:
1346 close_req = 1;
1347 c3cn_set_state(c3cn, C3CN_STATE_ACTIVE_CLOSE);
1348 break;
1349 case C3CN_STATE_PASSIVE_CLOSE:
1350 close_req = 1;
1351 c3cn_set_state(c3cn, C3CN_STATE_CLOSE_WAIT_2);
1352 break;
1353 }
1354
1355 if (close_req) {
1356 if (data_lost)
1357 /* Unread data was tossed, zap the connection. */
1358 send_abort_req(c3cn);
1359 else
1360 send_close_req(c3cn);
1361 }
1362
1363 spin_unlock_bh(&c3cn->lock);
1364 c3cn_put(c3cn);
1365}
1366
1367/**
1368 * cxgb3i_c3cn_release - close and release an iscsi tcp connection and any
1369 * resource held
1370 * @c3cn: the iscsi tcp connection
1371 */
1372void cxgb3i_c3cn_release(struct s3_conn *c3cn)
1373{
1374 c3cn_conn_debug("c3cn 0x%p, s %u, f 0x%lx.\n",
1375 c3cn, c3cn->state, c3cn->flags);
1376 if (likely(c3cn->state != C3CN_STATE_CONNECTING))
1377 c3cn_active_close(c3cn);
1378 else
1379 c3cn_set_flag(c3cn, C3CN_ACTIVE_CLOSE_NEEDED);
1380 c3cn_put(c3cn);
1381}
1382
1383static int is_cxgb3_dev(struct net_device *dev)
1384{
1385 struct cxgb3i_sdev_data *cdata;
1386
1387 write_lock(&cdata_rwlock);
1388 list_for_each_entry(cdata, &cdata_list, list) {
1389 struct adap_ports *ports = &cdata->ports;
1390 int i;
1391
1392 for (i = 0; i < ports->nports; i++)
1393 if (dev == ports->lldevs[i]) {
1394 write_unlock(&cdata_rwlock);
1395 return 1;
1396 }
1397 }
1398 write_unlock(&cdata_rwlock);
1399 return 0;
1400}
1401
1402/**
1403 * cxgb3_egress_dev - return the cxgb3 egress device
1404 * @root_dev: the root device anchoring the search
1405 * @c3cn: the connection used to determine egress port in bonding mode
1406 * @context: in bonding mode, indicates a connection set up or failover
1407 *
1408 * Return egress device or NULL if the egress device isn't one of our ports.
1409 */
1410static struct net_device *cxgb3_egress_dev(struct net_device *root_dev,
1411 struct s3_conn *c3cn,
1412 int context)
1413{
1414 while (root_dev) {
1415 if (root_dev->priv_flags & IFF_802_1Q_VLAN)
1416 root_dev = vlan_dev_real_dev(root_dev);
1417 else if (is_cxgb3_dev(root_dev))
1418 return root_dev;
1419 else
1420 return NULL;
1421 }
1422 return NULL;
1423}
1424
1425static struct rtable *find_route(__be32 saddr, __be32 daddr,
1426 __be16 sport, __be16 dport)
1427{
1428 struct rtable *rt;
1429 struct flowi fl = {
1430 .oif = 0,
1431 .nl_u = {
1432 .ip4_u = {
1433 .daddr = daddr,
1434 .saddr = saddr,
1435 .tos = 0 } },
1436 .proto = IPPROTO_TCP,
1437 .uli_u = {
1438 .ports = {
1439 .sport = sport,
1440 .dport = dport } } };
1441
1442 if (ip_route_output_flow(&init_net, &rt, &fl, NULL, 0))
1443 return NULL;
1444 return rt;
1445}
1446
1447/*
1448 * Assign offload parameters to some connection fields.
1449 */
1450static void init_offload_conn(struct s3_conn *c3cn,
1451 struct t3cdev *cdev,
1452 struct dst_entry *dst)
1453{
1454 BUG_ON(c3cn->cdev != cdev);
1455 c3cn->wr_max = c3cn->wr_avail = T3C_DATA(cdev)->max_wrs;
1456 c3cn->wr_unacked = 0;
1457 c3cn->mss_idx = select_mss(c3cn, dst_mtu(dst));
1458
1459 reset_wr_list(c3cn);
1460}
1461
1462static int initiate_act_open(struct s3_conn *c3cn, struct net_device *dev)
1463{
1464 struct cxgb3i_sdev_data *cdata = NDEV2CDATA(dev);
1465 struct t3cdev *cdev = cdata->cdev;
1466 struct dst_entry *dst = c3cn->dst_cache;
1467 struct sk_buff *skb;
1468
1469 c3cn_conn_debug("c3cn 0x%p, state %u, flag 0x%lx.\n",
1470 c3cn, c3cn->state, c3cn->flags);
1471 /*
1472 * Initialize connection data. Note that the flags and ULP mode are
1473 * initialized higher up ...
1474 */
1475 c3cn->dev = dev;
1476 c3cn->cdev = cdev;
1477 c3cn->tid = cxgb3_alloc_atid(cdev, cdata->client, c3cn);
1478 if (c3cn->tid < 0)
1479 goto out_err;
1480
1481 c3cn->qset = 0;
1482 c3cn->l2t = t3_l2t_get(cdev, dst->neighbour, dev);
1483 if (!c3cn->l2t)
1484 goto free_tid;
1485
1486 skb = alloc_skb(sizeof(struct cpl_act_open_req), GFP_KERNEL);
1487 if (!skb)
1488 goto free_l2t;
1489
1490 skb->sk = (struct sock *)c3cn;
1491 set_arp_failure_handler(skb, act_open_req_arp_failure);
1492
1493 c3cn_hold(c3cn);
1494
1495 init_offload_conn(c3cn, cdev, dst);
1496 c3cn->err = 0;
1497
1498 make_act_open_req(c3cn, skb, c3cn->tid, c3cn->l2t);
1499 l2t_send(cdev, skb, c3cn->l2t);
1500 return 0;
1501
1502free_l2t:
1503 l2t_release(L2DATA(cdev), c3cn->l2t);
1504free_tid:
1505 s3_free_atid(cdev, c3cn->tid);
1506 c3cn->tid = 0;
1507out_err:
1508 return -1;
1509}
1510
1511
1512/**
1513 * cxgb3i_c3cn_connect - initiates an iscsi tcp connection to a given address
1514 * @c3cn: the iscsi tcp connection
1515 * @usin: destination address
1516 *
1517 * return 0 if active open request is sent, < 0 otherwise.
1518 */
1519int cxgb3i_c3cn_connect(struct s3_conn *c3cn, struct sockaddr_in *usin)
1520{
1521 struct rtable *rt;
1522 struct net_device *dev;
1523 struct cxgb3i_sdev_data *cdata;
1524 struct t3cdev *cdev;
1525 __be32 sipv4;
1526 int err;
1527
1528 if (usin->sin_family != AF_INET)
1529 return -EAFNOSUPPORT;
1530
1531 c3cn->daddr.sin_port = usin->sin_port;
1532 c3cn->daddr.sin_addr.s_addr = usin->sin_addr.s_addr;
1533
1534 rt = find_route(c3cn->saddr.sin_addr.s_addr,
1535 c3cn->daddr.sin_addr.s_addr,
1536 c3cn->saddr.sin_port,
1537 c3cn->daddr.sin_port);
1538 if (rt == NULL) {
1539 c3cn_conn_debug("NO route to 0x%x, port %u.\n",
1540 c3cn->daddr.sin_addr.s_addr,
1541 ntohs(c3cn->daddr.sin_port));
1542 return -ENETUNREACH;
1543 }
1544
1545 if (rt->rt_flags & (RTCF_MULTICAST | RTCF_BROADCAST)) {
1546 c3cn_conn_debug("multi-cast route to 0x%x, port %u.\n",
1547 c3cn->daddr.sin_addr.s_addr,
1548 ntohs(c3cn->daddr.sin_port));
1549 ip_rt_put(rt);
1550 return -ENETUNREACH;
1551 }
1552
1553 if (!c3cn->saddr.sin_addr.s_addr)
1554 c3cn->saddr.sin_addr.s_addr = rt->rt_src;
1555
1556 /* now commit destination to connection */
1557 c3cn->dst_cache = &rt->u.dst;
1558
1559 /* try to establish an offloaded connection */
1560 dev = cxgb3_egress_dev(c3cn->dst_cache->dev, c3cn, 0);
1561 if (dev == NULL) {
1562 c3cn_conn_debug("c3cn 0x%p, egress dev NULL.\n", c3cn);
1563 return -ENETUNREACH;
1564 }
1565 cdata = NDEV2CDATA(dev);
1566 cdev = cdata->cdev;
1567
1568 /* get a source port if one hasn't been provided */
1569 err = c3cn_get_port(c3cn, cdata);
1570 if (err)
1571 return err;
1572
1573 c3cn_conn_debug("c3cn 0x%p get port %u.\n",
1574 c3cn, ntohs(c3cn->saddr.sin_port));
1575
1576 sipv4 = cxgb3i_get_private_ipv4addr(dev);
1577 if (!sipv4) {
1578 c3cn_conn_debug("c3cn 0x%p, iscsi ip not configured.\n", c3cn);
1579 sipv4 = c3cn->saddr.sin_addr.s_addr;
1580 cxgb3i_set_private_ipv4addr(dev, sipv4);
1581 } else
1582 c3cn->saddr.sin_addr.s_addr = sipv4;
1583
1584 c3cn_conn_debug("c3cn 0x%p, %u.%u.%u.%u,%u-%u.%u.%u.%u,%u SYN_SENT.\n",
1585 c3cn, NIPQUAD(c3cn->saddr.sin_addr.s_addr),
1586 ntohs(c3cn->saddr.sin_port),
1587 NIPQUAD(c3cn->daddr.sin_addr.s_addr),
1588 ntohs(c3cn->daddr.sin_port));
1589
1590 c3cn_set_state(c3cn, C3CN_STATE_CONNECTING);
1591 if (!initiate_act_open(c3cn, dev))
1592 return 0;
1593
1594 /*
1595 * If we get here, we don't have an offload connection so simply
1596 * return a failure.
1597 */
1598 err = -ENOTSUPP;
1599
1600 /*
1601 * This trashes the connection and releases the local port,
1602 * if necessary.
1603 */
1604 c3cn_conn_debug("c3cn 0x%p -> CLOSED.\n", c3cn);
1605 c3cn_set_state(c3cn, C3CN_STATE_CLOSED);
1606 ip_rt_put(rt);
1607 c3cn_put_port(c3cn);
1608 c3cn->daddr.sin_port = 0;
1609 return err;
1610}
1611
1612/**
1613 * cxgb3i_c3cn_rx_credits - ack received tcp data.
1614 * @c3cn: iscsi tcp connection
1615 * @copied: # of bytes processed
1616 *
1617 * Called after some received data has been read. It returns RX credits
1618 * to the HW for the amount of data processed.
1619 */
1620void cxgb3i_c3cn_rx_credits(struct s3_conn *c3cn, int copied)
1621{
1622 struct t3cdev *cdev;
1623 int must_send;
1624 u32 credits, dack = 0;
1625
1626 if (c3cn->state != C3CN_STATE_ESTABLISHED)
1627 return;
1628
1629 credits = c3cn->copied_seq - c3cn->rcv_wup;
1630 if (unlikely(!credits))
1631 return;
1632
1633 cdev = c3cn->cdev;
1634
1635 if (unlikely(cxgb3_rx_credit_thres == 0))
1636 return;
1637
1638 dack = F_RX_DACK_CHANGE | V_RX_DACK_MODE(1);
1639
1640 /*
1641 * For coalescing to work effectively ensure the receive window has
1642 * at least 16KB left.
1643 */
1644 must_send = credits + 16384 >= cxgb3_rcv_win;
1645
1646 if (must_send || credits >= cxgb3_rx_credit_thres)
1647 c3cn->rcv_wup += send_rx_credits(c3cn, credits, dack);
1648}
1649
1650/**
1651 * cxgb3i_c3cn_send_pdus - send the skbs containing iscsi pdus
1652 * @c3cn: iscsi tcp connection
1653 * @skb: skb contains the iscsi pdu
1654 *
1655 * Add a list of skbs to a connection send queue. The skbs must comply with
1656 * the max size limit of the device and have a headroom of at least
1657 * TX_HEADER_LEN bytes.
1658 * Return # of bytes queued.
1659 */
1660int cxgb3i_c3cn_send_pdus(struct s3_conn *c3cn, struct sk_buff *skb)
1661{
1662 struct sk_buff *next;
1663 int err, copied = 0;
1664
1665 spin_lock_bh(&c3cn->lock);
1666
1667 if (c3cn->state != C3CN_STATE_ESTABLISHED) {
1668 c3cn_tx_debug("c3cn 0x%p, not in est. state %u.\n",
1669 c3cn, c3cn->state);
1670 err = -EAGAIN;
1671 goto out_err;
1672 }
1673
1674 err = -EPIPE;
1675 if (c3cn->err) {
1676 c3cn_tx_debug("c3cn 0x%p, err %d.\n", c3cn, c3cn->err);
1677 goto out_err;
1678 }
1679
1680 while (skb) {
1681 int frags = skb_shinfo(skb)->nr_frags +
1682 (skb->len != skb->data_len);
1683
1684 if (unlikely(skb_headroom(skb) < TX_HEADER_LEN)) {
1685 c3cn_tx_debug("c3cn 0x%p, skb head.\n", c3cn);
1686 err = -EINVAL;
1687 goto out_err;
1688 }
1689
1690 if (frags >= SKB_WR_LIST_SIZE) {
1691 cxgb3i_log_error("c3cn 0x%p, tx frags %d, len %u,%u.\n",
1692 c3cn, skb_shinfo(skb)->nr_frags,
1693 skb->len, skb->data_len);
1694 err = -EINVAL;
1695 goto out_err;
1696 }
1697
1698 next = skb->next;
1699 skb->next = NULL;
1700 skb_entail(c3cn, skb, C3CB_FLAG_NO_APPEND | C3CB_FLAG_NEED_HDR);
1701 copied += skb->len;
1702 c3cn->write_seq += skb->len + ulp_extra_len(skb);
1703 skb = next;
1704 }
1705done:
1706 if (likely(skb_queue_len(&c3cn->write_queue)))
1707 c3cn_push_tx_frames(c3cn, 1);
1708 spin_unlock_bh(&c3cn->lock);
1709 return copied;
1710
1711out_err:
1712 if (copied == 0 && err == -EPIPE)
1713 copied = c3cn->err ? c3cn->err : -EPIPE;
1714 goto done;
1715}
1716
1717static void sdev_data_cleanup(struct cxgb3i_sdev_data *cdata)
1718{
1719 struct adap_ports *ports = &cdata->ports;
1720 int i;
1721
1722 for (i = 0; i < ports->nports; i++)
1723 NDEV2CDATA(ports->lldevs[i]) = NULL;
1724 cxgb3i_free_big_mem(cdata);
1725}
1726
1727void cxgb3i_sdev_cleanup(void)
1728{
1729 struct cxgb3i_sdev_data *cdata;
1730
1731 write_lock(&cdata_rwlock);
1732 list_for_each_entry(cdata, &cdata_list, list) {
1733 list_del(&cdata->list);
1734 sdev_data_cleanup(cdata);
1735 }
1736 write_unlock(&cdata_rwlock);
1737}
1738
1739int cxgb3i_sdev_init(cxgb3_cpl_handler_func *cpl_handlers)
1740{
1741 cpl_handlers[CPL_ACT_ESTABLISH] = do_act_establish;
1742 cpl_handlers[CPL_ACT_OPEN_RPL] = do_act_open_rpl;
1743 cpl_handlers[CPL_PEER_CLOSE] = do_peer_close;
1744 cpl_handlers[CPL_ABORT_REQ_RSS] = do_abort_req;
1745 cpl_handlers[CPL_ABORT_RPL_RSS] = do_abort_rpl;
1746 cpl_handlers[CPL_CLOSE_CON_RPL] = do_close_con_rpl;
1747 cpl_handlers[CPL_TX_DMA_ACK] = do_wr_ack;
1748 cpl_handlers[CPL_ISCSI_HDR] = do_iscsi_hdr;
1749
1750 if (cxgb3_max_connect > CXGB3I_MAX_CONN)
1751 cxgb3_max_connect = CXGB3I_MAX_CONN;
1752 return 0;
1753}
1754
1755/**
1756 * cxgb3i_sdev_add - allocate and initialize resources for each adapter found
1757 * @cdev: t3cdev adapter
1758 * @client: cxgb3 driver client
1759 */
1760void cxgb3i_sdev_add(struct t3cdev *cdev, struct cxgb3_client *client)
1761{
1762 struct cxgb3i_sdev_data *cdata;
1763 struct ofld_page_info rx_page_info;
1764 unsigned int wr_len;
1765 int mapsize = DIV_ROUND_UP(cxgb3_max_connect,
1766 8 * sizeof(unsigned long));
1767 int i;
1768
1769 cdata = cxgb3i_alloc_big_mem(sizeof(*cdata) + mapsize, GFP_KERNEL);
1770 if (!cdata)
1771 return;
1772
1773 if (cdev->ctl(cdev, GET_WR_LEN, &wr_len) < 0 ||
1774 cdev->ctl(cdev, GET_PORTS, &cdata->ports) < 0 ||
1775 cdev->ctl(cdev, GET_RX_PAGE_INFO, &rx_page_info) < 0)
1776 goto free_cdata;
1777
1778 s3_init_wr_tab(wr_len);
1779
1780 INIT_LIST_HEAD(&cdata->list);
1781 cdata->cdev = cdev;
1782 cdata->client = client;
1783
1784 for (i = 0; i < cdata->ports.nports; i++)
1785 NDEV2CDATA(cdata->ports.lldevs[i]) = cdata;
1786
1787 write_lock(&cdata_rwlock);
1788 list_add_tail(&cdata->list, &cdata_list);
1789 write_unlock(&cdata_rwlock);
1790
1791 return;
1792
1793free_cdata:
1794 cxgb3i_free_big_mem(cdata);
1795}
1796
1797/**
1798 * cxgb3i_sdev_remove - free the allocated resources for the adapter
1799 * @cdev: t3cdev adapter
1800 */
1801void cxgb3i_sdev_remove(struct t3cdev *cdev)
1802{
1803 struct cxgb3i_sdev_data *cdata = CXGB3_SDEV_DATA(cdev);
1804
1805 write_lock(&cdata_rwlock);
1806 list_del(&cdata->list);
1807 write_unlock(&cdata_rwlock);
1808
1809 sdev_data_cleanup(cdata);
1810}
Linux 6.x block layer and io_uring tree(s)