Commit | Line | Data |
---|---|---|
a6779341 AG |
1 | /* |
2 | * Copyright (c) 2018 Chelsio Communications, Inc. | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or modify | |
5 | * it under the terms of the GNU General Public License version 2 as | |
6 | * published by the Free Software Foundation. | |
7 | */ | |
8 | ||
9 | #ifndef __CHTLS_H__ | |
10 | #define __CHTLS_H__ | |
11 | ||
12 | #include <crypto/aes.h> | |
13 | #include <crypto/algapi.h> | |
14 | #include <crypto/hash.h> | |
15 | #include <crypto/sha.h> | |
16 | #include <crypto/authenc.h> | |
17 | #include <crypto/ctr.h> | |
18 | #include <crypto/gf128mul.h> | |
19 | #include <crypto/internal/aead.h> | |
20 | #include <crypto/null.h> | |
21 | #include <crypto/internal/skcipher.h> | |
22 | #include <crypto/aead.h> | |
23 | #include <crypto/scatterwalk.h> | |
24 | #include <crypto/internal/hash.h> | |
25 | #include <linux/tls.h> | |
26 | #include <net/tls.h> | |
27 | ||
28 | #include "t4fw_api.h" | |
29 | #include "t4_msg.h" | |
30 | #include "cxgb4.h" | |
31 | #include "cxgb4_uld.h" | |
32 | #include "l2t.h" | |
33 | #include "chcr_algo.h" | |
34 | #include "chcr_core.h" | |
35 | #include "chcr_crypto.h" | |
36 | ||
37 | #define MAX_IVS_PAGE 256 | |
38 | #define TLS_KEY_CONTEXT_SZ 64 | |
39 | #define CIPHER_BLOCK_SIZE 16 | |
40 | #define GCM_TAG_SIZE 16 | |
41 | #define KEY_ON_MEM_SZ 16 | |
42 | #define AEAD_EXPLICIT_DATA_SIZE 8 | |
43 | #define TLS_HEADER_LENGTH 5 | |
44 | #define SCMD_CIPH_MODE_AES_GCM 2 | |
45 | /* Any MFS size should work and come from openssl */ | |
46 | #define TLS_MFS 16384 | |
47 | ||
48 | #define RSS_HDR sizeof(struct rss_header) | |
49 | #define TLS_WR_CPL_LEN \ | |
50 | (sizeof(struct fw_tlstx_data_wr) + sizeof(struct cpl_tx_tls_sfo)) | |
51 | ||
52 | enum { | |
53 | CHTLS_KEY_CONTEXT_DSGL, | |
54 | CHTLS_KEY_CONTEXT_IMM, | |
55 | CHTLS_KEY_CONTEXT_DDR, | |
56 | }; | |
57 | ||
58 | enum { | |
59 | CHTLS_LISTEN_START, | |
60 | CHTLS_LISTEN_STOP, | |
61 | }; | |
62 | ||
63 | /* Flags for return value of CPL message handlers */ | |
64 | enum { | |
65 | CPL_RET_BUF_DONE = 1, /* buffer processing done */ | |
66 | CPL_RET_BAD_MSG = 2, /* bad CPL message */ | |
67 | CPL_RET_UNKNOWN_TID = 4 /* unexpected unknown TID */ | |
68 | }; | |
69 | ||
a6779341 AG |
70 | #define LISTEN_INFO_HASH_SIZE 32 |
71 | #define RSPQ_HASH_BITS 5 | |
72 | struct listen_info { | |
73 | struct listen_info *next; /* Link to next entry */ | |
74 | struct sock *sk; /* The listening socket */ | |
75 | unsigned int stid; /* The server TID */ | |
76 | }; | |
77 | ||
78 | enum { | |
79 | T4_LISTEN_START_PENDING, | |
80 | T4_LISTEN_STARTED | |
81 | }; | |
82 | ||
83 | enum csk_flags { | |
84 | CSK_CALLBACKS_CHKD, /* socket callbacks have been sanitized */ | |
85 | CSK_ABORT_REQ_RCVD, /* received one ABORT_REQ_RSS message */ | |
86 | CSK_TX_MORE_DATA, /* sending ULP data; don't set SHOVE bit */ | |
87 | CSK_TX_WAIT_IDLE, /* suspend Tx until in-flight data is ACKed */ | |
88 | CSK_ABORT_SHUTDOWN, /* shouldn't send more abort requests */ | |
89 | CSK_ABORT_RPL_PENDING, /* expecting an abort reply */ | |
90 | CSK_CLOSE_CON_REQUESTED,/* we've sent a close_conn_req */ | |
91 | CSK_TX_DATA_SENT, /* sent a TX_DATA WR on this connection */ | |
92 | CSK_TX_FAILOVER, /* Tx traffic failing over */ | |
93 | CSK_UPDATE_RCV_WND, /* Need to update rcv window */ | |
94 | CSK_RST_ABORTED, /* outgoing RST was aborted */ | |
95 | CSK_TLS_HANDSHK, /* TLS Handshake */ | |
96 | CSK_CONN_INLINE, /* Connection on HW */ | |
97 | }; | |
98 | ||
99 | struct listen_ctx { | |
100 | struct sock *lsk; | |
101 | struct chtls_dev *cdev; | |
102 | struct sk_buff_head synq; | |
103 | u32 state; | |
104 | }; | |
105 | ||
106 | struct key_map { | |
107 | unsigned long *addr; | |
108 | unsigned int start; | |
109 | unsigned int available; | |
110 | unsigned int size; | |
111 | spinlock_t lock; /* lock for key id request from map */ | |
112 | } __packed; | |
113 | ||
114 | struct tls_scmd { | |
115 | u32 seqno_numivs; | |
116 | u32 ivgen_hdrlen; | |
117 | }; | |
118 | ||
119 | struct chtls_dev { | |
120 | struct tls_device tlsdev; | |
121 | struct list_head list; | |
122 | struct cxgb4_lld_info *lldi; | |
123 | struct pci_dev *pdev; | |
124 | struct listen_info *listen_hash_tab[LISTEN_INFO_HASH_SIZE]; | |
125 | spinlock_t listen_lock; /* lock for listen list */ | |
126 | struct net_device **ports; | |
127 | struct tid_info *tids; | |
128 | unsigned int pfvf; | |
129 | const unsigned short *mtus; | |
130 | ||
131 | struct idr hwtid_idr; | |
132 | struct idr stid_idr; | |
133 | ||
134 | spinlock_t idr_lock ____cacheline_aligned_in_smp; | |
135 | ||
136 | struct net_device *egr_dev[NCHAN * 2]; | |
137 | struct sk_buff *rspq_skb_cache[1 << RSPQ_HASH_BITS]; | |
138 | struct sk_buff *askb; | |
139 | ||
140 | struct sk_buff_head deferq; | |
141 | struct work_struct deferq_task; | |
142 | ||
143 | struct list_head list_node; | |
144 | struct list_head rcu_node; | |
145 | struct list_head na_node; | |
146 | unsigned int send_page_order; | |
147 | struct key_map kmap; | |
148 | }; | |
149 | ||
150 | struct chtls_hws { | |
151 | struct sk_buff_head sk_recv_queue; | |
152 | u8 txqid; | |
153 | u8 ofld; | |
154 | u16 type; | |
155 | u16 rstate; | |
156 | u16 keyrpl; | |
157 | u16 pldlen; | |
158 | u16 rcvpld; | |
159 | u16 compute; | |
160 | u16 expansion; | |
161 | u16 keylen; | |
162 | u16 pdus; | |
163 | u16 adjustlen; | |
164 | u16 ivsize; | |
165 | u16 txleft; | |
166 | u32 mfs; | |
167 | s32 txkey; | |
168 | s32 rxkey; | |
169 | u32 fcplenmax; | |
170 | u32 copied_seq; | |
171 | u64 tx_seq_no; | |
172 | struct tls_scmd scmd; | |
173 | struct tls12_crypto_info_aes_gcm_128 crypto_info; | |
174 | }; | |
175 | ||
176 | struct chtls_sock { | |
177 | struct sock *sk; | |
178 | struct chtls_dev *cdev; | |
179 | struct l2t_entry *l2t_entry; /* pointer to the L2T entry */ | |
180 | struct net_device *egress_dev; /* TX_CHAN for act open retry */ | |
181 | ||
182 | struct sk_buff_head txq; | |
183 | struct sk_buff *wr_skb_head; | |
184 | struct sk_buff *wr_skb_tail; | |
185 | struct sk_buff *ctrl_skb_cache; | |
186 | struct sk_buff *txdata_skb_cache; /* abort path messages */ | |
187 | struct kref kref; | |
188 | unsigned long flags; | |
189 | u32 opt2; | |
190 | u32 wr_credits; | |
191 | u32 wr_unacked; | |
192 | u32 wr_max_credits; | |
193 | u32 wr_nondata; | |
194 | u32 hwtid; /* TCP Control Block ID */ | |
195 | u32 txq_idx; | |
196 | u32 rss_qid; | |
197 | u32 tid; | |
198 | u32 idr; | |
199 | u32 mss; | |
200 | u32 ulp_mode; | |
201 | u32 tx_chan; | |
202 | u32 rx_chan; | |
203 | u32 sndbuf; | |
204 | u32 txplen_max; | |
205 | u32 mtu_idx; /* MTU table index */ | |
206 | u32 smac_idx; | |
207 | u8 port_id; | |
208 | u8 tos; | |
209 | u16 resv2; | |
210 | u32 delack_mode; | |
211 | u32 delack_seq; | |
212 | ||
213 | void *passive_reap_next; /* placeholder for passive */ | |
214 | struct chtls_hws tlshws; | |
215 | struct synq { | |
216 | struct sk_buff *next; | |
217 | struct sk_buff *prev; | |
218 | } synq; | |
219 | struct listen_ctx *listen_ctx; | |
220 | }; | |
221 | ||
222 | struct tls_hdr { | |
223 | u8 type; | |
224 | u16 version; | |
225 | u16 length; | |
226 | } __packed; | |
227 | ||
228 | struct tlsrx_cmp_hdr { | |
229 | u8 type; | |
230 | u16 version; | |
231 | u16 length; | |
232 | ||
233 | u64 tls_seq; | |
234 | u16 reserved1; | |
235 | u8 res_to_mac_error; | |
236 | } __packed; | |
237 | ||
238 | /* res_to_mac_error fields */ | |
239 | #define TLSRX_HDR_PKT_INT_ERROR_S 4 | |
240 | #define TLSRX_HDR_PKT_INT_ERROR_M 0x1 | |
241 | #define TLSRX_HDR_PKT_INT_ERROR_V(x) \ | |
242 | ((x) << TLSRX_HDR_PKT_INT_ERROR_S) | |
243 | #define TLSRX_HDR_PKT_INT_ERROR_G(x) \ | |
244 | (((x) >> TLSRX_HDR_PKT_INT_ERROR_S) & TLSRX_HDR_PKT_INT_ERROR_M) | |
245 | #define TLSRX_HDR_PKT_INT_ERROR_F TLSRX_HDR_PKT_INT_ERROR_V(1U) | |
246 | ||
247 | #define TLSRX_HDR_PKT_SPP_ERROR_S 3 | |
248 | #define TLSRX_HDR_PKT_SPP_ERROR_M 0x1 | |
249 | #define TLSRX_HDR_PKT_SPP_ERROR_V(x) ((x) << TLSRX_HDR_PKT_SPP_ERROR) | |
250 | #define TLSRX_HDR_PKT_SPP_ERROR_G(x) \ | |
251 | (((x) >> TLSRX_HDR_PKT_SPP_ERROR_S) & TLSRX_HDR_PKT_SPP_ERROR_M) | |
252 | #define TLSRX_HDR_PKT_SPP_ERROR_F TLSRX_HDR_PKT_SPP_ERROR_V(1U) | |
253 | ||
254 | #define TLSRX_HDR_PKT_CCDX_ERROR_S 2 | |
255 | #define TLSRX_HDR_PKT_CCDX_ERROR_M 0x1 | |
256 | #define TLSRX_HDR_PKT_CCDX_ERROR_V(x) ((x) << TLSRX_HDR_PKT_CCDX_ERROR_S) | |
257 | #define TLSRX_HDR_PKT_CCDX_ERROR_G(x) \ | |
258 | (((x) >> TLSRX_HDR_PKT_CCDX_ERROR_S) & TLSRX_HDR_PKT_CCDX_ERROR_M) | |
259 | #define TLSRX_HDR_PKT_CCDX_ERROR_F TLSRX_HDR_PKT_CCDX_ERROR_V(1U) | |
260 | ||
261 | #define TLSRX_HDR_PKT_PAD_ERROR_S 1 | |
262 | #define TLSRX_HDR_PKT_PAD_ERROR_M 0x1 | |
263 | #define TLSRX_HDR_PKT_PAD_ERROR_V(x) ((x) << TLSRX_HDR_PKT_PAD_ERROR_S) | |
264 | #define TLSRX_HDR_PKT_PAD_ERROR_G(x) \ | |
265 | (((x) >> TLSRX_HDR_PKT_PAD_ERROR_S) & TLSRX_HDR_PKT_PAD_ERROR_M) | |
266 | #define TLSRX_HDR_PKT_PAD_ERROR_F TLSRX_HDR_PKT_PAD_ERROR_V(1U) | |
267 | ||
268 | #define TLSRX_HDR_PKT_MAC_ERROR_S 0 | |
269 | #define TLSRX_HDR_PKT_MAC_ERROR_M 0x1 | |
270 | #define TLSRX_HDR_PKT_MAC_ERROR_V(x) ((x) << TLSRX_HDR_PKT_MAC_ERROR) | |
271 | #define TLSRX_HDR_PKT_MAC_ERROR_G(x) \ | |
272 | (((x) >> S_TLSRX_HDR_PKT_MAC_ERROR_S) & TLSRX_HDR_PKT_MAC_ERROR_M) | |
273 | #define TLSRX_HDR_PKT_MAC_ERROR_F TLSRX_HDR_PKT_MAC_ERROR_V(1U) | |
274 | ||
275 | #define TLSRX_HDR_PKT_ERROR_M 0x1F | |
17a7d24a | 276 | #define CONTENT_TYPE_ERROR 0x7F |
a6779341 AG |
277 | |
278 | struct ulp_mem_rw { | |
279 | __be32 cmd; | |
280 | __be32 len16; /* command length */ | |
281 | __be32 dlen; /* data length in 32-byte units */ | |
282 | __be32 lock_addr; | |
283 | }; | |
284 | ||
285 | struct tls_key_wr { | |
286 | __be32 op_to_compl; | |
287 | __be32 flowid_len16; | |
288 | __be32 ftid; | |
289 | u8 reneg_to_write_rx; | |
290 | u8 protocol; | |
291 | __be16 mfs; | |
292 | }; | |
293 | ||
294 | struct tls_key_req { | |
295 | struct tls_key_wr wr; | |
296 | struct ulp_mem_rw req; | |
297 | struct ulptx_idata sc_imm; | |
298 | }; | |
299 | ||
300 | /* | |
301 | * This lives in skb->cb and is used to chain WRs in a linked list. | |
302 | */ | |
303 | struct wr_skb_cb { | |
304 | struct l2t_skb_cb l2t; /* reserve space for l2t CB */ | |
305 | struct sk_buff *next_wr; /* next write request */ | |
306 | }; | |
307 | ||
308 | /* Per-skb backlog handler. Run when a socket's backlog is processed. */ | |
309 | struct blog_skb_cb { | |
310 | void (*backlog_rcv)(struct sock *sk, struct sk_buff *skb); | |
311 | struct chtls_dev *cdev; | |
312 | }; | |
313 | ||
314 | /* | |
315 | * Similar to tcp_skb_cb but with ULP elements added to support TLS, | |
316 | * etc. | |
317 | */ | |
318 | struct ulp_skb_cb { | |
319 | struct wr_skb_cb wr; /* reserve space for write request */ | |
320 | u16 flags; /* TCP-like flags */ | |
321 | u8 psh; | |
322 | u8 ulp_mode; /* ULP mode/submode of sk_buff */ | |
323 | u32 seq; /* TCP sequence number */ | |
324 | union { /* ULP-specific fields */ | |
325 | struct { | |
326 | u8 type; | |
327 | u8 ofld; | |
328 | u8 iv; | |
329 | } tls; | |
330 | } ulp; | |
331 | }; | |
332 | ||
333 | #define ULP_SKB_CB(skb) ((struct ulp_skb_cb *)&((skb)->cb[0])) | |
334 | #define BLOG_SKB_CB(skb) ((struct blog_skb_cb *)(skb)->cb) | |
335 | ||
336 | /* | |
337 | * Flags for ulp_skb_cb.flags. | |
338 | */ | |
339 | enum { | |
340 | ULPCB_FLAG_NEED_HDR = 1 << 0, /* packet needs a TX_DATA_WR header */ | |
341 | ULPCB_FLAG_NO_APPEND = 1 << 1, /* don't grow this skb */ | |
342 | ULPCB_FLAG_BARRIER = 1 << 2, /* set TX_WAIT_IDLE after sending */ | |
343 | ULPCB_FLAG_HOLD = 1 << 3, /* skb not ready for Tx yet */ | |
344 | ULPCB_FLAG_COMPL = 1 << 4, /* request WR completion */ | |
345 | ULPCB_FLAG_URG = 1 << 5, /* urgent data */ | |
17a7d24a AG |
346 | ULPCB_FLAG_TLS_HDR = 1 << 6, /* payload with tls hdr */ |
347 | ULPCB_FLAG_NO_HDR = 1 << 7, /* not a ofld wr */ | |
a6779341 AG |
348 | }; |
349 | ||
350 | /* The ULP mode/submode of an skbuff */ | |
351 | #define skb_ulp_mode(skb) (ULP_SKB_CB(skb)->ulp_mode) | |
352 | #define TCP_PAGE(sk) (sk->sk_frag.page) | |
353 | #define TCP_OFF(sk) (sk->sk_frag.offset) | |
354 | ||
355 | static inline struct chtls_dev *to_chtls_dev(struct tls_device *tlsdev) | |
356 | { | |
357 | return container_of(tlsdev, struct chtls_dev, tlsdev); | |
358 | } | |
359 | ||
360 | static inline void csk_set_flag(struct chtls_sock *csk, | |
361 | enum csk_flags flag) | |
362 | { | |
363 | __set_bit(flag, &csk->flags); | |
364 | } | |
365 | ||
366 | static inline void csk_reset_flag(struct chtls_sock *csk, | |
367 | enum csk_flags flag) | |
368 | { | |
369 | __clear_bit(flag, &csk->flags); | |
370 | } | |
371 | ||
372 | static inline bool csk_conn_inline(const struct chtls_sock *csk) | |
373 | { | |
374 | return test_bit(CSK_CONN_INLINE, &csk->flags); | |
375 | } | |
376 | ||
377 | static inline int csk_flag(const struct sock *sk, enum csk_flags flag) | |
378 | { | |
379 | struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); | |
380 | ||
381 | if (!csk_conn_inline(csk)) | |
382 | return 0; | |
383 | return test_bit(flag, &csk->flags); | |
384 | } | |
385 | ||
386 | static inline int csk_flag_nochk(const struct chtls_sock *csk, | |
387 | enum csk_flags flag) | |
388 | { | |
389 | return test_bit(flag, &csk->flags); | |
390 | } | |
391 | ||
392 | static inline void *cplhdr(struct sk_buff *skb) | |
393 | { | |
394 | return skb->data; | |
395 | } | |
396 | ||
397 | static inline int is_neg_adv(unsigned int status) | |
398 | { | |
399 | return status == CPL_ERR_RTX_NEG_ADVICE || | |
400 | status == CPL_ERR_KEEPALV_NEG_ADVICE || | |
401 | status == CPL_ERR_PERSIST_NEG_ADVICE; | |
402 | } | |
403 | ||
404 | static inline void process_cpl_msg(void (*fn)(struct sock *, struct sk_buff *), | |
405 | struct sock *sk, | |
406 | struct sk_buff *skb) | |
407 | { | |
408 | skb_reset_mac_header(skb); | |
409 | skb_reset_network_header(skb); | |
410 | skb_reset_transport_header(skb); | |
411 | ||
412 | bh_lock_sock(sk); | |
413 | if (unlikely(sock_owned_by_user(sk))) { | |
414 | BLOG_SKB_CB(skb)->backlog_rcv = fn; | |
415 | __sk_add_backlog(sk, skb); | |
416 | } else { | |
417 | fn(sk, skb); | |
418 | } | |
419 | bh_unlock_sock(sk); | |
420 | } | |
421 | ||
422 | static inline void chtls_sock_free(struct kref *ref) | |
423 | { | |
424 | struct chtls_sock *csk = container_of(ref, struct chtls_sock, | |
425 | kref); | |
426 | kfree(csk); | |
427 | } | |
428 | ||
429 | static inline void __chtls_sock_put(const char *fn, struct chtls_sock *csk) | |
430 | { | |
431 | kref_put(&csk->kref, chtls_sock_free); | |
432 | } | |
433 | ||
434 | static inline void __chtls_sock_get(const char *fn, | |
435 | struct chtls_sock *csk) | |
436 | { | |
437 | kref_get(&csk->kref); | |
438 | } | |
439 | ||
440 | static inline void send_or_defer(struct sock *sk, struct tcp_sock *tp, | |
441 | struct sk_buff *skb, int through_l2t) | |
442 | { | |
443 | struct chtls_sock *csk = rcu_dereference_sk_user_data(sk); | |
444 | ||
445 | if (through_l2t) { | |
446 | /* send through L2T */ | |
447 | cxgb4_l2t_send(csk->egress_dev, skb, csk->l2t_entry); | |
448 | } else { | |
449 | /* send directly */ | |
450 | cxgb4_ofld_send(csk->egress_dev, skb); | |
451 | } | |
452 | } | |
453 | ||
454 | typedef int (*chtls_handler_func)(struct chtls_dev *, struct sk_buff *); | |
455 | extern chtls_handler_func chtls_handlers[NUM_CPL_CMDS]; | |
456 | void chtls_install_cpl_ops(struct sock *sk); | |
457 | int chtls_init_kmap(struct chtls_dev *cdev, struct cxgb4_lld_info *lldi); | |
458 | void chtls_listen_stop(struct chtls_dev *cdev, struct sock *sk); | |
459 | int chtls_listen_start(struct chtls_dev *cdev, struct sock *sk); | |
460 | void chtls_close(struct sock *sk, long timeout); | |
461 | int chtls_disconnect(struct sock *sk, int flags); | |
462 | void chtls_shutdown(struct sock *sk, int how); | |
463 | void chtls_destroy_sock(struct sock *sk); | |
464 | int chtls_sendmsg(struct sock *sk, struct msghdr *msg, size_t size); | |
465 | int chtls_recvmsg(struct sock *sk, struct msghdr *msg, | |
466 | size_t len, int nonblock, int flags, int *addr_len); | |
467 | int chtls_sendpage(struct sock *sk, struct page *page, | |
468 | int offset, size_t size, int flags); | |
469 | int send_tx_flowc_wr(struct sock *sk, int compl, | |
470 | u32 snd_nxt, u32 rcv_nxt); | |
471 | void chtls_tcp_push(struct sock *sk, int flags); | |
472 | int chtls_push_frames(struct chtls_sock *csk, int comp); | |
473 | int chtls_set_tcb_tflag(struct sock *sk, unsigned int bit_pos, int val); | |
474 | int chtls_setkey(struct chtls_sock *csk, u32 keylen, u32 mode); | |
475 | void skb_entail(struct sock *sk, struct sk_buff *skb, int flags); | |
476 | unsigned int keyid_to_addr(int start_addr, int keyid); | |
477 | void free_tls_keyid(struct sock *sk); | |
478 | #endif |