1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 // Copyright (c) 2019 Mellanox Technologies.
7 #include "en_accel/ktls.h"
10 MLX5E_STATIC_PARAMS_CONTEXT_TLS_1_2 = 0x2,
14 MLX5E_ENCRYPTION_STANDARD_TLS = 0x1,
17 #define EXTRACT_INFO_FIELDS do { \
19 rec_seq = info->rec_seq; \
20 salt_sz = sizeof(info->salt); \
21 rec_seq_sz = sizeof(info->rec_seq); \
25 fill_static_params_ctx(void *ctx, struct mlx5e_ktls_offload_context_tx *priv_tx)
27 struct tls12_crypto_info_aes_gcm_128 *info = &priv_tx->crypto_info;
28 char *initial_rn, *gcm_iv;
29 u16 salt_sz, rec_seq_sz;
35 gcm_iv = MLX5_ADDR_OF(tls_static_params, ctx, gcm_iv);
36 initial_rn = MLX5_ADDR_OF(tls_static_params, ctx, initial_record_number);
38 memcpy(gcm_iv, salt, salt_sz);
39 memcpy(initial_rn, rec_seq, rec_seq_sz);
41 tls_version = MLX5E_STATIC_PARAMS_CONTEXT_TLS_1_2;
43 MLX5_SET(tls_static_params, ctx, tls_version, tls_version);
44 MLX5_SET(tls_static_params, ctx, const_1, 1);
45 MLX5_SET(tls_static_params, ctx, const_2, 2);
46 MLX5_SET(tls_static_params, ctx, encryption_standard,
47 MLX5E_ENCRYPTION_STANDARD_TLS);
48 MLX5_SET(tls_static_params, ctx, dek_index, priv_tx->key_id);
52 build_static_params(struct mlx5e_umr_wqe *wqe, u16 pc, u32 sqn,
53 struct mlx5e_ktls_offload_context_tx *priv_tx,
56 struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl;
57 struct mlx5_wqe_umr_ctrl_seg *ucseg = &wqe->uctrl;
59 #define STATIC_PARAMS_DS_CNT \
60 DIV_ROUND_UP(MLX5E_KTLS_STATIC_UMR_WQE_SZ, MLX5_SEND_WQE_DS)
62 cseg->opmod_idx_opcode = cpu_to_be32((pc << 8) | MLX5_OPCODE_UMR |
63 (MLX5_OPC_MOD_TLS_TIS_STATIC_PARAMS << 24));
64 cseg->qpn_ds = cpu_to_be32((sqn << MLX5_WQE_CTRL_QPN_SHIFT) |
65 STATIC_PARAMS_DS_CNT);
66 cseg->fm_ce_se = fence ? MLX5_FENCE_MODE_INITIATOR_SMALL : 0;
67 cseg->tisn = cpu_to_be32(priv_tx->tisn << 8);
69 ucseg->flags = MLX5_UMR_INLINE;
70 ucseg->bsf_octowords = cpu_to_be16(MLX5_ST_SZ_BYTES(tls_static_params) / 16);
72 fill_static_params_ctx(wqe->tls_static_params_ctx, priv_tx);
76 fill_progress_params_ctx(void *ctx, struct mlx5e_ktls_offload_context_tx *priv_tx)
78 MLX5_SET(tls_progress_params, ctx, tisn, priv_tx->tisn);
79 MLX5_SET(tls_progress_params, ctx, record_tracker_state,
80 MLX5E_TLS_PROGRESS_PARAMS_RECORD_TRACKER_STATE_START);
81 MLX5_SET(tls_progress_params, ctx, auth_state,
82 MLX5E_TLS_PROGRESS_PARAMS_AUTH_STATE_NO_OFFLOAD);
86 build_progress_params(struct mlx5e_tx_wqe *wqe, u16 pc, u32 sqn,
87 struct mlx5e_ktls_offload_context_tx *priv_tx,
90 struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl;
92 #define PROGRESS_PARAMS_DS_CNT \
93 DIV_ROUND_UP(MLX5E_KTLS_PROGRESS_WQE_SZ, MLX5_SEND_WQE_DS)
95 cseg->opmod_idx_opcode =
96 cpu_to_be32((pc << 8) | MLX5_OPCODE_SET_PSV |
97 (MLX5_OPC_MOD_TLS_TIS_PROGRESS_PARAMS << 24));
98 cseg->qpn_ds = cpu_to_be32((sqn << MLX5_WQE_CTRL_QPN_SHIFT) |
99 PROGRESS_PARAMS_DS_CNT);
100 cseg->fm_ce_se = fence ? MLX5_FENCE_MODE_INITIATOR_SMALL : 0;
102 fill_progress_params_ctx(wqe->tls_progress_params_ctx, priv_tx);
105 static void tx_fill_wi(struct mlx5e_txqsq *sq,
106 u16 pi, u8 num_wqebbs, u32 num_bytes,
109 struct mlx5e_tx_wqe_info *wi = &sq->db.wqe_info[pi];
111 memset(wi, 0, sizeof(*wi));
112 wi->num_wqebbs = num_wqebbs;
113 wi->num_bytes = num_bytes;
114 wi->resync_dump_frag_page = page;
117 void mlx5e_ktls_tx_offload_set_pending(struct mlx5e_ktls_offload_context_tx *priv_tx)
119 priv_tx->ctx_post_pending = true;
123 mlx5e_ktls_tx_offload_test_and_clear_pending(struct mlx5e_ktls_offload_context_tx *priv_tx)
125 bool ret = priv_tx->ctx_post_pending;
127 priv_tx->ctx_post_pending = false;
133 post_static_params(struct mlx5e_txqsq *sq,
134 struct mlx5e_ktls_offload_context_tx *priv_tx,
137 struct mlx5e_umr_wqe *umr_wqe;
140 pi = mlx5_wq_cyc_ctr2ix(&sq->wq, sq->pc);
141 umr_wqe = MLX5E_TLS_FETCH_UMR_WQE(sq, pi);
142 build_static_params(umr_wqe, sq->pc, sq->sqn, priv_tx, fence);
143 tx_fill_wi(sq, pi, MLX5E_KTLS_STATIC_WQEBBS, 0, NULL);
144 sq->pc += MLX5E_KTLS_STATIC_WQEBBS;
148 post_progress_params(struct mlx5e_txqsq *sq,
149 struct mlx5e_ktls_offload_context_tx *priv_tx,
152 struct mlx5e_tx_wqe *wqe;
155 pi = mlx5_wq_cyc_ctr2ix(&sq->wq, sq->pc);
156 wqe = MLX5E_TLS_FETCH_PROGRESS_WQE(sq, pi);
157 build_progress_params(wqe, sq->pc, sq->sqn, priv_tx, fence);
158 tx_fill_wi(sq, pi, MLX5E_KTLS_PROGRESS_WQEBBS, 0, NULL);
159 sq->pc += MLX5E_KTLS_PROGRESS_WQEBBS;
163 mlx5e_ktls_tx_post_param_wqes(struct mlx5e_txqsq *sq,
164 struct mlx5e_ktls_offload_context_tx *priv_tx,
165 bool skip_static_post, bool fence_first_post)
167 bool progress_fence = skip_static_post || !fence_first_post;
169 mlx5e_txqsq_get_next_pi(sq, MLX5E_KTLS_STATIC_WQEBBS + MLX5E_KTLS_PROGRESS_WQEBBS);
171 if (!skip_static_post)
172 post_static_params(sq, priv_tx, fence_first_post);
174 post_progress_params(sq, priv_tx, progress_fence);
177 struct tx_sync_info {
181 skb_frag_t frags[MAX_SKB_FRAGS];
184 enum mlx5e_ktls_sync_retval {
185 MLX5E_KTLS_SYNC_DONE,
186 MLX5E_KTLS_SYNC_FAIL,
187 MLX5E_KTLS_SYNC_SKIP_NO_DATA,
190 static enum mlx5e_ktls_sync_retval
191 tx_sync_info_get(struct mlx5e_ktls_offload_context_tx *priv_tx,
192 u32 tcp_seq, int datalen, struct tx_sync_info *info)
194 struct tls_offload_context_tx *tx_ctx = priv_tx->tx_ctx;
195 enum mlx5e_ktls_sync_retval ret = MLX5E_KTLS_SYNC_DONE;
196 struct tls_record_info *record;
197 int remaining, i = 0;
201 spin_lock_irqsave(&tx_ctx->lock, flags);
202 record = tls_get_record(tx_ctx, tcp_seq, &info->rcd_sn);
204 if (unlikely(!record)) {
205 ret = MLX5E_KTLS_SYNC_FAIL;
209 /* There are the following cases:
210 * 1. packet ends before start marker: bypass offload.
211 * 2. packet starts before start marker and ends after it: drop,
212 * not supported, breaks contract with kernel.
213 * 3. packet ends before tls record info starts: drop,
214 * this packet was already acknowledged and its record info
217 ends_before = before(tcp_seq + datalen - 1, tls_record_start_seq(record));
219 if (unlikely(tls_record_is_start_marker(record))) {
220 ret = ends_before ? MLX5E_KTLS_SYNC_SKIP_NO_DATA : MLX5E_KTLS_SYNC_FAIL;
222 } else if (ends_before) {
223 ret = MLX5E_KTLS_SYNC_FAIL;
227 info->sync_len = tcp_seq - tls_record_start_seq(record);
228 remaining = info->sync_len;
229 while (remaining > 0) {
230 skb_frag_t *frag = &record->frags[i];
232 get_page(skb_frag_page(frag));
233 remaining -= skb_frag_size(frag);
234 info->frags[i++] = *frag;
236 /* reduce the part which will be sent with the original SKB */
238 skb_frag_size_add(&info->frags[i - 1], remaining);
241 spin_unlock_irqrestore(&tx_ctx->lock, flags);
246 tx_post_resync_params(struct mlx5e_txqsq *sq,
247 struct mlx5e_ktls_offload_context_tx *priv_tx,
250 struct tls12_crypto_info_aes_gcm_128 *info = &priv_tx->crypto_info;
251 __be64 rn_be = cpu_to_be64(rcd_sn);
252 bool skip_static_post;
256 rec_seq = info->rec_seq;
257 rec_seq_sz = sizeof(info->rec_seq);
259 skip_static_post = !memcmp(rec_seq, &rn_be, rec_seq_sz);
260 if (!skip_static_post)
261 memcpy(rec_seq, &rn_be, rec_seq_sz);
263 mlx5e_ktls_tx_post_param_wqes(sq, priv_tx, skip_static_post, true);
267 tx_post_resync_dump(struct mlx5e_txqsq *sq, skb_frag_t *frag, u32 tisn, bool first)
269 struct mlx5_wqe_ctrl_seg *cseg;
270 struct mlx5_wqe_data_seg *dseg;
271 struct mlx5e_dump_wqe *wqe;
272 dma_addr_t dma_addr = 0;
277 pi = mlx5_wq_cyc_ctr2ix(&sq->wq, sq->pc);
278 wqe = MLX5E_TLS_FETCH_DUMP_WQE(sq, pi);
280 ds_cnt = sizeof(*wqe) / MLX5_SEND_WQE_DS;
285 cseg->opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | MLX5_OPCODE_DUMP);
286 cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt);
287 cseg->tisn = cpu_to_be32(tisn << 8);
288 cseg->fm_ce_se = first ? MLX5_FENCE_MODE_INITIATOR_SMALL : 0;
290 fsz = skb_frag_size(frag);
291 dma_addr = skb_frag_dma_map(sq->pdev, frag, 0, fsz,
293 if (unlikely(dma_mapping_error(sq->pdev, dma_addr)))
296 dseg->addr = cpu_to_be64(dma_addr);
297 dseg->lkey = sq->mkey_be;
298 dseg->byte_count = cpu_to_be32(fsz);
299 mlx5e_dma_push(sq, dma_addr, fsz, MLX5E_DMA_MAP_PAGE);
301 tx_fill_wi(sq, pi, MLX5E_KTLS_DUMP_WQEBBS, fsz, skb_frag_page(frag));
302 sq->pc += MLX5E_KTLS_DUMP_WQEBBS;
307 void mlx5e_ktls_tx_handle_resync_dump_comp(struct mlx5e_txqsq *sq,
308 struct mlx5e_tx_wqe_info *wi,
311 struct mlx5e_sq_stats *stats;
312 struct mlx5e_sq_dma *dma;
314 if (!wi->resync_dump_frag_page)
317 dma = mlx5e_dma_get(sq, (*dma_fifo_cc)++);
320 mlx5e_tx_dma_unmap(sq->pdev, dma);
321 put_page(wi->resync_dump_frag_page);
322 stats->tls_dump_packets++;
323 stats->tls_dump_bytes += wi->num_bytes;
326 static void tx_post_fence_nop(struct mlx5e_txqsq *sq)
328 struct mlx5_wq_cyc *wq = &sq->wq;
329 u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
331 tx_fill_wi(sq, pi, 1, 0, NULL);
333 mlx5e_post_nop_fence(wq, sq->sqn, &sq->pc);
336 static enum mlx5e_ktls_sync_retval
337 mlx5e_ktls_tx_handle_ooo(struct mlx5e_ktls_offload_context_tx *priv_tx,
338 struct mlx5e_txqsq *sq,
342 struct mlx5e_sq_stats *stats = sq->stats;
343 enum mlx5e_ktls_sync_retval ret;
344 struct tx_sync_info info = {};
348 ret = tx_sync_info_get(priv_tx, seq, datalen, &info);
349 if (unlikely(ret != MLX5E_KTLS_SYNC_DONE)) {
350 if (ret == MLX5E_KTLS_SYNC_SKIP_NO_DATA) {
351 stats->tls_skip_no_sync_data++;
352 return MLX5E_KTLS_SYNC_SKIP_NO_DATA;
354 /* We might get here if a retransmission reaches the driver
355 * after the relevant record is acked.
356 * It should be safe to drop the packet in this case
358 stats->tls_drop_no_sync_data++;
364 tx_post_resync_params(sq, priv_tx, info.rcd_sn);
366 /* If no dump WQE was sent, we need to have a fence NOP WQE before the
369 if (!info.nr_frags) {
370 tx_post_fence_nop(sq);
371 return MLX5E_KTLS_SYNC_DONE;
374 num_wqebbs = mlx5e_ktls_dumps_num_wqebbs(sq, info.nr_frags, info.sync_len);
375 mlx5e_txqsq_get_next_pi(sq, num_wqebbs);
377 for (; i < info.nr_frags; i++) {
378 unsigned int orig_fsz, frag_offset = 0, n = 0;
379 skb_frag_t *f = &info.frags[i];
381 orig_fsz = skb_frag_size(f);
384 bool fence = !(i || frag_offset);
388 fsz = min_t(unsigned int, sq->hw_mtu, orig_fsz - frag_offset);
389 skb_frag_size_set(f, fsz);
390 if (tx_post_resync_dump(sq, f, priv_tx->tisn, fence)) {
391 page_ref_add(skb_frag_page(f), n - 1);
395 skb_frag_off_add(f, fsz);
397 } while (frag_offset < orig_fsz);
399 page_ref_add(skb_frag_page(f), n - 1);
402 return MLX5E_KTLS_SYNC_DONE;
405 for (; i < info.nr_frags; i++)
406 /* The put_page() here undoes the page ref obtained in tx_sync_info_get().
407 * Page refs obtained for the DUMP WQEs above (by page_ref_add) will be
408 * released only upon their completions (or in mlx5e_free_txqsq_descs,
409 * if channel closes).
411 put_page(skb_frag_page(&info.frags[i]));
413 return MLX5E_KTLS_SYNC_FAIL;
416 bool mlx5e_ktls_handle_tx_skb(struct net_device *netdev, struct mlx5e_txqsq *sq,
417 struct sk_buff *skb, struct mlx5e_tx_wqe **wqe,
420 struct mlx5e_ktls_offload_context_tx *priv_tx;
421 struct mlx5e_sq_stats *stats = sq->stats;
422 struct mlx5_wqe_ctrl_seg *cseg;
423 struct tls_context *tls_ctx;
427 if (!skb->sk || !tls_is_sk_tx_device_offloaded(skb->sk))
430 datalen = skb->len - (skb_transport_offset(skb) + tcp_hdrlen(skb));
434 tls_ctx = tls_get_ctx(skb->sk);
435 if (WARN_ON_ONCE(tls_ctx->netdev != netdev))
438 priv_tx = mlx5e_get_ktls_tx_priv_ctx(tls_ctx);
440 if (unlikely(mlx5e_ktls_tx_offload_test_and_clear_pending(priv_tx))) {
441 mlx5e_ktls_tx_post_param_wqes(sq, priv_tx, false, false);
442 *pi = mlx5_wq_cyc_ctr2ix(&sq->wq, sq->pc);
443 *wqe = MLX5E_TX_FETCH_WQE(sq, *pi);
447 seq = ntohl(tcp_hdr(skb)->seq);
448 if (unlikely(priv_tx->expected_seq != seq)) {
449 enum mlx5e_ktls_sync_retval ret =
450 mlx5e_ktls_tx_handle_ooo(priv_tx, sq, datalen, seq);
453 case MLX5E_KTLS_SYNC_DONE:
454 *pi = mlx5_wq_cyc_ctr2ix(&sq->wq, sq->pc);
455 *wqe = MLX5E_TX_FETCH_WQE(sq, *pi);
457 case MLX5E_KTLS_SYNC_SKIP_NO_DATA:
458 if (likely(!skb->decrypted))
462 default: /* MLX5E_KTLS_SYNC_FAIL */
467 priv_tx->expected_seq = seq + datalen;
469 cseg = &(*wqe)->ctrl;
470 cseg->tisn = cpu_to_be32(priv_tx->tisn << 8);
472 stats->tls_encrypted_packets += skb_is_gso(skb) ? skb_shinfo(skb)->gso_segs : 1;
473 stats->tls_encrypted_bytes += datalen;
479 dev_kfree_skb_any(skb);