net/mlx5e: IPsec: Add Connect-X IPsec Rx data path offload
authorRaed Salem <raeds@mellanox.com>
Thu, 24 Oct 2019 13:11:28 +0000 (16:11 +0300)
committerSaeed Mahameed <saeedm@mellanox.com>
Thu, 16 Jul 2020 23:36:49 +0000 (16:36 -0700)
On receive flow inspect received packets for IPsec offload indication
using the cqe, for IPsec offloaded packets propagate offload status
and stack handle to stack for further processing.

Supported statuses:
- Offload ok.
- Authentication failure.
- Bad trailer indication.

Connect-X IPsec does not use mlx5e_ipsec_handle_rx_cqe.

For RX only offload, we see the BW gain. Below is the iperf3
performance report on two server of 24 cores Intel(R) Xeon(R)
CPU E5-2620 v3 @ 2.40GHz with ConnectX6-DX.
We use one thread per IPsec tunnel.

---------------------------------------------------------------------
Mode          |  Num tunnel | BW     | Send CPU util | Recv CPU util
              |             | (Gbps) | (Average %)   | (Average %)
---------------------------------------------------------------------
Cryto offload | 1           | 4.6    | 4.2           | 14.5
---------------------------------------------------------------------
Cryto offload | 24          | 38     | 73            | 63
---------------------------------------------------------------------
Non-offload   | 1           | 4      | 4             | 13
---------------------------------------------------------------------
Non-offload   | 24          | 23     | 52            | 67

Signed-off-by: Raed Salem <raeds@mellanox.com>
Reviewed-by: Boris Pismenny <borisp@mellanox.com>
Reviewed-by: Tariq Toukan <tariqt@mellanox.com>
Signed-off-by: Saeed Mahameed <saeedm@mellanox.com>
drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.c
drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_rxtx.h
drivers/net/ethernet/mellanox/mlx5/core/en_main.c
drivers/net/ethernet/mellanox/mlx5/core/en_rx.c

index 824b87ac8f9eece197550f24de0206085ca20a90..93a8d68815adeefb2c66bee05c1636ff3d7ff643 100644 (file)
@@ -360,6 +360,62 @@ struct sk_buff *mlx5e_ipsec_handle_rx_skb(struct net_device *netdev,
        return skb;
 }
 
+enum {
+       MLX5E_IPSEC_OFFLOAD_RX_SYNDROME_DECRYPTED,
+       MLX5E_IPSEC_OFFLOAD_RX_SYNDROME_AUTH_FAILED,
+       MLX5E_IPSEC_OFFLOAD_RX_SYNDROME_BAD_TRAILER,
+};
+
+void mlx5e_ipsec_offload_handle_rx_skb(struct net_device *netdev,
+                                      struct sk_buff *skb,
+                                      struct mlx5_cqe64 *cqe)
+{
+       u32 ipsec_meta_data = be32_to_cpu(cqe->ft_metadata);
+       u8 ipsec_syndrome = ipsec_meta_data & 0xFF;
+       struct mlx5e_priv *priv;
+       struct xfrm_offload *xo;
+       struct xfrm_state *xs;
+       struct sec_path *sp;
+       u32  sa_handle;
+
+       sa_handle = MLX5_IPSEC_METADATA_HANDLE(ipsec_meta_data);
+       priv = netdev_priv(netdev);
+       sp = secpath_set(skb);
+       if (unlikely(!sp)) {
+               atomic64_inc(&priv->ipsec->sw_stats.ipsec_rx_drop_sp_alloc);
+               return;
+       }
+
+       xs = mlx5e_ipsec_sadb_rx_lookup(priv->ipsec, sa_handle);
+       if (unlikely(!xs)) {
+               atomic64_inc(&priv->ipsec->sw_stats.ipsec_rx_drop_sadb_miss);
+               return;
+       }
+
+       sp = skb_sec_path(skb);
+       sp->xvec[sp->len++] = xs;
+       sp->olen++;
+
+       xo = xfrm_offload(skb);
+       xo->flags = CRYPTO_DONE;
+
+       switch (ipsec_syndrome & MLX5_IPSEC_METADATA_SYNDROM_MASK) {
+       case MLX5E_IPSEC_OFFLOAD_RX_SYNDROME_DECRYPTED:
+               xo->status = CRYPTO_SUCCESS;
+               if (WARN_ON_ONCE(priv->ipsec->no_trailer))
+                       xo->flags |= XFRM_ESP_NO_TRAILER;
+               break;
+       case MLX5E_IPSEC_OFFLOAD_RX_SYNDROME_AUTH_FAILED:
+               xo->status = CRYPTO_TUNNEL_ESP_AUTH_FAILED;
+               break;
+       case MLX5E_IPSEC_OFFLOAD_RX_SYNDROME_BAD_TRAILER:
+               xo->status = CRYPTO_INVALID_PACKET_SYNTAX;
+               break;
+       default:
+               atomic64_inc(&priv->ipsec->sw_stats.ipsec_rx_drop_syndrome);
+       }
+}
+
 bool mlx5e_ipsec_feature_check(struct sk_buff *skb, struct net_device *netdev,
                               netdev_features_t features)
 {
index ba02643586a540593109951640daeea4ae037777..2a47673da5a4e081fd092bdbc58b4c7cacf585d5 100644 (file)
 #ifndef __MLX5E_IPSEC_RXTX_H__
 #define __MLX5E_IPSEC_RXTX_H__
 
-#ifdef CONFIG_MLX5_EN_IPSEC
-
 #include <linux/skbuff.h>
 #include <net/xfrm.h>
 #include "en.h"
 #include "en/txrx.h"
 
+#define MLX5_IPSEC_METADATA_MARKER_MASK      (0x80)
+#define MLX5_IPSEC_METADATA_SYNDROM_MASK     (0x7F)
+#define MLX5_IPSEC_METADATA_HANDLE(metadata) (((metadata) >> 8) & 0xFF)
+
+#ifdef CONFIG_MLX5_EN_IPSEC
+
 struct sk_buff *mlx5e_ipsec_handle_rx_skb(struct net_device *netdev,
                                          struct sk_buff *skb, u32 *cqe_bcnt);
 void mlx5e_ipsec_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe);
@@ -55,7 +59,21 @@ void mlx5e_ipsec_set_iv(struct sk_buff *skb, struct xfrm_state *x,
 bool mlx5e_ipsec_handle_tx_skb(struct mlx5e_priv *priv,
                               struct mlx5_wqe_eth_seg *eseg,
                               struct sk_buff *skb);
+void mlx5e_ipsec_offload_handle_rx_skb(struct net_device *netdev,
+                                      struct sk_buff *skb,
+                                      struct mlx5_cqe64 *cqe);
+static inline bool mlx5_ipsec_is_rx_flow(struct mlx5_cqe64 *cqe)
+{
+       return !!(MLX5_IPSEC_METADATA_MARKER_MASK & be32_to_cpu(cqe->ft_metadata));
+}
+#else
+static inline
+void mlx5e_ipsec_offload_handle_rx_skb(struct net_device *netdev,
+                                      struct sk_buff *skb,
+                                      struct mlx5_cqe64 *cqe)
+{}
 
+static inline bool mlx5_ipsec_is_rx_flow(struct mlx5_cqe64 *cqe) { return false; }
 #endif /* CONFIG_MLX5_EN_IPSEC */
 
 #endif /* __MLX5E_IPSEC_RXTX_H__ */
index 4e5d83f6334a489c2fad499fde3bc6568cb009b0..88ea1908cb14ab97fa4fcbedc6e3281ef67a95d4 100644 (file)
@@ -65,6 +65,7 @@
 #include "en/hv_vhca_stats.h"
 #include "en/devlink.h"
 #include "lib/mlx5.h"
+#include "fpga/ipsec.h"
 
 bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev)
 {
@@ -496,7 +497,8 @@ static int mlx5e_alloc_rq(struct mlx5e_channel *c,
                rq->dealloc_wqe = mlx5e_dealloc_rx_wqe;
 
 #ifdef CONFIG_MLX5_EN_IPSEC
-               if (c->priv->ipsec)
+               if ((mlx5_fpga_ipsec_device_caps(mdev) & MLX5_ACCEL_IPSEC_CAP_DEVICE) &&
+                   c->priv->ipsec)
                        rq->handle_rx_cqe = mlx5e_ipsec_handle_rx_cqe;
                else
 #endif
index 350f9c54e508f3d88ff9e8eaf249d7ac9aab3e6b..8b24e44f860a8e79b062f5f635f2250f9e773204 100644 (file)
@@ -973,9 +973,14 @@ static inline void mlx5e_handle_csum(struct net_device *netdev,
                goto csum_unnecessary;
 
        if (likely(is_last_ethertype_ip(skb, &network_depth, &proto))) {
-               if (unlikely(get_ip_proto(skb, network_depth, proto) == IPPROTO_SCTP))
+               u8 ipproto = get_ip_proto(skb, network_depth, proto);
+
+               if (unlikely(ipproto == IPPROTO_SCTP))
                        goto csum_unnecessary;
 
+               if (unlikely(mlx5_ipsec_is_rx_flow(cqe)))
+                       goto csum_none;
+
                stats->csum_complete++;
                skb->ip_summed = CHECKSUM_COMPLETE;
                skb->csum = csum_unfold((__force __sum16)cqe->check_sum);
@@ -1021,6 +1026,9 @@ static inline void mlx5e_build_rx_skb(struct mlx5_cqe64 *cqe,
 
        mlx5e_tls_handle_rx_skb(rq, skb, cqe, &cqe_bcnt);
 
+       if (unlikely(mlx5_ipsec_is_rx_flow(cqe)))
+               mlx5e_ipsec_offload_handle_rx_skb(netdev, skb, cqe);
+
        if (lro_num_seg > 1) {
                mlx5e_lro_update_hdr(skb, cqe, cqe_bcnt);
                skb_shinfo(skb)->gso_size = DIV_ROUND_UP(cqe_bcnt, lro_num_seg);