projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'sched-core-2024-09-19' of git://git.kernel.org/pub/scm/linux/kernel/git...
[linux-2.6-block.git] / drivers / net / ethernet / mellanox / mlx5 / core / en_main.c
CommitLineData
f62b8bb8 1/*
b3f63c3d 2 * Copyright (c) 2015-2016, Mellanox Technologies. All rights reserved.
f62b8bb8
AV		 3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
eca1e8a6 33#include <linux/dim.h>
e8f887ac 34#include <net/tc_act/tc_gact.h>
86d722ad 35#include <linux/mlx5/fs.h>
b3f63c3d 36#include <net/vxlan.h>
e3cfc7e6 37#include <net/geneve.h>
86994156 38#include <linux/bpf.h>
288eca60 39#include <linux/debugfs.h>
4b89251d 40#include <linux/if_bridge.h>
7d714ff1 41#include <linux/filter.h>
7b66ae53 42#include <net/netdev_queues.h>
a9ca9f9c 43#include <net/page_pool/types.h>
9adafe2b 44#include <net/pkt_sched.h>
39d6443c 45#include <net/xdp_sock_drv.h>
1d447a39 46#include "eswitch.h"
f62b8bb8 47#include "en.h"
7ec56914 48#include "en/dim.h"
542578c6 49#include "en/txrx.h"
e8f887ac 50#include "en_tc.h"
1d447a39 51#include "en_rep.h"
547eede0 52#include "en_accel/ipsec.h"
8ff0ac5b 53#include "en_accel/macsec.h"
e3cfc7e6 54#include "en_accel/en_accel.h"
943aa7bd 55#include "en_accel/ktls.h"
358aa5ce 56#include "lib/vxlan.h"
6dbc80ca 57#include "lib/clock.h"
2c81bfd5 58#include "en/port.h"
159d2131 59#include "en/xdp.h"
f2f3df55 60#include "lib/eq.h"
5c7e8bbb 61#include "en/monitor_stats.h"
4edc17fd 62#include "en/health.h"
9a22d5d8 63#include "en/params.h"
1742b3d5 64#include "en/xsk/pool.h"
db05815b
MM		 65#include "en/xsk/setup.h"
66#include "en/xsk/rx.h"
67#include "en/xsk/tx.h"
cef35af3 68#include "en/hv_vhca_stats.h"
c6acd629 69#include "en/devlink.h"
71c6eaeb 70#include "lib/mlx5.h"
145e5637 71#include "en/ptp.h"
462b0059 72#include "en/htb.h"
214baf22 73#include "qos.h"
70038b73 74#include "en/trap.h"
bf11485f 75#include "lib/devcom.h"
381978d2 76#include "lib/sd.h"
f62b8bb8 77
99be5617
YZ		 78static bool mlx5e_hw_gro_supported(struct mlx5_core_dev *mdev)
79{
80 if (!MLX5_CAP_GEN(mdev, shampo))
81 return false;
82
83 /* Our HW-GRO implementation relies on "KSM Mkey" for
84 * SHAMPO headers buffer mapping
85 */
86 if (!MLX5_CAP_GEN(mdev, fixed_buffer_size))
87 return false;
88
89 if (!MLX5_CAP_GEN_2(mdev, min_mkey_log_entity_size_fixed_buffer_valid))
90 return false;
91
92 if (MLX5_CAP_GEN_2(mdev, min_mkey_log_entity_size_fixed_buffer) >
93 MLX5E_SHAMPO_LOG_HEADER_ENTRY_SIZE)
94 return false;
95
96 return true;
97}
98
6470d2e7 99bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev, u8 page_shift,
168723c1 100 enum mlx5e_mpwrq_umr_mode umr_mode)
2fc4bfb7 101{
e5a3cc83
MM		 102 u16 umr_wqebbs, max_wqebbs;
103 bool striding_rq_umr;
ea3886ca 104
c27bd171
AL		 105 striding_rq_umr = MLX5_CAP_GEN(mdev, striding_rq) && MLX5_CAP_GEN(mdev, umr_ptr_rlky) &&
106 MLX5_CAP_ETH(mdev, reg_umr_sq);
ea3886ca
TT		 107 if (!striding_rq_umr)
108 return false;
e5a3cc83 109
168723c1 110 umr_wqebbs = mlx5e_mpwrq_umr_wqebbs(mdev, page_shift, umr_mode);
e5a3cc83
MM		 111 max_wqebbs = mlx5e_get_max_sq_aligned_wqebbs(mdev);
112 /* Sanity check; should never happen, because mlx5e_mpwrq_umr_wqebbs is
113 * calculated from mlx5e_get_max_sq_aligned_wqebbs.
114 */
115 if (WARN_ON(umr_wqebbs > max_wqebbs))
ea3886ca 116 return false;
e5a3cc83 117
ea3886ca 118 return true;
2fc4bfb7
SM		 119}
120
b36cdb42 121void mlx5e_update_carrier(struct mlx5e_priv *priv)
f62b8bb8
AV		 122{
123 struct mlx5_core_dev *mdev = priv->mdev;
124 u8 port_state;
490dceca 125 bool up;
f62b8bb8
AV		 126
127 port_state = mlx5_query_vport_state(mdev,
cc9c82a8 128 MLX5_VPORT_STATE_OP_MOD_VNIC_VPORT,
e53eef63 129 0);
f62b8bb8 130
490dceca
JK		 131 up = port_state == VPORT_STATE_UP;
132 if (up == netif_carrier_ok(priv->netdev))
133 netif_carrier_event(priv->netdev);
134 if (up) {
87424ad5 135 netdev_info(priv->netdev, "Link up\n");
f62b8bb8 136 netif_carrier_on(priv->netdev);
87424ad5
SD		 137 } else {
138 netdev_info(priv->netdev, "Link down\n");
f62b8bb8 139 netif_carrier_off(priv->netdev);
87424ad5 140 }
f62b8bb8
AV		 141}
142
143static void mlx5e_update_carrier_work(struct work_struct *work)
144{
145 struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv,
146 update_carrier_work);
147
148 mutex_lock(&priv->state_lock);
149 if (test_bit(MLX5E_STATE_OPENED, &priv->state))
7ca42c80
ES		 150 if (priv->profile->update_carrier)
151 priv->profile->update_carrier(priv);
f62b8bb8
AV		 152 mutex_unlock(&priv->state_lock);
153}
154
303211b4 155static void mlx5e_update_stats_work(struct work_struct *work)
f62b8bb8 156{
cdeef2b1 157 struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv,
f62b8bb8 158 update_stats_work);
ed56c519 159
f62b8bb8 160 mutex_lock(&priv->state_lock);
ed56c519 161 priv->profile->update_stats(priv);
f62b8bb8
AV		 162 mutex_unlock(&priv->state_lock);
163}
164
cdeef2b1
SM		 165void mlx5e_queue_update_stats(struct mlx5e_priv *priv)
166{
167 if (!priv->profile->update_stats)
168 return;
169
170 if (unlikely(test_bit(MLX5E_STATE_DESTROYING, &priv->state)))
171 return;
172
173 queue_work(priv->wq, &priv->update_stats_work);
174}
175
7cffaddd 176static int async_event(struct notifier_block *nb, unsigned long event, void *data)
f62b8bb8 177{
7cffaddd
SM		 178 struct mlx5e_priv *priv = container_of(nb, struct mlx5e_priv, events_nb);
179 struct mlx5_eqe *eqe = data;
daa21560 180
7cffaddd
SM		 181 if (event != MLX5_EVENT_TYPE_PORT_CHANGE)
182 return NOTIFY_DONE;
daa21560 183
7cffaddd
SM		 184 switch (eqe->sub_type) {
185 case MLX5_PORT_CHANGE_SUBTYPE_DOWN:
186 case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE:
7bb29755 187 queue_work(priv->wq, &priv->update_carrier_work);
f62b8bb8 188 break;
f62b8bb8 189 default:
7cffaddd 190 return NOTIFY_DONE;
f62b8bb8 191 }
7cffaddd
SM		 192
193 return NOTIFY_OK;
f62b8bb8
AV		 194}
195
f62b8bb8
AV		 196static void mlx5e_enable_async_events(struct mlx5e_priv *priv)
197{
7cffaddd
SM		 198 priv->events_nb.notifier_call = async_event;
199 mlx5_notifier_register(priv->mdev, &priv->events_nb);
f62b8bb8
AV		 200}
201
202static void mlx5e_disable_async_events(struct mlx5e_priv *priv)
203{
7cffaddd 204 mlx5_notifier_unregister(priv->mdev, &priv->events_nb);
f62b8bb8
AV		 205}
206
bf11485f
PH		 207static int mlx5e_devcom_event_mpv(int event, void *my_data, void *event_data)
208{
209 struct mlx5e_priv *slave_priv = my_data;
210
82f9378c
PH		 211 switch (event) {
212 case MPV_DEVCOM_MASTER_UP:
213 mlx5_devcom_comp_set_ready(slave_priv->devcom, true);
214 break;
215 case MPV_DEVCOM_MASTER_DOWN:
216 /* no need for comp set ready false since we unregister after
217 * and it hurts cleanup flow.
218 */
219 break;
220 case MPV_DEVCOM_IPSEC_MASTER_UP:
221 case MPV_DEVCOM_IPSEC_MASTER_DOWN:
222 mlx5e_ipsec_handle_mpv_event(event, my_data, event_data);
223 break;
224 }
bf11485f
PH		 225
226 return 0;
227}
228
229static int mlx5e_devcom_init_mpv(struct mlx5e_priv *priv, u64 *data)
230{
231 priv->devcom = mlx5_devcom_register_component(priv->mdev->priv.devc,
232 MLX5_DEVCOM_MPV,
233 *data,
234 mlx5e_devcom_event_mpv,
235 priv);
aa4ac90d
TT		 236 if (IS_ERR(priv->devcom))
237 return PTR_ERR(priv->devcom);
bf11485f 238
82f9378c 239 if (mlx5_core_is_mp_master(priv->mdev)) {
bf11485f
PH		 240 mlx5_devcom_send_event(priv->devcom, MPV_DEVCOM_MASTER_UP,
241 MPV_DEVCOM_MASTER_UP, priv);
82f9378c
PH		 242 mlx5e_ipsec_send_event(priv, MPV_DEVCOM_IPSEC_MASTER_UP);
243 }
bf11485f
PH		 244
245 return 0;
246}
247
248static void mlx5e_devcom_cleanup_mpv(struct mlx5e_priv *priv)
249{
82f9378c
PH		 250 if (IS_ERR_OR_NULL(priv->devcom))
251 return;
252
253 if (mlx5_core_is_mp_master(priv->mdev)) {
254 mlx5_devcom_send_event(priv->devcom, MPV_DEVCOM_MASTER_DOWN,
255 MPV_DEVCOM_MASTER_DOWN, priv);
256 mlx5e_ipsec_send_event(priv, MPV_DEVCOM_IPSEC_MASTER_DOWN);
257 }
258
bf11485f
PH		 259 mlx5_devcom_unregister_component(priv->devcom);
260}
261
70038b73
AL		 262static int blocking_event(struct notifier_block *nb, unsigned long event, void *data)
263{
264 struct mlx5e_priv *priv = container_of(nb, struct mlx5e_priv, blocking_events_nb);
3f26a315 265 struct mlx5_devlink_trap_event_ctx *trap_event_ctx = data;
70038b73
AL		 266 int err;
267
268 switch (event) {
269 case MLX5_DRIVER_EVENT_TYPE_TRAP:
3f26a315
JP		 270 err = mlx5e_handle_trap_event(priv, trap_event_ctx->trap);
271 if (err) {
272 trap_event_ctx->err = err;
273 return NOTIFY_BAD;
274 }
70038b73 275 break;
bf11485f
PH		 276 case MLX5_DRIVER_EVENT_AFFILIATION_DONE:
277 if (mlx5e_devcom_init_mpv(priv, data))
278 return NOTIFY_BAD;
279 break;
280 case MLX5_DRIVER_EVENT_AFFILIATION_REMOVED:
281 mlx5e_devcom_cleanup_mpv(priv);
282 break;
70038b73 283 default:
3f26a315 284 return NOTIFY_DONE;
70038b73 285 }
3f26a315 286 return NOTIFY_OK;
70038b73
AL		 287}
288
289static void mlx5e_enable_blocking_events(struct mlx5e_priv *priv)
290{
291 priv->blocking_events_nb.notifier_call = blocking_event;
292 mlx5_blocking_notifier_register(priv->mdev, &priv->blocking_events_nb);
293}
294
295static void mlx5e_disable_blocking_events(struct mlx5e_priv *priv)
296{
297 mlx5_blocking_notifier_unregister(priv->mdev, &priv->blocking_events_nb);
298}
299
168723c1
MM		 300static u16 mlx5e_mpwrq_umr_octowords(u32 entries, enum mlx5e_mpwrq_umr_mode umr_mode)
301{
9f123f74 302 u8 umr_entry_size = mlx5e_mpwrq_umr_entry_size(umr_mode);
3e874cb1 303 u32 sz;
9f123f74 304
02648b4b 305 sz = ALIGN(entries * umr_entry_size, MLX5_UMR_FLEX_ALIGNMENT);
9f123f74 306
3e874cb1 307 return sz / MLX5_OCTWORD;
168723c1
MM		 308}
309
31391048
SM		 310static inline void mlx5e_build_umr_wqe(struct mlx5e_rq *rq,
311 struct mlx5e_icosq *sq,
b8a98a4c 312 struct mlx5e_umr_wqe *wqe)
7e426671
TT		 313{
314 struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl;
315 struct mlx5_wqe_umr_ctrl_seg *ucseg = &wqe->uctrl;
6470d2e7 316 u16 octowords;
997ce6af
MM		 317 u8 ds_cnt;
318
6470d2e7 319 ds_cnt = DIV_ROUND_UP(mlx5e_mpwrq_umr_wqe_sz(rq->mdev, rq->mpwqe.page_shift,
168723c1 320 rq->mpwqe.umr_mode),
997ce6af 321 MLX5_SEND_WQE_DS);
7e426671
TT		 322
323 cseg->qpn_ds = cpu_to_be32((sq->sqn << MLX5_WQE_CTRL_QPN_SHIFT) |
324 ds_cnt);
ecc7ad2e 325 cseg->umr_mkey = rq->mpwqe.umr_mkey_be;
7e426671 326
ea3886ca 327 ucseg->flags = MLX5_UMR_TRANSLATION_OFFSET_EN | MLX5_UMR_INLINE;
168723c1 328 octowords = mlx5e_mpwrq_umr_octowords(rq->mpwqe.pages_per_wqe, rq->mpwqe.umr_mode);
6470d2e7 329 ucseg->xlt_octowords = cpu_to_be16(octowords);
7e426671 330 ucseg->mkey_mask = cpu_to_be64(MLX5_MKEY_MASK_FREE);
7e426671
TT		 331}
332
e5ca8fb0
BBI		 333static int mlx5e_rq_shampo_hd_alloc(struct mlx5e_rq *rq, int node)
334{
335 rq->mpwqe.shampo = kvzalloc_node(sizeof(*rq->mpwqe.shampo),
336 GFP_KERNEL, node);
337 if (!rq->mpwqe.shampo)
338 return -ENOMEM;
339 return 0;
340}
341
342static void mlx5e_rq_shampo_hd_free(struct mlx5e_rq *rq)
343{
344 kvfree(rq->mpwqe.shampo);
345}
346
347static int mlx5e_rq_shampo_hd_info_alloc(struct mlx5e_rq *rq, int node)
348{
349 struct mlx5e_shampo_hd *shampo = rq->mpwqe.shampo;
350
351 shampo->bitmap = bitmap_zalloc_node(shampo->hd_per_wq, GFP_KERNEL,
352 node);
e5ca8fb0
BBI		 353 shampo->info = kvzalloc_node(array_size(shampo->hd_per_wq,
354 sizeof(*shampo->info)),
355 GFP_KERNEL, node);
ca6ef9f0
DT		 356 shampo->pages = kvzalloc_node(array_size(shampo->hd_per_wq,
357 sizeof(*shampo->pages)),
358 GFP_KERNEL, node);
359 if (!shampo->bitmap || !shampo->info || !shampo->pages)
360 goto err_nomem;
361
e5ca8fb0 362 return 0;
ca6ef9f0
DT		 363
364err_nomem:
365 kvfree(shampo->info);
366 kvfree(shampo->bitmap);
367 kvfree(shampo->pages);
368
369 return -ENOMEM;
e5ca8fb0
BBI		 370}
371
372static void mlx5e_rq_shampo_hd_info_free(struct mlx5e_rq *rq)
373{
374 kvfree(rq->mpwqe.shampo->bitmap);
375 kvfree(rq->mpwqe.shampo->info);
ca6ef9f0 376 kvfree(rq->mpwqe.shampo->pages);
e5ca8fb0
BBI		 377}
378
ea886000 379static int mlx5e_rq_alloc_mpwqe_info(struct mlx5e_rq *rq, int node)
7e426671 380{
422d4c40 381 int wq_sz = mlx5_wq_ll_get_size(&rq->mpwqe.wq);
258e655c 382 size_t alloc_size;
7e426671 383
d39092ca 384 alloc_size = array_size(wq_sz, struct_size(rq->mpwqe.info,
6f574284 385 alloc_units.frag_pages,
997ce6af 386 rq->mpwqe.pages_per_wqe));
258e655c
MM		 387
388 rq->mpwqe.info = kvzalloc_node(alloc_size, GFP_KERNEL, node);
21c59685 389 if (!rq->mpwqe.info)
ea3886ca 390 return -ENOMEM;
7e426671 391
4c2a1323
DT		 392 /* For deferred page release (release right before alloc), make sure
393 * that on first round release is not called.
394 */
395 for (int i = 0; i < wq_sz; i++) {
396 struct mlx5e_mpw_info *wi = mlx5e_get_mpw_info(rq, i);
397
398 bitmap_fill(wi->skip_release_bitmap, rq->mpwqe.pages_per_wqe);
399 }
400
ea886000 401 mlx5e_build_umr_wqe(rq, rq->icosq, &rq->mpwqe.umr_wqe);
7e426671
TT		 402
403 return 0;
7e426671
TT		 404}
405
168723c1
MM		 406
407static u8 mlx5e_mpwrq_access_mode(enum mlx5e_mpwrq_umr_mode umr_mode)
408{
409 switch (umr_mode) {
410 case MLX5E_MPWRQ_UMR_MODE_ALIGNED:
411 return MLX5_MKC_ACCESS_MODE_MTT;
412 case MLX5E_MPWRQ_UMR_MODE_UNALIGNED:
413 return MLX5_MKC_ACCESS_MODE_KSM;
13921345
MM		 414 case MLX5E_MPWRQ_UMR_MODE_OVERSIZED:
415 return MLX5_MKC_ACCESS_MODE_KLMS;
c2c9e31d
MM		 416 case MLX5E_MPWRQ_UMR_MODE_TRIPLE:
417 return MLX5_MKC_ACCESS_MODE_KSM;
168723c1
MM		 418 }
419 WARN_ONCE(1, "MPWRQ UMR mode %d is not known\n", umr_mode);
420 return 0;
421}
422
6470d2e7
MM		 423static int mlx5e_create_umr_mkey(struct mlx5_core_dev *mdev,
424 u32 npages, u8 page_shift, u32 *umr_mkey,
168723c1 425 dma_addr_t filler_addr,
13921345
MM		 426 enum mlx5e_mpwrq_umr_mode umr_mode,
427 u32 xsk_chunk_size)
3608ae77 428{
c3c94023 429 struct mlx5_mtt *mtt;
6470d2e7 430 struct mlx5_ksm *ksm;
13921345 431 struct mlx5_klm *klm;
6470d2e7 432 u32 octwords;
c3c94023 433 int inlen;
3608ae77
TT		 434 void *mkc;
435 u32 *in;
436 int err;
c3c94023
AL		 437 int i;
438
c2c9e31d
MM		 439 if ((umr_mode == MLX5E_MPWRQ_UMR_MODE_UNALIGNED ||
440 umr_mode == MLX5E_MPWRQ_UMR_MODE_TRIPLE) &&
168723c1 441 !MLX5_CAP_GEN(mdev, fixed_buffer_size)) {
6470d2e7
MM		 442 mlx5_core_warn(mdev, "Unaligned AF_XDP requires fixed_buffer_size capability\n");
443 return -EINVAL;
444 }
445
168723c1
MM		 446 octwords = mlx5e_mpwrq_umr_octowords(npages, umr_mode);
447
6470d2e7 448 inlen = MLX5_FLEXIBLE_INLEN(mdev, MLX5_ST_SZ_BYTES(create_mkey_in),
168723c1 449 MLX5_OCTWORD, octwords);
6470d2e7
MM		 450 if (inlen < 0)
451 return inlen;
3608ae77 452
1b9a07ee 453 in = kvzalloc(inlen, GFP_KERNEL);
3608ae77
TT		 454 if (!in)
455 return -ENOMEM;
456
457 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
458
3608ae77
TT		 459 MLX5_SET(mkc, mkc, free, 1);
460 MLX5_SET(mkc, mkc, umr_en, 1);
461 MLX5_SET(mkc, mkc, lw, 1);
462 MLX5_SET(mkc, mkc, lr, 1);
168723c1 463 MLX5_SET(mkc, mkc, access_mode_1_0, mlx5e_mpwrq_access_mode(umr_mode));
17347d54 464 mlx5e_mkey_set_relaxed_ordering(mdev, mkc);
3608ae77 465 MLX5_SET(mkc, mkc, qpn, 0xffffff);
c276aae8 466 MLX5_SET(mkc, mkc, pd, mdev->mlx5e_res.hw_objs.pdn);
ec8b9981 467 MLX5_SET64(mkc, mkc, len, npages << page_shift);
6470d2e7 468 MLX5_SET(mkc, mkc, translations_octword_size, octwords);
c2c9e31d
MM		 469 if (umr_mode == MLX5E_MPWRQ_UMR_MODE_TRIPLE)
470 MLX5_SET(mkc, mkc, log_page_size, page_shift - 2);
471 else if (umr_mode != MLX5E_MPWRQ_UMR_MODE_OVERSIZED)
13921345 472 MLX5_SET(mkc, mkc, log_page_size, page_shift);
6470d2e7 473 MLX5_SET(create_mkey_in, in, translations_octword_actual_size, octwords);
c3c94023
AL		 474
475 /* Initialize the mkey with all MTTs pointing to a default
476 * page (filler_addr). When the channels are activated, UMR
477 * WQEs will redirect the RX WQEs to the actual memory from
478 * the RQ's pool, while the gaps (wqe_overflow) remain mapped
479 * to the default page.
480 */
168723c1 481 switch (umr_mode) {
13921345
MM		 482 case MLX5E_MPWRQ_UMR_MODE_OVERSIZED:
483 klm = MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt);
484 for (i = 0; i < npages; i++) {
485 klm[i << 1] = (struct mlx5_klm) {
486 .va = cpu_to_be64(filler_addr),
487 .bcount = cpu_to_be32(xsk_chunk_size),
488 .key = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey),
489 };
490 klm[(i << 1) + 1] = (struct mlx5_klm) {
491 .va = cpu_to_be64(filler_addr),
492 .bcount = cpu_to_be32((1 << page_shift) - xsk_chunk_size),
493 .key = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey),
494 };
495 }
496 break;
168723c1 497 case MLX5E_MPWRQ_UMR_MODE_UNALIGNED:
6470d2e7
MM		 498 ksm = MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt);
499 for (i = 0; i < npages; i++)
500 ksm[i] = (struct mlx5_ksm) {
501 .key = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey),
502 .va = cpu_to_be64(filler_addr),
503 };
168723c1
MM		 504 break;
505 case MLX5E_MPWRQ_UMR_MODE_ALIGNED:
6470d2e7
MM		 506 mtt = MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt);
507 for (i = 0; i < npages; i++)
508 mtt[i] = (struct mlx5_mtt) {
509 .ptag = cpu_to_be64(filler_addr),
510 };
168723c1 511 break;
c2c9e31d
MM		 512 case MLX5E_MPWRQ_UMR_MODE_TRIPLE:
513 ksm = MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt);
514 for (i = 0; i < npages * 4; i++) {
515 ksm[i] = (struct mlx5_ksm) {
516 .key = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey),
517 .va = cpu_to_be64(filler_addr),
518 };
519 }
520 break;
6470d2e7 521 }
3608ae77 522
ec8b9981 523 err = mlx5_core_create_mkey(mdev, umr_mkey, in, inlen);
3608ae77
TT		 524
525 kvfree(in);
526 return err;
527}
528
758191c9
YZ		 529static int mlx5e_create_umr_ksm_mkey(struct mlx5_core_dev *mdev,
530 u64 nentries, u8 log_entry_size,
573bce9e 531 u32 *umr_mkey)
e5ca8fb0
BBI		 532{
533 int inlen;
534 void *mkc;
535 u32 *in;
536 int err;
537
538 inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
539
540 in = kvzalloc(inlen, GFP_KERNEL);
541 if (!in)
542 return -ENOMEM;
543
544 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
545
546 MLX5_SET(mkc, mkc, free, 1);
547 MLX5_SET(mkc, mkc, umr_en, 1);
548 MLX5_SET(mkc, mkc, lw, 1);
549 MLX5_SET(mkc, mkc, lr, 1);
758191c9 550 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_KSM);
e5ca8fb0
BBI		 551 mlx5e_mkey_set_relaxed_ordering(mdev, mkc);
552 MLX5_SET(mkc, mkc, qpn, 0xffffff);
553 MLX5_SET(mkc, mkc, pd, mdev->mlx5e_res.hw_objs.pdn);
554 MLX5_SET(mkc, mkc, translations_octword_size, nentries);
758191c9
YZ		 555 MLX5_SET(mkc, mkc, log_page_size, log_entry_size);
556 MLX5_SET64(mkc, mkc, len, nentries << log_entry_size);
e5ca8fb0
BBI		 557 err = mlx5_core_create_mkey(mdev, umr_mkey, in, inlen);
558
559 kvfree(in);
560 return err;
561}
562
a43b25da 563static int mlx5e_create_rq_umr_mkey(struct mlx5_core_dev *mdev, struct mlx5e_rq *rq)
ec8b9981 564{
13921345 565 u32 xsk_chunk_size = rq->xsk_pool ? rq->xsk_pool->chunk_size : 0;
6470d2e7
MM		 566 u32 wq_size = mlx5_wq_ll_get_size(&rq->mpwqe.wq);
567 u32 num_entries, max_num_entries;
ecc7ad2e
MM		 568 u32 umr_mkey;
569 int err;
ec8b9981 570
168723c1 571 max_num_entries = mlx5e_mpwrq_max_num_entries(mdev, rq->mpwqe.umr_mode);
6470d2e7
MM		 572
573 /* Shouldn't overflow, the result is at most MLX5E_MAX_RQ_NUM_MTTS. */
574 if (WARN_ON_ONCE(check_mul_overflow(wq_size, (u32)rq->mpwqe.mtts_per_wqe,
575 &num_entries) ||
576 num_entries > max_num_entries))
577 mlx5_core_err(mdev, "%s: multiplication overflow: %u * %u > %u\n",
578 __func__, wq_size, rq->mpwqe.mtts_per_wqe,
579 max_num_entries);
580
581 err = mlx5e_create_umr_mkey(mdev, num_entries, rq->mpwqe.page_shift,
582 &umr_mkey, rq->wqe_overflow.addr,
13921345 583 rq->mpwqe.umr_mode, xsk_chunk_size);
ecc7ad2e
MM		 584 rq->mpwqe.umr_mkey_be = cpu_to_be32(umr_mkey);
585 return err;
e5ca8fb0
BBI		 586}
587
588static int mlx5e_create_rq_hd_umr_mkey(struct mlx5_core_dev *mdev,
589 struct mlx5e_rq *rq)
590{
758191c9 591 u32 max_ksm_size = BIT(MLX5_CAP_GEN(mdev, log_max_klm_list_size));
e5ca8fb0 592
758191c9
YZ		 593 if (max_ksm_size < rq->mpwqe.shampo->hd_per_wq) {
594 mlx5_core_err(mdev, "max ksm list size 0x%x is smaller than shampo header buffer list size 0x%x\n",
595 max_ksm_size, rq->mpwqe.shampo->hd_per_wq);
e5ca8fb0
BBI		 596 return -EINVAL;
597 }
758191c9
YZ		 598
599 return mlx5e_create_umr_ksm_mkey(mdev, rq->mpwqe.shampo->hd_per_wq,
600 MLX5E_SHAMPO_LOG_HEADER_ENTRY_SIZE,
e5ca8fb0 601 &rq->mpwqe.shampo->mkey);
ec8b9981
TT		 602}
603
069d1146
TT		 604static void mlx5e_init_frags_partition(struct mlx5e_rq *rq)
605{
60d60c8f
QC		 606 struct mlx5e_wqe_frag_info next_frag = {};
607 struct mlx5e_wqe_frag_info *prev = NULL;
069d1146
TT		 608 int i;
609
8fb1814f 610 WARN_ON(rq->xsk_pool);
069d1146 611
6f574284 612 next_frag.frag_page = &rq->wqe.alloc_units->frag_pages[0];
259bbc64 613
3f93f829
DT		 614 /* Skip first release due to deferred release. */
615 next_frag.flags = BIT(MLX5E_WQE_FRAG_SKIP_RELEASE);
069d1146
TT		 616
617 for (i = 0; i < mlx5_wq_cyc_get_size(&rq->wqe.wq); i++) {
618 struct mlx5e_rq_frag_info *frag_info = &rq->wqe.info.arr[0];
619 struct mlx5e_wqe_frag_info *frag =
620 &rq->wqe.frags[i << rq->wqe.info.log_num_frags];
621 int f;
622
623 for (f = 0; f < rq->wqe.info.num_frags; f++, frag++) {
624 if (next_frag.offset + frag_info[f].frag_stride > PAGE_SIZE) {
8fb1814f 625 /* Pages are assigned at runtime. */
6f574284 626 next_frag.frag_page++;
069d1146
TT		 627 next_frag.offset = 0;
628 if (prev)
625dff29 629 prev->flags |= BIT(MLX5E_WQE_FRAG_LAST_IN_PAGE);
069d1146
TT		 630 }
631 *frag = next_frag;
632
633 /* prepare next */
634 next_frag.offset += frag_info[f].frag_stride;
635 prev = frag;
636 }
637 }
638
639 if (prev)
625dff29 640 prev->flags |= BIT(MLX5E_WQE_FRAG_LAST_IN_PAGE);
069d1146
TT		 641}
642
8fb1814f
DT		 643static void mlx5e_init_xsk_buffs(struct mlx5e_rq *rq)
644{
645 int i;
646
647 /* Assumptions used by XSK batched allocator. */
648 WARN_ON(rq->wqe.info.num_frags != 1);
649 WARN_ON(rq->wqe.info.log_num_frags != 0);
650 WARN_ON(rq->wqe.info.arr[0].frag_stride != PAGE_SIZE);
651
652 /* Considering the above assumptions a fragment maps to a single
653 * xsk_buff.
654 */
3f93f829 655 for (i = 0; i < mlx5_wq_cyc_get_size(&rq->wqe.wq); i++) {
8fb1814f 656 rq->wqe.frags[i].xskp = &rq->wqe.alloc_units->xsk_buffs[i];
3f93f829
DT		 657
658 /* Skip first release due to deferred release as WQES are
659 * not allocated yet.
660 */
661 rq->wqe.frags[i].flags |= BIT(MLX5E_WQE_FRAG_SKIP_RELEASE);
662 }
069d1146
TT		 663}
664
8fb1814f 665static int mlx5e_init_wqe_alloc_info(struct mlx5e_rq *rq, int node)
069d1146 666{
8fb1814f 667 int wq_sz = mlx5_wq_cyc_get_size(&rq->wqe.wq);
069d1146 668 int len = wq_sz << rq->wqe.info.log_num_frags;
8fb1814f
DT		 669 struct mlx5e_wqe_frag_info *frags;
670 union mlx5e_alloc_units *aus;
671 int aus_sz;
069d1146 672
8fb1814f
DT		 673 if (rq->xsk_pool)
674 aus_sz = sizeof(*aus->xsk_buffs);
675 else
6f574284 676 aus_sz = sizeof(*aus->frag_pages);
8fb1814f
DT		 677
678 aus = kvzalloc_node(array_size(len, aus_sz), GFP_KERNEL, node);
679 if (!aus)
069d1146
TT		 680 return -ENOMEM;
681
8fb1814f
DT		 682 frags = kvzalloc_node(array_size(len, sizeof(*frags)), GFP_KERNEL, node);
683 if (!frags) {
684 kvfree(aus);
069d1146 685 return -ENOMEM;
8fb1814f
DT		 686 }
687
688 rq->wqe.alloc_units = aus;
689 rq->wqe.frags = frags;
069d1146 690
8fb1814f
DT		 691 if (rq->xsk_pool)
692 mlx5e_init_xsk_buffs(rq);
693 else
694 mlx5e_init_frags_partition(rq);
069d1146
TT		 695
696 return 0;
697}
698
8fb1814f 699static void mlx5e_free_wqe_alloc_info(struct mlx5e_rq *rq)
069d1146 700{
8fb1814f 701 kvfree(rq->wqe.frags);
79008676 702 kvfree(rq->wqe.alloc_units);
069d1146
TT		 703}
704
8276ea13
AL		 705static void mlx5e_rq_err_cqe_work(struct work_struct *recover_work)
706{
707 struct mlx5e_rq *rq = container_of(recover_work, struct mlx5e_rq, recover_work);
708
709 mlx5e_reporter_rq_cqe_err(rq);
710}
711
c3c94023
AL		 712static int mlx5e_alloc_mpwqe_rq_drop_page(struct mlx5e_rq *rq)
713{
714 rq->wqe_overflow.page = alloc_page(GFP_KERNEL);
715 if (!rq->wqe_overflow.page)
716 return -ENOMEM;
717
718 rq->wqe_overflow.addr = dma_map_page(rq->pdev, rq->wqe_overflow.page, 0,
719 PAGE_SIZE, rq->buff.map_dir);
720 if (dma_mapping_error(rq->pdev, rq->wqe_overflow.addr)) {
721 __free_page(rq->wqe_overflow.page);
722 return -ENOMEM;
723 }
724 return 0;
725}
726
727static void mlx5e_free_mpwqe_rq_drop_page(struct mlx5e_rq *rq)
728{
729 dma_unmap_page(rq->pdev, rq->wqe_overflow.addr, PAGE_SIZE,
730 rq->buff.map_dir);
731 __free_page(rq->wqe_overflow.page);
732}
733
869c5f92 734static int mlx5e_init_rxq_rq(struct mlx5e_channel *c, struct mlx5e_params *params,
4e7401fc 735 u32 xdp_frag_size, struct mlx5e_rq *rq)
869c5f92
AL		 736{
737 struct mlx5_core_dev *mdev = c->mdev;
738 int err;
739
740 rq->wq_type = params->rq_wq_type;
741 rq->pdev = c->pdev;
742 rq->netdev = c->netdev;
743 rq->priv = c->priv;
744 rq->tstamp = c->tstamp;
745 rq->clock = &mdev->clock;
746 rq->icosq = &c->icosq;
747 rq->ix = c->ix;
2e642afb 748 rq->channel = c;
869c5f92 749 rq->mdev = mdev;
1e662209
AF		 750 rq->hw_mtu =
751 MLX5E_SW2HW_MTU(params, params->sw_mtu) - ETH_FCS_LEN * !params->scatter_fcs_en;
869c5f92 752 rq->xdpsq = &c->rq_xdpsq;
be98737a 753 rq->stats = &c->priv->channel_stats[c->ix]->rq;
869c5f92
AL		 754 rq->ptp_cyc2time = mlx5_rq_ts_translator(mdev);
755 err = mlx5e_rq_set_handlers(rq, params, NULL);
756 if (err)
757 return err;
758
4e7401fc
MM		 759 return __xdp_rxq_info_reg(&rq->xdp_rxq, rq->netdev, rq->ix, c->napi.napi_id,
760 xdp_frag_size);
869c5f92
AL		 761}
762
e5ca8fb0
BBI		 763static int mlx5_rq_shampo_alloc(struct mlx5_core_dev *mdev,
764 struct mlx5e_params *params,
765 struct mlx5e_rq_param *rqp,
766 struct mlx5e_rq *rq,
767 u32 *pool_size,
768 int node)
769{
770 void *wqc = MLX5_ADDR_OF(rqc, rqp->rqc, wq);
771 int wq_size;
772 int err;
773
774 if (!test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state))
775 return 0;
776 err = mlx5e_rq_shampo_hd_alloc(rq, node);
777 if (err)
778 goto out;
779 rq->mpwqe.shampo->hd_per_wq =
780 mlx5e_shampo_hd_per_wq(mdev, params, rqp);
781 err = mlx5e_create_rq_hd_umr_mkey(mdev, rq);
782 if (err)
783 goto err_shampo_hd;
784 err = mlx5e_rq_shampo_hd_info_alloc(rq, node);
785 if (err)
786 goto err_shampo_info;
92552d3a
KM		 787 rq->hw_gro_data = kvzalloc_node(sizeof(*rq->hw_gro_data), GFP_KERNEL, node);
788 if (!rq->hw_gro_data) {
789 err = -ENOMEM;
790 goto err_hw_gro_data;
791 }
e5ca8fb0 792 rq->mpwqe.shampo->key =
573bce9e 793 cpu_to_be32(rq->mpwqe.shampo->mkey);
e5ca8fb0
BBI		 794 rq->mpwqe.shampo->hd_per_wqe =
795 mlx5e_shampo_hd_per_wqe(mdev, params, rqp);
796 wq_size = BIT(MLX5_GET(wq, wqc, log_wq_sz));
797 *pool_size += (rq->mpwqe.shampo->hd_per_wqe * wq_size) /
798 MLX5E_SHAMPO_WQ_HEADER_PER_PAGE;
799 return 0;
800
92552d3a
KM		 801err_hw_gro_data:
802 mlx5e_rq_shampo_hd_info_free(rq);
e5ca8fb0 803err_shampo_info:
573bce9e 804 mlx5_core_destroy_mkey(mdev, rq->mpwqe.shampo->mkey);
e5ca8fb0
BBI		 805err_shampo_hd:
806 mlx5e_rq_shampo_hd_free(rq);
807out:
808 return err;
809}
810
811static void mlx5e_rq_free_shampo(struct mlx5e_rq *rq)
812{
813 if (!test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state))
814 return;
92552d3a
KM		 815
816 kvfree(rq->hw_gro_data);
e5ca8fb0 817 mlx5e_rq_shampo_hd_info_free(rq);
573bce9e 818 mlx5_core_destroy_mkey(rq->mdev, rq->mpwqe.shampo->mkey);
e5ca8fb0
BBI		 819 mlx5e_rq_shampo_hd_free(rq);
820}
821
869c5f92 822static int mlx5e_alloc_rq(struct mlx5e_params *params,
db05815b 823 struct mlx5e_xsk_param *xsk,
6a9764ef 824 struct mlx5e_rq_param *rqp,
869c5f92 825 int node, struct mlx5e_rq *rq)
f62b8bb8 826{
869c5f92 827 struct mlx5_core_dev *mdev = rq->mdev;
6a9764ef 828 void *rqc = rqp->rqc;
f62b8bb8 829 void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq);
069d1146 830 u32 pool_size;
f62b8bb8
AV		 831 int wq_sz;
832 int err;
833 int i;
834
869c5f92 835 rqp->wq.db_numa_node = node;
8276ea13 836 INIT_WORK(&rq->recover_work, mlx5e_rq_err_cqe_work);
97bc402d 837
85192dbf
AN		 838 if (params->xdp_prog)
839 bpf_prog_inc(params->xdp_prog);
fe45386a 840 RCU_INIT_POINTER(rq->xdp_prog, params->xdp_prog);
7e426671 841
fe45386a 842 rq->buff.map_dir = params->xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE;
db05815b 843 rq->buff.headroom = mlx5e_get_rq_headroom(mdev, params, xsk);
60bbf7ee 844 pool_size = 1 << params->log_rq_mtu_frames;
b5503b99 845
ecc7ad2e
MM		 846 rq->mkey_be = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey);
847
6a9764ef 848 switch (rq->wq_type) {
461017cb 849 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
422d4c40
TT		 850 err = mlx5_wq_ll_create(mdev, &rqp->wq, rqc_wq, &rq->mpwqe.wq,
851 &rq->wq_ctrl);
852 if (err)
869c5f92 853 goto err_rq_xdp_prog;
422d4c40 854
c3c94023 855 err = mlx5e_alloc_mpwqe_rq_drop_page(rq);
422d4c40 856 if (err)
e692139e 857 goto err_rq_wq_destroy;
422d4c40
TT		 858
859 rq->mpwqe.wq.db = &rq->mpwqe.wq.db[MLX5_RCV_DBR];
860
861 wq_sz = mlx5_wq_ll_get_size(&rq->mpwqe.wq);
60bbf7ee 862
e5a3cc83 863 rq->mpwqe.page_shift = mlx5e_mpwrq_page_shift(mdev, xsk);
168723c1 864 rq->mpwqe.umr_mode = mlx5e_mpwrq_umr_mode(mdev, xsk);
e5a3cc83 865 rq->mpwqe.pages_per_wqe =
6470d2e7 866 mlx5e_mpwrq_pages_per_wqe(mdev, rq->mpwqe.page_shift,
168723c1 867 rq->mpwqe.umr_mode);
e5a3cc83 868 rq->mpwqe.umr_wqebbs =
6470d2e7 869 mlx5e_mpwrq_umr_wqebbs(mdev, rq->mpwqe.page_shift,
168723c1 870 rq->mpwqe.umr_mode);
e5a3cc83 871 rq->mpwqe.mtts_per_wqe =
6470d2e7 872 mlx5e_mpwrq_mtts_per_wqe(mdev, rq->mpwqe.page_shift,
168723c1 873 rq->mpwqe.umr_mode);
997ce6af
MM		 874
875 pool_size = rq->mpwqe.pages_per_wqe <<
e5a3cc83 876 mlx5e_mpwqe_get_log_rq_size(mdev, params, xsk);
422d4c40 877
f52ac702
TT		 878 if (!mlx5e_rx_mpwqe_is_linear_skb(mdev, params, xsk) && params->xdp_prog)
879 pool_size *= 2; /* additional page per packet for the linear part */
880
db05815b
MM		 881 rq->mpwqe.log_stride_sz = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk);
882 rq->mpwqe.num_strides =
883 BIT(mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk));
4b5fba4a 884 rq->mpwqe.min_wqe_bulk = mlx5e_mpwqe_get_min_wqe_bulk(wq_sz);
1bfecfca 885
d628ee4f
JDB		 886 rq->buff.frame0_sz = (1 << rq->mpwqe.log_stride_sz);
887
a43b25da 888 err = mlx5e_create_rq_umr_mkey(mdev, rq);
7e426671 889 if (err)
c3c94023 890 goto err_rq_drop_page;
ec8b9981 891
869c5f92 892 err = mlx5e_rq_alloc_mpwqe_info(rq, node);
ec8b9981 893 if (err)
08a762ce 894 goto err_rq_mkey;
e5ca8fb0
BBI		 895
896 err = mlx5_rq_shampo_alloc(mdev, params, rqp, rq, &pool_size, node);
897 if (err)
8f5ed1c1 898 goto err_free_mpwqe_info;
e5ca8fb0 899
461017cb 900 break;
99cbfa93
TT		 901 default: /* MLX5_WQ_TYPE_CYCLIC */
902 err = mlx5_wq_cyc_create(mdev, &rqp->wq, rqc_wq, &rq->wqe.wq,
903 &rq->wq_ctrl);
422d4c40 904 if (err)
869c5f92 905 goto err_rq_xdp_prog;
422d4c40
TT		 906
907 rq->wqe.wq.db = &rq->wqe.wq.db[MLX5_RCV_DBR];
908
99cbfa93 909 wq_sz = mlx5_wq_cyc_get_size(&rq->wqe.wq);
422d4c40 910
069d1146 911 rq->wqe.info = rqp->frags_info;
d628ee4f
JDB		 912 rq->buff.frame0_sz = rq->wqe.info.arr[0].frag_stride;
913
8fb1814f 914 err = mlx5e_init_wqe_alloc_info(rq, node);
069d1146 915 if (err)
8fb1814f 916 goto err_rq_wq_destroy;
461017cb 917 }
f62b8bb8 918
db05815b 919 if (xsk) {
db05815b 920 err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq,
39d6443c 921 MEM_TYPE_XSK_BUFF_POOL, NULL);
c4655761 922 xsk_pool_set_rxq_info(rq->xsk_pool, &rq->xdp_rxq);
db05815b
MM		 923 } else {
924 /* Create a page_pool and register it with rxq */
4a5c5e25
DT		 925 struct page_pool_params pp_params = { 0 };
926
db05815b 927 pp_params.order = 0;
09d96ee5 928 pp_params.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV;
db05815b 929 pp_params.pool_size = pool_size;
869c5f92
AL		 930 pp_params.nid = node;
931 pp_params.dev = rq->pdev;
a880f814 932 pp_params.napi = rq->cq.napi;
7cc9e6d7 933 pp_params.netdev = rq->netdev;
db05815b 934 pp_params.dma_dir = rq->buff.map_dir;
4a5c5e25 935 pp_params.max_len = PAGE_SIZE;
db05815b
MM		 936
937 /* page_pool can be used even when there is no rq->xdp_prog,
938 * given page_pool does not handle DMA mapping there is no
939 * required state to clear. And page_pool gracefully handle
940 * elevated refcnt.
941 */
942 rq->page_pool = page_pool_create(&pp_params);
943 if (IS_ERR(rq->page_pool)) {
944 err = PTR_ERR(rq->page_pool);
945 rq->page_pool = NULL;
8f5ed1c1 946 goto err_free_by_rq_type;
db05815b 947 }
5b232ea9
AL		 948 if (xdp_rxq_info_is_reg(&rq->xdp_rxq))
949 err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq,
950 MEM_TYPE_PAGE_POOL, rq->page_pool);
84f5e3fb 951 }
db05815b 952 if (err)
8f5ed1c1 953 goto err_destroy_page_pool;
84f5e3fb 954
f62b8bb8 955 for (i = 0; i < wq_sz; i++) {
4c2af5cc 956 if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) {
99cbfa93 957 struct mlx5e_rx_wqe_ll *wqe =
422d4c40 958 mlx5_wq_ll_get_wqe(&rq->mpwqe.wq, i);
069d1146
TT		 959 u32 byte_count =
960 rq->mpwqe.num_strides << rq->mpwqe.log_stride_sz;
997ce6af
MM		 961 u64 dma_offset = mul_u32_u32(i, rq->mpwqe.mtts_per_wqe) <<
962 rq->mpwqe.page_shift;
e5ca8fb0
BBI		 963 u16 headroom = test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state) ?
964 0 : rq->buff.headroom;
4c2af5cc 965
e5ca8fb0 966 wqe->data[0].addr = cpu_to_be64(dma_offset + headroom);
99cbfa93 967 wqe->data[0].byte_count = cpu_to_be32(byte_count);
ecc7ad2e 968 wqe->data[0].lkey = rq->mpwqe.umr_mkey_be;
422d4c40 969 } else {
99cbfa93
TT		 970 struct mlx5e_rx_wqe_cyc *wqe =
971 mlx5_wq_cyc_get_wqe(&rq->wqe.wq, i);
069d1146
TT		 972 int f;
973
974 for (f = 0; f < rq->wqe.info.num_frags; f++) {
975 u32 frag_size = rq->wqe.info.arr[f].frag_size |
976 MLX5_HW_START_PADDING;
977
978 wqe->data[f].byte_count = cpu_to_be32(frag_size);
979 wqe->data[f].lkey = rq->mkey_be;
980 }
981 /* check if num_frags is not a pow of two */
982 if (rq->wqe.info.num_frags < (1 << rq->wqe.info.log_num_frags)) {
983 wqe->data[f].byte_count = 0;
1db1f21c 984 wqe->data[f].lkey = params->terminate_lkey_be;
069d1146
TT		 985 wqe->data[f].addr = 0;
986 }
422d4c40 987 }
f62b8bb8
AV		 988 }
989
f62b8bb8
AV		 990 return 0;
991
8f5ed1c1
MM		 992err_destroy_page_pool:
993 page_pool_destroy(rq->page_pool);
08a762ce 994err_free_by_rq_type:
069d1146
TT		 995 switch (rq->wq_type) {
996 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
8f5ed1c1
MM		 997 mlx5e_rq_free_shampo(rq);
998err_free_mpwqe_info:
ca11b798 999 kvfree(rq->mpwqe.info);
08a762ce 1000err_rq_mkey:
ecc7ad2e 1001 mlx5_core_destroy_mkey(mdev, be32_to_cpu(rq->mpwqe.umr_mkey_be));
c3c94023
AL		 1002err_rq_drop_page:
1003 mlx5e_free_mpwqe_rq_drop_page(rq);
069d1146
TT		 1004 break;
1005 default: /* MLX5_WQ_TYPE_CYCLIC */
8fb1814f 1006 mlx5e_free_wqe_alloc_info(rq);
069d1146 1007 }
f62b8bb8 1008err_rq_wq_destroy:
08a762ce 1009 mlx5_wq_destroy(&rq->wq_ctrl);
08a762ce 1010err_rq_xdp_prog:
fe45386a
MM		 1011 if (params->xdp_prog)
1012 bpf_prog_put(params->xdp_prog);
f62b8bb8
AV		 1013
1014 return err;
1015}
1016
3b77235b 1017static void mlx5e_free_rq(struct mlx5e_rq *rq)
f62b8bb8 1018{
cc181291
RR		 1019 kvfree(rq->dim);
1020 page_pool_destroy(rq->page_pool);
86994156 1021
461017cb
TT		 1022 switch (rq->wq_type) {
1023 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
cc181291 1024 mlx5e_rq_free_shampo(rq);
ca11b798 1025 kvfree(rq->mpwqe.info);
ecc7ad2e 1026 mlx5_core_destroy_mkey(rq->mdev, be32_to_cpu(rq->mpwqe.umr_mkey_be));
c3c94023 1027 mlx5e_free_mpwqe_rq_drop_page(rq);
461017cb 1028 break;
99cbfa93 1029 default: /* MLX5_WQ_TYPE_CYCLIC */
8fb1814f 1030 mlx5e_free_wqe_alloc_info(rq);
4415a031 1031 }
29b006a6 1032
f62b8bb8 1033 mlx5_wq_destroy(&rq->wq_ctrl);
cc181291
RR		 1034
1035 if (xdp_rxq_info_is_reg(&rq->xdp_rxq)) {
1036 struct bpf_prog *old_prog;
1037
1038 old_prog = rcu_dereference_protected(rq->xdp_prog,
1039 lockdep_is_held(&rq->priv->state_lock));
1040 if (old_prog)
1041 bpf_prog_put(old_prog);
1042 }
1043 xdp_rxq_info_unreg(&rq->xdp_rxq);
f62b8bb8
AV		 1044}
1045
7f525acb 1046int mlx5e_create_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param, u16 q_counter)
f62b8bb8 1047{
a43b25da 1048 struct mlx5_core_dev *mdev = rq->mdev;
432119de 1049 u8 ts_format;
f62b8bb8
AV		 1050 void *in;
1051 void *rqc;
1052 void *wq;
1053 int inlen;
1054 int err;
1055
1056 inlen = MLX5_ST_SZ_BYTES(create_rq_in) +
1057 sizeof(u64) * rq->wq_ctrl.buf.npages;
1b9a07ee 1058 in = kvzalloc(inlen, GFP_KERNEL);
f62b8bb8
AV		 1059 if (!in)
1060 return -ENOMEM;
1061
432119de 1062 ts_format = mlx5_is_real_time_rq(mdev) ?
9a1ac95a
AL		 1063 MLX5_TIMESTAMP_FORMAT_REAL_TIME :
1064 MLX5_TIMESTAMP_FORMAT_FREE_RUNNING;
f62b8bb8
AV		 1065 rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
1066 wq = MLX5_ADDR_OF(rqc, rqc, wq);
1067
1068 memcpy(rqc, param->rqc, sizeof(param->rqc));
1069
97de9f31 1070 MLX5_SET(rqc, rqc, cqn, rq->cq.mcq.cqn);
f62b8bb8 1071 MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST);
432119de 1072 MLX5_SET(rqc, rqc, ts_format, ts_format);
7f525acb 1073 MLX5_SET(rqc, rqc, counter_set_id, q_counter);
f62b8bb8 1074 MLX5_SET(wq, wq, log_wq_pg_sz, rq->wq_ctrl.buf.page_shift -
68cdf5d6 1075 MLX5_ADAPTER_PAGE_SHIFT);
f62b8bb8
AV		 1076 MLX5_SET64(wq, wq, dbr_addr, rq->wq_ctrl.db.dma);
1077
e5ca8fb0
BBI		 1078 if (test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state)) {
1079 MLX5_SET(wq, wq, log_headers_buffer_entry_num,
1080 order_base_2(rq->mpwqe.shampo->hd_per_wq));
573bce9e 1081 MLX5_SET(wq, wq, headers_mkey, rq->mpwqe.shampo->mkey);
e5ca8fb0
BBI		 1082 }
1083
3a2f7033
TT		 1084 mlx5_fill_page_frag_array(&rq->wq_ctrl.buf,
1085 (__be64 *)MLX5_ADDR_OF(wq, wq, pas));
f62b8bb8 1086
7db22ffb 1087 err = mlx5_core_create_rq(mdev, in, inlen, &rq->rqn);
f62b8bb8
AV		 1088
1089 kvfree(in);
1090
1091 return err;
1092}
1093
d9ba64de 1094static int mlx5e_modify_rq_state(struct mlx5e_rq *rq, int curr_state, int next_state)
f62b8bb8 1095{
7cbaf9a3 1096 struct mlx5_core_dev *mdev = rq->mdev;
f62b8bb8
AV		 1097
1098 void *in;
1099 void *rqc;
1100 int inlen;
1101 int err;
1102
1103 inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
1b9a07ee 1104 in = kvzalloc(inlen, GFP_KERNEL);
f62b8bb8
AV		 1105 if (!in)
1106 return -ENOMEM;
1107
5ee090ed
AL		 1108 if (curr_state == MLX5_RQC_STATE_RST && next_state == MLX5_RQC_STATE_RDY)
1109 mlx5e_rqwq_reset(rq);
1110
f62b8bb8
AV		 1111 rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
1112
1113 MLX5_SET(modify_rq_in, in, rq_state, curr_state);
1114 MLX5_SET(rqc, rqc, state, next_state);
1115
e0b4b472 1116 err = mlx5_core_modify_rq(mdev, rq->rqn, in);
f62b8bb8
AV		 1117
1118 kvfree(in);
1119
1120 return err;
1121}
1122
39646d9b
DT		 1123static void mlx5e_flush_rq_cq(struct mlx5e_rq *rq)
1124{
1125 struct mlx5_cqwq *cqwq = &rq->cq.wq;
1126 struct mlx5_cqe64 *cqe;
1127
1128 if (test_bit(MLX5E_RQ_STATE_MINI_CQE_ENHANCED, &rq->state)) {
1129 while ((cqe = mlx5_cqwq_get_cqe_enahnced_comp(cqwq)))
1130 mlx5_cqwq_pop(cqwq);
1131 } else {
1132 while ((cqe = mlx5_cqwq_get_cqe(cqwq)))
1133 mlx5_cqwq_pop(cqwq);
1134 }
1135
1136 mlx5_cqwq_update_db_record(cqwq);
1137}
1138
1139int mlx5e_flush_rq(struct mlx5e_rq *rq, int curr_state)
d9ba64de
MM		 1140{
1141 struct net_device *dev = rq->netdev;
1142 int err;
1143
1144 err = mlx5e_modify_rq_state(rq, curr_state, MLX5_RQC_STATE_RST);
1145 if (err) {
1146 netdev_err(dev, "Failed to move rq 0x%x to reset\n", rq->rqn);
1147 return err;
1148 }
39646d9b
DT		 1149
1150 mlx5e_free_rx_descs(rq);
1151 mlx5e_flush_rq_cq(rq);
1152
d9ba64de
MM		 1153 err = mlx5e_modify_rq_state(rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY);
1154 if (err) {
1155 netdev_err(dev, "Failed to move rq 0x%x to ready\n", rq->rqn);
1156 return err;
1157 }
1158
1159 return 0;
1160}
1161
36350114
GP		 1162static int mlx5e_modify_rq_vsd(struct mlx5e_rq *rq, bool vsd)
1163{
521f31af 1164 struct mlx5_core_dev *mdev = rq->mdev;
36350114
GP		 1165 void *in;
1166 void *rqc;
1167 int inlen;
1168 int err;
1169
1170 inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
1b9a07ee 1171 in = kvzalloc(inlen, GFP_KERNEL);
36350114
GP		 1172 if (!in)
1173 return -ENOMEM;
1174
1175 rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
1176
1177 MLX5_SET(modify_rq_in, in, rq_state, MLX5_RQC_STATE_RDY);
83b502a1
AV		 1178 MLX5_SET64(modify_rq_in, in, modify_bitmask,
1179 MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_VSD);
36350114
GP		 1180 MLX5_SET(rqc, rqc, vsd, vsd);
1181 MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RDY);
1182
e0b4b472 1183 err = mlx5_core_modify_rq(mdev, rq->rqn, in);
36350114
GP		 1184
1185 kvfree(in);
1186
1187 return err;
1188}
1189
5543e989 1190void mlx5e_destroy_rq(struct mlx5e_rq *rq)
f62b8bb8 1191{
a43b25da 1192 mlx5_core_destroy_rq(rq->mdev, rq->rqn);
f62b8bb8
AV		 1193}
1194
db05815b 1195int mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq, int wait_time)
f62b8bb8 1196{
1e7477ae 1197 unsigned long exp_time = jiffies + msecs_to_jiffies(wait_time);
a43b25da 1198
422d4c40 1199 u16 min_wqes = mlx5_min_rx_wqes(rq->wq_type, mlx5e_rqwq_get_size(rq));
f62b8bb8 1200
1e7477ae 1201 do {
422d4c40 1202 if (mlx5e_rqwq_get_cur_sz(rq) >= min_wqes)
f62b8bb8
AV		 1203 return 0;
1204
1205 msleep(20);
1e7477ae
EBE		 1206 } while (time_before(jiffies, exp_time));
1207
521f31af
AL		 1208 netdev_warn(rq->netdev, "Failed to get min RX wqes on Channel[%d] RQN[0x%x] wq cur_sz(%d) min_rx_wqes(%d)\n",
1209 rq->ix, rq->rqn, mlx5e_rqwq_get_cur_sz(rq), min_wqes);
f62b8bb8 1210
32c57fb2 1211 mlx5e_reporter_rx_timeout(rq);
f62b8bb8
AV		 1212 return -ETIMEDOUT;
1213}
1214
4c2a1323 1215void mlx5e_free_rx_missing_descs(struct mlx5e_rq *rq)
e239c6d6
AL		 1216{
1217 struct mlx5_wq_ll *wq;
1218 u16 head;
1219 int i;
1220
1221 if (rq->wq_type != MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ)
1222 return;
1223
1224 wq = &rq->mpwqe.wq;
1225 head = wq->head;
1226
4c2a1323
DT		 1227 /* Release WQEs that are in missing state: they have been
1228 * popped from the list after completion but were not freed
1229 * due to deferred release.
1230 * Also free the linked-list reserved entry, hence the "+ 1".
1231 */
1232 for (i = 0; i < mlx5_wq_ll_missing(wq) + 1; i++) {
e239c6d6
AL		 1233 rq->dealloc_wqe(rq, head);
1234 head = mlx5_wq_ll_get_wqe_next_ix(wq, head);
1235 }
1236
1237 rq->mpwqe.actual_wq_head = wq->head;
1238 rq->mpwqe.umr_in_progress = 0;
1239 rq->mpwqe.umr_completed = 0;
94e52193
DT		 1240
1241 if (test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state)) {
1242 struct mlx5e_shampo_hd *shampo = rq->mpwqe.shampo;
1243 u16 len;
1244
1245 len = (shampo->pi - shampo->ci) & shampo->hd_per_wq;
1246 mlx5e_shampo_fill_umr(rq, len);
1247 }
e239c6d6
AL		 1248}
1249
be5323c8 1250void mlx5e_free_rx_descs(struct mlx5e_rq *rq)
f2fde18c 1251{
f2fde18c
SM		 1252 __be16 wqe_ix_be;
1253 u16 wqe_ix;
1254
422d4c40
TT		 1255 if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) {
1256 struct mlx5_wq_ll *wq = &rq->mpwqe.wq;
1257
4c2a1323 1258 mlx5e_free_rx_missing_descs(rq);
422d4c40
TT		 1259
1260 while (!mlx5_wq_ll_is_empty(wq)) {
99cbfa93 1261 struct mlx5e_rx_wqe_ll *wqe;
422d4c40
TT		 1262
1263 wqe_ix_be = *wq->tail_next;
1264 wqe_ix = be16_to_cpu(wqe_ix_be);
1265 wqe = mlx5_wq_ll_get_wqe(wq, wqe_ix);
1266 rq->dealloc_wqe(rq, wqe_ix);
1267 mlx5_wq_ll_pop(wq, wqe_ix_be,
1268 &wqe->next.next_wqe_index);
1269 }
e5ca8fb0
BBI		 1270
1271 if (test_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state))
e839ac9a 1272 mlx5e_shampo_dealloc_hd(rq);
422d4c40 1273 } else {
99cbfa93 1274 struct mlx5_wq_cyc *wq = &rq->wqe.wq;
3f93f829
DT		 1275 u16 missing = mlx5_wq_cyc_missing(wq);
1276 u16 head = mlx5_wq_cyc_get_head(wq);
422d4c40 1277
99cbfa93
TT		 1278 while (!mlx5_wq_cyc_is_empty(wq)) {
1279 wqe_ix = mlx5_wq_cyc_get_tail(wq);
422d4c40 1280 rq->dealloc_wqe(rq, wqe_ix);
99cbfa93 1281 mlx5_wq_cyc_pop(wq);
422d4c40 1282 }
3f93f829
DT		 1283 /* Missing slots might also contain unreleased pages due to
1284 * deferred release.
1285 */
1286 while (missing--) {
1287 wqe_ix = mlx5_wq_cyc_ctr2ix(wq, head++);
1288 rq->dealloc_wqe(rq, wqe_ix);
1289 }
accd5883 1290 }
069d1146 1291
f2fde18c
SM		 1292}
1293
869c5f92 1294int mlx5e_open_rq(struct mlx5e_params *params, struct mlx5e_rq_param *param,
7f525acb 1295 struct mlx5e_xsk_param *xsk, int node, u16 q_counter,
869c5f92 1296 struct mlx5e_rq *rq)
f62b8bb8 1297{
869c5f92 1298 struct mlx5_core_dev *mdev = rq->mdev;
f62b8bb8
AV		 1299 int err;
1300
e5ca8fb0
BBI		 1301 if (params->packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO)
1302 __set_bit(MLX5E_RQ_STATE_SHAMPO, &rq->state);
1303
869c5f92 1304 err = mlx5e_alloc_rq(params, xsk, param, node, rq);
f62b8bb8
AV		 1305 if (err)
1306 return err;
1307
7f525acb 1308 err = mlx5e_create_rq(rq, param, q_counter);
f62b8bb8 1309 if (err)
3b77235b 1310 goto err_free_rq;
f62b8bb8 1311
36350114 1312 err = mlx5e_modify_rq_state(rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY);
f62b8bb8 1313 if (err)
3b77235b 1314 goto err_destroy_rq;
f62b8bb8 1315
869c5f92
AL		 1316 if (MLX5_CAP_ETH(mdev, cqe_checksum_full))
1317 __set_bit(MLX5E_RQ_STATE_CSUM_FULL, &rq->state);
db849faa 1318
445a25f6
RR		 1319 if (rq->channel && !params->rx_dim_enabled) {
1320 rq->channel->rx_cq_moder = params->rx_cq_moderation;
1321 } else if (rq->channel) {
1322 u8 cq_period_mode;
1323
1324 cq_period_mode = params->rx_moder_use_cqe_mode ?
1325 DIM_CQ_PERIOD_MODE_START_FROM_CQE :
1326 DIM_CQ_PERIOD_MODE_START_FROM_EQE;
1327 mlx5e_reset_rx_moderation(&rq->channel->rx_cq_moder, cq_period_mode,
1328 params->rx_dim_enabled);
1329
1330 err = mlx5e_dim_rx_change(rq, params->rx_dim_enabled);
1331 if (err)
1332 goto err_destroy_rq;
1333 }
cb3c7fd4 1334
5d0bb3ba
SM		 1335 /* We disable csum_complete when XDP is enabled since
1336 * XDP programs might manipulate packets which will render
1337 * skb->checksum incorrect.
1338 */
869c5f92
AL		 1339 if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_NO_CSUM_COMPLETE) || params->xdp_prog)
1340 __set_bit(MLX5E_RQ_STATE_NO_CSUM_COMPLETE, &rq->state);
b856df28 1341
b7cf0806
OL		 1342 /* For CQE compression on striding RQ, use stride index provided by
1343 * HW if capability is supported.
1344 */
1345 if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_STRIDING_RQ) &&
869c5f92
AL		 1346 MLX5_CAP_GEN(mdev, mini_cqe_resp_stride_index))
1347 __set_bit(MLX5E_RQ_STATE_MINI_CQE_HW_STRIDX, &rq->state);
b7cf0806 1348
2c925db0
OL		 1349 /* For enhanced CQE compression packet processing. decompress
1350 * session according to the enhanced layout.
1351 */
1352 if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS) &&
1353 MLX5_CAP_GEN(mdev, enhanced_cqe_compression))
1354 __set_bit(MLX5E_RQ_STATE_MINI_CQE_ENHANCED, &rq->state);
1355
f62b8bb8
AV		 1356 return 0;
1357
f62b8bb8
AV		 1358err_destroy_rq:
1359 mlx5e_destroy_rq(rq);
3b77235b
SM		 1360err_free_rq:
1361 mlx5e_free_rq(rq);
f62b8bb8
AV		 1362
1363 return err;
1364}
1365
be5323c8 1366void mlx5e_activate_rq(struct mlx5e_rq *rq)
acc6c595 1367{
acc6c595 1368 set_bit(MLX5E_RQ_STATE_ENABLED, &rq->state);
acc6c595
SM		 1369}
1370
db05815b 1371void mlx5e_deactivate_rq(struct mlx5e_rq *rq)
f62b8bb8 1372{
c0f1147d 1373 clear_bit(MLX5E_RQ_STATE_ENABLED, &rq->state);
4d6e6b0c 1374 synchronize_net(); /* Sync with NAPI to prevent mlx5e_post_rx_wqes. */
acc6c595 1375}
cb3c7fd4 1376
db05815b 1377void mlx5e_close_rq(struct mlx5e_rq *rq)
acc6c595 1378{
445a25f6
RR		 1379 if (rq->dim)
1380 cancel_work_sync(&rq->dim->work);
8276ea13 1381 cancel_work_sync(&rq->recover_work);
f62b8bb8 1382 mlx5e_destroy_rq(rq);
3b77235b
SM		 1383 mlx5e_free_rx_descs(rq);
1384 mlx5e_free_rq(rq);
f62b8bb8
AV		 1385}
1386
b25bd37c
TT		 1387u32 mlx5e_profile_get_tisn(struct mlx5_core_dev *mdev,
1388 struct mlx5e_priv *priv,
1389 const struct mlx5e_profile *profile,
1390 u8 lag_port, u8 tc)
1391{
1392 if (profile->get_tisn)
1393 return profile->get_tisn(mdev, priv, lag_port, tc);
1394
1395 return mdev->mlx5e_res.hw_objs.tisn[lag_port][tc];
1396}
1397
31391048 1398static void mlx5e_free_xdpsq_db(struct mlx5e_xdpsq *sq)
b5503b99 1399{
fea28dd6 1400 kvfree(sq->db.xdpi_fifo.xi);
1feeab80 1401 kvfree(sq->db.wqe_info);
fea28dd6
TT		 1402}
1403
1404static int mlx5e_alloc_xdpsq_fifo(struct mlx5e_xdpsq *sq, int numa)
1405{
1406 struct mlx5e_xdp_info_fifo *xdpi_fifo = &sq->db.xdpi_fifo;
1407 int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
34a79876 1408 int entries;
51984c9e 1409 size_t size;
fea28dd6 1410
34a79876
DT		 1411 /* upper bound for maximum num of entries of all xmit_modes. */
1412 entries = roundup_pow_of_two(wq_sz * MLX5_SEND_WQEBB_NUM_DS *
1413 MLX5E_XDP_FIFO_ENTRIES2DS_MAX_RATIO);
1414
3f734b8c 1415 size = array_size(sizeof(*xdpi_fifo->xi), entries);
51984c9e 1416 xdpi_fifo->xi = kvzalloc_node(size, GFP_KERNEL, numa);
fea28dd6
TT		 1417 if (!xdpi_fifo->xi)
1418 return -ENOMEM;
1419
1420 xdpi_fifo->pc = &sq->xdpi_fifo_pc;
1421 xdpi_fifo->cc = &sq->xdpi_fifo_cc;
3f734b8c 1422 xdpi_fifo->mask = entries - 1;
fea28dd6
TT		 1423
1424 return 0;
b5503b99
SM		 1425}
1426
31391048 1427static int mlx5e_alloc_xdpsq_db(struct mlx5e_xdpsq *sq, int numa)
b5503b99 1428{
1feeab80 1429 int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
51984c9e 1430 size_t size;
fea28dd6 1431 int err;
b5503b99 1432
51984c9e
GS		 1433 size = array_size(sizeof(*sq->db.wqe_info), wq_sz);
1434 sq->db.wqe_info = kvzalloc_node(size, GFP_KERNEL, numa);
1feeab80
TT		 1435 if (!sq->db.wqe_info)
1436 return -ENOMEM;
1437
fea28dd6
TT		 1438 err = mlx5e_alloc_xdpsq_fifo(sq, numa);
1439 if (err) {
31391048 1440 mlx5e_free_xdpsq_db(sq);
fea28dd6 1441 return err;
b5503b99
SM		 1442 }
1443
1444 return 0;
1445}
1446
31391048 1447static int mlx5e_alloc_xdpsq(struct mlx5e_channel *c,
6a9764ef 1448 struct mlx5e_params *params,
1742b3d5 1449 struct xsk_buff_pool *xsk_pool,
31391048 1450 struct mlx5e_sq_param *param,
58b99ee3
TT		 1451 struct mlx5e_xdpsq *sq,
1452 bool is_redirect)
31391048
SM		 1453{
1454 void *sqc_wq = MLX5_ADDR_OF(sqc, param->sqc, wq);
a43b25da 1455 struct mlx5_core_dev *mdev = c->mdev;
ddf385e3 1456 struct mlx5_wq_cyc *wq = &sq->wq;
31391048
SM		 1457 int err;
1458
1459 sq->pdev = c->pdev;
1460 sq->mkey_be = c->mkey_be;
1461 sq->channel = c;
c276aae8 1462 sq->uar_map = mdev->mlx5e_res.hw_objs.bfreg.map;
6a9764ef 1463 sq->min_inline_mode = params->tx_min_inline_mode;
1e267ab8 1464 sq->hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu) - ETH_FCS_LEN;
1742b3d5 1465 sq->xsk_pool = xsk_pool;
db05815b 1466
1742b3d5 1467 sq->stats = sq->xsk_pool ?
be98737a 1468 &c->priv->channel_stats[c->ix]->xsksq :
db05815b 1469 is_redirect ?
be98737a
TT		 1470 &c->priv->channel_stats[c->ix]->xdpsq :
1471 &c->priv->channel_stats[c->ix]->rq_xdpsq;
21a0502d
MM		 1472 sq->stop_room = param->is_mpw ? mlx5e_stop_room_for_mpwqe(mdev) :
1473 mlx5e_stop_room_for_max_wqe(mdev);
ed5c92ff 1474 sq->max_sq_mpw_wqebbs = mlx5e_get_max_sq_aligned_wqebbs(mdev);
31391048 1475
231243c8 1476 param->wq.db_numa_node = cpu_to_node(c->cpu);
ddf385e3 1477 err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, wq, &sq->wq_ctrl);
31391048
SM		 1478 if (err)
1479 return err;
ddf385e3 1480 wq->db = &wq->db[MLX5_SND_DBR];
31391048 1481
231243c8 1482 err = mlx5e_alloc_xdpsq_db(sq, cpu_to_node(c->cpu));
31391048
SM		 1483 if (err)
1484 goto err_sq_wq_destroy;
1485
1486 return 0;
1487
1488err_sq_wq_destroy:
1489 mlx5_wq_destroy(&sq->wq_ctrl);
1490
1491 return err;
1492}
1493
1494static void mlx5e_free_xdpsq(struct mlx5e_xdpsq *sq)
1495{
1496 mlx5e_free_xdpsq_db(sq);
1497 mlx5_wq_destroy(&sq->wq_ctrl);
1498}
1499
1500static void mlx5e_free_icosq_db(struct mlx5e_icosq *sq)
f62b8bb8 1501{
7d42c8e9 1502 kvfree(sq->db.wqe_info);
f62b8bb8
AV		 1503}
1504
31391048 1505static int mlx5e_alloc_icosq_db(struct mlx5e_icosq *sq, int numa)
f10b7cc7 1506{
fd9b4be8 1507 int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
7d42c8e9 1508 size_t size;
f10b7cc7 1509
7d42c8e9
MM		 1510 size = array_size(wq_sz, sizeof(*sq->db.wqe_info));
1511 sq->db.wqe_info = kvzalloc_node(size, GFP_KERNEL, numa);
1512 if (!sq->db.wqe_info)
f10b7cc7
SM		 1513 return -ENOMEM;
1514
1515 return 0;
1516}
1517
be5323c8
AL		 1518static void mlx5e_icosq_err_cqe_work(struct work_struct *recover_work)
1519{
1520 struct mlx5e_icosq *sq = container_of(recover_work, struct mlx5e_icosq,
1521 recover_work);
1522
1523 mlx5e_reporter_icosq_cqe_err(sq);
1524}
1525
19c4aba2
MM		 1526static void mlx5e_async_icosq_err_cqe_work(struct work_struct *recover_work)
1527{
1528 struct mlx5e_icosq *sq = container_of(recover_work, struct mlx5e_icosq,
1529 recover_work);
1530
1531 /* Not implemented yet. */
1532
1533 netdev_warn(sq->channel->netdev, "async_icosq recovery is not implemented\n");
1534}
1535
31391048 1536static int mlx5e_alloc_icosq(struct mlx5e_channel *c,
31391048 1537 struct mlx5e_sq_param *param,
19c4aba2
MM		 1538 struct mlx5e_icosq *sq,
1539 work_func_t recover_work_func)
f10b7cc7 1540{
31391048 1541 void *sqc_wq = MLX5_ADDR_OF(sqc, param->sqc, wq);
a43b25da 1542 struct mlx5_core_dev *mdev = c->mdev;
ddf385e3 1543 struct mlx5_wq_cyc *wq = &sq->wq;
31391048 1544 int err;
f10b7cc7 1545
31391048 1546 sq->channel = c;
c276aae8 1547 sq->uar_map = mdev->mlx5e_res.hw_objs.bfreg.map;
3ff3874f 1548 sq->reserved_room = param->stop_room;
f62b8bb8 1549
231243c8 1550 param->wq.db_numa_node = cpu_to_node(c->cpu);
ddf385e3 1551 err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, wq, &sq->wq_ctrl);
31391048
SM		 1552 if (err)
1553 return err;
ddf385e3 1554 wq->db = &wq->db[MLX5_SND_DBR];
f62b8bb8 1555
231243c8 1556 err = mlx5e_alloc_icosq_db(sq, cpu_to_node(c->cpu));
31391048
SM		 1557 if (err)
1558 goto err_sq_wq_destroy;
1559
19c4aba2 1560 INIT_WORK(&sq->recover_work, recover_work_func);
be5323c8 1561
f62b8bb8 1562 return 0;
31391048
SM		 1563
1564err_sq_wq_destroy:
1565 mlx5_wq_destroy(&sq->wq_ctrl);
1566
1567 return err;
f62b8bb8
AV		 1568}
1569
31391048 1570static void mlx5e_free_icosq(struct mlx5e_icosq *sq)
f10b7cc7 1571{
31391048
SM		 1572 mlx5e_free_icosq_db(sq);
1573 mlx5_wq_destroy(&sq->wq_ctrl);
f10b7cc7
SM		 1574}
1575
145e5637 1576void mlx5e_free_txqsq_db(struct mlx5e_txqsq *sq)
f10b7cc7 1577{
ca11b798 1578 kvfree(sq->db.wqe_info);
0b676aae 1579 kvfree(sq->db.skb_fifo.fifo);
ca11b798 1580 kvfree(sq->db.dma_fifo);
f10b7cc7
SM		 1581}
1582
145e5637 1583int mlx5e_alloc_txqsq_db(struct mlx5e_txqsq *sq, int numa)
b5503b99 1584{
31391048
SM		 1585 int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
1586 int df_sz = wq_sz * MLX5_SEND_WQEBB_NUM_DS;
1587
eec4edc9
KC		 1588 sq->db.dma_fifo = kvzalloc_node(array_size(df_sz,
1589 sizeof(*sq->db.dma_fifo)),
ca11b798 1590 GFP_KERNEL, numa);
0b676aae
EBE		 1591 sq->db.skb_fifo.fifo = kvzalloc_node(array_size(df_sz,
1592 sizeof(*sq->db.skb_fifo.fifo)),
338c46c6 1593 GFP_KERNEL, numa);
eec4edc9
KC		 1594 sq->db.wqe_info = kvzalloc_node(array_size(wq_sz,
1595 sizeof(*sq->db.wqe_info)),
ca11b798 1596 GFP_KERNEL, numa);
0b676aae 1597 if (!sq->db.dma_fifo || !sq->db.skb_fifo.fifo || !sq->db.wqe_info) {
31391048
SM		 1598 mlx5e_free_txqsq_db(sq);
1599 return -ENOMEM;
b5503b99 1600 }
31391048
SM		 1601
1602 sq->dma_fifo_mask = df_sz - 1;
0b676aae
EBE		 1603
1604 sq->db.skb_fifo.pc = &sq->skb_fifo_pc;
1605 sq->db.skb_fifo.cc = &sq->skb_fifo_cc;
1606 sq->db.skb_fifo.mask = df_sz - 1;
31391048
SM		 1607
1608 return 0;
b5503b99
SM		 1609}
1610
31391048 1611static int mlx5e_alloc_txqsq(struct mlx5e_channel *c,
acc6c595 1612 int txq_ix,
6a9764ef 1613 struct mlx5e_params *params,
31391048 1614 struct mlx5e_sq_param *param,
05909bab
EBE		 1615 struct mlx5e_txqsq *sq,
1616 int tc)
f62b8bb8 1617{
31391048 1618 void *sqc_wq = MLX5_ADDR_OF(sqc, param->sqc, wq);
a43b25da 1619 struct mlx5_core_dev *mdev = c->mdev;
ddf385e3 1620 struct mlx5_wq_cyc *wq = &sq->wq;
f62b8bb8
AV		 1621 int err;
1622
f10b7cc7 1623 sq->pdev = c->pdev;
7c39afb3 1624 sq->clock = &mdev->clock;
f10b7cc7 1625 sq->mkey_be = c->mkey_be;
4ad40d8e
EBE		 1626 sq->netdev = c->netdev;
1627 sq->mdev = c->mdev;
79efecb4 1628 sq->channel = c;
4ad40d8e 1629 sq->priv = c->priv;
57c70d87 1630 sq->ch_ix = c->ix;
acc6c595 1631 sq->txq_ix = txq_ix;
c276aae8 1632 sq->uar_map = mdev->mlx5e_res.hw_objs.bfreg.map;
6a9764ef 1633 sq->min_inline_mode = params->tx_min_inline_mode;
84d1bb2b 1634 sq->hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu);
ed5c92ff 1635 sq->max_sq_mpw_wqebbs = mlx5e_get_max_sq_aligned_wqebbs(mdev);
de8650a8 1636 INIT_WORK(&sq->recover_work, mlx5e_tx_err_cqe_work);
b431302e
TT		 1637 if (!MLX5_CAP_ETH(mdev, wqe_vlan_insert))
1638 set_bit(MLX5E_SQ_STATE_VLAN_NEED_L2_INLINE, &sq->state);
2451da08 1639 if (mlx5_ipsec_device_caps(c->priv->mdev))
2ac9cfe7 1640 set_bit(MLX5E_SQ_STATE_IPSEC, &sq->state);
5af75c74
MM		 1641 if (param->is_mpw)
1642 set_bit(MLX5E_SQ_STATE_MPWQE, &sq->state);
579524c6 1643 sq->stop_room = param->stop_room;
183532b7 1644 sq->ptp_cyc2time = mlx5_sq_ts_translator(mdev);
f10b7cc7 1645
231243c8 1646 param->wq.db_numa_node = cpu_to_node(c->cpu);
ddf385e3 1647 err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, wq, &sq->wq_ctrl);
f62b8bb8 1648 if (err)
aff26157 1649 return err;
ddf385e3 1650 wq->db = &wq->db[MLX5_SND_DBR];
f62b8bb8 1651
231243c8 1652 err = mlx5e_alloc_txqsq_db(sq, cpu_to_node(c->cpu));
7ec0bb22 1653 if (err)
f62b8bb8 1654 goto err_sq_wq_destroy;
cbce4f44 1655
f62b8bb8
AV		 1656 return 0;
1657
1658err_sq_wq_destroy:
1659 mlx5_wq_destroy(&sq->wq_ctrl);
1660
f62b8bb8
AV		 1661 return err;
1662}
1663
145e5637 1664void mlx5e_free_txqsq(struct mlx5e_txqsq *sq)
f62b8bb8 1665{
a5e89a3f 1666 kvfree(sq->dim);
31391048 1667 mlx5e_free_txqsq_db(sq);
f62b8bb8 1668 mlx5_wq_destroy(&sq->wq_ctrl);
f62b8bb8
AV		 1669}
1670
a43b25da 1671static int mlx5e_create_sq(struct mlx5_core_dev *mdev,
33ad9711
SM		 1672 struct mlx5e_sq_param *param,
1673 struct mlx5e_create_sq_param *csp,
1674 u32 *sqn)
f62b8bb8 1675{
432119de 1676 u8 ts_format;
f62b8bb8
AV		 1677 void *in;
1678 void *sqc;
1679 void *wq;
1680 int inlen;
1681 int err;
1682
1683 inlen = MLX5_ST_SZ_BYTES(create_sq_in) +
33ad9711 1684 sizeof(u64) * csp->wq_ctrl->buf.npages;
1b9a07ee 1685 in = kvzalloc(inlen, GFP_KERNEL);
f62b8bb8
AV		 1686 if (!in)
1687 return -ENOMEM;
1688
432119de 1689 ts_format = mlx5_is_real_time_sq(mdev) ?
9a1ac95a
AL		 1690 MLX5_TIMESTAMP_FORMAT_REAL_TIME :
1691 MLX5_TIMESTAMP_FORMAT_FREE_RUNNING;
f62b8bb8
AV		 1692 sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
1693 wq = MLX5_ADDR_OF(sqc, sqc, wq);
1694
1695 memcpy(sqc, param->sqc, sizeof(param->sqc));
33ad9711
SM		 1696 MLX5_SET(sqc, sqc, tis_lst_sz, csp->tis_lst_sz);
1697 MLX5_SET(sqc, sqc, tis_num_0, csp->tisn);
1698 MLX5_SET(sqc, sqc, cqn, csp->cqn);
1880bc4e 1699 MLX5_SET(sqc, sqc, ts_cqe_to_dest_cqn, csp->ts_cqe_to_dest_cqn);
432119de
AL		 1700 MLX5_SET(sqc, sqc, ts_format, ts_format);
1701
a6f402e4
SM		 1702
1703 if (MLX5_CAP_ETH(mdev, wqe_inline_mode) == MLX5_CAP_INLINE_MODE_VPORT_CONTEXT)
33ad9711 1704 MLX5_SET(sqc, sqc, min_wqe_inline_mode, csp->min_inline_mode);
a6f402e4 1705
33ad9711 1706 MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RST);
db75373c 1707 MLX5_SET(sqc, sqc, flush_in_error_en, 1);
f62b8bb8
AV		 1708
1709 MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC);
c276aae8 1710 MLX5_SET(wq, wq, uar_page, mdev->mlx5e_res.hw_objs.bfreg.index);
33ad9711 1711 MLX5_SET(wq, wq, log_wq_pg_sz, csp->wq_ctrl->buf.page_shift -
68cdf5d6 1712 MLX5_ADAPTER_PAGE_SHIFT);
33ad9711 1713 MLX5_SET64(wq, wq, dbr_addr, csp->wq_ctrl->db.dma);
f62b8bb8 1714
3a2f7033
TT		 1715 mlx5_fill_page_frag_array(&csp->wq_ctrl->buf,
1716 (__be64 *)MLX5_ADDR_OF(wq, wq, pas));
f62b8bb8 1717
33ad9711 1718 err = mlx5_core_create_sq(mdev, in, inlen, sqn);
f62b8bb8
AV		 1719
1720 kvfree(in);
1721
1722 return err;
1723}
1724
de8650a8
EBE		 1725int mlx5e_modify_sq(struct mlx5_core_dev *mdev, u32 sqn,
1726 struct mlx5e_modify_sq_param *p)
f62b8bb8 1727{
214baf22 1728 u64 bitmask = 0;
f62b8bb8
AV		 1729 void *in;
1730 void *sqc;
1731 int inlen;
1732 int err;
1733
1734 inlen = MLX5_ST_SZ_BYTES(modify_sq_in);
1b9a07ee 1735 in = kvzalloc(inlen, GFP_KERNEL);
f62b8bb8
AV		 1736 if (!in)
1737 return -ENOMEM;
1738
1739 sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx);
1740
33ad9711
SM		 1741 MLX5_SET(modify_sq_in, in, sq_state, p->curr_state);
1742 MLX5_SET(sqc, sqc, state, p->next_state);
1743 if (p->rl_update && p->next_state == MLX5_SQC_STATE_RDY) {
214baf22
MM		 1744 bitmask |= 1;
1745 MLX5_SET(sqc, sqc, packet_pacing_rate_limit_index, p->rl_index);
507f0c81 1746 }
214baf22
MM		 1747 if (p->qos_update && p->next_state == MLX5_SQC_STATE_RDY) {
1748 bitmask |= 1 << 2;
1749 MLX5_SET(sqc, sqc, qos_queue_group_id, p->qos_queue_group_id);
1750 }
1751 MLX5_SET64(modify_sq_in, in, modify_bitmask, bitmask);
f62b8bb8 1752
e0b4b472 1753 err = mlx5_core_modify_sq(mdev, sqn, in);
f62b8bb8
AV		 1754
1755 kvfree(in);
1756
1757 return err;
1758}
1759
a43b25da 1760static void mlx5e_destroy_sq(struct mlx5_core_dev *mdev, u32 sqn)
33ad9711 1761{
a43b25da 1762 mlx5_core_destroy_sq(mdev, sqn);
f62b8bb8
AV		 1763}
1764
145e5637
EBE		 1765int mlx5e_create_sq_rdy(struct mlx5_core_dev *mdev,
1766 struct mlx5e_sq_param *param,
1767 struct mlx5e_create_sq_param *csp,
214baf22 1768 u16 qos_queue_group_id,
145e5637 1769 u32 *sqn)
f62b8bb8 1770{
33ad9711 1771 struct mlx5e_modify_sq_param msp = {0};
31391048
SM		 1772 int err;
1773
a43b25da 1774 err = mlx5e_create_sq(mdev, param, csp, sqn);
31391048
SM		 1775 if (err)
1776 return err;
1777
1778 msp.curr_state = MLX5_SQC_STATE_RST;
1779 msp.next_state = MLX5_SQC_STATE_RDY;
214baf22
MM		 1780 if (qos_queue_group_id) {
1781 msp.qos_update = true;
1782 msp.qos_queue_group_id = qos_queue_group_id;
1783 }
a43b25da 1784 err = mlx5e_modify_sq(mdev, *sqn, &msp);
31391048 1785 if (err)
a43b25da 1786 mlx5e_destroy_sq(mdev, *sqn);
31391048
SM		 1787
1788 return err;
1789}
1790
7f859ecf
SM		 1791static int mlx5e_set_sq_maxrate(struct net_device *dev,
1792 struct mlx5e_txqsq *sq, u32 rate);
1793
214baf22
MM		 1794int mlx5e_open_txqsq(struct mlx5e_channel *c, u32 tisn, int txq_ix,
1795 struct mlx5e_params *params, struct mlx5e_sq_param *param,
e0ee6891
TT		 1796 struct mlx5e_txqsq *sq, int tc, u16 qos_queue_group_id,
1797 struct mlx5e_sq_stats *sq_stats)
31391048
SM		 1798{
1799 struct mlx5e_create_sq_param csp = {};
7f859ecf 1800 u32 tx_rate;
f62b8bb8
AV		 1801 int err;
1802
05909bab 1803 err = mlx5e_alloc_txqsq(c, txq_ix, params, param, sq, tc);
f62b8bb8
AV		 1804 if (err)
1805 return err;
1806
e0ee6891 1807 sq->stats = sq_stats;
214baf22 1808
a43b25da 1809 csp.tisn = tisn;
31391048 1810 csp.tis_lst_sz = 1;
33ad9711
SM		 1811 csp.cqn = sq->cq.mcq.cqn;
1812 csp.wq_ctrl = &sq->wq_ctrl;
1813 csp.min_inline_mode = sq->min_inline_mode;
214baf22 1814 err = mlx5e_create_sq_rdy(c->mdev, param, &csp, qos_queue_group_id, &sq->sqn);
f62b8bb8 1815 if (err)
31391048 1816 goto err_free_txqsq;
f62b8bb8 1817
a43b25da 1818 tx_rate = c->priv->tx_rates[sq->txq_ix];
7f859ecf 1819 if (tx_rate)
a43b25da 1820 mlx5e_set_sq_maxrate(c->netdev, sq, tx_rate);
7f859ecf 1821
445a25f6
RR		 1822 if (sq->channel && !params->tx_dim_enabled) {
1823 sq->channel->tx_cq_moder = params->tx_cq_moderation;
1824 } else if (sq->channel) {
1825 u8 cq_period_mode;
1826
1827 cq_period_mode = params->tx_moder_use_cqe_mode ?
1828 DIM_CQ_PERIOD_MODE_START_FROM_CQE :
1829 DIM_CQ_PERIOD_MODE_START_FROM_EQE;
1830 mlx5e_reset_tx_moderation(&sq->channel->tx_cq_moder,
1831 cq_period_mode,
1832 params->tx_dim_enabled);
1833
1834 err = mlx5e_dim_tx_change(sq, params->tx_dim_enabled);
1835 if (err)
1836 goto err_destroy_sq;
1837 }
cbce4f44 1838
f62b8bb8
AV		 1839 return 0;
1840
445a25f6
RR		 1841err_destroy_sq:
1842 mlx5e_destroy_sq(c->mdev, sq->sqn);
31391048 1843err_free_txqsq:
31391048 1844 mlx5e_free_txqsq(sq);
f62b8bb8
AV		 1845
1846 return err;
1847}
1848
de8650a8 1849void mlx5e_activate_txqsq(struct mlx5e_txqsq *sq)
acc6c595 1850{
4ad40d8e 1851 sq->txq = netdev_get_tx_queue(sq->netdev, sq->txq_ix);
acc6c595
SM		 1852 set_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
1853 netdev_tx_reset_queue(sq->txq);
1854 netif_tx_start_queue(sq->txq);
f25e7b82 1855 netif_queue_set_napi(sq->netdev, sq->txq_ix, NETDEV_QUEUE_TYPE_TX, sq->cq.napi);
acc6c595
SM		 1856}
1857
de8650a8 1858void mlx5e_tx_disable_queue(struct netdev_queue *txq)
f62b8bb8
AV		 1859{
1860 __netif_tx_lock_bh(txq);
1861 netif_tx_stop_queue(txq);
1862 __netif_tx_unlock_bh(txq);
1863}
1864
145e5637 1865void mlx5e_deactivate_txqsq(struct mlx5e_txqsq *sq)
f62b8bb8 1866{
ddf385e3 1867 struct mlx5_wq_cyc *wq = &sq->wq;
33ad9711 1868
f25e7b82 1869 netif_queue_set_napi(sq->netdev, sq->txq_ix, NETDEV_QUEUE_TYPE_TX, NULL);
c0f1147d 1870 clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
4d6e6b0c 1871 synchronize_net(); /* Sync with NAPI to prevent netif_tx_wake_queue. */
29429f33 1872
de8650a8 1873 mlx5e_tx_disable_queue(sq->txq);
f62b8bb8 1874
31391048 1875 /* last doorbell out, godspeed .. */
ddf385e3
TT		 1876 if (mlx5e_wqc_has_room_for(wq, sq->cc, sq->pc, 1)) {
1877 u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
31391048 1878 struct mlx5e_tx_wqe *nop;
864b2d71 1879
41a8e4eb
TT		 1880 sq->db.wqe_info[pi] = (struct mlx5e_tx_wqe_info) {
1881 .num_wqebbs = 1,
1882 };
0c258dec 1883
ddf385e3
TT		 1884 nop = mlx5e_post_nop(wq, sq->sqn, &sq->pc);
1885 mlx5e_notify_hw(wq, sq->pc, sq->uar_map, &nop->ctrl);
29429f33 1886 }
acc6c595
SM		 1887}
1888
214baf22 1889void mlx5e_close_txqsq(struct mlx5e_txqsq *sq)
acc6c595 1890{
4ad40d8e 1891 struct mlx5_core_dev *mdev = sq->mdev;
05d3ac97 1892 struct mlx5_rate_limit rl = {0};
f62b8bb8 1893
445a25f6
RR		 1894 if (sq->dim)
1895 cancel_work_sync(&sq->dim->work);
de8650a8 1896 cancel_work_sync(&sq->recover_work);
a43b25da 1897 mlx5e_destroy_sq(mdev, sq->sqn);
05d3ac97
BW		 1898 if (sq->rate_limit) {
1899 rl.rate = sq->rate_limit;
1900 mlx5_rl_remove_rate(mdev, &rl);
1901 }
31391048
SM		 1902 mlx5e_free_txqsq_descs(sq);
1903 mlx5e_free_txqsq(sq);
1904}
1905
145e5637 1906void mlx5e_tx_err_cqe_work(struct work_struct *recover_work)
db75373c 1907{
de8650a8
EBE		 1908 struct mlx5e_txqsq *sq = container_of(recover_work, struct mlx5e_txqsq,
1909 recover_work);
30e5c2c6 1910
06293ae4 1911 mlx5e_reporter_tx_err_cqe(sq);
db75373c
EBE		 1912}
1913
445a25f6
RR		 1914static struct dim_cq_moder mlx5e_get_def_tx_moderation(u8 cq_period_mode)
1915{
1916 return (struct dim_cq_moder) {
1917 .cq_period_mode = cq_period_mode,
1918 .pkts = MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_PKTS,
1919 .usec = cq_period_mode == DIM_CQ_PERIOD_MODE_START_FROM_CQE ?
1920 MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC_FROM_CQE :
1921 MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC,
1922 };
1923}
1924
1925bool mlx5e_reset_tx_moderation(struct dim_cq_moder *cq_moder, u8 cq_period_mode,
1926 bool dim_enabled)
1927{
1928 bool reset_needed = cq_moder->cq_period_mode != cq_period_mode;
1929
1930 if (dim_enabled)
1931 *cq_moder = net_dim_get_def_tx_moderation(cq_period_mode);
1932 else
1933 *cq_moder = mlx5e_get_def_tx_moderation(cq_period_mode);
1934
1935 return reset_needed;
1936}
1937
1938bool mlx5e_reset_tx_channels_moderation(struct mlx5e_channels *chs, u8 cq_period_mode,
1939 bool dim_enabled, bool keep_dim_state)
1940{
1941 bool reset = false;
1942 int i, tc;
1943
1944 for (i = 0; i < chs->num; i++) {
1945 for (tc = 0; tc < mlx5e_get_dcb_num_tc(&chs->params); tc++) {
1946 if (keep_dim_state)
1947 dim_enabled = !!chs->c[i]->sq[tc].dim;
1948
1949 reset |= mlx5e_reset_tx_moderation(&chs->c[i]->tx_cq_moder,
1950 cq_period_mode, dim_enabled);
1951 }
1952 }
1953
1954 return reset;
1955}
1956
19c4aba2
MM		 1957static int mlx5e_open_icosq(struct mlx5e_channel *c, struct mlx5e_params *params,
1958 struct mlx5e_sq_param *param, struct mlx5e_icosq *sq,
1959 work_func_t recover_work_func)
31391048
SM		 1960{
1961 struct mlx5e_create_sq_param csp = {};
1962 int err;
1963
19c4aba2 1964 err = mlx5e_alloc_icosq(c, param, sq, recover_work_func);
31391048
SM		 1965 if (err)
1966 return err;
1967
1968 csp.cqn = sq->cq.mcq.cqn;
1969 csp.wq_ctrl = &sq->wq_ctrl;
6a9764ef 1970 csp.min_inline_mode = params->tx_min_inline_mode;
214baf22 1971 err = mlx5e_create_sq_rdy(c->mdev, param, &csp, 0, &sq->sqn);
31391048
SM		 1972 if (err)
1973 goto err_free_icosq;
1974
e9ce991b
TT		 1975 if (param->is_tls) {
1976 sq->ktls_resync = mlx5e_ktls_rx_resync_create_resp_list();
1977 if (IS_ERR(sq->ktls_resync)) {
1978 err = PTR_ERR(sq->ktls_resync);
1979 goto err_destroy_icosq;
1980 }
1981 }
31391048
SM		 1982 return 0;
1983
e9ce991b
TT		 1984err_destroy_icosq:
1985 mlx5e_destroy_sq(c->mdev, sq->sqn);
31391048 1986err_free_icosq:
31391048
SM		 1987 mlx5e_free_icosq(sq);
1988
1989 return err;
1990}
1991
be5323c8 1992void mlx5e_activate_icosq(struct mlx5e_icosq *icosq)
31391048 1993{
9d18b514
AL		 1994 set_bit(MLX5E_SQ_STATE_ENABLED, &icosq->state);
1995}
31391048 1996
be5323c8 1997void mlx5e_deactivate_icosq(struct mlx5e_icosq *icosq)
9d18b514 1998{
9d18b514 1999 clear_bit(MLX5E_SQ_STATE_ENABLED, &icosq->state);
4d6e6b0c 2000 synchronize_net(); /* Sync with NAPI. */
9d18b514
AL		 2001}
2002
19c4aba2 2003static void mlx5e_close_icosq(struct mlx5e_icosq *sq)
9d18b514
AL		 2004{
2005 struct mlx5e_channel *c = sq->channel;
31391048 2006
e9ce991b
TT		 2007 if (sq->ktls_resync)
2008 mlx5e_ktls_rx_resync_destroy_resp_list(sq->ktls_resync);
a43b25da 2009 mlx5e_destroy_sq(c->mdev, sq->sqn);
1182f365 2010 mlx5e_free_icosq_descs(sq);
31391048
SM		 2011 mlx5e_free_icosq(sq);
2012}
2013
db05815b 2014int mlx5e_open_xdpsq(struct mlx5e_channel *c, struct mlx5e_params *params,
1742b3d5 2015 struct mlx5e_sq_param *param, struct xsk_buff_pool *xsk_pool,
db05815b 2016 struct mlx5e_xdpsq *sq, bool is_redirect)
31391048 2017{
31391048 2018 struct mlx5e_create_sq_param csp = {};
31391048 2019 int err;
31391048 2020
1742b3d5 2021 err = mlx5e_alloc_xdpsq(c, params, xsk_pool, param, sq, is_redirect);
31391048
SM		 2022 if (err)
2023 return err;
2024
2025 csp.tis_lst_sz = 1;
b25bd37c
TT		 2026 csp.tisn = mlx5e_profile_get_tisn(c->mdev, c->priv, c->priv->profile,
2027 c->lag_port, 0); /* tc = 0 */
31391048
SM		 2028 csp.cqn = sq->cq.mcq.cqn;
2029 csp.wq_ctrl = &sq->wq_ctrl;
2030 csp.min_inline_mode = sq->min_inline_mode;
2031 set_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
9ded70fa 2032
c1783e74 2033 if (param->is_xdp_mb)
9ded70fa
MM		 2034 set_bit(MLX5E_SQ_STATE_XDP_MULTIBUF, &sq->state);
2035
214baf22 2036 err = mlx5e_create_sq_rdy(c->mdev, param, &csp, 0, &sq->sqn);
31391048
SM		 2037 if (err)
2038 goto err_free_xdpsq;
2039
5e0d2eef
TT		 2040 mlx5e_set_xmit_fp(sq, param->is_mpw);
2041
9ded70fa 2042 if (!param->is_mpw && !test_bit(MLX5E_SQ_STATE_XDP_MULTIBUF, &sq->state)) {
08c34e95 2043 unsigned int ds_cnt = MLX5E_TX_WQE_EMPTY_DS_COUNT + 1;
5e0d2eef
TT		 2044 unsigned int inline_hdr_sz = 0;
2045 int i;
31391048 2046
5e0d2eef
TT		 2047 if (sq->min_inline_mode != MLX5_INLINE_MODE_NONE) {
2048 inline_hdr_sz = MLX5E_XDP_MIN_INLINE;
2049 ds_cnt++;
2050 }
2051
2052 /* Pre initialize fixed WQE fields */
2053 for (i = 0; i < mlx5_wq_cyc_get_size(&sq->wq); i++) {
5e0d2eef
TT		 2054 struct mlx5e_tx_wqe *wqe = mlx5_wq_cyc_get_wqe(&sq->wq, i);
2055 struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl;
2056 struct mlx5_wqe_eth_seg *eseg = &wqe->eth;
31391048 2057
41a8e4eb
TT		 2058 sq->db.wqe_info[i] = (struct mlx5e_xdp_wqe_info) {
2059 .num_wqebbs = 1,
2060 .num_pkts = 1,
2061 };
2062
5e0d2eef
TT		 2063 cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt);
2064 eseg->inline_hdr.sz = cpu_to_be16(inline_hdr_sz);
5e0d2eef 2065 }
31391048
SM		 2066 }
2067
2068 return 0;
2069
2070err_free_xdpsq:
2071 clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
2072 mlx5e_free_xdpsq(sq);
2073
2074 return err;
2075}
2076
db05815b 2077void mlx5e_close_xdpsq(struct mlx5e_xdpsq *sq)
31391048
SM		 2078{
2079 struct mlx5e_channel *c = sq->channel;
2080
2081 clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
4d6e6b0c 2082 synchronize_net(); /* Sync with NAPI. */
31391048 2083
a43b25da 2084 mlx5e_destroy_sq(c->mdev, sq->sqn);
b9673cf5 2085 mlx5e_free_xdpsq_descs(sq);
31391048 2086 mlx5e_free_xdpsq(sq);
f62b8bb8
AV		 2087}
2088
db52aa6d
TT		 2089static int mlx5e_alloc_cq_common(struct mlx5_core_dev *mdev,
2090 struct net_device *netdev,
2091 struct workqueue_struct *workqueue,
95b6c6a5
EBE		 2092 struct mlx5e_cq_param *param,
2093 struct mlx5e_cq *cq)
f62b8bb8 2094{
f62b8bb8 2095 struct mlx5_core_cq *mcq = &cq->mcq;
f62b8bb8
AV		 2096 int err;
2097 u32 i;
2098
f62b8bb8
AV		 2099 err = mlx5_cqwq_create(mdev, &param->wq, param->cqc, &cq->wq,
2100 &cq->wq_ctrl);
2101 if (err)
2102 return err;
2103
f62b8bb8
AV		 2104 mcq->cqe_sz = 64;
2105 mcq->set_ci_db = cq->wq_ctrl.db.db;
2106 mcq->arm_db = cq->wq_ctrl.db.db + 1;
2107 *mcq->set_ci_db = 0;
2108 *mcq->arm_db = 0;
2109 mcq->vector = param->eq_ix;
2110 mcq->comp = mlx5e_completion_event;
2111 mcq->event = mlx5e_cq_error_event;
f62b8bb8
AV		 2112
2113 for (i = 0; i < mlx5_cqwq_get_size(&cq->wq); i++) {
2114 struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(&cq->wq, i);
2115
2116 cqe->op_own = 0xf1;
2c925db0 2117 cqe->validity_iteration_count = 0xff;
f62b8bb8
AV		 2118 }
2119
a43b25da 2120 cq->mdev = mdev;
db52aa6d
TT		 2121 cq->netdev = netdev;
2122 cq->workqueue = workqueue;
f62b8bb8
AV		 2123
2124 return 0;
2125}
2126
db52aa6d 2127static int mlx5e_alloc_cq(struct mlx5_core_dev *mdev,
95b6c6a5 2128 struct mlx5e_cq_param *param,
4d0b7ef9 2129 struct mlx5e_create_cq_param *ccp,
95b6c6a5
EBE		 2130 struct mlx5e_cq *cq)
2131{
95b6c6a5
EBE		 2132 int err;
2133
4d0b7ef9
AL		 2134 param->wq.buf_numa_node = ccp->node;
2135 param->wq.db_numa_node = ccp->node;
2136 param->eq_ix = ccp->ix;
95b6c6a5 2137
db52aa6d 2138 err = mlx5e_alloc_cq_common(mdev, ccp->netdev, ccp->wq, param, cq);
95b6c6a5 2139
4d0b7ef9
AL		 2140 cq->napi = ccp->napi;
2141 cq->ch_stats = ccp->ch_stats;
95b6c6a5
EBE		 2142
2143 return err;
2144}
2145
3b77235b 2146static void mlx5e_free_cq(struct mlx5e_cq *cq)
f62b8bb8 2147{
3a2f7033 2148 mlx5_wq_destroy(&cq->wq_ctrl);
f62b8bb8
AV		 2149}
2150
3b77235b 2151static int mlx5e_create_cq(struct mlx5e_cq *cq, struct mlx5e_cq_param *param)
f62b8bb8 2152{
38164b77 2153 u32 out[MLX5_ST_SZ_DW(create_cq_out)];
a43b25da 2154 struct mlx5_core_dev *mdev = cq->mdev;
f62b8bb8
AV		 2155 struct mlx5_core_cq *mcq = &cq->mcq;
2156
2157 void *in;
2158 void *cqc;
2159 int inlen;
f62b8bb8
AV		 2160 int eqn;
2161 int err;
2162
f14c1a14 2163 err = mlx5_comp_eqn_get(mdev, param->eq_ix, &eqn);
a1f240f1
YA		 2164 if (err)
2165 return err;
2166
f62b8bb8 2167 inlen = MLX5_ST_SZ_BYTES(create_cq_in) +
3a2f7033 2168 sizeof(u64) * cq->wq_ctrl.buf.npages;
1b9a07ee 2169 in = kvzalloc(inlen, GFP_KERNEL);
f62b8bb8
AV		 2170 if (!in)
2171 return -ENOMEM;
2172
2173 cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context);
2174
2175 memcpy(cqc, param->cqc, sizeof(param->cqc));
2176
3a2f7033 2177 mlx5_fill_page_frag_array(&cq->wq_ctrl.buf,
1c1b5228 2178 (__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas));
f62b8bb8 2179
445a25f6
RR		 2180 MLX5_SET(cqc, cqc, cq_period_mode, mlx5e_cq_period_mode(param->cq_period_mode));
2181
616d5769 2182 MLX5_SET(cqc, cqc, c_eqn_or_apu_element, eqn);
30aa60b3 2183 MLX5_SET(cqc, cqc, uar_page, mdev->priv.uar->index);
3a2f7033 2184 MLX5_SET(cqc, cqc, log_page_size, cq->wq_ctrl.buf.page_shift -
68cdf5d6 2185 MLX5_ADAPTER_PAGE_SHIFT);
f62b8bb8
AV		 2186 MLX5_SET64(cqc, cqc, dbr_addr, cq->wq_ctrl.db.dma);
2187
38164b77 2188 err = mlx5_core_create_cq(mdev, mcq, in, inlen, out, sizeof(out));
f62b8bb8
AV		 2189
2190 kvfree(in);
2191
2192 if (err)
2193 return err;
2194
2195 mlx5e_cq_arm(cq);
2196
2197 return 0;
2198}
2199
3b77235b 2200static void mlx5e_destroy_cq(struct mlx5e_cq *cq)
f62b8bb8 2201{
a43b25da 2202 mlx5_core_destroy_cq(cq->mdev, &cq->mcq);
f62b8bb8
AV		 2203}
2204
db52aa6d 2205int mlx5e_open_cq(struct mlx5_core_dev *mdev, struct dim_cq_moder moder,
4d0b7ef9
AL		 2206 struct mlx5e_cq_param *param, struct mlx5e_create_cq_param *ccp,
2207 struct mlx5e_cq *cq)
f62b8bb8
AV		 2208{
2209 int err;
f62b8bb8 2210
db52aa6d 2211 err = mlx5e_alloc_cq(mdev, param, ccp, cq);
f62b8bb8
AV		 2212 if (err)
2213 return err;
2214
3b77235b 2215 err = mlx5e_create_cq(cq, param);
f62b8bb8 2216 if (err)
3b77235b 2217 goto err_free_cq;
f62b8bb8 2218
445a25f6
RR		 2219 if (MLX5_CAP_GEN(mdev, cq_moderation) &&
2220 MLX5_CAP_GEN(mdev, cq_period_mode_modify))
2221 mlx5e_modify_cq_moderation(mdev, &cq->mcq, moder.usec, moder.pkts,
2222 mlx5e_cq_period_mode(moder.cq_period_mode));
f62b8bb8
AV		 2223 return 0;
2224
3b77235b
SM		 2225err_free_cq:
2226 mlx5e_free_cq(cq);
f62b8bb8
AV		 2227
2228 return err;
2229}
2230
db05815b 2231void mlx5e_close_cq(struct mlx5e_cq *cq)
f62b8bb8 2232{
f62b8bb8 2233 mlx5e_destroy_cq(cq);
3b77235b 2234 mlx5e_free_cq(cq);
f62b8bb8
AV		 2235}
2236
445a25f6
RR		 2237int mlx5e_modify_cq_period_mode(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq,
2238 u8 cq_period_mode)
2239{
2240 u32 in[MLX5_ST_SZ_DW(modify_cq_in)] = {};
2241 void *cqc;
2242
2243 MLX5_SET(modify_cq_in, in, cqn, cq->cqn);
2244 cqc = MLX5_ADDR_OF(modify_cq_in, in, cq_context);
2245 MLX5_SET(cqc, cqc, cq_period_mode, mlx5e_cq_period_mode(cq_period_mode));
2246 MLX5_SET(modify_cq_in, in,
2247 modify_field_select_resize_field_select.modify_field_select.modify_field_select,
2248 MLX5_CQ_MODIFY_PERIOD_MODE);
2249
2250 return mlx5_core_modify_cq(dev, cq, in, sizeof(in));
2251}
2252
2253int mlx5e_modify_cq_moderation(struct mlx5_core_dev *dev, struct mlx5_core_cq *cq,
2254 u16 cq_period, u16 cq_max_count, u8 cq_period_mode)
2255{
2256 u32 in[MLX5_ST_SZ_DW(modify_cq_in)] = {};
2257 void *cqc;
2258
2259 MLX5_SET(modify_cq_in, in, cqn, cq->cqn);
2260 cqc = MLX5_ADDR_OF(modify_cq_in, in, cq_context);
2261 MLX5_SET(cqc, cqc, cq_period, cq_period);
2262 MLX5_SET(cqc, cqc, cq_max_count, cq_max_count);
2263 MLX5_SET(cqc, cqc, cq_period_mode, cq_period_mode);
2264 MLX5_SET(modify_cq_in, in,
2265 modify_field_select_resize_field_select.modify_field_select.modify_field_select,
2266 MLX5_CQ_MODIFY_PERIOD | MLX5_CQ_MODIFY_COUNT | MLX5_CQ_MODIFY_PERIOD_MODE);
2267
2268 return mlx5_core_modify_cq(dev, cq, in, sizeof(in));
2269}
2270
f62b8bb8 2271static int mlx5e_open_tx_cqs(struct mlx5e_channel *c,
6a9764ef 2272 struct mlx5e_params *params,
4d0b7ef9 2273 struct mlx5e_create_cq_param *ccp,
f62b8bb8
AV		 2274 struct mlx5e_channel_param *cparam)
2275{
f62b8bb8
AV		 2276 int err;
2277 int tc;
2278
2279 for (tc = 0; tc < c->num_tc; tc++) {
db52aa6d 2280 err = mlx5e_open_cq(c->mdev, params->tx_cq_moderation, &cparam->txq_sq.cqp,
4d0b7ef9 2281 ccp, &c->sq[tc].cq);
f62b8bb8
AV		 2282 if (err)
2283 goto err_close_tx_cqs;
f62b8bb8
AV		 2284 }
2285
2286 return 0;
2287
2288err_close_tx_cqs:
2289 for (tc--; tc >= 0; tc--)
2290 mlx5e_close_cq(&c->sq[tc].cq);
2291
2292 return err;
2293}
2294
2295static void mlx5e_close_tx_cqs(struct mlx5e_channel *c)
2296{
2297 int tc;
2298
2299 for (tc = 0; tc < c->num_tc; tc++)
2300 mlx5e_close_cq(&c->sq[tc].cq);
2301}
2302
80743c4f
TT		 2303static int mlx5e_mqprio_txq_to_tc(struct netdev_tc_txq *tc_to_txq, unsigned int txq)
2304{
2305 int tc;
2306
2307 for (tc = 0; tc < TC_MAX_QUEUE; tc++)
2308 if (txq - tc_to_txq[tc].offset < tc_to_txq[tc].count)
2309 return tc;
2310
2311 WARN(1, "Unexpected TCs configuration. No match found for txq %u", txq);
2312 return -ENOENT;
2313}
2314
2315static int mlx5e_txq_get_qos_node_hw_id(struct mlx5e_params *params, int txq_ix,
2316 u32 *hw_id)
2317{
2318 int tc;
2319
0bb7228f 2320 if (params->mqprio.mode != TC_MQPRIO_MODE_CHANNEL) {
80743c4f
TT		 2321 *hw_id = 0;
2322 return 0;
2323 }
2324
2325 tc = mlx5e_mqprio_txq_to_tc(params->mqprio.tc_to_txq, txq_ix);
2326 if (tc < 0)
2327 return tc;
2328
0bb7228f
MT		 2329 if (tc >= params->mqprio.num_tc) {
2330 WARN(1, "Unexpected TCs configuration. tc %d is out of range of %u",
2331 tc, params->mqprio.num_tc);
2332 return -EINVAL;
2333 }
2334
2335 *hw_id = params->mqprio.channel.hw_id[tc];
2336 return 0;
80743c4f
TT		 2337}
2338
f62b8bb8 2339static int mlx5e_open_sqs(struct mlx5e_channel *c,
6a9764ef 2340 struct mlx5e_params *params,
f62b8bb8
AV		 2341 struct mlx5e_channel_param *cparam)
2342{
694826e3 2343 int err, tc;
f62b8bb8 2344
86d747a3 2345 for (tc = 0; tc < mlx5e_get_dcb_num_tc(params); tc++) {
c55d8b10 2346 int txq_ix = c->ix + tc * params->num_channels;
80743c4f 2347 u32 qos_queue_group_id;
b25bd37c 2348 u32 tisn;
80743c4f 2349
b25bd37c
TT		 2350 tisn = mlx5e_profile_get_tisn(c->mdev, c->priv, c->priv->profile,
2351 c->lag_port, tc);
80743c4f
TT		 2352 err = mlx5e_txq_get_qos_node_hw_id(params, txq_ix, &qos_queue_group_id);
2353 if (err)
2354 goto err_close_sqs;
acc6c595 2355
b25bd37c 2356 err = mlx5e_open_txqsq(c, tisn, txq_ix,
80743c4f
TT		 2357 params, &cparam->txq_sq, &c->sq[tc], tc,
2358 qos_queue_group_id,
be98737a 2359 &c->priv->channel_stats[c->ix]->sq[tc]);
f62b8bb8
AV		 2360 if (err)
2361 goto err_close_sqs;
2362 }
2363
2364 return 0;
2365
2366err_close_sqs:
2367 for (tc--; tc >= 0; tc--)
31391048 2368 mlx5e_close_txqsq(&c->sq[tc]);
f62b8bb8
AV		 2369
2370 return err;
2371}
2372
2373static void mlx5e_close_sqs(struct mlx5e_channel *c)
2374{
2375 int tc;
2376
2377 for (tc = 0; tc < c->num_tc; tc++)
31391048 2378 mlx5e_close_txqsq(&c->sq[tc]);
f62b8bb8
AV		 2379}
2380
507f0c81 2381static int mlx5e_set_sq_maxrate(struct net_device *dev,
31391048 2382 struct mlx5e_txqsq *sq, u32 rate)
507f0c81
YP		 2383{
2384 struct mlx5e_priv *priv = netdev_priv(dev);
2385 struct mlx5_core_dev *mdev = priv->mdev;
33ad9711 2386 struct mlx5e_modify_sq_param msp = {0};
05d3ac97 2387 struct mlx5_rate_limit rl = {0};
507f0c81
YP		 2388 u16 rl_index = 0;
2389 int err;
2390
2391 if (rate == sq->rate_limit)
2392 /* nothing to do */
2393 return 0;
2394
05d3ac97
BW		 2395 if (sq->rate_limit) {
2396 rl.rate = sq->rate_limit;
507f0c81 2397 /* remove current rl index to free space to next ones */
05d3ac97
BW		 2398 mlx5_rl_remove_rate(mdev, &rl);
2399 }
507f0c81
YP		 2400
2401 sq->rate_limit = 0;
2402
2403 if (rate) {
05d3ac97
BW		 2404 rl.rate = rate;
2405 err = mlx5_rl_add_rate(mdev, &rl_index, &rl);
507f0c81
YP		 2406 if (err) {
2407 netdev_err(dev, "Failed configuring rate %u: %d\n",
2408 rate, err);
2409 return err;
2410 }
2411 }
2412
33ad9711
SM		 2413 msp.curr_state = MLX5_SQC_STATE_RDY;
2414 msp.next_state = MLX5_SQC_STATE_RDY;
2415 msp.rl_index = rl_index;
2416 msp.rl_update = true;
a43b25da 2417 err = mlx5e_modify_sq(mdev, sq->sqn, &msp);
507f0c81
YP		 2418 if (err) {
2419 netdev_err(dev, "Failed configuring rate %u: %d\n",
2420 rate, err);
2421 /* remove the rate from the table */
2422 if (rate)
05d3ac97 2423 mlx5_rl_remove_rate(mdev, &rl);
507f0c81
YP		 2424 return err;
2425 }
2426
2427 sq->rate_limit = rate;
2428 return 0;
2429}
2430
2431static int mlx5e_set_tx_maxrate(struct net_device *dev, int index, u32 rate)
2432{
2433 struct mlx5e_priv *priv = netdev_priv(dev);
2434 struct mlx5_core_dev *mdev = priv->mdev;
acc6c595 2435 struct mlx5e_txqsq *sq = priv->txq2sq[index];
507f0c81
YP		 2436 int err = 0;
2437
2438 if (!mlx5_rl_is_supported(mdev)) {
2439 netdev_err(dev, "Rate limiting is not supported on this device\n");
2440 return -EINVAL;
2441 }
2442
2443 /* rate is given in Mb/sec, HW config is in Kb/sec */
2444 rate = rate << 10;
2445
2446 /* Check whether rate in valid range, 0 is always valid */
2447 if (rate && !mlx5_rl_is_in_range(mdev, rate)) {
2448 netdev_err(dev, "TX rate %u, is not in range\n", rate);
2449 return -ERANGE;
2450 }
2451
2452 mutex_lock(&priv->state_lock);
2453 if (test_bit(MLX5E_STATE_OPENED, &priv->state))
2454 err = mlx5e_set_sq_maxrate(dev, sq, rate);
2455 if (!err)
2456 priv->tx_rates[index] = rate;
2457 mutex_unlock(&priv->state_lock);
2458
2459 return err;
2460}
2461
869c5f92
AL		 2462static int mlx5e_open_rxq_rq(struct mlx5e_channel *c, struct mlx5e_params *params,
2463 struct mlx5e_rq_param *rq_params)
2464{
7f525acb 2465 u16 q_counter = c->priv->q_counter[c->sd_ix];
869c5f92
AL		 2466 int err;
2467
4e7401fc 2468 err = mlx5e_init_rxq_rq(c, params, rq_params->xdp_frag_size, &c->rq);
869c5f92
AL		 2469 if (err)
2470 return err;
2471
7f525acb 2472 return mlx5e_open_rq(params, rq_params, NULL, cpu_to_node(c->cpu), q_counter, &c->rq);
869c5f92
AL		 2473}
2474
0a06382f
MM		 2475static int mlx5e_open_queues(struct mlx5e_channel *c,
2476 struct mlx5e_params *params,
2477 struct mlx5e_channel_param *cparam)
f62b8bb8 2478{
8960b389 2479 struct dim_cq_moder icocq_moder = {0, 0};
2f6b379c 2480 struct mlx5e_create_cq_param ccp;
f62b8bb8 2481 int err;
f62b8bb8 2482
2f6b379c 2483 mlx5e_build_create_cq_param(&ccp, c);
4d0b7ef9 2484
db52aa6d 2485 err = mlx5e_open_cq(c->mdev, icocq_moder, &cparam->async_icosq.cqp, &ccp,
4d0b7ef9 2486 &c->async_icosq.cq);
f62b8bb8 2487 if (err)
0a06382f 2488 return err;
f62b8bb8 2489
db52aa6d 2490 err = mlx5e_open_cq(c->mdev, icocq_moder, &cparam->icosq.cqp, &ccp,
4d0b7ef9 2491 &c->icosq.cq);
8d94b590
TT		 2492 if (err)
2493 goto err_close_async_icosq_cq;
2494
4d0b7ef9 2495 err = mlx5e_open_tx_cqs(c, params, &ccp, cparam);
d3c9bc27
TT		 2496 if (err)
2497 goto err_close_icosq_cq;
2498
db52aa6d 2499 err = mlx5e_open_cq(c->mdev, params->tx_cq_moderation, &cparam->xdp_sq.cqp, &ccp,
4d0b7ef9 2500 &c->xdpsq.cq);
f62b8bb8
AV		 2501 if (err)
2502 goto err_close_tx_cqs;
f62b8bb8 2503
db52aa6d 2504 err = mlx5e_open_cq(c->mdev, params->rx_cq_moderation, &cparam->rq.cqp, &ccp,
4d0b7ef9 2505 &c->rq.cq);
58b99ee3
TT		 2506 if (err)
2507 goto err_close_xdp_tx_cqs;
2508
db52aa6d 2509 err = c->xdp ? mlx5e_open_cq(c->mdev, params->tx_cq_moderation, &cparam->xdp_sq.cqp,
4d0b7ef9 2510 &ccp, &c->rq_xdpsq.cq) : 0;
d7a0ecab
SM		 2511 if (err)
2512 goto err_close_rx_cq;
2513
8d94b590
TT		 2514 spin_lock_init(&c->async_icosq_lock);
2515
19c4aba2
MM		 2516 err = mlx5e_open_icosq(c, params, &cparam->async_icosq, &c->async_icosq,
2517 mlx5e_async_icosq_err_cqe_work);
f62b8bb8 2518 if (err)
7637e499 2519 goto err_close_xdpsq_cq;
f62b8bb8 2520
17958d7c
MM		 2521 mutex_init(&c->icosq_recovery_lock);
2522
19c4aba2
MM		 2523 err = mlx5e_open_icosq(c, params, &cparam->icosq, &c->icosq,
2524 mlx5e_icosq_err_cqe_work);
8d94b590
TT		 2525 if (err)
2526 goto err_close_async_icosq;
2527
6a9764ef 2528 err = mlx5e_open_sqs(c, params, cparam);
d3c9bc27
TT		 2529 if (err)
2530 goto err_close_icosq;
2531
8ba3e4c8
MM		 2532 err = mlx5e_open_rxq_rq(c, params, &cparam->rq);
2533 if (err)
2534 goto err_close_sqs;
2535
b9673cf5 2536 if (c->xdp) {
db05815b 2537 err = mlx5e_open_xdpsq(c, params, &cparam->xdp_sq, NULL,
b9673cf5
MM		 2538 &c->rq_xdpsq, false);
2539 if (err)
8ba3e4c8 2540 goto err_close_rq;
b9673cf5 2541 }
b5503b99 2542
db05815b 2543 err = mlx5e_open_xdpsq(c, params, &cparam->xdp_sq, NULL, &c->xdpsq, true);
58b99ee3 2544 if (err)
8ba3e4c8 2545 goto err_close_xdp_sq;
58b99ee3 2546
f62b8bb8 2547 return 0;
58b99ee3 2548
b5503b99 2549err_close_xdp_sq:
d7a0ecab 2550 if (c->xdp)
b9673cf5 2551 mlx5e_close_xdpsq(&c->rq_xdpsq);
f62b8bb8 2552
8ba3e4c8
MM		 2553err_close_rq:
2554 mlx5e_close_rq(&c->rq);
2555
f62b8bb8
AV		 2556err_close_sqs:
2557 mlx5e_close_sqs(c);
2558
d3c9bc27 2559err_close_icosq:
31391048 2560 mlx5e_close_icosq(&c->icosq);
d3c9bc27 2561
8d94b590
TT		 2562err_close_async_icosq:
2563 mlx5e_close_icosq(&c->async_icosq);
2564
7637e499 2565err_close_xdpsq_cq:
d7a0ecab 2566 if (c->xdp)
b9673cf5 2567 mlx5e_close_cq(&c->rq_xdpsq.cq);
d7a0ecab
SM		 2568
2569err_close_rx_cq:
f62b8bb8
AV		 2570 mlx5e_close_cq(&c->rq.cq);
2571
58b99ee3
TT		 2572err_close_xdp_tx_cqs:
2573 mlx5e_close_cq(&c->xdpsq.cq);
2574
f62b8bb8
AV		 2575err_close_tx_cqs:
2576 mlx5e_close_tx_cqs(c);
2577
d3c9bc27
TT		 2578err_close_icosq_cq:
2579 mlx5e_close_cq(&c->icosq.cq);
2580
8d94b590
TT		 2581err_close_async_icosq_cq:
2582 mlx5e_close_cq(&c->async_icosq.cq);
2583
0a06382f
MM		 2584 return err;
2585}
2586
2587static void mlx5e_close_queues(struct mlx5e_channel *c)
2588{
2589 mlx5e_close_xdpsq(&c->xdpsq);
0a06382f
MM		 2590 if (c->xdp)
2591 mlx5e_close_xdpsq(&c->rq_xdpsq);
17958d7c
MM		 2592 /* The same ICOSQ is used for UMRs for both RQ and XSKRQ. */
2593 cancel_work_sync(&c->icosq.recover_work);
8ba3e4c8 2594 mlx5e_close_rq(&c->rq);
0a06382f
MM		 2595 mlx5e_close_sqs(c);
2596 mlx5e_close_icosq(&c->icosq);
17958d7c 2597 mutex_destroy(&c->icosq_recovery_lock);
8d94b590 2598 mlx5e_close_icosq(&c->async_icosq);
0a06382f
MM		 2599 if (c->xdp)
2600 mlx5e_close_cq(&c->rq_xdpsq.cq);
2601 mlx5e_close_cq(&c->rq.cq);
2602 mlx5e_close_cq(&c->xdpsq.cq);
2603 mlx5e_close_tx_cqs(c);
2604 mlx5e_close_cq(&c->icosq.cq);
8d94b590 2605 mlx5e_close_cq(&c->async_icosq.cq);
0a06382f
MM		 2606}
2607
45f171b1
MM		 2608static u8 mlx5e_enumerate_lag_port(struct mlx5_core_dev *mdev, int ix)
2609{
2610 u16 port_aff_bias = mlx5_core_is_pf(mdev) ? 0 : MLX5_CAP_GEN(mdev, vhca_id);
2611
2612 return (ix + port_aff_bias) % mlx5e_get_num_lag_ports(mdev);
2613}
2614
fa691d0c
LK		 2615static int mlx5e_channel_stats_alloc(struct mlx5e_priv *priv, int ix, int cpu)
2616{
2617 if (ix > priv->stats_nch) {
2618 netdev_warn(priv->netdev, "Unexpected channel stats index %d > %d\n", ix,
2619 priv->stats_nch);
2620 return -EINVAL;
2621 }
2622
2623 if (priv->channel_stats[ix])
2624 return 0;
2625
2626 /* Asymmetric dynamic memory allocation.
2627 * Freed in mlx5e_priv_arrays_free, not on channel closure.
2628 */
559f4c32 2629 netdev_dbg(priv->netdev, "Creating channel stats %d\n", ix);
fa691d0c
LK		 2630 priv->channel_stats[ix] = kvzalloc_node(sizeof(**priv->channel_stats),
2631 GFP_KERNEL, cpu_to_node(cpu));
2632 if (!priv->channel_stats[ix])
2633 return -ENOMEM;
2634 priv->stats_nch++;
2635
2636 return 0;
2637}
2638
2e642afb
MM		 2639void mlx5e_trigger_napi_icosq(struct mlx5e_channel *c)
2640{
2641 spin_lock_bh(&c->async_icosq_lock);
2642 mlx5e_trigger_irq(&c->async_icosq);
2643 spin_unlock_bh(&c->async_icosq_lock);
2644}
2645
2646void mlx5e_trigger_napi_sched(struct napi_struct *napi)
2647{
2648 local_bh_disable();
2649 napi_schedule(napi);
2650 local_bh_enable();
2651}
2652
0a06382f
MM		 2653static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
2654 struct mlx5e_params *params,
2655 struct mlx5e_channel_param *cparam,
1742b3d5 2656 struct xsk_buff_pool *xsk_pool,
0a06382f
MM		 2657 struct mlx5e_channel **cp)
2658{
0a06382f 2659 struct net_device *netdev = priv->netdev;
67936e13 2660 struct mlx5_core_dev *mdev;
db05815b 2661 struct mlx5e_xsk_param xsk;
0a06382f
MM		 2662 struct mlx5e_channel *c;
2663 unsigned int irq;
67936e13
TT		 2664 int vec_ix;
2665 int cpu;
0a06382f 2666 int err;
0a06382f 2667
67936e13
TT		 2668 mdev = mlx5_sd_ch_ix_get_dev(priv->mdev, ix);
2669 vec_ix = mlx5_sd_ch_ix_get_vec_ix(mdev, ix);
2670 cpu = mlx5_comp_vector_get_cpu(mdev, vec_ix);
2671
2672 err = mlx5_comp_irqn_get(mdev, vec_ix, &irq);
0a06382f
MM		 2673 if (err)
2674 return err;
2675
fa691d0c
LK		 2676 err = mlx5e_channel_stats_alloc(priv, ix, cpu);
2677 if (err)
2678 return err;
2679
0a06382f
MM		 2680 c = kvzalloc_node(sizeof(*c), GFP_KERNEL, cpu_to_node(cpu));
2681 if (!c)
2682 return -ENOMEM;
2683
2684 c->priv = priv;
67936e13 2685 c->mdev = mdev;
0a06382f
MM		 2686 c->tstamp = &priv->tstamp;
2687 c->ix = ix;
67936e13 2688 c->vec_ix = vec_ix;
7f525acb 2689 c->sd_ix = mlx5_sd_ch_ix_get_dev_ix(mdev, ix);
0a06382f 2690 c->cpu = cpu;
67936e13 2691 c->pdev = mlx5_core_dma_dev(mdev);
0a06382f 2692 c->netdev = priv->netdev;
67936e13 2693 c->mkey_be = cpu_to_be32(mdev->mlx5e_res.hw_objs.mkey);
86d747a3 2694 c->num_tc = mlx5e_get_dcb_num_tc(params);
0a06382f 2695 c->xdp = !!params->xdp_prog;
be98737a 2696 c->stats = &priv->channel_stats[ix]->ch;
ec7b37b6 2697 c->aff_mask = irq_get_effective_affinity_mask(irq);
67936e13 2698 c->lag_port = mlx5e_enumerate_lag_port(mdev, ix);
0a06382f 2699
b48b89f9 2700 netif_napi_add(netdev, &c->napi, mlx5e_napi_poll);
f25e7b82 2701 netif_napi_set_irq(&c->napi, irq);
0a06382f
MM		 2702
2703 err = mlx5e_open_queues(c, params, cparam);
2704 if (unlikely(err))
2705 goto err_napi_del;
2706
1742b3d5
MK		 2707 if (xsk_pool) {
2708 mlx5e_build_xsk_param(xsk_pool, &xsk);
2709 err = mlx5e_open_xsk(priv, params, &xsk, xsk_pool, c);
db05815b
MM		 2710 if (unlikely(err))
2711 goto err_close_queues;
2712 }
2713
0a06382f
MM		 2714 *cp = c;
2715
2716 return 0;
2717
db05815b
MM		 2718err_close_queues:
2719 mlx5e_close_queues(c);
2720
f62b8bb8
AV		 2721err_napi_del:
2722 netif_napi_del(&c->napi);
149e566f 2723
ca11b798 2724 kvfree(c);
f62b8bb8
AV		 2725
2726 return err;
2727}
2728
acc6c595
SM		 2729static void mlx5e_activate_channel(struct mlx5e_channel *c)
2730{
2731 int tc;
2732
7637e499
TT		 2733 napi_enable(&c->napi);
2734
acc6c595
SM		 2735 for (tc = 0; tc < c->num_tc; tc++)
2736 mlx5e_activate_txqsq(&c->sq[tc]);
9d18b514 2737 mlx5e_activate_icosq(&c->icosq);
8d94b590 2738 mlx5e_activate_icosq(&c->async_icosq);
db05815b
MM		 2739
2740 if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state))
2741 mlx5e_activate_xsk(c);
082a9edf
MM		 2742 else
2743 mlx5e_activate_rq(&c->rq);
f25e7b82
JD		 2744
2745 netif_queue_set_napi(c->netdev, c->ix, NETDEV_QUEUE_TYPE_RX, &c->napi);
acc6c595
SM		 2746}
2747
2748static void mlx5e_deactivate_channel(struct mlx5e_channel *c)
2749{
2750 int tc;
2751
f25e7b82
JD		 2752 netif_queue_set_napi(c->netdev, c->ix, NETDEV_QUEUE_TYPE_RX, NULL);
2753
db05815b
MM		 2754 if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state))
2755 mlx5e_deactivate_xsk(c);
082a9edf
MM		 2756 else
2757 mlx5e_deactivate_rq(&c->rq);
db05815b 2758
8d94b590 2759 mlx5e_deactivate_icosq(&c->async_icosq);
9d18b514 2760 mlx5e_deactivate_icosq(&c->icosq);
acc6c595
SM		 2761 for (tc = 0; tc < c->num_tc; tc++)
2762 mlx5e_deactivate_txqsq(&c->sq[tc]);
214baf22 2763 mlx5e_qos_deactivate_queues(c);
7637e499
TT		 2764
2765 napi_disable(&c->napi);
acc6c595
SM		 2766}
2767
f62b8bb8
AV		 2768static void mlx5e_close_channel(struct mlx5e_channel *c)
2769{
db05815b
MM		 2770 if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state))
2771 mlx5e_close_xsk(c);
0a06382f 2772 mlx5e_close_queues(c);
214baf22 2773 mlx5e_qos_close_queues(c);
f62b8bb8 2774 netif_napi_del(&c->napi);
7ae92ae5 2775
ca11b798 2776 kvfree(c);
f62b8bb8
AV		 2777}
2778
55c2503d
SM		 2779int mlx5e_open_channels(struct mlx5e_priv *priv,
2780 struct mlx5e_channels *chs)
f62b8bb8 2781{
6b87663f 2782 struct mlx5e_channel_param *cparam;
03289b88 2783 int err = -ENOMEM;
f62b8bb8 2784 int i;
f62b8bb8 2785
6a9764ef 2786 chs->num = chs->params.num_channels;
03289b88 2787
ff9c852f 2788 chs->c = kcalloc(chs->num, sizeof(struct mlx5e_channel *), GFP_KERNEL);
ca11b798 2789 cparam = kvzalloc(sizeof(struct mlx5e_channel_param), GFP_KERNEL);
acc6c595
SM		 2790 if (!chs->c || !cparam)
2791 goto err_free;
f62b8bb8 2792
7f525acb 2793 err = mlx5e_build_channel_param(priv->mdev, &chs->params, cparam);
6980ffa0
TT		 2794 if (err)
2795 goto err_free;
2796
ff9c852f 2797 for (i = 0; i < chs->num; i++) {
1742b3d5 2798 struct xsk_buff_pool *xsk_pool = NULL;
db05815b
MM		 2799
2800 if (chs->params.xdp_prog)
1742b3d5 2801 xsk_pool = mlx5e_xsk_get_pool(&chs->params, chs->params.xsk, i);
db05815b 2802
1742b3d5 2803 err = mlx5e_open_channel(priv, i, &chs->params, cparam, xsk_pool, &chs->c[i]);
f62b8bb8
AV		 2804 if (err)
2805 goto err_close_channels;
2806 }
2807
960fbfe2 2808 if (MLX5E_GET_PFLAG(&chs->params, MLX5E_PFLAG_TX_PORT_TS) || chs->params.ptp_rx) {
b0d35de4 2809 err = mlx5e_ptp_open(priv, &chs->params, chs->c[0]->lag_port, &chs->ptp);
145e5637
EBE		 2810 if (err)
2811 goto err_close_channels;
2812 }
2813
28df4a01
MT		 2814 if (priv->htb) {
2815 err = mlx5e_qos_open_queues(priv, chs);
2816 if (err)
2817 goto err_close_ptp;
2818 }
214baf22 2819
11af6a6d 2820 mlx5e_health_channels_update(priv);
ca11b798 2821 kvfree(cparam);
f62b8bb8
AV		 2822 return 0;
2823
214baf22 2824err_close_ptp:
b0d35de4
AL		 2825 if (chs->ptp)
2826 mlx5e_ptp_close(chs->ptp);
214baf22 2827
f62b8bb8
AV		 2828err_close_channels:
2829 for (i--; i >= 0; i--)
ff9c852f 2830 mlx5e_close_channel(chs->c[i]);
f62b8bb8 2831
acc6c595 2832err_free:
ff9c852f 2833 kfree(chs->c);
ca11b798 2834 kvfree(cparam);
ff9c852f 2835 chs->num = 0;
f62b8bb8
AV		 2836 return err;
2837}
2838
79efecb4 2839static void mlx5e_activate_channels(struct mlx5e_priv *priv, struct mlx5e_channels *chs)
f62b8bb8
AV		 2840{
2841 int i;
2842
acc6c595
SM		 2843 for (i = 0; i < chs->num; i++)
2844 mlx5e_activate_channel(chs->c[i]);
145e5637 2845
79efecb4
MM		 2846 if (priv->htb)
2847 mlx5e_qos_activate_queues(priv);
2848
2849 for (i = 0; i < chs->num; i++)
2850 mlx5e_trigger_napi_icosq(chs->c[i]);
2851
b0d35de4
AL		 2852 if (chs->ptp)
2853 mlx5e_ptp_activate_channel(chs->ptp);
acc6c595
SM		 2854}
2855
2856static int mlx5e_wait_channels_min_rx_wqes(struct mlx5e_channels *chs)
2857{
2858 int err = 0;
2859 int i;
2860
f8ebecf2
MM		 2861 for (i = 0; i < chs->num; i++) {
2862 int timeout = err ? 0 : MLX5E_RQ_WQES_TIMEOUT;
082a9edf
MM		 2863 struct mlx5e_channel *c = chs->c[i];
2864
2865 if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state))
2866 continue;
f8ebecf2 2867
082a9edf 2868 err |= mlx5e_wait_for_min_rx_wqes(&c->rq, timeout);
db05815b
MM		 2869
2870 /* Don't wait on the XSK RQ, because the newer xdpsock sample
2871 * doesn't provide any Fill Ring entries at the setup stage.
2872 */
f8ebecf2 2873 }
acc6c595 2874
1e7477ae 2875 return err ? -ETIMEDOUT : 0;
acc6c595
SM		 2876}
2877
2878static void mlx5e_deactivate_channels(struct mlx5e_channels *chs)
2879{
2880 int i;
2881
b0d35de4
AL		 2882 if (chs->ptp)
2883 mlx5e_ptp_deactivate_channel(chs->ptp);
145e5637 2884
acc6c595
SM		 2885 for (i = 0; i < chs->num; i++)
2886 mlx5e_deactivate_channel(chs->c[i]);
2887}
2888
55c2503d 2889void mlx5e_close_channels(struct mlx5e_channels *chs)
acc6c595
SM		 2890{
2891 int i;
c3b7c5c9 2892
eab0da38 2893 ASSERT_RTNL();
b0d35de4
AL		 2894 if (chs->ptp) {
2895 mlx5e_ptp_close(chs->ptp);
2896 chs->ptp = NULL;
1c2cdf0b 2897 }
ff9c852f
SM		 2898 for (i = 0; i < chs->num; i++)
2899 mlx5e_close_channel(chs->c[i]);
f62b8bb8 2900
ff9c852f
SM		 2901 kfree(chs->c);
2902 chs->num = 0;
f62b8bb8
AV		 2903}
2904
eaee12f0 2905static int mlx5e_modify_tirs_packet_merge(struct mlx5e_priv *priv)
5c50368f 2906{
3f22d6c7 2907 struct mlx5e_rx_res *res = priv->rx_res;
5c50368f 2908
eaee12f0 2909 return mlx5e_rx_res_packet_merge_set_param(res, &priv->channels.params.packet_merge);
5c50368f
AS		 2910}
2911
eaee12f0 2912static MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(mlx5e_modify_tirs_packet_merge);
b9ab5d0e 2913
472a1e44
TT		 2914static int mlx5e_set_mtu(struct mlx5_core_dev *mdev,
2915 struct mlx5e_params *params, u16 mtu)
40ab6a6e 2916{
472a1e44 2917 u16 hw_mtu = MLX5E_SW2HW_MTU(params, mtu);
40ab6a6e
AS		 2918 int err;
2919
cd255eff 2920 err = mlx5_set_port_mtu(mdev, hw_mtu, 1);
40ab6a6e
AS		 2921 if (err)
2922 return err;
2923
cd255eff
SM		 2924 /* Update vport context MTU */
2925 mlx5_modify_nic_vport_mtu(mdev, hw_mtu);
2926 return 0;
2927}
40ab6a6e 2928
472a1e44
TT		 2929static void mlx5e_query_mtu(struct mlx5_core_dev *mdev,
2930 struct mlx5e_params *params, u16 *mtu)
cd255eff 2931{
cd255eff
SM		 2932 u16 hw_mtu = 0;
2933 int err;
40ab6a6e 2934
cd255eff
SM		 2935 err = mlx5_query_nic_vport_mtu(mdev, &hw_mtu);
2936 if (err || !hw_mtu) /* fallback to port oper mtu */
2937 mlx5_query_port_oper_mtu(mdev, &hw_mtu, 1);
2938
472a1e44 2939 *mtu = MLX5E_HW2SW_MTU(params, hw_mtu);
cd255eff
SM		 2940}
2941
d9ee0491 2942int mlx5e_set_dev_port_mtu(struct mlx5e_priv *priv)
cd255eff 2943{
472a1e44 2944 struct mlx5e_params *params = &priv->channels.params;
2e20a151 2945 struct net_device *netdev = priv->netdev;
472a1e44 2946 struct mlx5_core_dev *mdev = priv->mdev;
cd255eff
SM		 2947 u16 mtu;
2948 int err;
2949
472a1e44 2950 err = mlx5e_set_mtu(mdev, params, params->sw_mtu);
cd255eff
SM		 2951 if (err)
2952 return err;
40ab6a6e 2953
472a1e44
TT		 2954 mlx5e_query_mtu(mdev, params, &mtu);
2955 if (mtu != params->sw_mtu)
cd255eff 2956 netdev_warn(netdev, "%s: VPort MTU %d is different than netdev mtu %d\n",
472a1e44 2957 __func__, mtu, params->sw_mtu);
40ab6a6e 2958
472a1e44 2959 params->sw_mtu = mtu;
40ab6a6e
AS		 2960 return 0;
2961}
2962
b9ab5d0e
MM		 2963MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(mlx5e_set_dev_port_mtu);
2964
6d7ee2ed
TT		 2965void mlx5e_set_netdev_mtu_boundaries(struct mlx5e_priv *priv)
2966{
2967 struct mlx5e_params *params = &priv->channels.params;
2968 struct net_device *netdev = priv->netdev;
2969 struct mlx5_core_dev *mdev = priv->mdev;
2970 u16 max_mtu;
2971
2972 /* MTU range: 68 - hw-specific max */
2973 netdev->min_mtu = ETH_MIN_MTU;
2974
2975 mlx5_query_port_max_mtu(mdev, &max_mtu, 1);
2976 netdev->max_mtu = min_t(unsigned int, MLX5E_HW2SW_MTU(params, max_mtu),
2977 ETH_MAX_MTU);
2978}
2979
ec60c458 2980static int mlx5e_netdev_set_tcs(struct net_device *netdev, u16 nch, u8 ntc,
7dbc849b 2981 struct netdev_tc_txq *tc_to_txq)
08fb1dac 2982{
21ecfcb8 2983 int tc, err;
08fb1dac
SM		 2984
2985 netdev_reset_tc(netdev);
2986
2987 if (ntc == 1)
21ecfcb8 2988 return 0;
08fb1dac 2989
21ecfcb8
TT		 2990 err = netdev_set_num_tc(netdev, ntc);
2991 if (err) {
2992 netdev_WARN(netdev, "netdev_set_num_tc failed (%d), ntc = %d\n", err, ntc);
2993 return err;
2994 }
08fb1dac 2995
ec60c458
TT		 2996 for (tc = 0; tc < ntc; tc++) {
2997 u16 count, offset;
2998
7dbc849b
TT		 2999 count = tc_to_txq[tc].count;
3000 offset = tc_to_txq[tc].offset;
ec60c458
TT		 3001 netdev_set_tc_queue(netdev, tc, count, offset);
3002 }
21ecfcb8
TT		 3003
3004 return 0;
08fb1dac
SM		 3005}
3006
214baf22
MM		 3007int mlx5e_update_tx_netdev_queues(struct mlx5e_priv *priv)
3008{
aaffda6b
SM		 3009 int nch, ntc, num_txqs, err;
3010 int qos_queues = 0;
214baf22 3011
aaffda6b 3012 if (priv->htb)
3685eed5 3013 qos_queues = mlx5e_htb_cur_leaf_nodes(priv->htb);
214baf22
MM		 3014
3015 nch = priv->channels.params.num_channels;
86d747a3 3016 ntc = mlx5e_get_dcb_num_tc(&priv->channels.params);
214baf22
MM		 3017 num_txqs = nch * ntc + qos_queues;
3018 if (MLX5E_GET_PFLAG(&priv->channels.params, MLX5E_PFLAG_TX_PORT_TS))
3019 num_txqs += ntc;
3020
559f4c32 3021 netdev_dbg(priv->netdev, "Setting num_txqs %d\n", num_txqs);
214baf22
MM		 3022 err = netif_set_real_num_tx_queues(priv->netdev, num_txqs);
3023 if (err)
3024 netdev_warn(priv->netdev, "netif_set_real_num_tx_queues failed, %d\n", err);
3025
3026 return err;
3027}
3028
bcee0937
CJ		 3029static void mlx5e_set_default_xps_cpumasks(struct mlx5e_priv *priv,
3030 struct mlx5e_params *params)
3031{
a07e953d 3032 int ix;
bcee0937
CJ		 3033
3034 for (ix = 0; ix < params->num_channels; ix++) {
a07e953d
CJ		 3035 int num_comp_vectors, irq, vec_ix;
3036 struct mlx5_core_dev *mdev;
3037
3038 mdev = mlx5_sd_ch_ix_get_dev(priv->mdev, ix);
3039 num_comp_vectors = mlx5_comp_vectors_max(mdev);
bcee0937 3040 cpumask_clear(priv->scratchpad.cpumask);
a07e953d 3041 vec_ix = mlx5_sd_ch_ix_get_vec_ix(mdev, ix);
bcee0937 3042
a07e953d 3043 for (irq = vec_ix; irq < num_comp_vectors; irq += params->num_channels) {
bcee0937
CJ		 3044 int cpu = mlx5_comp_vector_get_cpu(mdev, irq);
3045
3046 cpumask_set_cpu(cpu, priv->scratchpad.cpumask);
3047 }
3048
3049 netif_set_xps_queue(priv->netdev, priv->scratchpad.cpumask, ix);
3050 }
3051}
3052
3053static int mlx5e_update_tc_and_tx_queues(struct mlx5e_priv *priv)
c2c95271 3054{
7dbc849b 3055 struct netdev_tc_txq old_tc_to_txq[TC_MAX_QUEUE], *tc_to_txq;
c2c95271 3056 struct net_device *netdev = priv->netdev;
fa374877 3057 int old_num_txqs, old_ntc;
3db4c85c 3058 int nch, ntc;
fa374877 3059 int err;
7dbc849b 3060 int i;
fa374877
MM		 3061
3062 old_num_txqs = netdev->real_num_tx_queues;
5e7923ac 3063 old_ntc = netdev->num_tc ? : 1;
7dbc849b
TT		 3064 for (i = 0; i < ARRAY_SIZE(old_tc_to_txq); i++)
3065 old_tc_to_txq[i] = netdev->tc_to_txq[i];
c2c95271 3066
fa374877 3067 nch = priv->channels.params.num_channels;
7dbc849b 3068 ntc = priv->channels.params.mqprio.num_tc;
7dbc849b 3069 tc_to_txq = priv->channels.params.mqprio.tc_to_txq;
fa374877 3070
7dbc849b 3071 err = mlx5e_netdev_set_tcs(netdev, nch, ntc, tc_to_txq);
21ecfcb8
TT		 3072 if (err)
3073 goto err_out;
214baf22
MM		 3074 err = mlx5e_update_tx_netdev_queues(priv);
3075 if (err)
fa374877 3076 goto err_tcs;
bcee0937 3077 mlx5e_set_default_xps_cpumasks(priv, &priv->channels.params);
fa374877
MM		 3078
3079 return 0;
3080
fa374877 3081err_tcs:
7dbc849b
TT		 3082 WARN_ON_ONCE(mlx5e_netdev_set_tcs(netdev, old_num_txqs / old_ntc, old_ntc,
3083 old_tc_to_txq));
21ecfcb8 3084err_out:
fa374877 3085 return err;
c2c95271
MM		 3086}
3087
bcee0937 3088MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(mlx5e_update_tc_and_tx_queues);
3909a12e 3089
e9542221 3090static int mlx5e_num_channels_changed(struct mlx5e_priv *priv)
fe867cac
MM		 3091{
3092 u16 count = priv->channels.params.num_channels;
bcee0937
CJ		 3093 struct net_device *netdev = priv->netdev;
3094 int old_num_rxqs;
fa374877
MM		 3095 int err;
3096
bcee0937
CJ		 3097 old_num_rxqs = netdev->real_num_rx_queues;
3098 err = netif_set_real_num_rx_queues(netdev, count);
3099 if (err) {
3100 netdev_warn(netdev, "%s: netif_set_real_num_rx_queues failed, %d\n",
3101 __func__, err);
fa374877 3102 return err;
bcee0937
CJ		 3103 }
3104 err = mlx5e_update_tc_and_tx_queues(priv);
3105 if (err) {
3106 /* mlx5e_update_tc_and_tx_queues can fail if channels or TCs number increases.
3107 * Since channel number changed, it increased. That means, the call to
3108 * netif_set_real_num_rx_queues below should not fail, because it
3109 * decreases the number of RX queues.
3110 */
3111 WARN_ON_ONCE(netif_set_real_num_rx_queues(netdev, old_num_rxqs));
3112 return err;
3113 }
3909a12e 3114
3f22d6c7 3115 /* This function may be called on attach, before priv->rx_res is created. */
74a8dada
AF		 3116 if (priv->rx_res) {
3117 mlx5e_rx_res_rss_update_num_channels(priv->rx_res, count);
3118
3119 if (!netif_is_rxfh_configured(priv->netdev))
3120 mlx5e_rx_res_rss_set_indir_uniform(priv->rx_res, count);
3121 }
fe867cac
MM		 3122
3123 return 0;
3124}
3125
b9ab5d0e
MM		 3126MLX5E_DEFINE_PREACTIVATE_WRAPPER_CTX(mlx5e_num_channels_changed);
3127
c55d8b10 3128static void mlx5e_build_txq_maps(struct mlx5e_priv *priv)
acc6c595 3129{
145e5637 3130 int i, ch, tc, num_tc;
acc6c595 3131
c55d8b10 3132 ch = priv->channels.num;
86d747a3 3133 num_tc = mlx5e_get_dcb_num_tc(&priv->channels.params);
8bfaf07f 3134
c55d8b10 3135 for (i = 0; i < ch; i++) {
145e5637 3136 for (tc = 0; tc < num_tc; tc++) {
c55d8b10
EBE		 3137 struct mlx5e_channel *c = priv->channels.c[i];
3138 struct mlx5e_txqsq *sq = &c->sq[tc];
acc6c595 3139
acc6c595 3140 priv->txq2sq[sq->txq_ix] = sq;
0a3e5c1b 3141 priv->txq2sq_stats[sq->txq_ix] = sq->stats;
acc6c595
SM		 3142 }
3143 }
145e5637 3144
b0d35de4 3145 if (!priv->channels.ptp)
6ce204ea 3146 goto out;
145e5637 3147
24c22dd0 3148 if (!test_bit(MLX5E_PTP_STATE_TX, priv->channels.ptp->state))
6ce204ea 3149 goto out;
24c22dd0 3150
145e5637 3151 for (tc = 0; tc < num_tc; tc++) {
b0d35de4 3152 struct mlx5e_ptp *c = priv->channels.ptp;
145e5637
EBE		 3153 struct mlx5e_txqsq *sq = &c->ptpsq[tc].txqsq;
3154
3155 priv->txq2sq[sq->txq_ix] = sq;
0a3e5c1b 3156 priv->txq2sq_stats[sq->txq_ix] = sq->stats;
145e5637 3157 }
6ce204ea
MM		 3158
3159out:
3160 /* Make the change to txq2sq visible before the queue is started.
3161 * As mlx5e_xmit runs under a spinlock, there is an implicit ACQUIRE,
3162 * which pairs with this barrier.
3163 */
3164 smp_wmb();
145e5637
EBE		 3165}
3166
603f4a45 3167void mlx5e_activate_priv_channels(struct mlx5e_priv *priv)
acc6c595 3168{
c55d8b10 3169 mlx5e_build_txq_maps(priv);
79efecb4 3170 mlx5e_activate_channels(priv, &priv->channels);
407e17b1 3171 mlx5e_xdp_tx_enable(priv);
befa4177
MM		 3172
3173 /* dev_watchdog() wants all TX queues to be started when the carrier is
3174 * OK, including the ones in range real_num_tx_queues..num_tx_queues-1.
3175 * Make it happy to avoid TX timeout false alarms.
3176 */
acc6c595 3177 netif_tx_start_all_queues(priv->netdev);
9008ae07 3178
d9ee0491 3179 if (mlx5e_is_vport_rep(priv))
430e2d5e 3180 mlx5e_rep_activate_channels(priv);
9008ae07 3181
eab0da38
MS		 3182 set_bit(MLX5E_STATE_CHANNELS_ACTIVE, &priv->state);
3183
acc6c595 3184 mlx5e_wait_channels_min_rx_wqes(&priv->channels);
db05815b 3185
43ec0f41
MM		 3186 if (priv->rx_res)
3187 mlx5e_rx_res_channels_activate(priv->rx_res, &priv->channels);
acc6c595
SM		 3188}
3189
eab0da38
MS		 3190static void mlx5e_cancel_tx_timeout_work(struct mlx5e_priv *priv)
3191{
3192 WARN_ON_ONCE(test_bit(MLX5E_STATE_CHANNELS_ACTIVE, &priv->state));
3193 if (current_work() != &priv->tx_timeout_work)
3194 cancel_work_sync(&priv->tx_timeout_work);
3195}
3196
603f4a45 3197void mlx5e_deactivate_priv_channels(struct mlx5e_priv *priv)
acc6c595 3198{
43ec0f41
MM		 3199 if (priv->rx_res)
3200 mlx5e_rx_res_channels_deactivate(priv->rx_res);
9008ae07 3201
eab0da38
MS		 3202 clear_bit(MLX5E_STATE_CHANNELS_ACTIVE, &priv->state);
3203 mlx5e_cancel_tx_timeout_work(priv);
3204
d9ee0491 3205 if (mlx5e_is_vport_rep(priv))
430e2d5e 3206 mlx5e_rep_deactivate_channels(priv);
9008ae07 3207
befa4177
MM		 3208 /* The results of ndo_select_queue are unreliable, while netdev config
3209 * is being changed (real_num_tx_queues, num_tc). Stop all queues to
3210 * prevent ndo_start_xmit from being called, so that it can assume that
3211 * the selected queue is always valid.
acc6c595 3212 */
acc6c595 3213 netif_tx_disable(priv->netdev);
befa4177 3214
407e17b1 3215 mlx5e_xdp_tx_disable(priv);
acc6c595
SM		 3216 mlx5e_deactivate_channels(&priv->channels);
3217}
3218
b3b886cf
MM		 3219static int mlx5e_switch_priv_params(struct mlx5e_priv *priv,
3220 struct mlx5e_params *new_params,
3221 mlx5e_fp_preactivate preactivate,
3222 void *context)
3223{
3224 struct mlx5e_params old_params;
3225
3226 old_params = priv->channels.params;
3227 priv->channels.params = *new_params;
3228
3229 if (preactivate) {
3230 int err;
3231
3232 err = preactivate(priv, context);
3233 if (err) {
3234 priv->channels.params = old_params;
3235 return err;
3236 }
3237 }
3238
3239 return 0;
3240}
3241
35a78ed4
MM		 3242static int mlx5e_switch_priv_channels(struct mlx5e_priv *priv,
3243 struct mlx5e_channels *new_chs,
b9ab5d0e
MM		 3244 mlx5e_fp_preactivate preactivate,
3245 void *context)
55c2503d
SM		 3246{
3247 struct net_device *netdev = priv->netdev;
35a78ed4 3248 struct mlx5e_channels old_chs;
7ca42c80 3249 int carrier_ok;
35a78ed4 3250 int err = 0;
877662e2 3251
b3b886cf
MM		 3252 carrier_ok = netif_carrier_ok(netdev);
3253 netif_carrier_off(netdev);
55c2503d 3254
b3b886cf 3255 mlx5e_deactivate_priv_channels(priv);
55c2503d 3256
35a78ed4 3257 old_chs = priv->channels;
55c2503d
SM		 3258 priv->channels = *new_chs;
3259
dca147b3
MM		 3260 /* New channels are ready to roll, call the preactivate hook if needed
3261 * to modify HW settings or update kernel parameters.
3262 */
35a78ed4 3263 if (preactivate) {
b9ab5d0e 3264 err = preactivate(priv, context);
35a78ed4
MM		 3265 if (err) {
3266 priv->channels = old_chs;
3267 goto out;
3268 }
3269 }
2e20a151 3270
b3b886cf
MM		 3271 mlx5e_close_channels(&old_chs);
3272 priv->profile->update_rx(priv);
35a78ed4 3273
8bf30be7 3274 mlx5e_selq_apply(&priv->selq);
35a78ed4 3275out:
b3b886cf 3276 mlx5e_activate_priv_channels(priv);
55c2503d 3277
b3b886cf
MM		 3278 /* return carrier back if needed */
3279 if (carrier_ok)
3280 netif_carrier_on(netdev);
35a78ed4
MM		 3281
3282 return err;
55c2503d
SM		 3283}
3284
94872d4e
MM		 3285int mlx5e_safe_switch_params(struct mlx5e_priv *priv,
3286 struct mlx5e_params *params,
3287 mlx5e_fp_preactivate preactivate,
3288 void *context, bool reset)
877662e2 3289{
78028862 3290 struct mlx5e_channels *new_chs;
877662e2
TT		 3291 int err;
3292
b3b886cf
MM		 3293 reset &= test_bit(MLX5E_STATE_OPENED, &priv->state);
3294 if (!reset)
94872d4e 3295 return mlx5e_switch_priv_params(priv, params, preactivate, context);
877662e2 3296
78028862
AB		 3297 new_chs = kzalloc(sizeof(*new_chs), GFP_KERNEL);
3298 if (!new_chs)
3299 return -ENOMEM;
3300 new_chs->params = *params;
8bf30be7 3301
78028862 3302 mlx5e_selq_prepare_params(&priv->selq, &new_chs->params);
8bf30be7 3303
78028862 3304 err = mlx5e_open_channels(priv, new_chs);
b3b886cf 3305 if (err)
8bf30be7
MM		 3306 goto err_cancel_selq;
3307
78028862 3308 err = mlx5e_switch_priv_channels(priv, new_chs, preactivate, context);
35a78ed4 3309 if (err)
8bf30be7 3310 goto err_close;
35a78ed4 3311
78028862 3312 kfree(new_chs);
8bf30be7
MM		 3313 return 0;
3314
3315err_close:
78028862 3316 mlx5e_close_channels(new_chs);
8bf30be7
MM		 3317
3318err_cancel_selq:
3319 mlx5e_selq_cancel(&priv->selq);
78028862 3320 kfree(new_chs);
35a78ed4 3321 return err;
877662e2
TT		 3322}
3323
484c1ada
EBE		 3324int mlx5e_safe_reopen_channels(struct mlx5e_priv *priv)
3325{
94872d4e 3326 return mlx5e_safe_switch_params(priv, &priv->channels.params, NULL, NULL, true);
484c1ada
EBE		 3327}
3328
237f258c 3329void mlx5e_timestamp_init(struct mlx5e_priv *priv)
7c39afb3
FD		 3330{
3331 priv->tstamp.tx_type = HWTSTAMP_TX_OFF;
3332 priv->tstamp.rx_filter = HWTSTAMP_FILTER_NONE;
3333}
3334
7d0314b1
RD		 3335static void mlx5e_modify_admin_state(struct mlx5_core_dev *mdev,
3336 enum mlx5_port_status state)
3337{
3338 struct mlx5_eswitch *esw = mdev->priv.eswitch;
3339 int vport_admin_state;
3340
3341 mlx5_set_port_admin_status(mdev, state);
3342
9c9be85f
AL		 3343 if (mlx5_eswitch_mode(mdev) == MLX5_ESWITCH_OFFLOADS ||
3344 !MLX5_CAP_GEN(mdev, uplink_follow))
7d0314b1
RD		 3345 return;
3346
3347 if (state == MLX5_PORT_UP)
3348 vport_admin_state = MLX5_VPORT_ADMIN_STATE_AUTO;
3349 else
3350 vport_admin_state = MLX5_VPORT_ADMIN_STATE_DOWN;
3351
3352 mlx5_eswitch_set_vport_state(esw, MLX5_VPORT_UPLINK, vport_admin_state);
3353}
3354
40ab6a6e
AS		 3355int mlx5e_open_locked(struct net_device *netdev)
3356{
3357 struct mlx5e_priv *priv = netdev_priv(netdev);
40ab6a6e
AS		 3358 int err;
3359
4f8d1d3a 3360 mlx5e_selq_prepare_params(&priv->selq, &priv->channels.params);
8bf30be7 3361
40ab6a6e
AS		 3362 set_bit(MLX5E_STATE_OPENED, &priv->state);
3363
ff9c852f 3364 err = mlx5e_open_channels(priv, &priv->channels);
acc6c595 3365 if (err)
343b29f3 3366 goto err_clear_state_opened_flag;
40ab6a6e 3367
e74ae1fa
GT		 3368 err = priv->profile->update_rx(priv);
3369 if (err)
3370 goto err_close_channels;
3371
8bf30be7 3372 mlx5e_selq_apply(&priv->selq);
acc6c595 3373 mlx5e_activate_priv_channels(priv);
eb3862a0 3374 mlx5e_apply_traps(priv, true);
7ca42c80
ES		 3375 if (priv->profile->update_carrier)
3376 priv->profile->update_carrier(priv);
be4891af 3377
cdeef2b1 3378 mlx5e_queue_update_stats(priv);
9b37b07f 3379 return 0;
343b29f3 3380
e74ae1fa
GT		 3381err_close_channels:
3382 mlx5e_close_channels(&priv->channels);
343b29f3
AS		 3383err_clear_state_opened_flag:
3384 clear_bit(MLX5E_STATE_OPENED, &priv->state);
8bf30be7 3385 mlx5e_selq_cancel(&priv->selq);
343b29f3 3386 return err;
40ab6a6e
AS		 3387}
3388
cb67b832 3389int mlx5e_open(struct net_device *netdev)
40ab6a6e
AS		 3390{
3391 struct mlx5e_priv *priv = netdev_priv(netdev);
3392 int err;
3393
3394 mutex_lock(&priv->state_lock);
3395 err = mlx5e_open_locked(netdev);
63bfd399 3396 if (!err)
7d0314b1 3397 mlx5e_modify_admin_state(priv->mdev, MLX5_PORT_UP);
40ab6a6e
AS		 3398 mutex_unlock(&priv->state_lock);
3399
3400 return err;
3401}
3402
3403int mlx5e_close_locked(struct net_device *netdev)
3404{
3405 struct mlx5e_priv *priv = netdev_priv(netdev);
3406
a1985740
AS		 3407 /* May already be CLOSED in case a previous configuration operation
3408 * (e.g RX/TX queue size change) that involves close&open failed.
3409 */
3410 if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
3411 return 0;
3412
eb3862a0 3413 mlx5e_apply_traps(priv, false);
40ab6a6e
AS		 3414 clear_bit(MLX5E_STATE_OPENED, &priv->state);
3415
40ab6a6e 3416 netif_carrier_off(priv->netdev);
acc6c595
SM		 3417 mlx5e_deactivate_priv_channels(priv);
3418 mlx5e_close_channels(&priv->channels);
40ab6a6e
AS		 3419
3420 return 0;
3421}
3422
cb67b832 3423int mlx5e_close(struct net_device *netdev)
40ab6a6e
AS		 3424{
3425 struct mlx5e_priv *priv = netdev_priv(netdev);
3426 int err;
3427
26e59d80
MHY		 3428 if (!netif_device_present(netdev))
3429 return -ENODEV;
3430
40ab6a6e 3431 mutex_lock(&priv->state_lock);
7d0314b1 3432 mlx5e_modify_admin_state(priv->mdev, MLX5_PORT_DOWN);
40ab6a6e
AS		 3433 err = mlx5e_close_locked(netdev);
3434 mutex_unlock(&priv->state_lock);
3435
3436 return err;
3437}
3438
cdd3f236
TT		 3439static void mlx5e_free_drop_rq(struct mlx5e_rq *rq)
3440{
3441 mlx5_wq_destroy(&rq->wq_ctrl);
3442}
3443
a43b25da 3444static int mlx5e_alloc_drop_rq(struct mlx5_core_dev *mdev,
3b77235b
SM		 3445 struct mlx5e_rq *rq,
3446 struct mlx5e_rq_param *param)
40ab6a6e 3447{
40ab6a6e
AS		 3448 void *rqc = param->rqc;
3449 void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq);
3450 int err;
3451
3452 param->wq.db_numa_node = param->wq.buf_numa_node;
3453
99cbfa93
TT		 3454 err = mlx5_wq_cyc_create(mdev, &param->wq, rqc_wq, &rq->wqe.wq,
3455 &rq->wq_ctrl);
40ab6a6e
AS		 3456 if (err)
3457 return err;
3458
0ddf5432
JDB		 3459 /* Mark as unused given "Drop-RQ" packets never reach XDP */
3460 xdp_rxq_info_unused(&rq->xdp_rxq);
3461
a43b25da 3462 rq->mdev = mdev;
40ab6a6e
AS		 3463
3464 return 0;
3465}
3466
4d0b7ef9 3467static int mlx5e_alloc_drop_cq(struct mlx5e_priv *priv,
3b77235b
SM		 3468 struct mlx5e_cq *cq,
3469 struct mlx5e_cq_param *param)
40ab6a6e 3470{
4d0b7ef9
AL		 3471 struct mlx5_core_dev *mdev = priv->mdev;
3472
7be3412a
PP		 3473 param->wq.buf_numa_node = dev_to_node(mlx5_core_dma_dev(mdev));
3474 param->wq.db_numa_node = dev_to_node(mlx5_core_dma_dev(mdev));
2f0db879 3475
db52aa6d 3476 return mlx5e_alloc_cq_common(priv->mdev, priv->netdev, priv->wq, param, cq);
40ab6a6e
AS		 3477}
3478
1462e48d
RD		 3479int mlx5e_open_drop_rq(struct mlx5e_priv *priv,
3480 struct mlx5e_rq *drop_rq)
40ab6a6e 3481{
7cbaf9a3 3482 struct mlx5_core_dev *mdev = priv->mdev;
a43b25da
SM		 3483 struct mlx5e_cq_param cq_param = {};
3484 struct mlx5e_rq_param rq_param = {};
3485 struct mlx5e_cq *cq = &drop_rq->cq;
40ab6a6e
AS		 3486 int err;
3487
7f525acb 3488 mlx5e_build_drop_rq_param(mdev, &rq_param);
40ab6a6e 3489
4d0b7ef9 3490 err = mlx5e_alloc_drop_cq(priv, cq, &cq_param);
40ab6a6e
AS		 3491 if (err)
3492 return err;
3493
3b77235b 3494 err = mlx5e_create_cq(cq, &cq_param);
40ab6a6e 3495 if (err)
3b77235b 3496 goto err_free_cq;
40ab6a6e 3497
a43b25da 3498 err = mlx5e_alloc_drop_rq(mdev, drop_rq, &rq_param);
40ab6a6e 3499 if (err)
3b77235b 3500 goto err_destroy_cq;
40ab6a6e 3501
7f525acb 3502 err = mlx5e_create_rq(drop_rq, &rq_param, priv->drop_rq_q_counter);
40ab6a6e 3503 if (err)
3b77235b 3504 goto err_free_rq;
40ab6a6e 3505
7cbaf9a3
MS		 3506 err = mlx5e_modify_rq_state(drop_rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY);
3507 if (err)
3508 mlx5_core_warn(priv->mdev, "modify_rq_state failed, rx_if_down_packets won't be counted %d\n", err);
3509
40ab6a6e
AS		 3510 return 0;
3511
3b77235b 3512err_free_rq:
cdd3f236 3513 mlx5e_free_drop_rq(drop_rq);
40ab6a6e
AS		 3514
3515err_destroy_cq:
a43b25da 3516 mlx5e_destroy_cq(cq);
40ab6a6e 3517
3b77235b 3518err_free_cq:
a43b25da 3519 mlx5e_free_cq(cq);
3b77235b 3520
40ab6a6e
AS		 3521 return err;
3522}
3523
1462e48d 3524void mlx5e_close_drop_rq(struct mlx5e_rq *drop_rq)
40ab6a6e 3525{
a43b25da 3526 mlx5e_destroy_rq(drop_rq);
cdd3f236 3527 mlx5e_free_drop_rq(drop_rq);
a43b25da
SM		 3528 mlx5e_destroy_cq(&drop_rq->cq);
3529 mlx5e_free_cq(&drop_rq->cq);
40ab6a6e
AS		 3530}
3531
d9ee0491 3532static void mlx5e_cleanup_nic_tx(struct mlx5e_priv *priv)
40ab6a6e 3533{
0bb7228f
MT		 3534 if (priv->mqprio_rl) {
3535 mlx5e_mqprio_rl_cleanup(priv->mqprio_rl);
3536 mlx5e_mqprio_rl_free(priv->mqprio_rl);
3537 priv->mqprio_rl = NULL;
3538 }
c4dfe704 3539 mlx5e_accel_cleanup_tx(priv);
40ab6a6e
AS		 3540}
3541
f6d96a20 3542static int mlx5e_modify_channels_vsd(struct mlx5e_channels *chs, bool vsd)
36350114 3543{
a759f845 3544 int err;
36350114
GP		 3545 int i;
3546
ff9c852f
SM		 3547 for (i = 0; i < chs->num; i++) {
3548 err = mlx5e_modify_rq_vsd(&chs->c[i]->rq, vsd);
36350114
GP		 3549 if (err)
3550 return err;
3551 }
a759f845
AL		 3552 if (chs->ptp && test_bit(MLX5E_PTP_STATE_RX, chs->ptp->state))
3553 return mlx5e_modify_rq_vsd(&chs->ptp->rq, vsd);
36350114
GP		 3554
3555 return 0;
3556}
3557
7dbc849b
TT		 3558static void mlx5e_mqprio_build_default_tc_to_txq(struct netdev_tc_txq *tc_to_txq,
3559 int ntc, int nch)
3560{
3561 int tc;
3562
3563 memset(tc_to_txq, 0, sizeof(*tc_to_txq) * TC_MAX_QUEUE);
3564
3565 /* Map netdev TCs to offset 0.
3566 * We have our own UP to TXQ mapping for DCB mode of QoS
3567 */
3568 for (tc = 0; tc < ntc; tc++) {
3569 tc_to_txq[tc] = (struct netdev_tc_txq) {
3570 .count = nch,
3571 .offset = 0,
3572 };
3573 }
3574}
3575
3576static void mlx5e_mqprio_build_tc_to_txq(struct netdev_tc_txq *tc_to_txq,
3577 struct tc_mqprio_qopt *qopt)
3578{
3579 int tc;
3580
3581 for (tc = 0; tc < TC_MAX_QUEUE; tc++) {
3582 tc_to_txq[tc] = (struct netdev_tc_txq) {
3583 .count = qopt->count[tc],
3584 .offset = qopt->offset[tc],
3585 };
3586 }
3587}
3588
3589static void mlx5e_params_mqprio_dcb_set(struct mlx5e_params *params, u8 num_tc)
3590{
3591 params->mqprio.mode = TC_MQPRIO_MODE_DCB;
3592 params->mqprio.num_tc = num_tc;
3593 mlx5e_mqprio_build_default_tc_to_txq(params->mqprio.tc_to_txq, num_tc,
3594 params->num_channels);
3595}
3596
0bb7228f
MT		 3597static void mlx5e_mqprio_rl_update_params(struct mlx5e_params *params,
3598 struct mlx5e_mqprio_rl *rl)
3599{
3600 int tc;
3601
3602 for (tc = 0; tc < TC_MAX_QUEUE; tc++) {
3603 u32 hw_id = 0;
3604
3605 if (rl)
3606 mlx5e_mqprio_rl_get_node_hw_id(rl, tc, &hw_id);
3607 params->mqprio.channel.hw_id[tc] = hw_id;
3608 }
3609}
3610
7dbc849b 3611static void mlx5e_params_mqprio_channel_set(struct mlx5e_params *params,
0bb7228f 3612 struct tc_mqprio_qopt_offload *mqprio,
80743c4f 3613 struct mlx5e_mqprio_rl *rl)
7dbc849b 3614{
0bb7228f
MT		 3615 int tc;
3616
7dbc849b 3617 params->mqprio.mode = TC_MQPRIO_MODE_CHANNEL;
0bb7228f
MT		 3618 params->mqprio.num_tc = mqprio->qopt.num_tc;
3619
3620 for (tc = 0; tc < TC_MAX_QUEUE; tc++)
3621 params->mqprio.channel.max_rate[tc] = mqprio->max_rate[tc];
3622
3623 mlx5e_mqprio_rl_update_params(params, rl);
3624 mlx5e_mqprio_build_tc_to_txq(params->mqprio.tc_to_txq, &mqprio->qopt);
7dbc849b
TT		 3625}
3626
3627static void mlx5e_params_mqprio_reset(struct mlx5e_params *params)
3628{
3629 mlx5e_params_mqprio_dcb_set(params, 1);
3630}
3631
e2aeac44
TT		 3632static int mlx5e_setup_tc_mqprio_dcb(struct mlx5e_priv *priv,
3633 struct tc_mqprio_qopt *mqprio)
08fb1dac 3634{
94872d4e 3635 struct mlx5e_params new_params;
0cf0f6d3 3636 u8 tc = mqprio->num_tc;
e2aeac44 3637 int err;
08fb1dac 3638
0cf0f6d3
JP		 3639 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
3640
b25bd37c 3641 if (tc && tc != MLX5_MAX_NUM_TC)
08fb1dac
SM		 3642 return -EINVAL;
3643
94872d4e 3644 new_params = priv->channels.params;
7dbc849b 3645 mlx5e_params_mqprio_dcb_set(&new_params, tc ? tc : 1);
08fb1dac 3646
94872d4e 3647 err = mlx5e_safe_switch_params(priv, &new_params,
bcee0937 3648 mlx5e_update_tc_and_tx_queues_ctx, NULL, true);
08fb1dac 3649
0bb7228f
MT		 3650 if (!err && priv->mqprio_rl) {
3651 mlx5e_mqprio_rl_cleanup(priv->mqprio_rl);
3652 mlx5e_mqprio_rl_free(priv->mqprio_rl);
3653 priv->mqprio_rl = NULL;
3654 }
3655
5a2ba25a 3656 priv->max_opened_tc = max_t(u8, priv->max_opened_tc,
86d747a3 3657 mlx5e_get_dcb_num_tc(&priv->channels.params));
08fb1dac
SM		 3658 return err;
3659}
3660
ec60c458
TT		 3661static int mlx5e_mqprio_channel_validate(struct mlx5e_priv *priv,
3662 struct tc_mqprio_qopt_offload *mqprio)
3663{
3664 struct net_device *netdev = priv->netdev;
3bf1742f 3665 struct mlx5e_ptp *ptp_channel;
ec60c458
TT		 3666 int agg_count = 0;
3667 int i;
3668
3bf1742f
AL		 3669 ptp_channel = priv->channels.ptp;
3670 if (ptp_channel && test_bit(MLX5E_PTP_STATE_TX, ptp_channel->state)) {
3671 netdev_err(netdev,
3672 "Cannot activate MQPRIO mode channel since it conflicts with TX port TS\n");
3673 return -EINVAL;
3674 }
3675
ec60c458
TT		 3676 if (mqprio->qopt.offset[0] != 0 || mqprio->qopt.num_tc < 1 ||
3677 mqprio->qopt.num_tc > MLX5E_MAX_NUM_MQPRIO_CH_TC)
3678 return -EINVAL;
3679
3680 for (i = 0; i < mqprio->qopt.num_tc; i++) {
3681 if (!mqprio->qopt.count[i]) {
3682 netdev_err(netdev, "Zero size for queue-group (%d) is not supported\n", i);
3683 return -EINVAL;
3684 }
3685 if (mqprio->min_rate[i]) {
3686 netdev_err(netdev, "Min tx rate is not supported\n");
3687 return -EINVAL;
3688 }
80743c4f 3689
ec60c458 3690 if (mqprio->max_rate[i]) {
80743c4f
TT		 3691 int err;
3692
3693 err = mlx5e_qos_bytes_rate_check(priv->mdev, mqprio->max_rate[i]);
3694 if (err)
3695 return err;
ec60c458
TT		 3696 }
3697
3698 if (mqprio->qopt.offset[i] != agg_count) {
3699 netdev_err(netdev, "Discontinuous queues config is not supported\n");
3700 return -EINVAL;
3701 }
3702 agg_count += mqprio->qopt.count[i];
3703 }
3704
ca20dfda
TT		 3705 if (priv->channels.params.num_channels != agg_count) {
3706 netdev_err(netdev, "Num of queues (%d) does not match available (%d)\n",
ec60c458
TT		 3707 agg_count, priv->channels.params.num_channels);
3708 return -EINVAL;
3709 }
3710
3711 return 0;
3712}
3713
0bb7228f 3714static bool mlx5e_mqprio_rate_limit(u8 num_tc, u64 max_rate[])
80743c4f
TT		 3715{
3716 int tc;
3717
0bb7228f
MT		 3718 for (tc = 0; tc < num_tc; tc++)
3719 if (max_rate[tc])
80743c4f
TT		 3720 return true;
3721 return false;
3722}
3723
0bb7228f
MT		 3724static struct mlx5e_mqprio_rl *mlx5e_mqprio_rl_create(struct mlx5_core_dev *mdev,
3725 u8 num_tc, u64 max_rate[])
3726{
3727 struct mlx5e_mqprio_rl *rl;
3728 int err;
3729
3730 if (!mlx5e_mqprio_rate_limit(num_tc, max_rate))
3731 return NULL;
3732
3733 rl = mlx5e_mqprio_rl_alloc();
3734 if (!rl)
3735 return ERR_PTR(-ENOMEM);
3736
3737 err = mlx5e_mqprio_rl_init(rl, mdev, num_tc, max_rate);
3738 if (err) {
3739 mlx5e_mqprio_rl_free(rl);
3740 return ERR_PTR(err);
3741 }
3742
3743 return rl;
3744}
3745
ec60c458
TT		 3746static int mlx5e_setup_tc_mqprio_channel(struct mlx5e_priv *priv,
3747 struct tc_mqprio_qopt_offload *mqprio)
3748{
3749 struct mlx5e_params new_params;
80743c4f 3750 struct mlx5e_mqprio_rl *rl;
ec60c458
TT		 3751 int err;
3752
3753 err = mlx5e_mqprio_channel_validate(priv, mqprio);
3754 if (err)
3755 return err;
3756
0bb7228f
MT		 3757 rl = mlx5e_mqprio_rl_create(priv->mdev, mqprio->qopt.num_tc, mqprio->max_rate);
3758 if (IS_ERR(rl))
3759 return PTR_ERR(rl);
80743c4f 3760
ec60c458 3761 new_params = priv->channels.params;
0bb7228f 3762 mlx5e_params_mqprio_channel_set(&new_params, mqprio, rl);
ec60c458 3763
bcee0937
CJ		 3764 err = mlx5e_safe_switch_params(priv, &new_params,
3765 mlx5e_update_tc_and_tx_queues_ctx, NULL, true);
0bb7228f
MT		 3766 if (err) {
3767 if (rl) {
3768 mlx5e_mqprio_rl_cleanup(rl);
3769 mlx5e_mqprio_rl_free(rl);
3770 }
3771 return err;
80743c4f
TT		 3772 }
3773
0bb7228f
MT		 3774 if (priv->mqprio_rl) {
3775 mlx5e_mqprio_rl_cleanup(priv->mqprio_rl);
3776 mlx5e_mqprio_rl_free(priv->mqprio_rl);
3777 }
3778 priv->mqprio_rl = rl;
3779
3780 return 0;
ec60c458
TT		 3781}
3782
e2aeac44
TT		 3783static int mlx5e_setup_tc_mqprio(struct mlx5e_priv *priv,
3784 struct tc_mqprio_qopt_offload *mqprio)
3785{
3786 /* MQPRIO is another toplevel qdisc that can't be attached
3787 * simultaneously with the offloaded HTB.
3788 */
4f8d1d3a 3789 if (WARN_ON(mlx5e_selq_is_htb_enabled(&priv->selq)))
e2aeac44
TT		 3790 return -EINVAL;
3791
3792 switch (mqprio->mode) {
3793 case TC_MQPRIO_MODE_DCB:
3794 return mlx5e_setup_tc_mqprio_dcb(priv, &mqprio->qopt);
ec60c458
TT		 3795 case TC_MQPRIO_MODE_CHANNEL:
3796 return mlx5e_setup_tc_mqprio_channel(priv, mqprio);
e2aeac44
TT		 3797 default:
3798 return -EOPNOTSUPP;
3799 }
3800}
3801
955bcb6e
PNA		 3802static LIST_HEAD(mlx5e_block_cb_list);
3803
9afe9a53
OG		 3804static int mlx5e_setup_tc(struct net_device *dev, enum tc_setup_type type,
3805 void *type_data)
0cf0f6d3 3806{
4e95bc26 3807 struct mlx5e_priv *priv = netdev_priv(dev);
2ff349c5 3808 bool tc_unbind = false;
214baf22 3809 int err;
4e95bc26 3810
2ff349c5
RD		 3811 if (type == TC_SETUP_BLOCK &&
3812 ((struct flow_block_offload *)type_data)->command == FLOW_BLOCK_UNBIND)
3813 tc_unbind = true;
3814
3815 if (!netif_device_present(dev) && !tc_unbind)
3816 return -ENODEV;
3817
2572ac53 3818 switch (type) {
daa664a5
VB		 3819 case TC_SETUP_BLOCK: {
3820 struct flow_block_offload *f = type_data;
3821
c9f14470 3822 f->unlocked_driver_cb = true;
955bcb6e
PNA		 3823 return flow_block_cb_setup_simple(type_data,
3824 &mlx5e_block_cb_list,
4e95bc26
PNA		 3825 mlx5e_setup_tc_block_cb,
3826 priv, priv, true);
daa664a5 3827 }
575ed7d3 3828 case TC_SETUP_QDISC_MQPRIO:
e2aeac44
TT		 3829 mutex_lock(&priv->state_lock);
3830 err = mlx5e_setup_tc_mqprio(priv, type_data);
3831 mutex_unlock(&priv->state_lock);
3832 return err;
214baf22
MM		 3833 case TC_SETUP_QDISC_HTB:
3834 mutex_lock(&priv->state_lock);
efe31799 3835 err = mlx5e_htb_setup_tc(priv, type_data);
214baf22
MM		 3836 mutex_unlock(&priv->state_lock);
3837 return err;
e8f887ac
AV		 3838 default:
3839 return -EOPNOTSUPP;
3840 }
08fb1dac
SM		 3841}
3842
b832d4fd 3843void mlx5e_fold_sw_stats64(struct mlx5e_priv *priv, struct rtnl_link_stats64 *s)
9659e49a
SM		 3844{
3845 int i;
3846
9d758d4a 3847 for (i = 0; i < priv->stats_nch; i++) {
be98737a 3848 struct mlx5e_channel_stats *channel_stats = priv->channel_stats[i];
db05815b 3849 struct mlx5e_rq_stats *xskrq_stats = &channel_stats->xskrq;
9659e49a
SM		 3850 struct mlx5e_rq_stats *rq_stats = &channel_stats->rq;
3851 int j;
3852
db05815b
MM		 3853 s->rx_packets += rq_stats->packets + xskrq_stats->packets;
3854 s->rx_bytes += rq_stats->bytes + xskrq_stats->bytes;
47c97e6b 3855 s->multicast += rq_stats->mcast_packets + xskrq_stats->mcast_packets;
9659e49a
SM		 3856
3857 for (j = 0; j < priv->max_opened_tc; j++) {
3858 struct mlx5e_sq_stats *sq_stats = &channel_stats->sq[j];
3859
354521ee
AL		 3860 s->tx_packets += sq_stats->packets;
3861 s->tx_bytes += sq_stats->bytes;
3862 s->tx_dropped += sq_stats->dropped;
3863 }
3864 }
b0d35de4 3865 if (priv->tx_ptp_opened) {
354521ee 3866 for (i = 0; i < priv->max_opened_tc; i++) {
b0d35de4 3867 struct mlx5e_sq_stats *sq_stats = &priv->ptp_stats.sq[i];
354521ee 3868
9659e49a
SM		 3869 s->tx_packets += sq_stats->packets;
3870 s->tx_bytes += sq_stats->bytes;
3871 s->tx_dropped += sq_stats->dropped;
3872 }
3873 }
a28359e9
AL		 3874 if (priv->rx_ptp_opened) {
3875 struct mlx5e_rq_stats *rq_stats = &priv->ptp_stats.rq;
3876
3877 s->rx_packets += rq_stats->packets;
3878 s->rx_bytes += rq_stats->bytes;
3879 s->multicast += rq_stats->mcast_packets;
3880 }
9659e49a
SM		 3881}
3882
d9ee0491 3883void
f62b8bb8
AV		 3884mlx5e_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats)
3885{
3886 struct mlx5e_priv *priv = netdev_priv(dev);
269e6b3a 3887 struct mlx5e_pport_stats *pstats = &priv->stats.pport;
f62b8bb8 3888
2ff349c5
RD		 3889 if (!netif_device_present(dev))
3890 return;
3891
dcdf4ce0
ZY		 3892 /* In switchdev mode, monitor counters doesn't monitor
3893 * rx/tx stats of 802_3. The update stats mechanism
3894 * should keep the 802_3 layout counters updated
3895 */
3896 if (!mlx5e_monitor_counter_supported(priv) ||
3897 mlx5e_is_uplink_rep(priv)) {
5c7e8bbb
ED		 3898 /* update HW stats in background for next time */
3899 mlx5e_queue_update_stats(priv);
3900 }
ed56c519 3901
370bad0f 3902 if (mlx5e_is_uplink_rep(priv)) {
a0723108
HN		 3903 struct mlx5e_vport_stats *vstats = &priv->stats.vport;
3904
370bad0f
OG		 3905 stats->rx_packets = PPORT_802_3_GET(pstats, a_frames_received_ok);
3906 stats->rx_bytes = PPORT_802_3_GET(pstats, a_octets_received_ok);
3907 stats->tx_packets = PPORT_802_3_GET(pstats, a_frames_transmitted_ok);
3908 stats->tx_bytes = PPORT_802_3_GET(pstats, a_octets_transmitted_ok);
a0723108
HN		 3909
3910 /* vport multicast also counts packets that are dropped due to steering
3911 * or rx out of buffer
3912 */
3913 stats->multicast = VPORT_COUNTER_GET(vstats, received_eth_multicast.packets);
370bad0f 3914 } else {
9659e49a 3915 mlx5e_fold_sw_stats64(priv, stats);
370bad0f 3916 }
269e6b3a 3917
5c74195d 3918 stats->rx_missed_errors = priv->stats.qcnt.rx_out_of_buffer;
269e6b3a
GP		 3919
3920 stats->rx_length_errors =
9218b44d
GP		 3921 PPORT_802_3_GET(pstats, a_in_range_length_errors) +
3922 PPORT_802_3_GET(pstats, a_out_of_range_length_field) +
16ab85e7
GP		 3923 PPORT_802_3_GET(pstats, a_frame_too_long_errors) +
3924 VNIC_ENV_GET(&priv->stats.vnic, eth_wqe_too_small);
269e6b3a 3925 stats->rx_crc_errors =
9218b44d
GP		 3926 PPORT_802_3_GET(pstats, a_frame_check_sequence_errors);
3927 stats->rx_frame_errors = PPORT_802_3_GET(pstats, a_alignment_errors);
3928 stats->tx_aborted_errors = PPORT_2863_GET(pstats, if_out_discards);
269e6b3a
GP		 3929 stats->rx_errors = stats->rx_length_errors + stats->rx_crc_errors +
3930 stats->rx_frame_errors;
3931 stats->tx_errors = stats->tx_aborted_errors + stats->tx_carrier_errors;
f62b8bb8
AV		 3932}
3933
3fbf6120 3934static void mlx5e_nic_set_rx_mode(struct mlx5e_priv *priv)
1aa48ca6
RD		 3935{
3936 if (mlx5e_is_uplink_rep(priv))
3937 return; /* no rx mode for uplink rep */
3938
3939 queue_work(priv->wq, &priv->set_rx_mode_work);
3940}
3941
f62b8bb8
AV		 3942static void mlx5e_set_rx_mode(struct net_device *dev)
3943{
3944 struct mlx5e_priv *priv = netdev_priv(dev);
3945
1aa48ca6 3946 mlx5e_nic_set_rx_mode(priv);
f62b8bb8
AV		 3947}
3948
3949static int mlx5e_set_mac(struct net_device *netdev, void *addr)
3950{
3951 struct mlx5e_priv *priv = netdev_priv(netdev);
3952 struct sockaddr *saddr = addr;
3953
3954 if (!is_valid_ether_addr(saddr->sa_data))
3955 return -EADDRNOTAVAIL;
3956
3957 netif_addr_lock_bh(netdev);
f3956ebb 3958 eth_hw_addr_set(netdev, saddr->sa_data);
f62b8bb8
AV		 3959 netif_addr_unlock_bh(netdev);
3960
1aa48ca6 3961 mlx5e_nic_set_rx_mode(priv);
f62b8bb8
AV		 3962
3963 return 0;
3964}
3965
75b81ce7 3966#define MLX5E_SET_FEATURE(features, feature, enable) \
0e405443
GP		 3967 do { \
3968 if (enable) \
75b81ce7 3969 *features |= feature; \
0e405443 3970 else \
75b81ce7 3971 *features &= ~feature; \
0e405443
GP		 3972 } while (0)
3973
3974typedef int (*mlx5e_feature_handler)(struct net_device *netdev, bool enable);
3975
3976static int set_feature_lro(struct net_device *netdev, bool enable)
f62b8bb8
AV		 3977{
3978 struct mlx5e_priv *priv = netdev_priv(netdev);
619a8f2a 3979 struct mlx5_core_dev *mdev = priv->mdev;
8355060f 3980 struct mlx5e_params *cur_params;
94872d4e 3981 struct mlx5e_params new_params;
b3b886cf 3982 bool reset = true;
2e20a151 3983 int err = 0;
f62b8bb8
AV		 3984
3985 mutex_lock(&priv->state_lock);
f62b8bb8 3986
8355060f 3987 cur_params = &priv->channels.params;
94872d4e 3988 new_params = *cur_params;
2e20a151 3989
eaee12f0
KM		 3990 if (enable)
3991 new_params.packet_merge.type = MLX5E_PACKET_MERGE_LRO;
3992 else if (new_params.packet_merge.type == MLX5E_PACKET_MERGE_LRO)
3993 new_params.packet_merge.type = MLX5E_PACKET_MERGE_NONE;
3994 else
3995 goto out;
3996
3997 if (!(cur_params->packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO &&
3998 new_params.packet_merge.type == MLX5E_PACKET_MERGE_LRO)) {
3999 if (cur_params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) {
4000 if (mlx5e_rx_mpwqe_is_linear_skb(mdev, cur_params, NULL) ==
4001 mlx5e_rx_mpwqe_is_linear_skb(mdev, &new_params, NULL))
4002 reset = false;
4003 }
98e81b0a 4004 }
f62b8bb8 4005
94872d4e 4006 err = mlx5e_safe_switch_params(priv, &new_params,
eaee12f0 4007 mlx5e_modify_tirs_packet_merge_ctx, NULL, reset);
2e20a151 4008out:
9b37b07f 4009 mutex_unlock(&priv->state_lock);
0e405443
GP		 4010 return err;
4011}
4012
83439f3c
KM		 4013static int set_feature_hw_gro(struct net_device *netdev, bool enable)
4014{
4015 struct mlx5e_priv *priv = netdev_priv(netdev);
4016 struct mlx5e_params new_params;
4017 bool reset = true;
4018 int err = 0;
4019
4020 mutex_lock(&priv->state_lock);
4021 new_params = priv->channels.params;
4022
4023 if (enable) {
4024 new_params.packet_merge.type = MLX5E_PACKET_MERGE_SHAMPO;
4025 new_params.packet_merge.shampo.match_criteria_type =
4026 MLX5_RQC_SHAMPO_MATCH_CRITERIA_TYPE_EXTENDED;
4027 new_params.packet_merge.shampo.alignment_granularity =
4028 MLX5_RQC_SHAMPO_NO_MATCH_ALIGNMENT_GRANULARITY_STRIDE;
4029 } else if (new_params.packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO) {
4030 new_params.packet_merge.type = MLX5E_PACKET_MERGE_NONE;
4031 } else {
4032 goto out;
4033 }
4034
99a2b9be 4035 err = mlx5e_safe_switch_params(priv, &new_params, NULL, NULL, reset);
83439f3c
KM		 4036out:
4037 mutex_unlock(&priv->state_lock);
4038 return err;
4039}
4040
2b52a283 4041static int set_feature_cvlan_filter(struct net_device *netdev, bool enable)
0e405443
GP		 4042{
4043 struct mlx5e_priv *priv = netdev_priv(netdev);
4044
4045 if (enable)
d494dd2b
LK		 4046 mlx5e_enable_cvlan_filter(priv->fs,
4047 !!(priv->netdev->flags & IFF_PROMISC));
0e405443 4048 else
d494dd2b
LK		 4049 mlx5e_disable_cvlan_filter(priv->fs,
4050 !!(priv->netdev->flags & IFF_PROMISC));
0e405443
GP		 4051
4052 return 0;
4053}
4054
214baf22 4055static int set_feature_hw_tc(struct net_device *netdev, bool enable)
0e405443
GP		 4056{
4057 struct mlx5e_priv *priv = netdev_priv(netdev);
4f8d1d3a 4058 int err = 0;
f62b8bb8 4059
214baf22 4060#if IS_ENABLED(CONFIG_MLX5_CLS_ACT)
550f9643
MD		 4061 int tc_flag = mlx5e_is_uplink_rep(priv) ? MLX5_TC_FLAG(ESW_OFFLOAD) :
4062 MLX5_TC_FLAG(NIC_OFFLOAD);
4063 if (!enable && mlx5e_tc_num_filters(priv, tc_flag)) {
e8f887ac
AV		 4064 netdev_err(netdev,
4065 "Active offloaded tc filters, can't turn hw_tc_offload off\n");
4066 return -EINVAL;
4067 }
214baf22
MM		 4068#endif
4069
4f8d1d3a
MT		 4070 mutex_lock(&priv->state_lock);
4071 if (!enable && mlx5e_selq_is_htb_enabled(&priv->selq)) {
214baf22 4072 netdev_err(netdev, "Active HTB offload, can't turn hw_tc_offload off\n");
4f8d1d3a 4073 err = -EINVAL;
214baf22 4074 }
4f8d1d3a 4075 mutex_unlock(&priv->state_lock);
e8f887ac 4076
4f8d1d3a 4077 return err;
0e405443
GP		 4078}
4079
94cb1ebb
EBE		 4080static int set_feature_rx_all(struct net_device *netdev, bool enable)
4081{
4082 struct mlx5e_priv *priv = netdev_priv(netdev);
4083 struct mlx5_core_dev *mdev = priv->mdev;
4084
4085 return mlx5_set_port_fcs(mdev, !enable);
4086}
4087
445a25f6
RR		 4088static struct dim_cq_moder mlx5e_get_def_rx_moderation(u8 cq_period_mode)
4089{
4090 return (struct dim_cq_moder) {
4091 .cq_period_mode = cq_period_mode,
4092 .pkts = MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_PKTS,
4093 .usec = cq_period_mode == DIM_CQ_PERIOD_MODE_START_FROM_CQE ?
4094 MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC_FROM_CQE :
4095 MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC,
4096 };
4097}
4098
4099bool mlx5e_reset_rx_moderation(struct dim_cq_moder *cq_moder, u8 cq_period_mode,
4100 bool dim_enabled)
4101{
4102 bool reset_needed = cq_moder->cq_period_mode != cq_period_mode;
4103
4104 if (dim_enabled)
4105 *cq_moder = net_dim_get_def_rx_moderation(cq_period_mode);
4106 else
4107 *cq_moder = mlx5e_get_def_rx_moderation(cq_period_mode);
4108
4109 return reset_needed;
4110}
4111
4112bool mlx5e_reset_rx_channels_moderation(struct mlx5e_channels *chs, u8 cq_period_mode,
4113 bool dim_enabled, bool keep_dim_state)
4114{
4115 bool reset = false;
4116 int i;
4117
4118 for (i = 0; i < chs->num; i++) {
4119 if (keep_dim_state)
4120 dim_enabled = !!chs->c[i]->rq.dim;
4121
4122 reset |= mlx5e_reset_rx_moderation(&chs->c[i]->rx_cq_moder,
4123 cq_period_mode, dim_enabled);
4124 }
4125
4126 return reset;
4127}
4128
0bc73ad4
AL		 4129static int mlx5e_set_rx_port_ts(struct mlx5_core_dev *mdev, bool enable)
4130{
4131 u32 in[MLX5_ST_SZ_DW(pcmr_reg)] = {};
4132 bool supported, curr_state;
4133 int err;
4134
4135 if (!MLX5_CAP_GEN(mdev, ports_check))
4136 return 0;
4137
4138 err = mlx5_query_ports_check(mdev, in, sizeof(in));
4139 if (err)
4140 return err;
4141
4142 supported = MLX5_GET(pcmr_reg, in, rx_ts_over_crc_cap);
4143 curr_state = MLX5_GET(pcmr_reg, in, rx_ts_over_crc);
4144
4145 if (!supported || enable == curr_state)
4146 return 0;
4147
4148 MLX5_SET(pcmr_reg, in, local_port, 1);
4149 MLX5_SET(pcmr_reg, in, rx_ts_over_crc, enable);
4150
4151 return mlx5_set_ports_check(mdev, in, sizeof(in));
4152}
4153
1e662209
AF		 4154static int mlx5e_set_rx_port_ts_wrap(struct mlx5e_priv *priv, void *ctx)
4155{
4156 struct mlx5_core_dev *mdev = priv->mdev;
4157 bool enable = *(bool *)ctx;
4158
4159 return mlx5e_set_rx_port_ts(mdev, enable);
4160}
4161
102722fc
GE		 4162static int set_feature_rx_fcs(struct net_device *netdev, bool enable)
4163{
4164 struct mlx5e_priv *priv = netdev_priv(netdev);
0bc73ad4 4165 struct mlx5e_channels *chs = &priv->channels;
1e662209 4166 struct mlx5e_params new_params;
102722fc 4167 int err;
da6192ca 4168 bool rx_ts_over_crc = !enable;
102722fc
GE		 4169
4170 mutex_lock(&priv->state_lock);
4171
1e662209
AF		 4172 new_params = chs->params;
4173 new_params.scatter_fcs_en = enable;
4174 err = mlx5e_safe_switch_params(priv, &new_params, mlx5e_set_rx_port_ts_wrap,
da6192ca 4175 &rx_ts_over_crc, true);
0bc73ad4 4176 mutex_unlock(&priv->state_lock);
102722fc
GE		 4177 return err;
4178}
4179
36350114
GP		 4180static int set_feature_rx_vlan(struct net_device *netdev, bool enable)
4181{
4182 struct mlx5e_priv *priv = netdev_priv(netdev);
ff9c852f 4183 int err = 0;
36350114
GP		 4184
4185 mutex_lock(&priv->state_lock);
4186
f52f2fae 4187 mlx5e_fs_set_vlan_strip_disable(priv->fs, !enable);
6a9764ef 4188 priv->channels.params.vlan_strip_disable = !enable;
5b031add 4189
ff9c852f
SM		 4190 if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
4191 goto unlock;
4192
4193 err = mlx5e_modify_channels_vsd(&priv->channels, !enable);
5b031add 4194 if (err) {
f52f2fae 4195 mlx5e_fs_set_vlan_strip_disable(priv->fs, enable);
6a9764ef 4196 priv->channels.params.vlan_strip_disable = enable;
5b031add 4197 }
ff9c852f 4198unlock:
36350114
GP		 4199 mutex_unlock(&priv->state_lock);
4200
4201 return err;
4202}
4203
a02c07ea
LK		 4204int mlx5e_vlan_rx_add_vid(struct net_device *dev, __be16 proto, u16 vid)
4205{
4206 struct mlx5e_priv *priv = netdev_priv(dev);
4207 struct mlx5e_flow_steering *fs = priv->fs;
4208
4209 if (mlx5e_is_uplink_rep(priv))
4210 return 0; /* no vlan table for uplink rep */
4211
4212 return mlx5e_fs_vlan_rx_add_vid(fs, dev, proto, vid);
4213}
4214
4215int mlx5e_vlan_rx_kill_vid(struct net_device *dev, __be16 proto, u16 vid)
4216{
4217 struct mlx5e_priv *priv = netdev_priv(dev);
4218 struct mlx5e_flow_steering *fs = priv->fs;
4219
4220 if (mlx5e_is_uplink_rep(priv))
4221 return 0; /* no vlan table for uplink rep */
4222
4223 return mlx5e_fs_vlan_rx_kill_vid(fs, dev, proto, vid);
4224}
4225
ec080045 4226#ifdef CONFIG_MLX5_EN_ARFS
45bf454a
MG		 4227static int set_feature_arfs(struct net_device *netdev, bool enable)
4228{
4229 struct mlx5e_priv *priv = netdev_priv(netdev);
4230 int err;
4231
4232 if (enable)
45b83c6c 4233 err = mlx5e_arfs_enable(priv->fs);
45bf454a 4234 else
45b83c6c 4235 err = mlx5e_arfs_disable(priv->fs);
45bf454a
MG		 4236
4237 return err;
4238}
4239#endif
4240
0e405443 4241static int mlx5e_handle_feature(struct net_device *netdev,
75b81ce7 4242 netdev_features_t *features,
0e405443
GP		 4243 netdev_features_t feature,
4244 mlx5e_feature_handler feature_handler)
4245{
992d8a4e
GP		 4246 netdev_features_t changes = *features ^ netdev->features;
4247 bool enable = !!(*features & feature);
0e405443
GP		 4248 int err;
4249
4250 if (!(changes & feature))
4251 return 0;
4252
4253 err = feature_handler(netdev, enable);
4254 if (err) {
992d8a4e 4255 MLX5E_SET_FEATURE(features, feature, !enable);
b20eab15
GP		 4256 netdev_err(netdev, "%s feature %pNF failed, err %d\n",
4257 enable ? "Enable" : "Disable", &feature, err);
0e405443
GP		 4258 return err;
4259 }
4260
0e405443
GP		 4261 return 0;
4262}
4263
4d5ab0ad
LB		 4264void mlx5e_set_xdp_feature(struct net_device *netdev)
4265{
4266 struct mlx5e_priv *priv = netdev_priv(netdev);
4267 struct mlx5e_params *params = &priv->channels.params;
4268 xdp_features_t val;
4269
4270 if (params->packet_merge.type != MLX5E_PACKET_MERGE_NONE) {
4271 xdp_clear_features_flag(netdev);
4272 return;
4273 }
4274
4275 val = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT |
4276 NETDEV_XDP_ACT_XSK_ZEROCOPY |
f52ac702 4277 NETDEV_XDP_ACT_RX_SG |
c1783e74
TT		 4278 NETDEV_XDP_ACT_NDO_XMIT |
4279 NETDEV_XDP_ACT_NDO_XMIT_SG;
4d5ab0ad
LB		 4280 xdp_set_features_flag(netdev, val);
4281}
4282
d3cbd425 4283int mlx5e_set_features(struct net_device *netdev, netdev_features_t features)
0e405443 4284{
992d8a4e 4285 netdev_features_t oper_features = features;
be0f780b
GP		 4286 int err = 0;
4287
4288#define MLX5E_HANDLE_FEATURE(feature, handler) \
992d8a4e 4289 mlx5e_handle_feature(netdev, &oper_features, feature, handler)
0e405443 4290
a64bbd8c
DT		 4291 if (features & (NETIF_F_GRO_HW | NETIF_F_LRO)) {
4292 err |= MLX5E_HANDLE_FEATURE(NETIF_F_RXFCS, set_feature_rx_fcs);
4293 err |= MLX5E_HANDLE_FEATURE(NETIF_F_LRO, set_feature_lro);
4294 err |= MLX5E_HANDLE_FEATURE(NETIF_F_GRO_HW, set_feature_hw_gro);
4295 } else {
4296 err |= MLX5E_HANDLE_FEATURE(NETIF_F_LRO, set_feature_lro);
4297 err |= MLX5E_HANDLE_FEATURE(NETIF_F_GRO_HW, set_feature_hw_gro);
4298 err |= MLX5E_HANDLE_FEATURE(NETIF_F_RXFCS, set_feature_rx_fcs);
4299 }
be0f780b 4300 err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_VLAN_CTAG_FILTER,
2b52a283 4301 set_feature_cvlan_filter);
214baf22 4302 err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_TC, set_feature_hw_tc);
be0f780b 4303 err |= MLX5E_HANDLE_FEATURE(NETIF_F_RXALL, set_feature_rx_all);
be0f780b 4304 err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_VLAN_CTAG_RX, set_feature_rx_vlan);
ec080045 4305#ifdef CONFIG_MLX5_EN_ARFS
be0f780b 4306 err |= MLX5E_HANDLE_FEATURE(NETIF_F_NTUPLE, set_feature_arfs);
45bf454a 4307#endif
1182f365 4308 err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_TLS_RX, mlx5e_ktls_set_feature_rx);
0e405443 4309
75b81ce7
GP		 4310 if (err) {
4311 netdev->features = oper_features;
4312 return -EINVAL;
4313 }
4314
4d5ab0ad
LB		 4315 /* update XDP supported features */
4316 mlx5e_set_xdp_feature(netdev);
4317
75b81ce7 4318 return 0;
f62b8bb8
AV		 4319}
4320
90b22b9b
MD		 4321static netdev_features_t mlx5e_fix_uplink_rep_features(struct net_device *netdev,
4322 netdev_features_t features)
4323{
4324 features &= ~NETIF_F_HW_TLS_RX;
4325 if (netdev->features & NETIF_F_HW_TLS_RX)
4326 netdev_warn(netdev, "Disabling hw_tls_rx, not supported in switchdev mode\n");
4327
4328 features &= ~NETIF_F_HW_TLS_TX;
4329 if (netdev->features & NETIF_F_HW_TLS_TX)
4330 netdev_warn(netdev, "Disabling hw_tls_tx, not supported in switchdev mode\n");
4331
4332 features &= ~NETIF_F_NTUPLE;
4333 if (netdev->features & NETIF_F_NTUPLE)
4334 netdev_warn(netdev, "Disabling ntuple, not supported in switchdev mode\n");
4335
15a5078c
AL		 4336 features &= ~NETIF_F_GRO_HW;
4337 if (netdev->features & NETIF_F_GRO_HW)
4338 netdev_warn(netdev, "Disabling HW_GRO, not supported in switchdev mode\n");
4339
8974aa96
AT		 4340 features &= ~NETIF_F_HW_VLAN_CTAG_FILTER;
4341 if (netdev->features & NETIF_F_HW_VLAN_CTAG_FILTER)
4342 netdev_warn(netdev, "Disabling HW_VLAN CTAG FILTERING, not supported in switchdev mode\n");
4343
90b22b9b
MD		 4344 return features;
4345}
4346
7d92d580
GP		 4347static netdev_features_t mlx5e_fix_features(struct net_device *netdev,
4348 netdev_features_t features)
4349{
4350 struct mlx5e_priv *priv = netdev_priv(netdev);
f52f2fae 4351 struct mlx5e_vlan_table *vlan;
6c3a823e 4352 struct mlx5e_params *params;
7d92d580 4353
ab4b01bf
RN		 4354 if (!netif_device_present(netdev))
4355 return features;
4356
f52f2fae 4357 vlan = mlx5e_fs_get_vlan(priv->fs);
7d92d580 4358 mutex_lock(&priv->state_lock);
6c3a823e 4359 params = &priv->channels.params;
f52f2fae
LK		 4360 if (!vlan ||
4361 !bitmap_empty(mlx5e_vlan_get_active_svlans(vlan), VLAN_N_VID)) {
7d92d580
GP		 4362 /* HW strips the outer C-tag header, this is a problem
4363 * for S-tag traffic.
4364 */
4365 features &= ~NETIF_F_HW_VLAN_CTAG_RX;
6c3a823e 4366 if (!params->vlan_strip_disable)
7d92d580
GP		 4367 netdev_warn(netdev, "Dropping C-tag vlan stripping offload due to S-tag vlan\n");
4368 }
3ef14e46 4369
6c3a823e 4370 if (!MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_STRIDING_RQ)) {
842a2eb2 4371 if (features & NETIF_F_LRO) {
6c3a823e 4372 netdev_warn(netdev, "Disabling LRO, not supported in legacy RQ\n");
842a2eb2
HN		 4373 features &= ~NETIF_F_LRO;
4374 }
83439f3c
KM		 4375 if (features & NETIF_F_GRO_HW) {
4376 netdev_warn(netdev, "Disabling HW-GRO, not supported in legacy RQ\n");
4377 features &= ~NETIF_F_GRO_HW;
4378 }
6c3a823e
TT		 4379 }
4380
cf6e34c8
MM		 4381 if (params->xdp_prog) {
4382 if (features & NETIF_F_LRO) {
4383 netdev_warn(netdev, "LRO is incompatible with XDP\n");
4384 features &= ~NETIF_F_LRO;
4385 }
b0617e7b
MM		 4386 if (features & NETIF_F_GRO_HW) {
4387 netdev_warn(netdev, "HW GRO is incompatible with XDP\n");
4388 features &= ~NETIF_F_GRO_HW;
4389 }
4390 }
4391
4392 if (priv->xsk.refcnt) {
1c31cb92
MM		 4393 if (features & NETIF_F_LRO) {
4394 netdev_warn(netdev, "LRO is incompatible with AF_XDP (%u XSKs are active)\n",
4395 priv->xsk.refcnt);
4396 features &= ~NETIF_F_LRO;
4397 }
b0617e7b
MM		 4398 if (features & NETIF_F_GRO_HW) {
4399 netdev_warn(netdev, "HW GRO is incompatible with AF_XDP (%u XSKs are active)\n",
4400 priv->xsk.refcnt);
4401 features &= ~NETIF_F_GRO_HW;
4402 }
cf6e34c8
MM		 4403 }
4404
c0194e2d
SM		 4405 if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS)) {
4406 features &= ~NETIF_F_RXHASH;
4407 if (netdev->features & NETIF_F_RXHASH)
4408 netdev_warn(netdev, "Disabling rxhash, not supported when CQE compress is active\n");
b5f42903
GP		 4409
4410 if (features & NETIF_F_GRO_HW) {
4411 netdev_warn(netdev, "Disabling HW-GRO, not supported when CQE compress is active\n");
4412 features &= ~NETIF_F_GRO_HW;
4413 }
c0194e2d
SM		 4414 }
4415
c83172b0 4416 if (mlx5e_is_uplink_rep(priv)) {
90b22b9b 4417 features = mlx5e_fix_uplink_rep_features(netdev, features);
05c1280a 4418 netdev->netns_local = true;
c83172b0 4419 } else {
05c1280a 4420 netdev->netns_local = false;
c83172b0 4421 }
b38742e4 4422
7d92d580
GP		 4423 mutex_unlock(&priv->state_lock);
4424
4425 return features;
4426}
4427
db05815b
MM		 4428static bool mlx5e_xsk_validate_mtu(struct net_device *netdev,
4429 struct mlx5e_channels *chs,
4430 struct mlx5e_params *new_params,
4431 struct mlx5_core_dev *mdev)
4432{
4433 u16 ix;
4434
4435 for (ix = 0; ix < chs->params.num_channels; ix++) {
1742b3d5
MK		 4436 struct xsk_buff_pool *xsk_pool =
4437 mlx5e_xsk_get_pool(&chs->params, chs->params.xsk, ix);
db05815b 4438 struct mlx5e_xsk_param xsk;
78dee7be 4439 int max_xdp_mtu;
db05815b 4440
1742b3d5 4441 if (!xsk_pool)
db05815b
MM		 4442 continue;
4443
1742b3d5 4444 mlx5e_build_xsk_param(xsk_pool, &xsk);
78dee7be 4445 max_xdp_mtu = mlx5e_xdp_max_mtu(new_params, &xsk);
db05815b 4446
78dee7be
AF		 4447 /* Validate XSK params and XDP MTU in advance */
4448 if (!mlx5e_validate_xsk_param(new_params, &xsk, mdev) ||
4449 new_params->sw_mtu > max_xdp_mtu) {
db05815b
MM		 4450 u32 hr = mlx5e_get_linear_rq_headroom(new_params, &xsk);
4451 int max_mtu_frame, max_mtu_page, max_mtu;
4452
4453 /* Two criteria must be met:
4454 * 1. HW MTU + all headrooms <= XSK frame size.
4455 * 2. Size of SKBs allocated on XDP_PASS <= PAGE_SIZE.
4456 */
4457 max_mtu_frame = MLX5E_HW2SW_MTU(new_params, xsk.chunk_size - hr);
411295fb 4458 max_mtu_page = MLX5E_HW2SW_MTU(new_params, SKB_MAX_HEAD(0));
78dee7be 4459 max_mtu = min3(max_mtu_frame, max_mtu_page, max_xdp_mtu);
db05815b 4460
78dee7be 4461 netdev_err(netdev, "MTU %d is too big for an XSK running on channel %u or its redirection XDP program. Try MTU <= %d\n",
db05815b
MM		 4462 new_params->sw_mtu, ix, max_mtu);
4463 return false;
4464 }
4465 }
4466
4467 return true;
4468}
4469
e5a3cc83
MM		 4470static bool mlx5e_params_validate_xdp(struct net_device *netdev,
4471 struct mlx5_core_dev *mdev,
4472 struct mlx5e_params *params)
1b8a10bb
MM		 4473{
4474 bool is_linear;
4475
4476 /* No XSK params: AF_XDP can't be enabled yet at the point of setting
4477 * the XDP program.
4478 */
7fc06dd2
TT		 4479 is_linear = params->rq_wq_type == MLX5_WQ_TYPE_CYCLIC ?
4480 mlx5e_rx_is_linear_skb(mdev, params, NULL) :
4481 mlx5e_rx_mpwqe_is_linear_skb(mdev, params, NULL);
1b8a10bb 4482
f52ac702
TT		 4483 if (!is_linear) {
4484 if (!params->xdp_prog->aux->xdp_has_frags) {
4485 netdev_warn(netdev, "MTU(%d) > %d, too big for an XDP program not aware of multi buffer\n",
4486 params->sw_mtu,
4487 mlx5e_xdp_max_mtu(params, NULL));
4488 return false;
4489 }
4490 if (params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ &&
4491 !mlx5e_verify_params_rx_mpwqe_strides(mdev, params, NULL)) {
4492 netdev_warn(netdev, "XDP is not allowed with striding RQ and MTU(%d) > %d\n",
4493 params->sw_mtu,
4494 mlx5e_xdp_max_mtu(params, NULL));
4495 return false;
4496 }
1b8a10bb
MM		 4497 }
4498
4499 return true;
4500}
4501
250a42b6 4502int mlx5e_change_mtu(struct net_device *netdev, int new_mtu,
b9ab5d0e 4503 mlx5e_fp_preactivate preactivate)
f62b8bb8
AV		 4504{
4505 struct mlx5e_priv *priv = netdev_priv(netdev);
94872d4e 4506 struct mlx5e_params new_params;
472a1e44 4507 struct mlx5e_params *params;
b3b886cf 4508 bool reset = true;
98e81b0a 4509 int err = 0;
f62b8bb8 4510
f62b8bb8 4511 mutex_lock(&priv->state_lock);
98e81b0a 4512
472a1e44 4513 params = &priv->channels.params;
506753b0 4514
94872d4e
MM		 4515 new_params = *params;
4516 new_params.sw_mtu = new_mtu;
4517 err = mlx5e_validate_params(priv->mdev, &new_params);
579524c6
VT		 4518 if (err)
4519 goto out;
73281b78 4520
e5a3cc83
MM		 4521 if (new_params.xdp_prog && !mlx5e_params_validate_xdp(netdev, priv->mdev,
4522 &new_params)) {
a26a5bdf
TT		 4523 err = -EINVAL;
4524 goto out;
4525 }
4526
db05815b
MM		 4527 if (priv->xsk.refcnt &&
4528 !mlx5e_xsk_validate_mtu(netdev, &priv->channels,
94872d4e 4529 &new_params, priv->mdev)) {
a26a5bdf
TT		 4530 err = -EINVAL;
4531 goto out;
4532 }
4533
eaee12f0 4534 if (params->packet_merge.type == MLX5E_PACKET_MERGE_LRO)
b3b886cf 4535 reset = false;
69cc4185 4536
3904d2af
MM		 4537 if (params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ &&
4538 params->packet_merge.type != MLX5E_PACKET_MERGE_SHAMPO) {
69cc4185
MM		 4539 bool is_linear_old = mlx5e_rx_mpwqe_is_linear_skb(priv->mdev, params, NULL);
4540 bool is_linear_new = mlx5e_rx_mpwqe_is_linear_skb(priv->mdev,
94872d4e 4541 &new_params, NULL);
e5a3cc83
MM		 4542 u8 sz_old = mlx5e_mpwqe_get_log_rq_size(priv->mdev, params, NULL);
4543 u8 sz_new = mlx5e_mpwqe_get_log_rq_size(priv->mdev, &new_params, NULL);
db05815b 4544
69cc4185
MM		 4545 /* Always reset in linear mode - hw_mtu is used in data path.
4546 * Check that the mode was non-linear and didn't change.
4547 * If XSK is active, XSK RQs are linear.
3904d2af 4548 * Reset if the RQ size changed, even if it's non-linear.
69cc4185
MM		 4549 */
4550 if (!is_linear_old && !is_linear_new && !priv->xsk.refcnt &&
3904d2af 4551 sz_old == sz_new)
b3b886cf 4552 reset = false;
2e20a151 4553 }
98e81b0a 4554
94872d4e 4555 err = mlx5e_safe_switch_params(priv, &new_params, preactivate, NULL, reset);
f62b8bb8 4556
2e20a151 4557out:
1eb2cded 4558 WRITE_ONCE(netdev->mtu, params->sw_mtu);
2e20a151 4559 mutex_unlock(&priv->state_lock);
f62b8bb8
AV		 4560 return err;
4561}
4562
250a42b6
AN		 4563static int mlx5e_change_nic_mtu(struct net_device *netdev, int new_mtu)
4564{
b9ab5d0e 4565 return mlx5e_change_mtu(netdev, new_mtu, mlx5e_set_dev_port_mtu_ctx);
250a42b6
AN		 4566}
4567
885b8cfb 4568int mlx5e_ptp_rx_manage_fs_ctx(struct mlx5e_priv *priv, void *ctx)
960fbfe2
AL		 4569{
4570 bool set = *(bool *)ctx;
4571
4572 return mlx5e_ptp_rx_manage_fs(priv, set);
4573}
4574
256f79d1
AL		 4575static int mlx5e_hwstamp_config_no_ptp_rx(struct mlx5e_priv *priv, bool rx_filter)
4576{
4577 bool rx_cqe_compress_def = priv->channels.params.rx_cqe_compress_def;
4578 int err;
4579
4580 if (!rx_filter)
4581 /* Reset CQE compression to Admin default */
c91c1da7 4582 return mlx5e_modify_rx_cqe_compression_locked(priv, rx_cqe_compress_def, false);
256f79d1
AL		 4583
4584 if (!MLX5E_GET_PFLAG(&priv->channels.params, MLX5E_PFLAG_RX_CQE_COMPRESS))
4585 return 0;
4586
4587 /* Disable CQE compression */
4588 netdev_warn(priv->netdev, "Disabling RX cqe compression\n");
c91c1da7 4589 err = mlx5e_modify_rx_cqe_compression_locked(priv, false, true);
256f79d1
AL		 4590 if (err)
4591 netdev_err(priv->netdev, "Failed disabling cqe compression err=%d\n", err);
4592
4593 return err;
4594}
4595
4596static int mlx5e_hwstamp_config_ptp_rx(struct mlx5e_priv *priv, bool ptp_rx)
7c39afb3 4597{
94872d4e 4598 struct mlx5e_params new_params;
256f79d1
AL		 4599
4600 if (ptp_rx == priv->channels.params.ptp_rx)
4601 return 0;
4602
4603 new_params = priv->channels.params;
4604 new_params.ptp_rx = ptp_rx;
4605 return mlx5e_safe_switch_params(priv, &new_params, mlx5e_ptp_rx_manage_fs_ctx,
4606 &new_params.ptp_rx, true);
4607}
4608
4609int mlx5e_hwstamp_set(struct mlx5e_priv *priv, struct ifreq *ifr)
4610{
7c39afb3 4611 struct hwtstamp_config config;
960fbfe2 4612 bool rx_cqe_compress_def;
256f79d1 4613 bool ptp_rx;
7c39afb3
FD		 4614 int err;
4615
6dbc80ca
MS		 4616 if (!MLX5_CAP_GEN(priv->mdev, device_frequency_khz) ||
4617 (mlx5_clock_get_ptp_index(priv->mdev) == -1))
7c39afb3
FD		 4618 return -EOPNOTSUPP;
4619
4620 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
4621 return -EFAULT;
4622
4623 /* TX HW timestamp */
4624 switch (config.tx_type) {
4625 case HWTSTAMP_TX_OFF:
4626 case HWTSTAMP_TX_ON:
4627 break;
4628 default:
4629 return -ERANGE;
4630 }
4631
4632 mutex_lock(&priv->state_lock);
960fbfe2
AL		 4633 rx_cqe_compress_def = priv->channels.params.rx_cqe_compress_def;
4634
7c39afb3
FD		 4635 /* RX HW timestamp */
4636 switch (config.rx_filter) {
4637 case HWTSTAMP_FILTER_NONE:
256f79d1 4638 ptp_rx = false;
7c39afb3
FD		 4639 break;
4640 case HWTSTAMP_FILTER_ALL:
4641 case HWTSTAMP_FILTER_SOME:
4642 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
4643 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
4644 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
4645 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
4646 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
4647 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
4648 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
4649 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
4650 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
4651 case HWTSTAMP_FILTER_PTP_V2_EVENT:
4652 case HWTSTAMP_FILTER_PTP_V2_SYNC:
4653 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
4654 case HWTSTAMP_FILTER_NTP_ALL:
7c39afb3 4655 config.rx_filter = HWTSTAMP_FILTER_ALL;
256f79d1
AL		 4656 /* ptp_rx is set if both HW TS is set and CQE
4657 * compression is set
4658 */
4659 ptp_rx = rx_cqe_compress_def;
7c39afb3
FD		 4660 break;
4661 default:
256f79d1
AL		 4662 err = -ERANGE;
4663 goto err_unlock;
7c39afb3
FD		 4664 }
4665
6c72cb05 4666 if (!mlx5e_profile_feature_cap(priv->profile, PTP_RX))
256f79d1
AL		 4667 err = mlx5e_hwstamp_config_no_ptp_rx(priv,
4668 config.rx_filter != HWTSTAMP_FILTER_NONE);
4669 else
4670 err = mlx5e_hwstamp_config_ptp_rx(priv, ptp_rx);
4671 if (err)
4672 goto err_unlock;
960fbfe2 4673
7c39afb3
FD		 4674 memcpy(&priv->tstamp, &config, sizeof(config));
4675 mutex_unlock(&priv->state_lock);
4676
c0194e2d
SM		 4677 /* might need to fix some features */
4678 netdev_update_features(priv->netdev);
4679
7c39afb3
FD		 4680 return copy_to_user(ifr->ifr_data, &config,
4681 sizeof(config)) ? -EFAULT : 0;
256f79d1
AL		 4682err_unlock:
4683 mutex_unlock(&priv->state_lock);
4684 return err;
7c39afb3
FD		 4685}
4686
4687int mlx5e_hwstamp_get(struct mlx5e_priv *priv, struct ifreq *ifr)
4688{
4689 struct hwtstamp_config *cfg = &priv->tstamp;
4690
4691 if (!MLX5_CAP_GEN(priv->mdev, device_frequency_khz))
4692 return -EOPNOTSUPP;
4693
4694 return copy_to_user(ifr->ifr_data, cfg, sizeof(*cfg)) ? -EFAULT : 0;
4695}
4696
ef9814de
EBE		 4697static int mlx5e_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
4698{
1170fbd8
FD		 4699 struct mlx5e_priv *priv = netdev_priv(dev);
4700
ef9814de
EBE		 4701 switch (cmd) {
4702 case SIOCSHWTSTAMP:
1170fbd8 4703 return mlx5e_hwstamp_set(priv, ifr);
ef9814de 4704 case SIOCGHWTSTAMP:
1170fbd8 4705 return mlx5e_hwstamp_get(priv, ifr);
ef9814de
EBE		 4706 default:
4707 return -EOPNOTSUPP;
4708 }
4709}
4710
e80541ec 4711#ifdef CONFIG_MLX5_ESWITCH
073caf50 4712int mlx5e_set_vf_mac(struct net_device *dev, int vf, u8 *mac)
66e49ded
SM		 4713{
4714 struct mlx5e_priv *priv = netdev_priv(dev);
4715 struct mlx5_core_dev *mdev = priv->mdev;
4716
4717 return mlx5_eswitch_set_vport_mac(mdev->priv.eswitch, vf + 1, mac);
4718}
4719
79aab093
MS		 4720static int mlx5e_set_vf_vlan(struct net_device *dev, int vf, u16 vlan, u8 qos,
4721 __be16 vlan_proto)
66e49ded
SM		 4722{
4723 struct mlx5e_priv *priv = netdev_priv(dev);
4724 struct mlx5_core_dev *mdev = priv->mdev;
4725
79aab093
MS		 4726 if (vlan_proto != htons(ETH_P_8021Q))
4727 return -EPROTONOSUPPORT;
4728
66e49ded
SM		 4729 return mlx5_eswitch_set_vport_vlan(mdev->priv.eswitch, vf + 1,
4730 vlan, qos);
4731}
4732
f942380c
MHY		 4733static int mlx5e_set_vf_spoofchk(struct net_device *dev, int vf, bool setting)
4734{
4735 struct mlx5e_priv *priv = netdev_priv(dev);
4736 struct mlx5_core_dev *mdev = priv->mdev;
4737
4738 return mlx5_eswitch_set_vport_spoofchk(mdev->priv.eswitch, vf + 1, setting);
4739}
4740
1edc57e2
MHY		 4741static int mlx5e_set_vf_trust(struct net_device *dev, int vf, bool setting)
4742{
4743 struct mlx5e_priv *priv = netdev_priv(dev);
4744 struct mlx5_core_dev *mdev = priv->mdev;
4745
4746 return mlx5_eswitch_set_vport_trust(mdev->priv.eswitch, vf + 1, setting);
4747}
bd77bf1c 4748
073caf50
OG		 4749int mlx5e_set_vf_rate(struct net_device *dev, int vf, int min_tx_rate,
4750 int max_tx_rate)
bd77bf1c
MHY		 4751{
4752 struct mlx5e_priv *priv = netdev_priv(dev);
4753 struct mlx5_core_dev *mdev = priv->mdev;
4754
bd77bf1c 4755 return mlx5_eswitch_set_vport_rate(mdev->priv.eswitch, vf + 1,
c9497c98 4756 max_tx_rate, min_tx_rate);
bd77bf1c
MHY		 4757}
4758
66e49ded
SM		 4759static int mlx5_vport_link2ifla(u8 esw_link)
4760{
4761 switch (esw_link) {
cc9c82a8 4762 case MLX5_VPORT_ADMIN_STATE_DOWN:
66e49ded 4763 return IFLA_VF_LINK_STATE_DISABLE;
cc9c82a8 4764 case MLX5_VPORT_ADMIN_STATE_UP:
66e49ded
SM		 4765 return IFLA_VF_LINK_STATE_ENABLE;
4766 }
4767 return IFLA_VF_LINK_STATE_AUTO;
4768}
4769
4770static int mlx5_ifla_link2vport(u8 ifla_link)
4771{
4772 switch (ifla_link) {
4773 case IFLA_VF_LINK_STATE_DISABLE:
cc9c82a8 4774 return MLX5_VPORT_ADMIN_STATE_DOWN;
66e49ded 4775 case IFLA_VF_LINK_STATE_ENABLE:
cc9c82a8 4776 return MLX5_VPORT_ADMIN_STATE_UP;
66e49ded 4777 }
cc9c82a8 4778 return MLX5_VPORT_ADMIN_STATE_AUTO;
66e49ded
SM		 4779}
4780
4781static int mlx5e_set_vf_link_state(struct net_device *dev, int vf,
4782 int link_state)
4783{
4784 struct mlx5e_priv *priv = netdev_priv(dev);
4785 struct mlx5_core_dev *mdev = priv->mdev;
4786
1aa48ca6
RD		 4787 if (mlx5e_is_uplink_rep(priv))
4788 return -EOPNOTSUPP;
4789
66e49ded
SM		 4790 return mlx5_eswitch_set_vport_state(mdev->priv.eswitch, vf + 1,
4791 mlx5_ifla_link2vport(link_state));
4792}
4793
073caf50
OG		 4794int mlx5e_get_vf_config(struct net_device *dev,
4795 int vf, struct ifla_vf_info *ivi)
66e49ded
SM		 4796{
4797 struct mlx5e_priv *priv = netdev_priv(dev);
4798 struct mlx5_core_dev *mdev = priv->mdev;
4799 int err;
4800
2ff349c5
RD		 4801 if (!netif_device_present(dev))
4802 return -EOPNOTSUPP;
4803
66e49ded
SM		 4804 err = mlx5_eswitch_get_vport_config(mdev->priv.eswitch, vf + 1, ivi);
4805 if (err)
4806 return err;
4807 ivi->linkstate = mlx5_vport_link2ifla(ivi->linkstate);
4808 return 0;
4809}
4810
073caf50
OG		 4811int mlx5e_get_vf_stats(struct net_device *dev,
4812 int vf, struct ifla_vf_stats *vf_stats)
66e49ded
SM		 4813{
4814 struct mlx5e_priv *priv = netdev_priv(dev);
4815 struct mlx5_core_dev *mdev = priv->mdev;
4816
4817 return mlx5_eswitch_get_vport_stats(mdev->priv.eswitch, vf + 1,
4818 vf_stats);
4819}
ee526030
RD		 4820
4821static bool
4822mlx5e_has_offload_stats(const struct net_device *dev, int attr_id)
4823{
4824 struct mlx5e_priv *priv = netdev_priv(dev);
4825
2ff349c5
RD		 4826 if (!netif_device_present(dev))
4827 return false;
4828
ee526030
RD		 4829 if (!mlx5e_is_uplink_rep(priv))
4830 return false;
4831
4832 return mlx5e_rep_has_offload_stats(dev, attr_id);
4833}
4834
4835static int
4836mlx5e_get_offload_stats(int attr_id, const struct net_device *dev,
4837 void *sp)
4838{
4839 struct mlx5e_priv *priv = netdev_priv(dev);
4840
4841 if (!mlx5e_is_uplink_rep(priv))
4842 return -EOPNOTSUPP;
4843
4844 return mlx5e_rep_get_offload_stats(attr_id, dev, sp);
4845}
e80541ec 4846#endif
66e49ded 4847
c28e3bd4
AL		 4848static bool mlx5e_tunnel_proto_supported_tx(struct mlx5_core_dev *mdev, u8 proto_type)
4849{
4850 switch (proto_type) {
4851 case IPPROTO_GRE:
4852 return MLX5_CAP_ETH(mdev, tunnel_stateless_gre);
4853 case IPPROTO_IPIP:
4854 case IPPROTO_IPV6:
4855 return (MLX5_CAP_ETH(mdev, tunnel_stateless_ip_over_ip) ||
4856 MLX5_CAP_ETH(mdev, tunnel_stateless_ip_over_ip_tx));
4857 default:
4858 return false;
4859 }
4860}
4861
3d093bc2
AL		 4862static bool mlx5e_gre_tunnel_inner_proto_offload_supported(struct mlx5_core_dev *mdev,
4863 struct sk_buff *skb)
4864{
4865 switch (skb->inner_protocol) {
4866 case htons(ETH_P_IP):
4867 case htons(ETH_P_IPV6):
4868 case htons(ETH_P_TEB):
4869 return true;
4870 case htons(ETH_P_MPLS_UC):
4871 case htons(ETH_P_MPLS_MC):
4872 return MLX5_CAP_ETH(mdev, tunnel_stateless_mpls_over_gre);
4873 }
4874 return false;
4875}
4876
27299841
GP		 4877static netdev_features_t mlx5e_tunnel_features_check(struct mlx5e_priv *priv,
4878 struct sk_buff *skb,
4879 netdev_features_t features)
b3f63c3d 4880{
2989ad1e 4881 unsigned int offset = 0;
b3f63c3d 4882 struct udphdr *udph;
27299841
GP		 4883 u8 proto;
4884 u16 port;
b3f63c3d
MF		 4885
4886 switch (vlan_get_protocol(skb)) {
4887 case htons(ETH_P_IP):
4888 proto = ip_hdr(skb)->protocol;
4889 break;
4890 case htons(ETH_P_IPV6):
2989ad1e 4891 proto = ipv6_find_hdr(skb, &offset, -1, NULL, NULL);
b3f63c3d
MF		 4892 break;
4893 default:
4894 goto out;
4895 }
4896
27299841
GP		 4897 switch (proto) {
4898 case IPPROTO_GRE:
3d093bc2
AL		 4899 if (mlx5e_gre_tunnel_inner_proto_offload_supported(priv->mdev, skb))
4900 return features;
4901 break;
25948b87
MV		 4902 case IPPROTO_IPIP:
4903 case IPPROTO_IPV6:
c28e3bd4 4904 if (mlx5e_tunnel_proto_supported_tx(priv->mdev, IPPROTO_IPIP))
9c98f7ec
MV		 4905 return features;
4906 break;
27299841 4907 case IPPROTO_UDP:
b3f63c3d
MF		 4908 udph = udp_hdr(skb);
4909 port = be16_to_cpu(udph->dest);
b3f63c3d 4910
27299841 4911 /* Verify if UDP port is being offloaded by HW */
358aa5ce 4912 if (mlx5_vxlan_lookup_port(priv->mdev->vxlan, port))
791b4089 4913 return vxlan_features_check(skb, features);
e3cfc7e6
MS		 4914
4915#if IS_ENABLED(CONFIG_GENEVE)
4916 /* Support Geneve offload for default UDP port */
4917 if (port == GENEVE_UDP_PORT && mlx5_geneve_tx_allowed(priv->mdev))
4918 return features;
dd7cf00f
HN		 4919#endif
4920 break;
4921#ifdef CONFIG_MLX5_EN_IPSEC
4922 case IPPROTO_ESP:
4923 return mlx5e_ipsec_feature_check(skb, features);
e3cfc7e6 4924#endif
27299841 4925 }
b3f63c3d
MF		 4926
4927out:
a9dbb4ac 4928 /* Disable CSUM and GSO if skb cannot be offloaded by HW */
b3f63c3d
MF		 4929 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
4930}
4931
073caf50
OG		 4932netdev_features_t mlx5e_features_check(struct sk_buff *skb,
4933 struct net_device *netdev,
4934 netdev_features_t features)
b3f63c3d
MF		 4935{
4936 struct mlx5e_priv *priv = netdev_priv(netdev);
4937
4938 features = vlan_features_check(skb, features);
b3f63c3d
MF		 4939
4940 /* Validate if the tunneled packet is being offloaded by HW */
4941 if (skb->encapsulation &&
4942 (features & NETIF_F_CSUM_MASK || features & NETIF_F_GSO_MASK))
27299841 4943 return mlx5e_tunnel_features_check(priv, skb, features);
b3f63c3d
MF		 4944
4945 return features;
4946}
4947
bfc647d5 4948static void mlx5e_tx_timeout_work(struct work_struct *work)
3947ca18 4949{
bfc647d5
EBE		 4950 struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv,
4951 tx_timeout_work);
145e5637 4952 struct net_device *netdev = priv->netdev;
7d91126b 4953 int i;
3947ca18 4954
eab0da38
MS		 4955 /* Take rtnl_lock to ensure no change in netdev->real_num_tx_queues
4956 * through this flow. However, channel closing flows have to wait for
4957 * this work to finish while holding rtnl lock too. So either get the
4958 * lock or find that channels are being closed for other reason and
4959 * this work is not relevant anymore.
4960 */
4961 while (!rtnl_trylock()) {
4962 if (!test_bit(MLX5E_STATE_CHANNELS_ACTIVE, &priv->state))
4963 return;
4964 msleep(20);
4965 }
bfc647d5
EBE		 4966
4967 if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
4968 goto unlock;
3947ca18 4969
145e5637 4970 for (i = 0; i < netdev->real_num_tx_queues; i++) {
7d91126b 4971 struct netdev_queue *dev_queue =
145e5637 4972 netdev_get_tx_queue(netdev, i);
acc6c595 4973 struct mlx5e_txqsq *sq = priv->txq2sq[i];
3947ca18 4974
84990945 4975 if (!netif_xmit_stopped(dev_queue))
3947ca18 4976 continue;
bfc647d5 4977
06293ae4 4978 if (mlx5e_reporter_tx_timeout(sq))
e6205564
AL		 4979 /* break if tried to reopened channels */
4980 break;
3947ca18
DJ		 4981 }
4982
bfc647d5 4983unlock:
bfc647d5
EBE		 4984 rtnl_unlock();
4985}
4986
0290bd29 4987static void mlx5e_tx_timeout(struct net_device *dev, unsigned int txqueue)
bfc647d5
EBE		 4988{
4989 struct mlx5e_priv *priv = netdev_priv(dev);
4990
4991 netdev_err(dev, "TX timeout detected\n");
4992 queue_work(priv->wq, &priv->tx_timeout_work);
3947ca18
DJ		 4993}
4994
abd3f84e
TT		 4995static int mlx5e_xdp_allowed(struct net_device *netdev, struct mlx5_core_dev *mdev,
4996 struct mlx5e_params *params)
0ec13877 4997{
abd3f84e 4998 if (params->packet_merge.type != MLX5E_PACKET_MERGE_NONE) {
eaee12f0 4999 netdev_warn(netdev, "can't set XDP while HW-GRO/LRO is on, disable them first\n");
0ec13877
TT		 5000 return -EINVAL;
5001 }
5002
abd3f84e 5003 if (!mlx5e_params_validate_xdp(netdev, mdev, params))
a26a5bdf 5004 return -EINVAL;
a26a5bdf 5005
0ec13877
TT		 5006 return 0;
5007}
5008
fe45386a
MM		 5009static void mlx5e_rq_replace_xdp_prog(struct mlx5e_rq *rq, struct bpf_prog *prog)
5010{
5011 struct bpf_prog *old_prog;
5012
5013 old_prog = rcu_replace_pointer(rq->xdp_prog, prog,
521f31af 5014 lockdep_is_held(&rq->priv->state_lock));
fe45386a
MM		 5015 if (old_prog)
5016 bpf_prog_put(old_prog);
5017}
5018
86994156
RS		 5019static int mlx5e_xdp_set(struct net_device *netdev, struct bpf_prog *prog)
5020{
5021 struct mlx5e_priv *priv = netdev_priv(netdev);
94872d4e 5022 struct mlx5e_params new_params;
86994156 5023 struct bpf_prog *old_prog;
96d39502 5024 int err = 0;
b3b886cf 5025 bool reset;
86994156
RS		 5026 int i;
5027
5028 mutex_lock(&priv->state_lock);
5029
abd3f84e
TT		 5030 new_params = priv->channels.params;
5031 new_params.xdp_prog = prog;
5032
0ec13877 5033 if (prog) {
abd3f84e 5034 err = mlx5e_xdp_allowed(netdev, priv->mdev, &new_params);
0ec13877
TT		 5035 if (err)
5036 goto unlock;
547eede0
IT		 5037 }
5038
86994156 5039 /* no need for full reset when exchanging programs */
6a9764ef 5040 reset = (!priv->channels.params.xdp_prog || !prog);
86994156 5041
b3b886cf 5042 old_prog = priv->channels.params.xdp_prog;
e1895324 5043
94872d4e 5044 err = mlx5e_safe_switch_params(priv, &new_params, NULL, NULL, reset);
b3b886cf
MM		 5045 if (err)
5046 goto unlock;
e1895324 5047
86994156
RS		 5048 if (old_prog)
5049 bpf_prog_put(old_prog);
5050
b3b886cf 5051 if (!test_bit(MLX5E_STATE_OPENED, &priv->state) || reset)
86994156
RS		 5052 goto unlock;
5053
5054 /* exchanging programs w/o reset, we update ref counts on behalf
5055 * of the channels RQs here.
5056 */
b3b886cf 5057 bpf_prog_add(prog, priv->channels.num);
ff9c852f
SM		 5058 for (i = 0; i < priv->channels.num; i++) {
5059 struct mlx5e_channel *c = priv->channels.c[i];
86994156 5060
fe45386a 5061 mlx5e_rq_replace_xdp_prog(&c->rq, prog);
e5eb0134
MM		 5062 if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state)) {
5063 bpf_prog_inc(prog);
fe45386a 5064 mlx5e_rq_replace_xdp_prog(&c->xskrq, prog);
e5eb0134 5065 }
86994156
RS		 5066 }
5067
5068unlock:
5069 mutex_unlock(&priv->state_lock);
f6279f11
MM		 5070
5071 /* Need to fix some features. */
5072 if (!err)
5073 netdev_update_features(netdev);
5074
86994156
RS		 5075 return err;
5076}
5077
f4e63525 5078static int mlx5e_xdp(struct net_device *dev, struct netdev_bpf *xdp)
86994156
RS		 5079{
5080 switch (xdp->command) {
5081 case XDP_SETUP_PROG:
5082 return mlx5e_xdp_set(dev, xdp->prog);
1742b3d5
MK		 5083 case XDP_SETUP_XSK_POOL:
5084 return mlx5e_xsk_setup_pool(dev, xdp->xsk.pool,
db05815b 5085 xdp->xsk.queue_id);
86994156
RS		 5086 default:
5087 return -EINVAL;
5088 }
5089}
5090
4b89251d
HN		 5091#ifdef CONFIG_MLX5_ESWITCH
5092static int mlx5e_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq,
5093 struct net_device *dev, u32 filter_mask,
5094 int nlflags)
5095{
5096 struct mlx5e_priv *priv = netdev_priv(dev);
5097 struct mlx5_core_dev *mdev = priv->mdev;
5098 u8 mode, setting;
5099 int err;
5100
5101 err = mlx5_eswitch_get_vepa(mdev->priv.eswitch, &setting);
5102 if (err)
5103 return err;
5104 mode = setting ? BRIDGE_MODE_VEPA : BRIDGE_MODE_VEB;
5105 return ndo_dflt_bridge_getlink(skb, pid, seq, dev,
5106 mode,
5107 0, 0, nlflags, filter_mask, NULL);
5108}
5109
5110static int mlx5e_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh,
5111 u16 flags, struct netlink_ext_ack *extack)
5112{
5113 struct mlx5e_priv *priv = netdev_priv(dev);
5114 struct mlx5_core_dev *mdev = priv->mdev;
5115 struct nlattr *attr, *br_spec;
5116 u16 mode = BRIDGE_MODE_UNDEF;
5117 u8 setting;
5118 int rem;
5119
5120 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC);
5121 if (!br_spec)
5122 return -EINVAL;
5123
e8058a49 5124 nla_for_each_nested_type(attr, IFLA_BRIDGE_MODE, br_spec, rem) {
4b89251d
HN		 5125 mode = nla_get_u16(attr);
5126 if (mode > BRIDGE_MODE_VEPA)
5127 return -EINVAL;
5128
5129 break;
5130 }
5131
5132 if (mode == BRIDGE_MODE_UNDEF)
5133 return -EINVAL;
5134
5135 setting = (mode == BRIDGE_MODE_VEPA) ? 1 : 0;
5136 return mlx5_eswitch_set_vepa(mdev->priv.eswitch, setting);
5137}
5138#endif
5139
4d8fcf21 5140const struct net_device_ops mlx5e_netdev_ops = {
f62b8bb8
AV		 5141 .ndo_open = mlx5e_open,
5142 .ndo_stop = mlx5e_close,
5143 .ndo_start_xmit = mlx5e_xmit,
0cf0f6d3 5144 .ndo_setup_tc = mlx5e_setup_tc,
08fb1dac 5145 .ndo_select_queue = mlx5e_select_queue,
f62b8bb8
AV		 5146 .ndo_get_stats64 = mlx5e_get_stats,
5147 .ndo_set_rx_mode = mlx5e_set_rx_mode,
5148 .ndo_set_mac_address = mlx5e_set_mac,
b0eed40e
SM		 5149 .ndo_vlan_rx_add_vid = mlx5e_vlan_rx_add_vid,
5150 .ndo_vlan_rx_kill_vid = mlx5e_vlan_rx_kill_vid,
f62b8bb8 5151 .ndo_set_features = mlx5e_set_features,
7d92d580 5152 .ndo_fix_features = mlx5e_fix_features,
250a42b6 5153 .ndo_change_mtu = mlx5e_change_nic_mtu,
a7605370 5154 .ndo_eth_ioctl = mlx5e_ioctl,
507f0c81 5155 .ndo_set_tx_maxrate = mlx5e_set_tx_maxrate,
706b3583 5156 .ndo_features_check = mlx5e_features_check,
3947ca18 5157 .ndo_tx_timeout = mlx5e_tx_timeout,
f4e63525 5158 .ndo_bpf = mlx5e_xdp,
58b99ee3 5159 .ndo_xdp_xmit = mlx5e_xdp_xmit,
9116e5e2 5160 .ndo_xsk_wakeup = mlx5e_xsk_wakeup,
ec080045
SM		 5161#ifdef CONFIG_MLX5_EN_ARFS
5162 .ndo_rx_flow_steer = mlx5e_rx_flow_steer,
5163#endif
e80541ec 5164#ifdef CONFIG_MLX5_ESWITCH
4b89251d
HN		 5165 .ndo_bridge_setlink = mlx5e_bridge_setlink,
5166 .ndo_bridge_getlink = mlx5e_bridge_getlink,
5167
706b3583 5168 /* SRIOV E-Switch NDOs */
b0eed40e
SM		 5169 .ndo_set_vf_mac = mlx5e_set_vf_mac,
5170 .ndo_set_vf_vlan = mlx5e_set_vf_vlan,
f942380c 5171 .ndo_set_vf_spoofchk = mlx5e_set_vf_spoofchk,
1edc57e2 5172 .ndo_set_vf_trust = mlx5e_set_vf_trust,
bd77bf1c 5173 .ndo_set_vf_rate = mlx5e_set_vf_rate,
b0eed40e
SM		 5174 .ndo_get_vf_config = mlx5e_get_vf_config,
5175 .ndo_set_vf_link_state = mlx5e_set_vf_link_state,
5176 .ndo_get_vf_stats = mlx5e_get_vf_stats,
ee526030
RD		 5177 .ndo_has_offload_stats = mlx5e_has_offload_stats,
5178 .ndo_get_offload_stats = mlx5e_get_offload_stats,
e80541ec 5179#endif
f62b8bb8
AV		 5180};
5181
3ef14e46 5182void mlx5e_build_nic_params(struct mlx5e_priv *priv, struct mlx5e_xsk *xsk, u16 mtu)
f62b8bb8 5183{
3ef14e46 5184 struct mlx5e_params *params = &priv->channels.params;
57c7fce1 5185 struct mlx5_core_dev *mdev = priv->mdev;
2fc4bfb7 5186
472a1e44
TT		 5187 params->sw_mtu = mtu;
5188 params->hard_mtu = MLX5E_ETH_HARD_MTU;
57c7fce1
FL		 5189 params->num_channels = min_t(unsigned int, MLX5E_MAX_NUM_CHANNELS / 2,
5190 priv->max_nch);
7dbc849b 5191 mlx5e_params_mqprio_reset(params);
2b029556 5192
6a9764ef
SM		 5193 /* SQ */
5194 params->log_sq_size = is_kdump_kernel() ?
b4e029da
KH		 5195 MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE :
5196 MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE;
040ee617 5197 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_SKB_TX_MPWQE, mlx5e_tx_mpwqe_supported(mdev));
461017cb 5198
6277053a 5199 /* XDP SQ */
040ee617 5200 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_XDP_TX_MPWQE, mlx5e_tx_mpwqe_supported(mdev));
6277053a 5201
b797a684 5202 /* set CQE compression */
6a9764ef 5203 params->rx_cqe_compress_def = false;
b797a684 5204 if (MLX5_CAP_GEN(mdev, cqe_compression) &&
e53eef63 5205 MLX5_CAP_GEN(mdev, vport_group_manager))
0608d4db 5206 params->rx_cqe_compress_def = slow_pci_heuristic(mdev);
0f6e4cf6 5207
6a9764ef 5208 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS, params->rx_cqe_compress_def);
b856df28 5209 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_NO_CSUM_COMPLETE, false);
6a9764ef
SM		 5210
5211 /* RQ */
749359f4 5212 mlx5e_build_rq_params(mdev, params);
b797a684 5213
1db1f21c
DT		 5214 params->terminate_lkey_be = mlx5_core_get_terminate_scatter_list_mkey(mdev);
5215
eaee12f0 5216 params->packet_merge.timeout = mlx5e_choose_lro_timeout(mdev, MLX5E_DEFAULT_LRO_TIMEOUT);
b0d4660b 5217
6a9764ef 5218 /* CQ moderation params */
445a25f6
RR		 5219 params->rx_dim_enabled = MLX5_CAP_GEN(mdev, cq_moderation) &&
5220 MLX5_CAP_GEN(mdev, cq_period_mode_modify);
5221 params->tx_dim_enabled = MLX5_CAP_GEN(mdev, cq_moderation) &&
5222 MLX5_CAP_GEN(mdev, cq_period_mode_modify);
5223 params->rx_moder_use_cqe_mode = !!MLX5_CAP_GEN(mdev, cq_period_start_from_cqe);
5224 params->tx_moder_use_cqe_mode = false;
5225 mlx5e_reset_rx_moderation(&params->rx_cq_moderation, params->rx_moder_use_cqe_mode,
5226 params->rx_dim_enabled);
5227 mlx5e_reset_tx_moderation(&params->tx_cq_moderation, params->tx_moder_use_cqe_mode,
5228 params->tx_dim_enabled);
9908aa29 5229
6a9764ef 5230 /* TX inline */
b431302e 5231 mlx5_query_min_inline(mdev, &params->tx_min_inline_mode);
a6f402e4 5232
db05815b
MM		 5233 /* AF_XDP */
5234 params->xsk = xsk;
3ef14e46
SM		 5235
5236 /* Do not update netdev->features directly in here
5237 * on mlx5e_attach_netdev() we will call mlx5e_update_features()
5238 * To update netdev->features please modify mlx5e_fix_features()
5239 */
6a9764ef 5240}
f62b8bb8 5241
f62b8bb8
AV		 5242static void mlx5e_set_netdev_dev_addr(struct net_device *netdev)
5243{
5244 struct mlx5e_priv *priv = netdev_priv(netdev);
537e4d2e 5245 u8 addr[ETH_ALEN];
f62b8bb8 5246
537e4d2e
JK		 5247 mlx5_query_mac_address(priv->mdev, addr);
5248 if (is_zero_ether_addr(addr) &&
108805fc
SM		 5249 !MLX5_CAP_GEN(priv->mdev, vport_group_manager)) {
5250 eth_hw_addr_random(netdev);
5251 mlx5_core_info(priv->mdev, "Assigned random MAC address %pM\n", netdev->dev_addr);
537e4d2e 5252 return;
108805fc 5253 }
537e4d2e
JK		 5254
5255 eth_hw_addr_set(netdev, addr);
f62b8bb8
AV		 5256}
5257
18a2b7f9
JK		 5258static int mlx5e_vxlan_set_port(struct net_device *netdev, unsigned int table,
5259 unsigned int entry, struct udp_tunnel_info *ti)
5260{
5261 struct mlx5e_priv *priv = netdev_priv(netdev);
5262
5263 return mlx5_vxlan_add_port(priv->mdev->vxlan, ntohs(ti->port));
5264}
5265
5266static int mlx5e_vxlan_unset_port(struct net_device *netdev, unsigned int table,
5267 unsigned int entry, struct udp_tunnel_info *ti)
5268{
5269 struct mlx5e_priv *priv = netdev_priv(netdev);
5270
5271 return mlx5_vxlan_del_port(priv->mdev->vxlan, ntohs(ti->port));
5272}
5273
5274void mlx5e_vxlan_set_netdev_info(struct mlx5e_priv *priv)
5275{
5276 if (!mlx5_vxlan_allowed(priv->mdev->vxlan))
5277 return;
5278
5279 priv->nic_info.set_port = mlx5e_vxlan_set_port;
5280 priv->nic_info.unset_port = mlx5e_vxlan_unset_port;
5281 priv->nic_info.flags = UDP_TUNNEL_NIC_INFO_MAY_SLEEP |
5282 UDP_TUNNEL_NIC_INFO_STATIC_IANA_VXLAN;
5283 priv->nic_info.tables[0].tunnel_types = UDP_TUNNEL_TYPE_VXLAN;
5284 /* Don't count the space hard-coded to the IANA port */
5285 priv->nic_info.tables[0].n_entries =
5286 mlx5_vxlan_max_udp_ports(priv->mdev) - 1;
5287
5288 priv->netdev->udp_tunnel_nic_info = &priv->nic_info;
5289}
5290
c28e3bd4
AL		 5291static bool mlx5e_tunnel_any_tx_proto_supported(struct mlx5_core_dev *mdev)
5292{
5293 int tt;
5294
d443c6f6 5295 for (tt = 0; tt < MLX5_NUM_TUNNEL_TT; tt++) {
5fba089e 5296 if (mlx5e_tunnel_proto_supported_tx(mdev, mlx5_get_proto_by_tunnel_type(tt)))
c28e3bd4
AL		 5297 return true;
5298 }
5299 return (mlx5_vxlan_allowed(mdev->vxlan) || mlx5_geneve_tx_allowed(mdev));
5300}
5301
7b66ae53
JD		 5302static void mlx5e_get_queue_stats_rx(struct net_device *dev, int i,
5303 struct netdev_queue_stats_rx *stats)
5304{
5305 struct mlx5e_priv *priv = netdev_priv(dev);
5306 struct mlx5e_channel_stats *channel_stats;
5307 struct mlx5e_rq_stats *xskrq_stats;
5308 struct mlx5e_rq_stats *rq_stats;
5309
5310 ASSERT_RTNL();
0b4a4534 5311 if (mlx5e_is_uplink_rep(priv) || !priv->stats_nch)
7b66ae53
JD		 5312 return;
5313
5314 channel_stats = priv->channel_stats[i];
5315 xskrq_stats = &channel_stats->xskrq;
5316 rq_stats = &channel_stats->rq;
5317
5318 stats->packets = rq_stats->packets + xskrq_stats->packets;
5319 stats->bytes = rq_stats->bytes + xskrq_stats->bytes;
5320 stats->alloc_fail = rq_stats->buff_alloc_err +
5321 xskrq_stats->buff_alloc_err;
5322}
5323
5324static void mlx5e_get_queue_stats_tx(struct net_device *dev, int i,
5325 struct netdev_queue_stats_tx *stats)
5326{
5327 struct mlx5e_priv *priv = netdev_priv(dev);
5328 struct mlx5e_sq_stats *sq_stats;
5329
5330 ASSERT_RTNL();
0b4a4534
GP		 5331 if (!priv->stats_nch)
5332 return;
5333
7b66ae53
JD		 5334 /* no special case needed for ptp htb etc since txq2sq_stats is kept up
5335 * to date for active sq_stats, otherwise get_base_stats takes care of
5336 * inactive sqs.
5337 */
5338 sq_stats = priv->txq2sq_stats[i];
5339 stats->packets = sq_stats->packets;
5340 stats->bytes = sq_stats->bytes;
5341}
5342
5343static void mlx5e_get_base_stats(struct net_device *dev,
5344 struct netdev_queue_stats_rx *rx,
5345 struct netdev_queue_stats_tx *tx)
5346{
5347 struct mlx5e_priv *priv = netdev_priv(dev);
5348 struct mlx5e_ptp *ptp_channel;
5349 int i, tc;
5350
5351 ASSERT_RTNL();
5352 if (!mlx5e_is_uplink_rep(priv)) {
5353 rx->packets = 0;
5354 rx->bytes = 0;
5355 rx->alloc_fail = 0;
5356
5357 for (i = priv->channels.params.num_channels; i < priv->stats_nch; i++) {
5358 struct netdev_queue_stats_rx rx_i = {0};
5359
5360 mlx5e_get_queue_stats_rx(dev, i, &rx_i);
5361
5362 rx->packets += rx_i.packets;
5363 rx->bytes += rx_i.bytes;
5364 rx->alloc_fail += rx_i.alloc_fail;
5365 }
5366
5367 /* always report PTP RX stats from base as there is no
5368 * corresponding channel to report them under in
5369 * mlx5e_get_queue_stats_rx.
5370 */
5371 if (priv->rx_ptp_opened) {
5372 struct mlx5e_rq_stats *rq_stats = &priv->ptp_stats.rq;
5373
5374 rx->packets += rq_stats->packets;
5375 rx->bytes += rq_stats->bytes;
5376 }
5377 }
5378
5379 tx->packets = 0;
5380 tx->bytes = 0;
5381
5382 for (i = 0; i < priv->stats_nch; i++) {
5383 struct mlx5e_channel_stats *channel_stats = priv->channel_stats[i];
5384
5385 /* handle two cases:
5386 *
5387 * 1. channels which are active. In this case,
5388 * report only deactivated TCs on these channels.
5389 *
5390 * 2. channels which were deactivated
5391 * (i > priv->channels.params.num_channels)
5392 * must have all of their TCs [0 .. priv->max_opened_tc)
5393 * examined because deactivated channels will not be in the
5394 * range of [0..real_num_tx_queues) and will not have their
5395 * stats reported by mlx5e_get_queue_stats_tx.
5396 */
5397 if (i < priv->channels.params.num_channels)
5398 tc = mlx5e_get_dcb_num_tc(&priv->channels.params);
5399 else
5400 tc = 0;
5401
5402 for (; tc < priv->max_opened_tc; tc++) {
5403 struct mlx5e_sq_stats *sq_stats = &channel_stats->sq[tc];
5404
5405 tx->packets += sq_stats->packets;
5406 tx->bytes += sq_stats->bytes;
5407 }
5408 }
5409
5410 /* if PTP TX was opened at some point and has since either:
5411 * - been shutdown and set to NULL, or
5412 * - simply disabled (bit unset)
5413 *
5414 * report stats directly from the ptp_stats structures as these queues
5415 * are now unavailable and there is no txq index to retrieve these
5416 * stats via calls to mlx5e_get_queue_stats_tx.
5417 */
5418 ptp_channel = priv->channels.ptp;
5419 if (priv->tx_ptp_opened && (!ptp_channel || !test_bit(MLX5E_PTP_STATE_TX, ptp_channel->state))) {
5420 for (tc = 0; tc < priv->max_opened_tc; tc++) {
5421 struct mlx5e_sq_stats *sq_stats = &priv->ptp_stats.sq[tc];
5422
5423 tx->packets += sq_stats->packets;
5424 tx->bytes += sq_stats->bytes;
5425 }
5426 }
5427}
5428
5429static const struct netdev_stat_ops mlx5e_stat_ops = {
5430 .get_queue_stats_rx = mlx5e_get_queue_stats_rx,
5431 .get_queue_stats_tx = mlx5e_get_queue_stats_tx,
5432 .get_base_stats = mlx5e_get_base_stats,
5433};
5434
6bfd390b 5435static void mlx5e_build_nic_netdev(struct net_device *netdev)
f62b8bb8
AV		 5436{
5437 struct mlx5e_priv *priv = netdev_priv(netdev);
5438 struct mlx5_core_dev *mdev = priv->mdev;
94cb1ebb
EBE		 5439 bool fcs_supported;
5440 bool fcs_enabled;
f62b8bb8 5441
c42260f1 5442 SET_NETDEV_DEV(netdev, mdev->device);
f62b8bb8 5443
3fbf6120 5444 netdev->netdev_ops = &mlx5e_netdev_ops;
bc8d405b 5445 netdev->xdp_metadata_ops = &mlx5e_xdp_metadata_ops;
ec706a86 5446 netdev->xsk_tx_metadata_ops = &mlx5e_xsk_tx_metadata_ops;
e80541ec 5447
3f3ab178 5448 mlx5e_dcbnl_build_netdev(netdev);
66e49ded 5449
f62b8bb8
AV		 5450 netdev->watchdog_timeo = 15 * HZ;
5451
7b66ae53 5452 netdev->stat_ops = &mlx5e_stat_ops;
f62b8bb8
AV		 5453 netdev->ethtool_ops = &mlx5e_ethtool_ops;
5454
12be4b21 5455 netdev->vlan_features |= NETIF_F_SG;
e4683f35 5456 netdev->vlan_features |= NETIF_F_HW_CSUM;
339ccec8 5457 netdev->vlan_features |= NETIF_F_HW_MACSEC;
f62b8bb8
AV		 5458 netdev->vlan_features |= NETIF_F_GRO;
5459 netdev->vlan_features |= NETIF_F_TSO;
5460 netdev->vlan_features |= NETIF_F_TSO6;
5461 netdev->vlan_features |= NETIF_F_RXCSUM;
5462 netdev->vlan_features |= NETIF_F_RXHASH;
682adfa6 5463 netdev->vlan_features |= NETIF_F_GSO_PARTIAL;
f62b8bb8 5464
5dc9520b
AL		 5465 netdev->mpls_features |= NETIF_F_SG;
5466 netdev->mpls_features |= NETIF_F_HW_CSUM;
5467 netdev->mpls_features |= NETIF_F_TSO;
5468 netdev->mpls_features |= NETIF_F_TSO6;
5469
71186172
AH		 5470 netdev->hw_enc_features |= NETIF_F_HW_VLAN_CTAG_TX;
5471 netdev->hw_enc_features |= NETIF_F_HW_VLAN_CTAG_RX;
5472
26ab7b38
MM		 5473 /* Tunneled LRO is not supported in the driver, and the same RQs are
5474 * shared between inner and outer TIRs, so the driver can't disable LRO
5475 * for inner TIRs while having it enabled for outer TIRs. Due to this,
5476 * block LRO altogether if the firmware declares tunneled LRO support.
5477 */
6c3a823e 5478 if (!!MLX5_CAP_ETH(mdev, lro_cap) &&
26ab7b38
MM		 5479 !MLX5_CAP_ETH(mdev, tunnel_lro_vxlan) &&
5480 !MLX5_CAP_ETH(mdev, tunnel_lro_gre) &&
168723c1
MM		 5481 mlx5e_check_fragmented_striding_rq_cap(mdev, PAGE_SHIFT,
5482 MLX5E_MPWRQ_UMR_MODE_ALIGNED))
f62b8bb8
AV		 5483 netdev->vlan_features |= NETIF_F_LRO;
5484
5485 netdev->hw_features = netdev->vlan_features;
e4cf27bd 5486 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_TX;
f62b8bb8
AV		 5487 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX;
5488 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_FILTER;
4382c7b9 5489 netdev->hw_features |= NETIF_F_HW_VLAN_STAG_TX;
f62b8bb8 5490
99be5617
YZ		 5491 if (mlx5e_hw_gro_supported(mdev) &&
5492 mlx5e_check_fragmented_striding_rq_cap(mdev, PAGE_SHIFT,
5493 MLX5E_MPWRQ_UMR_MODE_ALIGNED))
5494 netdev->hw_features |= NETIF_F_GRO_HW;
5495
c28e3bd4 5496 if (mlx5e_tunnel_any_tx_proto_supported(mdev)) {
e4683f35 5497 netdev->hw_enc_features |= NETIF_F_HW_CSUM;
b3f63c3d
MF		 5498 netdev->hw_enc_features |= NETIF_F_TSO;
5499 netdev->hw_enc_features |= NETIF_F_TSO6;
27299841
GP		 5500 netdev->hw_enc_features |= NETIF_F_GSO_PARTIAL;
5501 }
5502
e3cfc7e6 5503 if (mlx5_vxlan_allowed(mdev->vxlan) || mlx5_geneve_tx_allowed(mdev)) {
64050cda
AL		 5504 netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL |
5505 NETIF_F_GSO_UDP_TUNNEL_CSUM;
5506 netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL |
5507 NETIF_F_GSO_UDP_TUNNEL_CSUM;
5508 netdev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM;
5509 netdev->vlan_features |= NETIF_F_GSO_UDP_TUNNEL |
5510 NETIF_F_GSO_UDP_TUNNEL_CSUM;
b3f63c3d
MF		 5511 }
5512
c28e3bd4 5513 if (mlx5e_tunnel_proto_supported_tx(mdev, IPPROTO_GRE)) {
01c3fd11
AL		 5514 netdev->hw_features |= NETIF_F_GSO_GRE |
5515 NETIF_F_GSO_GRE_CSUM;
5516 netdev->hw_enc_features |= NETIF_F_GSO_GRE |
5517 NETIF_F_GSO_GRE_CSUM;
5518 netdev->gso_partial_features |= NETIF_F_GSO_GRE |
5519 NETIF_F_GSO_GRE_CSUM;
27299841
GP		 5520 }
5521
c28e3bd4 5522 if (mlx5e_tunnel_proto_supported_tx(mdev, IPPROTO_IPIP)) {
25948b87
MV		 5523 netdev->hw_features |= NETIF_F_GSO_IPXIP4 |
5524 NETIF_F_GSO_IPXIP6;
5525 netdev->hw_enc_features |= NETIF_F_GSO_IPXIP4 |
5526 NETIF_F_GSO_IPXIP6;
5527 netdev->gso_partial_features |= NETIF_F_GSO_IPXIP4 |
5528 NETIF_F_GSO_IPXIP6;
5529 }
5530
3f44899e
BP		 5531 netdev->gso_partial_features |= NETIF_F_GSO_UDP_L4;
5532 netdev->hw_features |= NETIF_F_GSO_UDP_L4;
3f44899e 5533
94cb1ebb
EBE		 5534 mlx5_query_port_fcs(mdev, &fcs_supported, &fcs_enabled);
5535
5536 if (fcs_supported)
5537 netdev->hw_features |= NETIF_F_RXALL;
5538
102722fc
GE		 5539 if (MLX5_CAP_ETH(mdev, scatter_fcs))
5540 netdev->hw_features |= NETIF_F_RXFCS;
5541
9841d58f
MM		 5542 if (mlx5_qos_is_supported(mdev))
5543 netdev->hw_features |= NETIF_F_HW_TC;
5544
f62b8bb8 5545 netdev->features = netdev->hw_features;
f62b8bb8 5546
3ef14e46 5547 /* Defaults */
94cb1ebb
EBE		 5548 if (fcs_enabled)
5549 netdev->features &= ~NETIF_F_RXALL;
3ef14e46 5550 netdev->features &= ~NETIF_F_LRO;
83439f3c 5551 netdev->features &= ~NETIF_F_GRO_HW;
3ef14e46 5552 netdev->features &= ~NETIF_F_RXFCS;
c0194e2d 5553
e8f887ac
AV		 5554#define FT_CAP(f) MLX5_CAP_FLOWTABLE(mdev, flow_table_properties_nic_receive.f)
5555 if (FT_CAP(flow_modify_en) &&
5556 FT_CAP(modify_root) &&
5557 FT_CAP(identified_miss_table_mode) &&
1cabe6b0 5558 FT_CAP(flow_table_modify)) {
156878d0 5559#if IS_ENABLED(CONFIG_MLX5_CLS_ACT)
1cabe6b0 5560 netdev->hw_features |= NETIF_F_HW_TC;
077ecd78 5561#endif
3771266b 5562#if IS_ENABLED(CONFIG_MLX5_EN_ARFS)
1cabe6b0 5563 netdev->hw_features |= NETIF_F_NTUPLE;
3771266b
JK		 5564#elif IS_ENABLED(CONFIG_MLX5_EN_RXNFC)
5565 netdev->features |= NETIF_F_NTUPLE;
1cabe6b0
MG		 5566#endif
5567 }
e8f887ac 5568
f62b8bb8 5569 netdev->features |= NETIF_F_HIGHDMA;
7d92d580 5570 netdev->features |= NETIF_F_HW_VLAN_STAG_FILTER;
f62b8bb8
AV		 5571
5572 netdev->priv_flags |= IFF_UNICAST_FLT;
5573
de78960e 5574 netif_set_tso_max_size(netdev, GSO_MAX_SIZE);
4d5ab0ad 5575 mlx5e_set_xdp_feature(netdev);
f62b8bb8 5576 mlx5e_set_netdev_dev_addr(netdev);
8ff0ac5b 5577 mlx5e_macsec_build_netdev(priv);
547eede0 5578 mlx5e_ipsec_build_netdev(priv);
943aa7bd 5579 mlx5e_ktls_build_netdev(priv);
f62b8bb8
AV		 5580}
5581
1462e48d 5582void mlx5e_create_q_counters(struct mlx5e_priv *priv)
593cf338 5583{
66247fbb
LR		 5584 u32 out[MLX5_ST_SZ_DW(alloc_q_counter_out)] = {};
5585 u32 in[MLX5_ST_SZ_DW(alloc_q_counter_in)] = {};
593cf338 5586 struct mlx5_core_dev *mdev = priv->mdev;
7f525acb
TT		 5587 struct mlx5_core_dev *pos;
5588 int err, i;
593cf338 5589
66247fbb 5590 MLX5_SET(alloc_q_counter_in, in, opcode, MLX5_CMD_OP_ALLOC_Q_COUNTER);
7f525acb
TT		 5591
5592 mlx5_sd_for_each_dev(i, mdev, pos) {
5593 err = mlx5_cmd_exec_inout(pos, alloc_q_counter, in, out);
5594 if (!err)
5595 priv->q_counter[i] =
5596 MLX5_GET(alloc_q_counter_out, out, counter_set_id);
5597 }
7cbaf9a3 5598
66247fbb
LR		 5599 err = mlx5_cmd_exec_inout(mdev, alloc_q_counter, in, out);
5600 if (!err)
5601 priv->drop_rq_q_counter =
5602 MLX5_GET(alloc_q_counter_out, out, counter_set_id);
593cf338
RS		 5603}
5604
1462e48d 5605void mlx5e_destroy_q_counters(struct mlx5e_priv *priv)
593cf338 5606{
66247fbb 5607 u32 in[MLX5_ST_SZ_DW(dealloc_q_counter_in)] = {};
7f525acb
TT		 5608 struct mlx5_core_dev *pos;
5609 int i;
66247fbb
LR		 5610
5611 MLX5_SET(dealloc_q_counter_in, in, opcode,
5612 MLX5_CMD_OP_DEALLOC_Q_COUNTER);
7f525acb
TT		 5613 mlx5_sd_for_each_dev(i, priv->mdev, pos) {
5614 if (priv->q_counter[i]) {
5615 MLX5_SET(dealloc_q_counter_in, in, counter_set_id,
5616 priv->q_counter[i]);
5617 mlx5_cmd_exec_in(pos, dealloc_q_counter, in);
5618 }
66247fbb 5619 }
593cf338 5620
66247fbb
LR		 5621 if (priv->drop_rq_q_counter) {
5622 MLX5_SET(dealloc_q_counter_in, in, counter_set_id,
5623 priv->drop_rq_q_counter);
5624 mlx5_cmd_exec_in(priv->mdev, dealloc_q_counter, in);
5625 }
593cf338
RS		 5626}
5627
182570b2 5628static int mlx5e_nic_init(struct mlx5_core_dev *mdev,
3ef14e46 5629 struct net_device *netdev)
6bfd390b 5630{
72cc6549 5631 const bool take_rtnl = netdev->reg_state == NETREG_REGISTERED;
6bfd390b 5632 struct mlx5e_priv *priv = netdev_priv(netdev);
af8bbf73 5633 struct mlx5e_flow_steering *fs;
547eede0 5634 int err;
6bfd390b 5635
3ef14e46 5636 mlx5e_build_nic_params(priv, &priv->xsk, netdev->mtu);
84db6612 5637 mlx5e_vxlan_set_netdev_info(priv);
519a0bf5
SM		 5638
5639 mlx5e_timestamp_init(priv);
5640
c4c24fc3
JL		 5641 priv->dfs_root = debugfs_create_dir("nic",
5642 mlx5_debugfs_get_dev_root(mdev));
5643
5b031add 5644 fs = mlx5e_fs_init(priv->profile, mdev,
3a3da78d
GP		 5645 !test_bit(MLX5E_STATE_DESTROYING, &priv->state),
5646 priv->dfs_root);
af8bbf73
LK		 5647 if (!fs) {
5648 err = -ENOMEM;
68e66e1a 5649 mlx5_core_err(mdev, "FS initialization failed, %d\n", err);
c4c24fc3 5650 debugfs_remove_recursive(priv->dfs_root);
68e66e1a
MS		 5651 return err;
5652 }
af8bbf73 5653 priv->fs = fs;
68e66e1a 5654
943aa7bd 5655 err = mlx5e_ktls_init(priv);
43585a41
IL		 5656 if (err)
5657 mlx5_core_err(mdev, "TLS initialization failed, %d\n", err);
3ef14e46 5658
35f69867 5659 mlx5e_health_create_reporters(priv);
72cc6549
GP		 5660
5661 /* If netdev is already registered (e.g. move from uplink to nic profile),
5662 * RTNL lock must be held before triggering netdev notifiers.
5663 */
5664 if (take_rtnl)
5665 rtnl_lock();
5666
4d5ab0ad
LB		 5667 /* update XDP supported features */
5668 mlx5e_set_xdp_feature(netdev);
5669
72cc6549
GP		 5670 if (take_rtnl)
5671 rtnl_unlock();
5672
182570b2 5673 return 0;
6bfd390b
HHZ		 5674}
5675
5676static void mlx5e_nic_cleanup(struct mlx5e_priv *priv)
5677{
35f69867 5678 mlx5e_health_destroy_reporters(priv);
943aa7bd 5679 mlx5e_ktls_cleanup(priv);
af8bbf73 5680 mlx5e_fs_cleanup(priv->fs);
c4c24fc3 5681 debugfs_remove_recursive(priv->dfs_root);
0a6b069c 5682 priv->fs = NULL;
6bfd390b
HHZ		 5683}
5684
5685static int mlx5e_init_nic_rx(struct mlx5e_priv *priv)
5686{
5687 struct mlx5_core_dev *mdev = priv->mdev;
43ec0f41 5688 enum mlx5e_rx_res_features features;
6bfd390b 5689 int err;
6bfd390b 5690
1462e48d
RD		 5691 mlx5e_create_q_counters(priv);
5692
5693 err = mlx5e_open_drop_rq(priv, &priv->drop_rq);
5694 if (err) {
5695 mlx5_core_err(mdev, "open drop rq failed, %d\n", err);
5696 goto err_destroy_q_counters;
5697 }
5698
3db4c85c 5699 features = MLX5E_RX_RES_FEATURE_PTP;
9a92fe1d 5700 if (mlx5_tunnel_inner_ft_supported(mdev))
43ec0f41 5701 features |= MLX5E_RX_RES_FEATURE_INNER_FT;
40e6ad91
TT		 5702 if (mlx5_get_sd(priv->mdev))
5703 features |= MLX5E_RX_RES_FEATURE_MULTI_VHCA;
d90ea843
AF		 5704
5705 priv->rx_res = mlx5e_rx_res_create(priv->mdev, features, priv->max_nch, priv->drop_rq.rqn,
5706 &priv->channels.params.packet_merge,
5707 priv->channels.params.num_channels);
5708 if (IS_ERR(priv->rx_res)) {
5709 err = PTR_ERR(priv->rx_res);
5710 priv->rx_res = NULL;
5711 mlx5_core_err(mdev, "create rx resources failed, %d\n", err);
1462e48d 5712 goto err_close_drop_rq;
d90ea843 5713 }
6bfd390b 5714
d494dd2b
LK		 5715 err = mlx5e_create_flow_steering(priv->fs, priv->rx_res, priv->profile,
5716 priv->netdev);
6bfd390b
HHZ		 5717 if (err) {
5718 mlx5_core_warn(mdev, "create flow steering failed, %d\n", err);
43ec0f41 5719 goto err_destroy_rx_res;
6bfd390b
HHZ		 5720 }
5721
655dc3d2 5722 err = mlx5e_tc_nic_init(priv);
6bfd390b
HHZ		 5723 if (err)
5724 goto err_destroy_flow_steering;
5725
1182f365
TT		 5726 err = mlx5e_accel_init_rx(priv);
5727 if (err)
5728 goto err_tc_nic_cleanup;
5729
f4aebbfb
AL		 5730#ifdef CONFIG_MLX5_EN_ARFS
5731 priv->netdev->rx_cpu_rmap = mlx5_eq_table_get_rmap(priv->mdev);
5732#endif
5733
6bfd390b
HHZ		 5734 return 0;
5735
1182f365
TT		 5736err_tc_nic_cleanup:
5737 mlx5e_tc_nic_cleanup(priv);
6bfd390b 5738err_destroy_flow_steering:
3771266b 5739 mlx5e_destroy_flow_steering(priv->fs, mlx5e_fs_has_arfs(priv->netdev),
d494dd2b 5740 priv->profile);
43ec0f41
MM		 5741err_destroy_rx_res:
5742 mlx5e_rx_res_destroy(priv->rx_res);
d90ea843 5743 priv->rx_res = NULL;
1462e48d
RD		 5744err_close_drop_rq:
5745 mlx5e_close_drop_rq(&priv->drop_rq);
5746err_destroy_q_counters:
5747 mlx5e_destroy_q_counters(priv);
6bfd390b
HHZ		 5748 return err;
5749}
5750
5751static void mlx5e_cleanup_nic_rx(struct mlx5e_priv *priv)
5752{
1182f365 5753 mlx5e_accel_cleanup_rx(priv);
655dc3d2 5754 mlx5e_tc_nic_cleanup(priv);
3771266b 5755 mlx5e_destroy_flow_steering(priv->fs, mlx5e_fs_has_arfs(priv->netdev),
d494dd2b 5756 priv->profile);
43ec0f41 5757 mlx5e_rx_res_destroy(priv->rx_res);
d90ea843 5758 priv->rx_res = NULL;
1462e48d
RD		 5759 mlx5e_close_drop_rq(&priv->drop_rq);
5760 mlx5e_destroy_q_counters(priv);
6bfd390b
HHZ		 5761}
5762
0bb7228f
MT		 5763static void mlx5e_set_mqprio_rl(struct mlx5e_priv *priv)
5764{
5765 struct mlx5e_params *params;
5766 struct mlx5e_mqprio_rl *rl;
5767
5768 params = &priv->channels.params;
5769 if (params->mqprio.mode != TC_MQPRIO_MODE_CHANNEL)
5770 return;
5771
5772 rl = mlx5e_mqprio_rl_create(priv->mdev, params->mqprio.num_tc,
5773 params->mqprio.channel.max_rate);
5774 if (IS_ERR(rl))
5775 rl = NULL;
5776 priv->mqprio_rl = rl;
5777 mlx5e_mqprio_rl_update_params(params, rl);
5778}
5779
6bfd390b
HHZ		 5780static int mlx5e_init_nic_tx(struct mlx5e_priv *priv)
5781{
5782 int err;
5783
c4dfe704
TT		 5784 err = mlx5e_accel_init_tx(priv);
5785 if (err)
b25bd37c 5786 return err;
c4dfe704 5787
0bb7228f 5788 mlx5e_set_mqprio_rl(priv);
e207b7e9 5789 mlx5e_dcbnl_initialize(priv);
6bfd390b
HHZ		 5790 return 0;
5791}
5792
5793static void mlx5e_nic_enable(struct mlx5e_priv *priv)
5794{
5795 struct net_device *netdev = priv->netdev;
5796 struct mlx5_core_dev *mdev = priv->mdev;
3fd3fb6b 5797 int err;
2c3b5bee 5798
069448b2 5799 mlx5e_fs_init_l2_addr(priv->fs, netdev);
953d7715 5800 mlx5e_ipsec_init(priv);
2c3b5bee 5801
3fd3fb6b
EH		 5802 err = mlx5e_macsec_init(priv);
5803 if (err)
5804 mlx5_core_err(mdev, "MACsec initialization failed, %d\n", err);
5805
63bfd399
EBE		 5806 /* Marking the link as currently not needed by the Driver */
5807 if (!netif_running(netdev))
7d0314b1 5808 mlx5e_modify_admin_state(mdev, MLX5_PORT_DOWN);
63bfd399 5809
6d7ee2ed 5810 mlx5e_set_netdev_mtu_boundaries(priv);
2c3b5bee 5811 mlx5e_set_dev_port_mtu(priv);
6bfd390b 5812
8a66e458 5813 mlx5_lag_add_netdev(mdev, netdev);
7907f23a 5814
6bfd390b 5815 mlx5e_enable_async_events(priv);
70038b73 5816 mlx5e_enable_blocking_events(priv);
5c7e8bbb
ED		 5817 if (mlx5e_monitor_counter_supported(priv))
5818 mlx5e_monitor_counter_init(priv);
127ea380 5819
cef35af3 5820 mlx5e_hv_vhca_stats_create(priv);
610e89e0
SM		 5821 if (netdev->reg_state != NETREG_REGISTERED)
5822 return;
2a5e7a13 5823 mlx5e_dcbnl_init_app(priv);
610e89e0 5824
1aa48ca6 5825 mlx5e_nic_set_rx_mode(priv);
2c3b5bee
SM		 5826
5827 rtnl_lock();
5828 if (netif_running(netdev))
5829 mlx5e_open(netdev);
18a2b7f9 5830 udp_tunnel_nic_reset_ntf(priv->netdev);
2c3b5bee
SM		 5831 netif_device_attach(netdev);
5832 rtnl_unlock();
6bfd390b
HHZ		 5833}
5834
5835static void mlx5e_nic_disable(struct mlx5e_priv *priv)
5836{
3deef8ce 5837 struct mlx5_core_dev *mdev = priv->mdev;
3deef8ce 5838
2a5e7a13
HN		 5839 if (priv->netdev->reg_state == NETREG_REGISTERED)
5840 mlx5e_dcbnl_delete_app(priv);
2a5e7a13 5841
2c3b5bee
SM		 5842 rtnl_lock();
5843 if (netif_running(priv->netdev))
5844 mlx5e_close(priv->netdev);
5845 netif_device_detach(priv->netdev);
5846 rtnl_unlock();
5847
1aa48ca6 5848 mlx5e_nic_set_rx_mode(priv);
1d447a39 5849
cef35af3 5850 mlx5e_hv_vhca_stats_destroy(priv);
5c7e8bbb
ED		 5851 if (mlx5e_monitor_counter_supported(priv))
5852 mlx5e_monitor_counter_cleanup(priv);
5853
70038b73
AL		 5854 mlx5e_disable_blocking_events(priv);
5855 if (priv->en_trap) {
5856 mlx5e_deactivate_trap(priv);
5857 mlx5e_close_trap(priv->en_trap);
5858 priv->en_trap = NULL;
5859 }
6bfd390b 5860 mlx5e_disable_async_events(priv);
8a66e458 5861 mlx5_lag_remove_netdev(mdev, priv->netdev);
c5eb51ad 5862 mlx5_vxlan_reset_to_default(mdev->vxlan);
3fd3fb6b 5863 mlx5e_macsec_cleanup(priv);
953d7715 5864 mlx5e_ipsec_cleanup(priv);
6bfd390b
HHZ		 5865}
5866
958f56e4 5867static int mlx5e_update_nic_rx(struct mlx5e_priv *priv)
a90f88fe 5868{
80639b19 5869 return mlx5e_refresh_tirs(priv, false, false);
a90f88fe
GT		 5870}
5871
6bfd390b
HHZ		 5872static const struct mlx5e_profile mlx5e_nic_profile = {
5873 .init = mlx5e_nic_init,
5874 .cleanup = mlx5e_nic_cleanup,
5875 .init_rx = mlx5e_init_nic_rx,
5876 .cleanup_rx = mlx5e_cleanup_nic_rx,
5877 .init_tx = mlx5e_init_nic_tx,
5878 .cleanup_tx = mlx5e_cleanup_nic_tx,
5879 .enable = mlx5e_nic_enable,
5880 .disable = mlx5e_nic_disable,
a90f88fe 5881 .update_rx = mlx5e_update_nic_rx,
b521105b 5882 .update_stats = mlx5e_stats_update_ndo_stats,
7ca42c80 5883 .update_carrier = mlx5e_update_carrier,
5adf4c47 5884 .rx_handlers = &mlx5e_rx_handlers_nic,
b25bd37c 5885 .max_tc = MLX5_MAX_NUM_TC,
3460c184
SM		 5886 .stats_grps = mlx5e_nic_stats_grps,
5887 .stats_grps_num = mlx5e_nic_stats_grps_num,
1958c2bd
TT		 5888 .features = BIT(MLX5E_PROFILE_FEATURE_PTP_RX) |
5889 BIT(MLX5E_PROFILE_FEATURE_PTP_TX) |
454533aa
LK		 5890 BIT(MLX5E_PROFILE_FEATURE_QOS_HTB) |
5891 BIT(MLX5E_PROFILE_FEATURE_FS_VLAN) |
5892 BIT(MLX5E_PROFILE_FEATURE_FS_TC),
6bfd390b
HHZ		 5893};
5894
473baf2e
TT		 5895static int mlx5e_profile_max_num_channels(struct mlx5_core_dev *mdev,
5896 const struct mlx5e_profile *profile)
5897{
5898 int nch;
5899
5900 nch = mlx5e_get_max_num_channels(mdev);
5901
5902 if (profile->max_nch_limit)
5903 nch = min_t(int, nch, profile->max_nch_limit(mdev));
5904 return nch;
5905}
5906
9d758d4a
TT		 5907static unsigned int
5908mlx5e_calc_max_nch(struct mlx5_core_dev *mdev, struct net_device *netdev,
5909 const struct mlx5e_profile *profile)
5910
5911{
5912 unsigned int max_nch, tmp;
5913
5914 /* core resources */
473baf2e 5915 max_nch = mlx5e_profile_max_num_channels(mdev, profile);
9d758d4a
TT		 5916
5917 /* netdev rx queues */
3db4c85c 5918 max_nch = min_t(unsigned int, max_nch, netdev->num_rx_queues);
9d758d4a
TT		 5919
5920 /* netdev tx queues */
5921 tmp = netdev->num_tx_queues;
5922 if (mlx5_qos_is_supported(mdev))
5923 tmp -= mlx5e_qos_max_leaf_nodes(mdev);
5924 if (MLX5_CAP_GEN(mdev, ts_cqe_to_dest_cqn))
5925 tmp -= profile->max_tc;
5926 tmp = tmp / profile->max_tc;
5927 max_nch = min_t(unsigned int, max_nch, tmp);
5928
5929 return max_nch;
5930}
5931
6d0ba493
MT		 5932int mlx5e_get_pf_num_tirs(struct mlx5_core_dev *mdev)
5933{
5934 /* Indirect TIRS: 2 sets of TTCs (inner + outer steering)
5935 * and 1 set of direct TIRS
5936 */
5937 return 2 * MLX5E_NUM_INDIR_TIRS
5938 + mlx5e_profile_max_num_channels(mdev, &mlx5e_nic_profile);
5939}
5940
5b031add
LK		 5941void mlx5e_set_rx_mode_work(struct work_struct *work)
5942{
5943 struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv,
5944 set_rx_mode_work);
5945
5946 return mlx5e_fs_set_rx_mode_work(priv->fs, priv->netdev);
5947}
5948
2c3b5bee 5949/* mlx5e generic netdev management API (move to en_common.c) */
c9fd1e33 5950int mlx5e_priv_init(struct mlx5e_priv *priv,
9d758d4a 5951 const struct mlx5e_profile *profile,
c9fd1e33
RD		 5952 struct net_device *netdev,
5953 struct mlx5_core_dev *mdev)
182570b2 5954{
3ab45777 5955 int nch, num_txqs, node;
8bf30be7 5956 int err;
0246a57a
TT		 5957
5958 num_txqs = netdev->num_tx_queues;
5959 nch = mlx5e_calc_max_nch(mdev, netdev, profile);
5960 node = dev_to_node(mlx5_core_dma_dev(mdev));
5961
519a0bf5
SM		 5962 /* priv init */
5963 priv->mdev = mdev;
5964 priv->netdev = netdev;
0246a57a 5965 priv->max_nch = nch;
519a0bf5 5966 priv->max_opened_tc = 1;
182570b2 5967
3909a12e
MM		 5968 if (!alloc_cpumask_var(&priv->scratchpad.cpumask, GFP_KERNEL))
5969 return -ENOMEM;
5970
519a0bf5 5971 mutex_init(&priv->state_lock);
8bf30be7
MM		 5972
5973 err = mlx5e_selq_init(&priv->selq, &priv->state_lock);
5974 if (err)
5975 goto err_free_cpumask;
5976
519a0bf5
SM		 5977 INIT_WORK(&priv->update_carrier_work, mlx5e_update_carrier_work);
5978 INIT_WORK(&priv->set_rx_mode_work, mlx5e_set_rx_mode_work);
5979 INIT_WORK(&priv->tx_timeout_work, mlx5e_tx_timeout_work);
cdeef2b1 5980 INIT_WORK(&priv->update_stats_work, mlx5e_update_stats_work);
303211b4 5981
182570b2
FD		 5982 priv->wq = create_singlethread_workqueue("mlx5e");
5983 if (!priv->wq)
8bf30be7 5984 goto err_free_selq;
182570b2 5985
0246a57a
TT		 5986 priv->txq2sq = kcalloc_node(num_txqs, sizeof(*priv->txq2sq), GFP_KERNEL, node);
5987 if (!priv->txq2sq)
5988 goto err_destroy_workqueue;
5989
0a3e5c1b
JD		 5990 priv->txq2sq_stats = kcalloc_node(num_txqs, sizeof(*priv->txq2sq_stats), GFP_KERNEL, node);
5991 if (!priv->txq2sq_stats)
5992 goto err_free_txq2sq;
5993
0246a57a
TT		 5994 priv->tx_rates = kcalloc_node(num_txqs, sizeof(*priv->tx_rates), GFP_KERNEL, node);
5995 if (!priv->tx_rates)
0a3e5c1b 5996 goto err_free_txq2sq_stats;
0246a57a 5997
0246a57a
TT		 5998 priv->channel_stats =
5999 kcalloc_node(nch, sizeof(*priv->channel_stats), GFP_KERNEL, node);
6000 if (!priv->channel_stats)
3ab45777 6001 goto err_free_tx_rates;
0246a57a 6002
182570b2 6003 return 0;
3909a12e 6004
0246a57a
TT		 6005err_free_tx_rates:
6006 kfree(priv->tx_rates);
0a3e5c1b
JD		 6007err_free_txq2sq_stats:
6008 kfree(priv->txq2sq_stats);
0246a57a
TT		 6009err_free_txq2sq:
6010 kfree(priv->txq2sq);
6011err_destroy_workqueue:
6012 destroy_workqueue(priv->wq);
8bf30be7
MM		 6013err_free_selq:
6014 mlx5e_selq_cleanup(&priv->selq);
3909a12e
MM		 6015err_free_cpumask:
6016 free_cpumask_var(priv->scratchpad.cpumask);
3909a12e 6017 return -ENOMEM;
182570b2
FD		 6018}
6019
c9fd1e33 6020void mlx5e_priv_cleanup(struct mlx5e_priv *priv)
182570b2 6021{
214baf22
MM		 6022 int i;
6023
469549e4
RD		 6024 /* bail if change profile failed and also rollback failed */
6025 if (!priv->mdev)
6026 return;
6027
be98737a
TT		 6028 for (i = 0; i < priv->stats_nch; i++)
6029 kvfree(priv->channel_stats[i]);
0246a57a 6030 kfree(priv->channel_stats);
0246a57a 6031 kfree(priv->tx_rates);
0a3e5c1b 6032 kfree(priv->txq2sq_stats);
0246a57a 6033 kfree(priv->txq2sq);
182570b2 6034 destroy_workqueue(priv->wq);
8bf30be7 6035 mlx5e_selq_cleanup(&priv->selq);
3909a12e 6036 free_cpumask_var(priv->scratchpad.cpumask);
214baf22 6037
db83f24d
MT		 6038 for (i = 0; i < priv->htb_max_qos_sqs; i++)
6039 kfree(priv->htb_qos_sq_stats[i]);
6040 kvfree(priv->htb_qos_sq_stats);
469549e4 6041
1da839ea
JL		 6042 if (priv->mqprio_rl) {
6043 mlx5e_mqprio_rl_cleanup(priv->mqprio_rl);
6044 mlx5e_mqprio_rl_free(priv->mqprio_rl);
6045 }
6046
469549e4 6047 memset(priv, 0, sizeof(*priv));
182570b2
FD		 6048}
6049
1958c2bd
TT		 6050static unsigned int mlx5e_get_max_num_txqs(struct mlx5_core_dev *mdev,
6051 const struct mlx5e_profile *profile)
6052{
6053 unsigned int nch, ptp_txqs, qos_txqs;
6054
473baf2e 6055 nch = mlx5e_profile_max_num_channels(mdev, profile);
1958c2bd
TT		 6056
6057 ptp_txqs = MLX5_CAP_GEN(mdev, ts_cqe_to_dest_cqn) &&
6058 mlx5e_profile_feature_cap(profile, PTP_TX) ?
6059 profile->max_tc : 0;
6060
6061 qos_txqs = mlx5_qos_is_supported(mdev) &&
6062 mlx5e_profile_feature_cap(profile, QOS_HTB) ?
6063 mlx5e_qos_max_leaf_nodes(mdev) : 0;
6064
6065 return nch * profile->max_tc + ptp_txqs + qos_txqs;
6066}
6067
6068static unsigned int mlx5e_get_max_num_rxqs(struct mlx5_core_dev *mdev,
6069 const struct mlx5e_profile *profile)
6070{
3db4c85c 6071 return mlx5e_profile_max_num_channels(mdev, profile);
1958c2bd
TT		 6072}
6073
3ef14e46 6074struct net_device *
1958c2bd 6075mlx5e_create_netdev(struct mlx5_core_dev *mdev, const struct mlx5e_profile *profile)
f62b8bb8
AV		 6076{
6077 struct net_device *netdev;
1958c2bd 6078 unsigned int txqs, rxqs;
182570b2 6079 int err;
f62b8bb8 6080
1958c2bd
TT		 6081 txqs = mlx5e_get_max_num_txqs(mdev, profile);
6082 rxqs = mlx5e_get_max_num_rxqs(mdev, profile);
6083
3ef14e46 6084 netdev = alloc_etherdev_mqs(sizeof(struct mlx5e_priv), txqs, rxqs);
f62b8bb8
AV		 6085 if (!netdev) {
6086 mlx5_core_err(mdev, "alloc_etherdev_mqs() failed\n");
6087 return NULL;
6088 }
6089
9d758d4a 6090 err = mlx5e_priv_init(netdev_priv(netdev), profile, netdev, mdev);
182570b2 6091 if (err) {
c9fd1e33 6092 mlx5_core_err(mdev, "mlx5e_priv_init failed, err=%d\n", err);
182570b2
FD		 6093 goto err_free_netdev;
6094 }
26e59d80 6095
1227bbc5 6096 netif_carrier_off(netdev);
d08c6e2a 6097 netif_tx_disable(netdev);
3ef14e46 6098 dev_net_set(netdev, mlx5_core_net(mdev));
26e59d80
MHY		 6099
6100 return netdev;
6101
182570b2 6102err_free_netdev:
26e59d80
MHY		 6103 free_netdev(netdev);
6104
6105 return NULL;
6106}
6107
3ef14e46
SM		 6108static void mlx5e_update_features(struct net_device *netdev)
6109{
6110 if (netdev->reg_state != NETREG_REGISTERED)
6111 return; /* features will be updated on netdev registration */
6112
6113 rtnl_lock();
6114 netdev_update_features(netdev);
6115 rtnl_unlock();
6116}
6117
77ecd10d
SM		 6118static void mlx5e_reset_channels(struct net_device *netdev)
6119{
6120 netdev_reset_tc(netdev);
6121}
6122
2c3b5bee 6123int mlx5e_attach_netdev(struct mlx5e_priv *priv)
26e59d80 6124{
3909a12e 6125 const bool take_rtnl = priv->netdev->reg_state == NETREG_REGISTERED;
3ef14e46 6126 const struct mlx5e_profile *profile = priv->profile;
a1f240f1 6127 int max_nch;
26e59d80
MHY		 6128 int err;
6129
26e59d80 6130 clear_bit(MLX5E_STATE_DESTROYING, &priv->state);
5b031add 6131 if (priv->fs)
f52f2fae
LK		 6132 mlx5e_fs_set_state_destroy(priv->fs,
6133 !test_bit(MLX5E_STATE_DESTROYING, &priv->state));
7bb29755 6134
f9c955b4
MM		 6135 /* Validate the max_wqe_size_sq capability. */
6136 if (WARN_ON_ONCE(mlx5e_get_max_sq_wqebbs(priv->mdev) < MLX5E_MAX_TX_WQEBBS)) {
163c2c70
SM		 6137 mlx5_core_warn(priv->mdev, "MLX5E: Max SQ WQEBBs firmware capability: %u, needed %u\n",
6138 mlx5e_get_max_sq_wqebbs(priv->mdev), (unsigned int)MLX5E_MAX_TX_WQEBBS);
f9c955b4
MM		 6139 return -EIO;
6140 }
6141
a1f240f1 6142 /* max number of channels may have changed */
9d758d4a 6143 max_nch = mlx5e_calc_max_nch(priv->mdev, priv->netdev, profile);
a1f240f1
YA		 6144 if (priv->channels.params.num_channels > max_nch) {
6145 mlx5_core_warn(priv->mdev, "MLX5E: Reducing number of channels to %d\n", max_nch);
3909a12e
MM		 6146 /* Reducing the number of channels - RXFH has to be reset, and
6147 * mlx5e_num_channels_changed below will build the RQT.
6148 */
fe867cac 6149 priv->netdev->priv_flags &= ~IFF_RXFH_CONFIGURED;
a1f240f1 6150 priv->channels.params.num_channels = max_nch;
7dbc849b
TT		 6151 if (priv->channels.params.mqprio.mode == TC_MQPRIO_MODE_CHANNEL) {
6152 mlx5_core_warn(priv->mdev, "MLX5E: Disabling MQPRIO channel mode\n");
6153 mlx5e_params_mqprio_reset(&priv->channels.params);
6154 }
a1f240f1 6155 }
9d758d4a
TT		 6156 if (max_nch != priv->max_nch) {
6157 mlx5_core_warn(priv->mdev,
6158 "MLX5E: Updating max number of channels from %u to %u\n",
6159 priv->max_nch, max_nch);
6160 priv->max_nch = max_nch;
6161 }
6162
3909a12e
MM		 6163 /* 1. Set the real number of queues in the kernel the first time.
6164 * 2. Set our default XPS cpumask.
6165 * 3. Build the RQT.
6166 *
6167 * rtnl_lock is required by netif_set_real_num_*_queues in case the
6168 * netdev has been registered by this point (if this function was called
6169 * in the reload or resume flow).
6170 */
6171 if (take_rtnl)
6172 rtnl_lock();
fa374877 6173 err = mlx5e_num_channels_changed(priv);
3909a12e
MM		 6174 if (take_rtnl)
6175 rtnl_unlock();
fa374877
MM		 6176 if (err)
6177 goto out;
a1f240f1 6178
6bfd390b
HHZ		 6179 err = profile->init_tx(priv);
6180 if (err)
ec8b9981 6181 goto out;
5c50368f 6182
6bfd390b
HHZ		 6183 err = profile->init_rx(priv);
6184 if (err)
1462e48d 6185 goto err_cleanup_tx;
5c50368f 6186
6bfd390b
HHZ		 6187 if (profile->enable)
6188 profile->enable(priv);
f62b8bb8 6189
3ef14e46
SM		 6190 mlx5e_update_features(priv->netdev);
6191
26e59d80 6192 return 0;
5c50368f 6193
1462e48d 6194err_cleanup_tx:
6bfd390b 6195 profile->cleanup_tx(priv);
5c50368f 6196
26e59d80 6197out:
77ecd10d 6198 mlx5e_reset_channels(priv->netdev);
5cd39b6e 6199 set_bit(MLX5E_STATE_DESTROYING, &priv->state);
5b031add 6200 if (priv->fs)
f52f2fae
LK		 6201 mlx5e_fs_set_state_destroy(priv->fs,
6202 !test_bit(MLX5E_STATE_DESTROYING, &priv->state));
5cd39b6e 6203 cancel_work_sync(&priv->update_stats_work);
26e59d80 6204 return err;
f62b8bb8
AV		 6205}
6206
2c3b5bee 6207void mlx5e_detach_netdev(struct mlx5e_priv *priv)
26e59d80 6208{
26e59d80
MHY		 6209 const struct mlx5e_profile *profile = priv->profile;
6210
6211 set_bit(MLX5E_STATE_DESTROYING, &priv->state);
5b031add 6212 if (priv->fs)
f52f2fae
LK		 6213 mlx5e_fs_set_state_destroy(priv->fs,
6214 !test_bit(MLX5E_STATE_DESTROYING, &priv->state));
26e59d80 6215
37f304d1
SM		 6216 if (profile->disable)
6217 profile->disable(priv);
6218 flush_workqueue(priv->wq);
6219
26e59d80 6220 profile->cleanup_rx(priv);
26e59d80 6221 profile->cleanup_tx(priv);
77ecd10d 6222 mlx5e_reset_channels(priv->netdev);
cdeef2b1 6223 cancel_work_sync(&priv->update_stats_work);
26e59d80
MHY		 6224}
6225
c4d7eb57 6226static int
bdf27475
DL		 6227mlx5e_netdev_init_profile(struct net_device *netdev, struct mlx5_core_dev *mdev,
6228 const struct mlx5e_profile *new_profile, void *new_ppriv)
c4d7eb57 6229{
469549e4 6230 struct mlx5e_priv *priv = netdev_priv(netdev);
c4d7eb57
SM		 6231 int err;
6232
9d758d4a 6233 err = mlx5e_priv_init(priv, new_profile, netdev, mdev);
c4d7eb57 6234 if (err) {
c9fd1e33 6235 mlx5_core_err(mdev, "mlx5e_priv_init failed, err=%d\n", err);
c4d7eb57
SM		 6236 return err;
6237 }
1227bbc5 6238 netif_carrier_off(netdev);
c4d7eb57
SM		 6239 priv->profile = new_profile;
6240 priv->ppriv = new_ppriv;
6241 err = new_profile->init(priv->mdev, priv->netdev);
6242 if (err)
469549e4 6243 goto priv_cleanup;
bdf27475
DL		 6244
6245 return 0;
6246
6247priv_cleanup:
6248 mlx5e_priv_cleanup(priv);
6249 return err;
6250}
6251
6252static int
6253mlx5e_netdev_attach_profile(struct net_device *netdev, struct mlx5_core_dev *mdev,
6254 const struct mlx5e_profile *new_profile, void *new_ppriv)
6255{
6256 struct mlx5e_priv *priv = netdev_priv(netdev);
6257 int err;
6258
6259 err = mlx5e_netdev_init_profile(netdev, mdev, new_profile, new_ppriv);
6260 if (err)
6261 return err;
6262
c4d7eb57
SM		 6263 err = mlx5e_attach_netdev(priv);
6264 if (err)
469549e4
RD		 6265 goto profile_cleanup;
6266 return err;
6267
6268profile_cleanup:
6269 new_profile->cleanup(priv);
469549e4 6270 mlx5e_priv_cleanup(priv);
c4d7eb57
SM		 6271 return err;
6272}
6273
6274int mlx5e_netdev_change_profile(struct mlx5e_priv *priv,
6275 const struct mlx5e_profile *new_profile, void *new_ppriv)
6276{
c4d7eb57 6277 const struct mlx5e_profile *orig_profile = priv->profile;
469549e4
RD		 6278 struct net_device *netdev = priv->netdev;
6279 struct mlx5_core_dev *mdev = priv->mdev;
c4d7eb57
SM		 6280 void *orig_ppriv = priv->ppriv;
6281 int err, rollback_err;
6282
c4d7eb57
SM		 6283 /* cleanup old profile */
6284 mlx5e_detach_netdev(priv);
6285 priv->profile->cleanup(priv);
c9fd1e33 6286 mlx5e_priv_cleanup(priv);
c4d7eb57 6287
bdf27475
DL		 6288 if (mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) {
6289 mlx5e_netdev_init_profile(netdev, mdev, new_profile, new_ppriv);
6290 set_bit(MLX5E_STATE_DESTROYING, &priv->state);
6291 return -EIO;
6292 }
6293
469549e4 6294 err = mlx5e_netdev_attach_profile(netdev, mdev, new_profile, new_ppriv);
c4d7eb57 6295 if (err) { /* roll back to original profile */
469549e4 6296 netdev_warn(netdev, "%s: new profile init failed, %d\n", __func__, err);
c4d7eb57
SM		 6297 goto rollback;
6298 }
6299
6300 return 0;
6301
6302rollback:
469549e4
RD		 6303 rollback_err = mlx5e_netdev_attach_profile(netdev, mdev, orig_profile, orig_ppriv);
6304 if (rollback_err)
6305 netdev_err(netdev, "%s: failed to rollback to orig profile, %d\n",
c4d7eb57 6306 __func__, rollback_err);
c4d7eb57
SM		 6307 return err;
6308}
6309
7a9fb35e
RD		 6310void mlx5e_netdev_attach_nic_profile(struct mlx5e_priv *priv)
6311{
6312 mlx5e_netdev_change_profile(priv, &mlx5e_nic_profile, NULL);
6313}
6314
2c3b5bee
SM		 6315void mlx5e_destroy_netdev(struct mlx5e_priv *priv)
6316{
2c3b5bee
SM		 6317 struct net_device *netdev = priv->netdev;
6318
c9fd1e33 6319 mlx5e_priv_cleanup(priv);
2c3b5bee
SM		 6320 free_netdev(netdev);
6321}
6322
381978d2 6323static int _mlx5e_resume(struct auxiliary_device *adev)
26e59d80 6324{
912cebf4 6325 struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev);
ee75f1fc
JP		 6326 struct mlx5e_dev *mlx5e_dev = auxiliary_get_drvdata(adev);
6327 struct mlx5e_priv *priv = mlx5e_dev->priv;
26e59d80 6328 struct net_device *netdev = priv->netdev;
912cebf4 6329 struct mlx5_core_dev *mdev = edev->mdev;
846122b1
TT		 6330 struct mlx5_core_dev *pos, *to;
6331 int err, i;
26e59d80
MHY		 6332
6333 if (netif_device_present(netdev))
6334 return 0;
6335
846122b1
TT		 6336 mlx5_sd_for_each_dev(i, mdev, pos) {
6337 err = mlx5e_create_mdev_resources(pos, true);
6338 if (err)
6339 goto err_destroy_mdev_res;
6340 }
e2578b4f 6341
3fbf6120 6342 err = mlx5e_attach_netdev(priv);
846122b1
TT		 6343 if (err)
6344 goto err_destroy_mdev_res;
e2578b4f 6345
e2578b4f 6346 return 0;
846122b1
TT		 6347
6348err_destroy_mdev_res:
6349 to = pos;
6350 mlx5_sd_for_each_dev_to(i, mdev, to, pos)
6351 mlx5e_destroy_mdev_resources(pos);
6352 return err;
e2578b4f
TT		 6353}
6354
381978d2
TT		 6355static int mlx5e_resume(struct auxiliary_device *adev)
6356{
6357 struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev);
6358 struct mlx5_core_dev *mdev = edev->mdev;
6359 struct auxiliary_device *actual_adev;
6360 int err;
6361
6362 err = mlx5_sd_init(mdev);
6363 if (err)
6364 return err;
6365
6366 actual_adev = mlx5_sd_get_adev(mdev, adev, edev->idx);
6367 if (actual_adev)
6368 return _mlx5e_resume(actual_adev);
6369 return 0;
6370}
6371
3d591847 6372static int _mlx5e_suspend(struct auxiliary_device *adev, bool pre_netdev_reg)
26e59d80 6373{
ee75f1fc
JP		 6374 struct mlx5e_dev *mlx5e_dev = auxiliary_get_drvdata(adev);
6375 struct mlx5e_priv *priv = mlx5e_dev->priv;
26e59d80 6376 struct net_device *netdev = priv->netdev;
912cebf4 6377 struct mlx5_core_dev *mdev = priv->mdev;
846122b1
TT		 6378 struct mlx5_core_dev *pos;
6379 int i;
47c9d2c9 6380
3d591847 6381 if (!pre_netdev_reg && !netif_device_present(netdev)) {
bdf27475 6382 if (test_bit(MLX5E_STATE_DESTROYING, &priv->state))
846122b1
TT		 6383 mlx5_sd_for_each_dev(i, mdev, pos)
6384 mlx5e_destroy_mdev_resources(pos);
912cebf4 6385 return -ENODEV;
bdf27475 6386 }
26e59d80 6387
2c3b5bee 6388 mlx5e_detach_netdev(priv);
846122b1
TT		 6389 mlx5_sd_for_each_dev(i, mdev, pos)
6390 mlx5e_destroy_mdev_resources(pos);
6391
912cebf4 6392 return 0;
26e59d80
MHY		 6393}
6394
b1a33e65
TT		 6395static int mlx5e_suspend(struct auxiliary_device *adev, pm_message_t state)
6396{
381978d2
TT		 6397 struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev);
6398 struct mlx5_core_dev *mdev = edev->mdev;
6399 struct auxiliary_device *actual_adev;
6400 int err = 0;
6401
6402 actual_adev = mlx5_sd_get_adev(mdev, adev, edev->idx);
6403 if (actual_adev)
3d591847 6404 err = _mlx5e_suspend(actual_adev, false);
381978d2
TT		 6405
6406 mlx5_sd_cleanup(mdev);
6407 return err;
b1a33e65
TT		 6408}
6409
6410static int _mlx5e_probe(struct auxiliary_device *adev)
b50d292b 6411{
912cebf4 6412 struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev);
3fbf6120 6413 const struct mlx5e_profile *profile = &mlx5e_nic_profile;
912cebf4 6414 struct mlx5_core_dev *mdev = edev->mdev;
ee75f1fc 6415 struct mlx5e_dev *mlx5e_dev;
07c9f1e5 6416 struct net_device *netdev;
3ef14e46 6417 struct mlx5e_priv *priv;
26e59d80 6418 int err;
b50d292b 6419
de411a82 6420 mlx5e_dev = mlx5e_create_devlink(&adev->dev, mdev);
ee75f1fc
JP		 6421 if (IS_ERR(mlx5e_dev))
6422 return PTR_ERR(mlx5e_dev);
6423 auxiliary_set_drvdata(adev, mlx5e_dev);
6424
6d6e71e6
JP		 6425 err = mlx5e_devlink_port_register(mlx5e_dev, mdev);
6426 if (err) {
6427 mlx5_core_err(mdev, "mlx5e_devlink_port_register failed, %d\n", err);
6428 goto err_devlink_unregister;
6429 }
6430
1958c2bd 6431 netdev = mlx5e_create_netdev(mdev, profile);
26e59d80
MHY		 6432 if (!netdev) {
6433 mlx5_core_err(mdev, "mlx5e_create_netdev failed\n");
ee75f1fc 6434 err = -ENOMEM;
6d6e71e6 6435 goto err_devlink_port_unregister;
26e59d80 6436 }
6d6e71e6 6437 SET_NETDEV_DEVLINK_PORT(netdev, &mlx5e_dev->dl_port);
26e59d80 6438
3ef14e46
SM		 6439 mlx5e_build_nic_netdev(netdev);
6440
26e59d80 6441 priv = netdev_priv(netdev);
ee75f1fc 6442 mlx5e_dev->priv = priv;
26e59d80 6443
3ef14e46
SM		 6444 priv->profile = profile;
6445 priv->ppriv = NULL;
865d6d1c 6446
3ef14e46
SM		 6447 err = profile->init(mdev, netdev);
6448 if (err) {
6449 mlx5_core_err(mdev, "mlx5e_nic_profile init failed, %d\n", err);
6d6e71e6 6450 goto err_destroy_netdev;
3ef14e46
SM		 6451 }
6452
381978d2 6453 err = _mlx5e_resume(adev);
26e59d80 6454 if (err) {
381978d2 6455 mlx5_core_err(mdev, "_mlx5e_resume failed, %d\n", err);
3ef14e46 6456 goto err_profile_cleanup;
26e59d80
MHY		 6457 }
6458
31e87b39 6459 err = register_netdev(netdev);
c6acd629 6460 if (err) {
31e87b39 6461 mlx5_core_err(mdev, "register_netdev failed, %d\n", err);
912cebf4 6462 goto err_resume;
c6acd629
VT		 6463 }
6464
2a5e7a13 6465 mlx5e_dcbnl_init_app(priv);
c7d4e6ab 6466 mlx5_core_uplink_netdev_set(mdev, netdev);
38438d39 6467 mlx5e_params_print_info(mdev, &priv->channels.params);
912cebf4 6468 return 0;
26e59d80 6469
912cebf4 6470err_resume:
3d591847 6471 _mlx5e_suspend(adev, true);
3ef14e46
SM		 6472err_profile_cleanup:
6473 profile->cleanup(priv);
26e59d80 6474err_destroy_netdev:
2c3b5bee 6475 mlx5e_destroy_netdev(priv);
6d6e71e6
JP		 6476err_devlink_port_unregister:
6477 mlx5e_devlink_port_unregister(mlx5e_dev);
ee75f1fc
JP		 6478err_devlink_unregister:
6479 mlx5e_destroy_devlink(mlx5e_dev);
912cebf4 6480 return err;
b50d292b
HHZ		 6481}
6482
b1a33e65
TT		 6483static int mlx5e_probe(struct auxiliary_device *adev,
6484 const struct auxiliary_device_id *id)
6485{
381978d2
TT		 6486 struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev);
6487 struct mlx5_core_dev *mdev = edev->mdev;
6488 struct auxiliary_device *actual_adev;
6489 int err;
6490
6491 err = mlx5_sd_init(mdev);
6492 if (err)
6493 return err;
6494
6495 actual_adev = mlx5_sd_get_adev(mdev, adev, edev->idx);
6496 if (actual_adev)
6497 return _mlx5e_probe(actual_adev);
6498 return 0;
b1a33e65
TT		 6499}
6500
381978d2 6501static void _mlx5e_remove(struct auxiliary_device *adev)
b50d292b 6502{
381978d2 6503 struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev);
ee75f1fc
JP		 6504 struct mlx5e_dev *mlx5e_dev = auxiliary_get_drvdata(adev);
6505 struct mlx5e_priv *priv = mlx5e_dev->priv;
381978d2 6506 struct mlx5_core_dev *mdev = edev->mdev;
127ea380 6507
381978d2 6508 mlx5_core_uplink_netdev_set(mdev, NULL);
2a5e7a13 6509 mlx5e_dcbnl_delete_app(priv);
5e1e93c7 6510 unregister_netdev(priv->netdev);
3d591847 6511 _mlx5e_suspend(adev, false);
3ef14e46 6512 priv->profile->cleanup(priv);
2c3b5bee 6513 mlx5e_destroy_netdev(priv);
6d6e71e6 6514 mlx5e_devlink_port_unregister(mlx5e_dev);
ee75f1fc 6515 mlx5e_destroy_devlink(mlx5e_dev);
b50d292b
HHZ		 6516}
6517
381978d2
TT		 6518static void mlx5e_remove(struct auxiliary_device *adev)
6519{
6520 struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev);
6521 struct mlx5_core_dev *mdev = edev->mdev;
6522 struct auxiliary_device *actual_adev;
6523
6524 actual_adev = mlx5_sd_get_adev(mdev, adev, edev->idx);
6525 if (actual_adev)
6526 _mlx5e_remove(actual_adev);
6527
6528 mlx5_sd_cleanup(mdev);
6529}
6530
912cebf4
LR		 6531static const struct auxiliary_device_id mlx5e_id_table[] = {
6532 { .name = MLX5_ADEV_NAME ".eth", },
6533 {},
f62b8bb8
AV		 6534};
6535
912cebf4
LR		 6536MODULE_DEVICE_TABLE(auxiliary, mlx5e_id_table);
6537
6538static struct auxiliary_driver mlx5e_driver = {
6539 .name = "eth",
6540 .probe = mlx5e_probe,
6541 .remove = mlx5e_remove,
6542 .suspend = mlx5e_suspend,
6543 .resume = mlx5e_resume,
6544 .id_table = mlx5e_id_table,
6545};
6546
6547int mlx5e_init(void)
f62b8bb8 6548{
912cebf4
LR		 6549 int ret;
6550
665bc539 6551 mlx5e_build_ptys2ethtool_map();
fec2b4bb 6552 ret = auxiliary_driver_register(&mlx5e_driver);
912cebf4
LR		 6553 if (ret)
6554 return ret;
6555
fec2b4bb 6556 ret = mlx5e_rep_init();
912cebf4 6557 if (ret)
fec2b4bb 6558 auxiliary_driver_unregister(&mlx5e_driver);
912cebf4 6559 return ret;
f62b8bb8
AV		 6560}
6561
6562void mlx5e_cleanup(void)
6563{
912cebf4 6564 mlx5e_rep_cleanup();
fec2b4bb 6565 auxiliary_driver_unregister(&mlx5e_driver);
f62b8bb8 6566}
Linux 6.x block layer and io_uring tree(s)