projects / linux-2.6-block.git / blame_incremental
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (non-incremental) | history | HEAD
Merge tag 'probes-fixes-v6.16-rc6' of git://git.kernel.org/pub/scm/linux/kernel/git...
[linux-2.6-block.git] / drivers / infiniband / hw / mlx5 / main.c
... / ...
CommitLineData
1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2/*
3 * Copyright (c) 2013-2020, Mellanox Technologies inc. All rights reserved.
4 * Copyright (c) 2020, Intel Corporation. All rights reserved.
5 */
6
7#include <linux/debugfs.h>
8#include <linux/highmem.h>
9#include <linux/module.h>
10#include <linux/init.h>
11#include <linux/errno.h>
12#include <linux/pci.h>
13#include <linux/dma-mapping.h>
14#include <linux/slab.h>
15#include <linux/bitmap.h>
16#include <linux/sched.h>
17#include <linux/sched/mm.h>
18#include <linux/sched/task.h>
19#include <linux/delay.h>
20#include <rdma/ib_user_verbs.h>
21#include <rdma/ib_addr.h>
22#include <rdma/ib_cache.h>
23#include <linux/mlx5/port.h>
24#include <linux/mlx5/vport.h>
25#include <linux/mlx5/fs.h>
26#include <linux/mlx5/eswitch.h>
27#include <linux/mlx5/driver.h>
28#include <linux/list.h>
29#include <rdma/ib_smi.h>
30#include <rdma/ib_umem_odp.h>
31#include <rdma/lag.h>
32#include <linux/in.h>
33#include <linux/etherdevice.h>
34#include "mlx5_ib.h"
35#include "ib_rep.h"
36#include "cmd.h"
37#include "devx.h"
38#include "dm.h"
39#include "fs.h"
40#include "srq.h"
41#include "qp.h"
42#include "wr.h"
43#include "restrack.h"
44#include "counters.h"
45#include "umr.h"
46#include <rdma/uverbs_std_types.h>
47#include <rdma/uverbs_ioctl.h>
48#include <rdma/mlx5_user_ioctl_verbs.h>
49#include <rdma/mlx5_user_ioctl_cmds.h>
50#include <rdma/ib_ucaps.h>
51#include "macsec.h"
52#include "data_direct.h"
53
54#define UVERBS_MODULE_NAME mlx5_ib
55#include <rdma/uverbs_named_ioctl.h>
56
57MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>");
58MODULE_DESCRIPTION("Mellanox 5th generation network adapters (ConnectX series) IB driver");
59MODULE_LICENSE("Dual BSD/GPL");
60
61struct mlx5_ib_event_work {
62 struct work_struct work;
63 union {
64 struct mlx5_ib_dev *dev;
65 struct mlx5_ib_multiport_info *mpi;
66 };
67 bool is_slave;
68 unsigned int event;
69 void *param;
70};
71
72enum {
73 MLX5_ATOMIC_SIZE_QP_8BYTES = 1 << 3,
74};
75
76static struct workqueue_struct *mlx5_ib_event_wq;
77static LIST_HEAD(mlx5_ib_unaffiliated_port_list);
78static LIST_HEAD(mlx5_ib_dev_list);
79/*
80 * This mutex should be held when accessing either of the above lists
81 */
82static DEFINE_MUTEX(mlx5_ib_multiport_mutex);
83
84struct mlx5_ib_dev *mlx5_ib_get_ibdev_from_mpi(struct mlx5_ib_multiport_info *mpi)
85{
86 struct mlx5_ib_dev *dev;
87
88 mutex_lock(&mlx5_ib_multiport_mutex);
89 dev = mpi->ibdev;
90 mutex_unlock(&mlx5_ib_multiport_mutex);
91 return dev;
92}
93
94static enum rdma_link_layer
95mlx5_port_type_cap_to_rdma_ll(int port_type_cap)
96{
97 switch (port_type_cap) {
98 case MLX5_CAP_PORT_TYPE_IB:
99 return IB_LINK_LAYER_INFINIBAND;
100 case MLX5_CAP_PORT_TYPE_ETH:
101 return IB_LINK_LAYER_ETHERNET;
102 default:
103 return IB_LINK_LAYER_UNSPECIFIED;
104 }
105}
106
107static enum rdma_link_layer
108mlx5_ib_port_link_layer(struct ib_device *device, u32 port_num)
109{
110 struct mlx5_ib_dev *dev = to_mdev(device);
111 int port_type_cap = MLX5_CAP_GEN(dev->mdev, port_type);
112
113 return mlx5_port_type_cap_to_rdma_ll(port_type_cap);
114}
115
116static int get_port_state(struct ib_device *ibdev,
117 u32 port_num,
118 enum ib_port_state *state)
119{
120 struct ib_port_attr attr;
121 int ret;
122
123 memset(&attr, 0, sizeof(attr));
124 ret = ibdev->ops.query_port(ibdev, port_num, &attr);
125 if (!ret)
126 *state = attr.state;
127 return ret;
128}
129
130static struct mlx5_roce *mlx5_get_rep_roce(struct mlx5_ib_dev *dev,
131 struct net_device *ndev,
132 struct net_device *upper,
133 u32 *port_num)
134{
135 struct net_device *rep_ndev;
136 struct mlx5_ib_port *port;
137 int i;
138
139 for (i = 0; i < dev->num_ports; i++) {
140 port = &dev->port[i];
141 if (!port->rep)
142 continue;
143
144 if (upper == ndev && port->rep->vport == MLX5_VPORT_UPLINK) {
145 *port_num = i + 1;
146 return &port->roce;
147 }
148
149 if (upper && port->rep->vport == MLX5_VPORT_UPLINK)
150 continue;
151 rep_ndev = ib_device_get_netdev(&dev->ib_dev, i + 1);
152 if (rep_ndev && rep_ndev == ndev) {
153 dev_put(rep_ndev);
154 *port_num = i + 1;
155 return &port->roce;
156 }
157
158 dev_put(rep_ndev);
159 }
160
161 return NULL;
162}
163
164static bool mlx5_netdev_send_event(struct mlx5_ib_dev *dev,
165 struct net_device *ndev,
166 struct net_device *upper,
167 struct net_device *ib_ndev)
168{
169 if (!dev->ib_active)
170 return false;
171
172 /* Event is about our upper device */
173 if (upper == ndev)
174 return true;
175
176 /* RDMA device is not in lag and not in switchdev */
177 if (!dev->is_rep && !upper && ndev == ib_ndev)
178 return true;
179
180 /* RDMA devie is in switchdev */
181 if (dev->is_rep && ndev == ib_ndev)
182 return true;
183
184 return false;
185}
186
187static struct net_device *mlx5_ib_get_rep_uplink_netdev(struct mlx5_ib_dev *ibdev)
188{
189 struct mlx5_ib_port *port;
190 int i;
191
192 for (i = 0; i < ibdev->num_ports; i++) {
193 port = &ibdev->port[i];
194 if (port->rep && port->rep->vport == MLX5_VPORT_UPLINK) {
195 return ib_device_get_netdev(&ibdev->ib_dev, i + 1);
196 }
197 }
198
199 return NULL;
200}
201
202static int mlx5_netdev_event(struct notifier_block *this,
203 unsigned long event, void *ptr)
204{
205 struct mlx5_roce *roce = container_of(this, struct mlx5_roce, nb);
206 struct net_device *ndev = netdev_notifier_info_to_dev(ptr);
207 u32 port_num = roce->native_port_num;
208 struct net_device *ib_ndev = NULL;
209 struct mlx5_core_dev *mdev;
210 struct mlx5_ib_dev *ibdev;
211
212 ibdev = roce->dev;
213 mdev = mlx5_ib_get_native_port_mdev(ibdev, port_num, NULL);
214 if (!mdev)
215 return NOTIFY_DONE;
216
217 switch (event) {
218 case NETDEV_REGISTER:
219 /* Should already be registered during the load */
220 if (ibdev->is_rep)
221 break;
222
223 ib_ndev = ib_device_get_netdev(&ibdev->ib_dev, port_num);
224 /* Exit if already registered */
225 if (ib_ndev)
226 goto put_ndev;
227
228 if (ndev->dev.parent == mdev->device)
229 ib_device_set_netdev(&ibdev->ib_dev, ndev, port_num);
230 break;
231
232 case NETDEV_UNREGISTER:
233 /* In case of reps, ib device goes away before the netdevs */
234 if (ibdev->is_rep)
235 break;
236 ib_ndev = ib_device_get_netdev(&ibdev->ib_dev, port_num);
237 if (ib_ndev == ndev)
238 ib_device_set_netdev(&ibdev->ib_dev, NULL, port_num);
239 goto put_ndev;
240
241 case NETDEV_CHANGE:
242 case NETDEV_UP:
243 case NETDEV_DOWN: {
244 struct net_device *upper = NULL;
245
246 if (!netif_is_lag_master(ndev) && !netif_is_lag_port(ndev) &&
247 !mlx5_core_mp_enabled(mdev))
248 return NOTIFY_DONE;
249
250 if (mlx5_lag_is_roce(mdev) || mlx5_lag_is_sriov(mdev)) {
251 struct net_device *lag_ndev;
252
253 if(mlx5_lag_is_roce(mdev))
254 lag_ndev = ib_device_get_netdev(&ibdev->ib_dev, 1);
255 else /* sriov lag */
256 lag_ndev = mlx5_ib_get_rep_uplink_netdev(ibdev);
257
258 if (lag_ndev) {
259 upper = netdev_master_upper_dev_get(lag_ndev);
260 dev_put(lag_ndev);
261 } else {
262 goto done;
263 }
264 }
265
266 if (ibdev->is_rep)
267 roce = mlx5_get_rep_roce(ibdev, ndev, upper, &port_num);
268 if (!roce)
269 return NOTIFY_DONE;
270
271 ib_ndev = ib_device_get_netdev(&ibdev->ib_dev, port_num);
272
273 if (mlx5_netdev_send_event(ibdev, ndev, upper, ib_ndev)) {
274 struct ib_event ibev = { };
275 enum ib_port_state port_state;
276
277 if (get_port_state(&ibdev->ib_dev, port_num,
278 &port_state))
279 goto put_ndev;
280
281 if (roce->last_port_state == port_state)
282 goto put_ndev;
283
284 roce->last_port_state = port_state;
285 ibev.device = &ibdev->ib_dev;
286 if (port_state == IB_PORT_DOWN)
287 ibev.event = IB_EVENT_PORT_ERR;
288 else if (port_state == IB_PORT_ACTIVE)
289 ibev.event = IB_EVENT_PORT_ACTIVE;
290 else
291 goto put_ndev;
292
293 ibev.element.port_num = port_num;
294 ib_dispatch_event(&ibev);
295 }
296 break;
297 }
298
299 default:
300 break;
301 }
302put_ndev:
303 dev_put(ib_ndev);
304done:
305 mlx5_ib_put_native_port_mdev(ibdev, port_num);
306 return NOTIFY_DONE;
307}
308
309struct mlx5_core_dev *mlx5_ib_get_native_port_mdev(struct mlx5_ib_dev *ibdev,
310 u32 ib_port_num,
311 u32 *native_port_num)
312{
313 enum rdma_link_layer ll = mlx5_ib_port_link_layer(&ibdev->ib_dev,
314 ib_port_num);
315 struct mlx5_core_dev *mdev = NULL;
316 struct mlx5_ib_multiport_info *mpi;
317 struct mlx5_ib_port *port;
318
319 if (ibdev->ib_dev.type == RDMA_DEVICE_TYPE_SMI) {
320 if (native_port_num)
321 *native_port_num = smi_to_native_portnum(ibdev,
322 ib_port_num);
323 return ibdev->mdev;
324
325 }
326
327 if (!mlx5_core_mp_enabled(ibdev->mdev) ||
328 ll != IB_LINK_LAYER_ETHERNET) {
329 if (native_port_num)
330 *native_port_num = ib_port_num;
331 return ibdev->mdev;
332 }
333
334 if (native_port_num)
335 *native_port_num = 1;
336
337 port = &ibdev->port[ib_port_num - 1];
338 spin_lock(&port->mp.mpi_lock);
339 mpi = ibdev->port[ib_port_num - 1].mp.mpi;
340 if (mpi && !mpi->unaffiliate) {
341 mdev = mpi->mdev;
342 /* If it's the master no need to refcount, it'll exist
343 * as long as the ib_dev exists.
344 */
345 if (!mpi->is_master)
346 mpi->mdev_refcnt++;
347 }
348 spin_unlock(&port->mp.mpi_lock);
349
350 return mdev;
351}
352
353void mlx5_ib_put_native_port_mdev(struct mlx5_ib_dev *ibdev, u32 port_num)
354{
355 enum rdma_link_layer ll = mlx5_ib_port_link_layer(&ibdev->ib_dev,
356 port_num);
357 struct mlx5_ib_multiport_info *mpi;
358 struct mlx5_ib_port *port;
359
360 if (!mlx5_core_mp_enabled(ibdev->mdev) || ll != IB_LINK_LAYER_ETHERNET)
361 return;
362
363 port = &ibdev->port[port_num - 1];
364
365 spin_lock(&port->mp.mpi_lock);
366 mpi = ibdev->port[port_num - 1].mp.mpi;
367 if (mpi->is_master)
368 goto out;
369
370 mpi->mdev_refcnt--;
371 if (mpi->unaffiliate)
372 complete(&mpi->unref_comp);
373out:
374 spin_unlock(&port->mp.mpi_lock);
375}
376
377static int translate_eth_legacy_proto_oper(u32 eth_proto_oper,
378 u16 *active_speed, u8 *active_width)
379{
380 switch (eth_proto_oper) {
381 case MLX5E_PROT_MASK(MLX5E_1000BASE_CX_SGMII):
382 case MLX5E_PROT_MASK(MLX5E_1000BASE_KX):
383 case MLX5E_PROT_MASK(MLX5E_100BASE_TX):
384 case MLX5E_PROT_MASK(MLX5E_1000BASE_T):
385 *active_width = IB_WIDTH_1X;
386 *active_speed = IB_SPEED_SDR;
387 break;
388 case MLX5E_PROT_MASK(MLX5E_10GBASE_T):
389 case MLX5E_PROT_MASK(MLX5E_10GBASE_CX4):
390 case MLX5E_PROT_MASK(MLX5E_10GBASE_KX4):
391 case MLX5E_PROT_MASK(MLX5E_10GBASE_KR):
392 case MLX5E_PROT_MASK(MLX5E_10GBASE_CR):
393 case MLX5E_PROT_MASK(MLX5E_10GBASE_SR):
394 case MLX5E_PROT_MASK(MLX5E_10GBASE_ER):
395 *active_width = IB_WIDTH_1X;
396 *active_speed = IB_SPEED_QDR;
397 break;
398 case MLX5E_PROT_MASK(MLX5E_25GBASE_CR):
399 case MLX5E_PROT_MASK(MLX5E_25GBASE_KR):
400 case MLX5E_PROT_MASK(MLX5E_25GBASE_SR):
401 *active_width = IB_WIDTH_1X;
402 *active_speed = IB_SPEED_EDR;
403 break;
404 case MLX5E_PROT_MASK(MLX5E_40GBASE_CR4):
405 case MLX5E_PROT_MASK(MLX5E_40GBASE_KR4):
406 case MLX5E_PROT_MASK(MLX5E_40GBASE_SR4):
407 case MLX5E_PROT_MASK(MLX5E_40GBASE_LR4):
408 *active_width = IB_WIDTH_4X;
409 *active_speed = IB_SPEED_QDR;
410 break;
411 case MLX5E_PROT_MASK(MLX5E_50GBASE_CR2):
412 case MLX5E_PROT_MASK(MLX5E_50GBASE_KR2):
413 case MLX5E_PROT_MASK(MLX5E_50GBASE_SR2):
414 *active_width = IB_WIDTH_1X;
415 *active_speed = IB_SPEED_HDR;
416 break;
417 case MLX5E_PROT_MASK(MLX5E_56GBASE_R4):
418 *active_width = IB_WIDTH_4X;
419 *active_speed = IB_SPEED_FDR;
420 break;
421 case MLX5E_PROT_MASK(MLX5E_100GBASE_CR4):
422 case MLX5E_PROT_MASK(MLX5E_100GBASE_SR4):
423 case MLX5E_PROT_MASK(MLX5E_100GBASE_KR4):
424 case MLX5E_PROT_MASK(MLX5E_100GBASE_LR4):
425 *active_width = IB_WIDTH_4X;
426 *active_speed = IB_SPEED_EDR;
427 break;
428 default:
429 return -EINVAL;
430 }
431
432 return 0;
433}
434
435static int translate_eth_ext_proto_oper(u32 eth_proto_oper, u16 *active_speed,
436 u8 *active_width)
437{
438 switch (eth_proto_oper) {
439 case MLX5E_PROT_MASK(MLX5E_SGMII_100M):
440 case MLX5E_PROT_MASK(MLX5E_1000BASE_X_SGMII):
441 *active_width = IB_WIDTH_1X;
442 *active_speed = IB_SPEED_SDR;
443 break;
444 case MLX5E_PROT_MASK(MLX5E_5GBASE_R):
445 *active_width = IB_WIDTH_1X;
446 *active_speed = IB_SPEED_DDR;
447 break;
448 case MLX5E_PROT_MASK(MLX5E_10GBASE_XFI_XAUI_1):
449 *active_width = IB_WIDTH_1X;
450 *active_speed = IB_SPEED_QDR;
451 break;
452 case MLX5E_PROT_MASK(MLX5E_40GBASE_XLAUI_4_XLPPI_4):
453 *active_width = IB_WIDTH_4X;
454 *active_speed = IB_SPEED_QDR;
455 break;
456 case MLX5E_PROT_MASK(MLX5E_25GAUI_1_25GBASE_CR_KR):
457 *active_width = IB_WIDTH_1X;
458 *active_speed = IB_SPEED_EDR;
459 break;
460 case MLX5E_PROT_MASK(MLX5E_50GAUI_2_LAUI_2_50GBASE_CR2_KR2):
461 *active_width = IB_WIDTH_2X;
462 *active_speed = IB_SPEED_EDR;
463 break;
464 case MLX5E_PROT_MASK(MLX5E_50GAUI_1_LAUI_1_50GBASE_CR_KR):
465 *active_width = IB_WIDTH_1X;
466 *active_speed = IB_SPEED_HDR;
467 break;
468 case MLX5E_PROT_MASK(MLX5E_CAUI_4_100GBASE_CR4_KR4):
469 *active_width = IB_WIDTH_4X;
470 *active_speed = IB_SPEED_EDR;
471 break;
472 case MLX5E_PROT_MASK(MLX5E_100GAUI_2_100GBASE_CR2_KR2):
473 *active_width = IB_WIDTH_2X;
474 *active_speed = IB_SPEED_HDR;
475 break;
476 case MLX5E_PROT_MASK(MLX5E_100GAUI_1_100GBASE_CR_KR):
477 *active_width = IB_WIDTH_1X;
478 *active_speed = IB_SPEED_NDR;
479 break;
480 case MLX5E_PROT_MASK(MLX5E_200GAUI_4_200GBASE_CR4_KR4):
481 *active_width = IB_WIDTH_4X;
482 *active_speed = IB_SPEED_HDR;
483 break;
484 case MLX5E_PROT_MASK(MLX5E_200GAUI_2_200GBASE_CR2_KR2):
485 *active_width = IB_WIDTH_2X;
486 *active_speed = IB_SPEED_NDR;
487 break;
488 case MLX5E_PROT_MASK(MLX5E_200GAUI_1_200GBASE_CR1_KR1):
489 *active_width = IB_WIDTH_1X;
490 *active_speed = IB_SPEED_XDR;
491 break;
492 case MLX5E_PROT_MASK(MLX5E_400GAUI_8_400GBASE_CR8):
493 *active_width = IB_WIDTH_8X;
494 *active_speed = IB_SPEED_HDR;
495 break;
496 case MLX5E_PROT_MASK(MLX5E_400GAUI_4_400GBASE_CR4_KR4):
497 *active_width = IB_WIDTH_4X;
498 *active_speed = IB_SPEED_NDR;
499 break;
500 case MLX5E_PROT_MASK(MLX5E_400GAUI_2_400GBASE_CR2_KR2):
501 *active_width = IB_WIDTH_2X;
502 *active_speed = IB_SPEED_XDR;
503 break;
504 case MLX5E_PROT_MASK(MLX5E_800GAUI_8_800GBASE_CR8_KR8):
505 *active_width = IB_WIDTH_8X;
506 *active_speed = IB_SPEED_NDR;
507 break;
508 case MLX5E_PROT_MASK(MLX5E_800GAUI_4_800GBASE_CR4_KR4):
509 *active_width = IB_WIDTH_4X;
510 *active_speed = IB_SPEED_XDR;
511 break;
512 default:
513 return -EINVAL;
514 }
515
516 return 0;
517}
518
519static int translate_eth_proto_oper(u32 eth_proto_oper, u16 *active_speed,
520 u8 *active_width, bool ext)
521{
522 return ext ?
523 translate_eth_ext_proto_oper(eth_proto_oper, active_speed,
524 active_width) :
525 translate_eth_legacy_proto_oper(eth_proto_oper, active_speed,
526 active_width);
527}
528
529static int mlx5_query_port_roce(struct ib_device *device, u32 port_num,
530 struct ib_port_attr *props)
531{
532 struct mlx5_ib_dev *dev = to_mdev(device);
533 u32 out[MLX5_ST_SZ_DW(ptys_reg)] = {0};
534 struct mlx5_core_dev *mdev;
535 struct net_device *ndev, *upper;
536 enum ib_mtu ndev_ib_mtu;
537 bool put_mdev = true;
538 u32 eth_prot_oper;
539 u32 mdev_port_num;
540 bool ext;
541 int err;
542
543 mdev = mlx5_ib_get_native_port_mdev(dev, port_num, &mdev_port_num);
544 if (!mdev) {
545 /* This means the port isn't affiliated yet. Get the
546 * info for the master port instead.
547 */
548 put_mdev = false;
549 mdev = dev->mdev;
550 mdev_port_num = 1;
551 port_num = 1;
552 }
553
554 /* Possible bad flows are checked before filling out props so in case
555 * of an error it will still be zeroed out.
556 * Use native port in case of reps
557 */
558 if (dev->is_rep)
559 err = mlx5_query_port_ptys(mdev, out, sizeof(out), MLX5_PTYS_EN,
560 1, 0);
561 else
562 err = mlx5_query_port_ptys(mdev, out, sizeof(out), MLX5_PTYS_EN,
563 mdev_port_num, 0);
564 if (err)
565 goto out;
566 ext = !!MLX5_GET_ETH_PROTO(ptys_reg, out, true, eth_proto_capability);
567 eth_prot_oper = MLX5_GET_ETH_PROTO(ptys_reg, out, ext, eth_proto_oper);
568
569 props->active_width = IB_WIDTH_4X;
570 props->active_speed = IB_SPEED_QDR;
571
572 translate_eth_proto_oper(eth_prot_oper, &props->active_speed,
573 &props->active_width, ext);
574
575 if (!dev->is_rep && dev->mdev->roce.roce_en) {
576 u16 qkey_viol_cntr;
577
578 props->port_cap_flags |= IB_PORT_CM_SUP;
579 props->ip_gids = true;
580 props->gid_tbl_len = MLX5_CAP_ROCE(dev->mdev,
581 roce_address_table_size);
582 mlx5_query_nic_vport_qkey_viol_cntr(mdev, &qkey_viol_cntr);
583 props->qkey_viol_cntr = qkey_viol_cntr;
584 }
585 props->max_mtu = IB_MTU_4096;
586 props->max_msg_sz = 1 << MLX5_CAP_GEN(dev->mdev, log_max_msg);
587 props->pkey_tbl_len = 1;
588 props->state = IB_PORT_DOWN;
589 props->phys_state = IB_PORT_PHYS_STATE_DISABLED;
590
591 /* If this is a stub query for an unaffiliated port stop here */
592 if (!put_mdev)
593 goto out;
594
595 ndev = ib_device_get_netdev(device, port_num);
596 if (!ndev)
597 goto out;
598
599 if (mlx5_lag_is_roce(mdev) || mlx5_lag_is_sriov(mdev)) {
600 rcu_read_lock();
601 upper = netdev_master_upper_dev_get_rcu(ndev);
602 if (upper) {
603 dev_put(ndev);
604 ndev = upper;
605 dev_hold(ndev);
606 }
607 rcu_read_unlock();
608 }
609
610 if (netif_running(ndev) && netif_carrier_ok(ndev)) {
611 props->state = IB_PORT_ACTIVE;
612 props->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
613 }
614
615 ndev_ib_mtu = iboe_get_mtu(ndev->mtu);
616
617 dev_put(ndev);
618
619 props->active_mtu = min(props->max_mtu, ndev_ib_mtu);
620out:
621 if (put_mdev)
622 mlx5_ib_put_native_port_mdev(dev, port_num);
623 return err;
624}
625
626int set_roce_addr(struct mlx5_ib_dev *dev, u32 port_num,
627 unsigned int index, const union ib_gid *gid,
628 const struct ib_gid_attr *attr)
629{
630 enum ib_gid_type gid_type;
631 u16 vlan_id = 0xffff;
632 u8 roce_version = 0;
633 u8 roce_l3_type = 0;
634 u8 mac[ETH_ALEN];
635 int ret;
636
637 gid_type = attr->gid_type;
638 if (gid) {
639 ret = rdma_read_gid_l2_fields(attr, &vlan_id, &mac[0]);
640 if (ret)
641 return ret;
642 }
643
644 switch (gid_type) {
645 case IB_GID_TYPE_ROCE:
646 roce_version = MLX5_ROCE_VERSION_1;
647 break;
648 case IB_GID_TYPE_ROCE_UDP_ENCAP:
649 roce_version = MLX5_ROCE_VERSION_2;
650 if (gid && ipv6_addr_v4mapped((void *)gid))
651 roce_l3_type = MLX5_ROCE_L3_TYPE_IPV4;
652 else
653 roce_l3_type = MLX5_ROCE_L3_TYPE_IPV6;
654 break;
655
656 default:
657 mlx5_ib_warn(dev, "Unexpected GID type %u\n", gid_type);
658 }
659
660 return mlx5_core_roce_gid_set(dev->mdev, index, roce_version,
661 roce_l3_type, gid->raw, mac,
662 vlan_id < VLAN_CFI_MASK, vlan_id,
663 port_num);
664}
665
666static int mlx5_ib_add_gid(const struct ib_gid_attr *attr,
667 __always_unused void **context)
668{
669 int ret;
670
671 ret = mlx5r_add_gid_macsec_operations(attr);
672 if (ret)
673 return ret;
674
675 return set_roce_addr(to_mdev(attr->device), attr->port_num,
676 attr->index, &attr->gid, attr);
677}
678
679static int mlx5_ib_del_gid(const struct ib_gid_attr *attr,
680 __always_unused void **context)
681{
682 int ret;
683
684 ret = set_roce_addr(to_mdev(attr->device), attr->port_num,
685 attr->index, NULL, attr);
686 if (ret)
687 return ret;
688
689 mlx5r_del_gid_macsec_operations(attr);
690 return 0;
691}
692
693__be16 mlx5_get_roce_udp_sport_min(const struct mlx5_ib_dev *dev,
694 const struct ib_gid_attr *attr)
695{
696 if (attr->gid_type != IB_GID_TYPE_ROCE_UDP_ENCAP)
697 return 0;
698
699 return cpu_to_be16(MLX5_CAP_ROCE(dev->mdev, r_roce_min_src_udp_port));
700}
701
702static int mlx5_use_mad_ifc(struct mlx5_ib_dev *dev)
703{
704 if (MLX5_CAP_GEN(dev->mdev, port_type) == MLX5_CAP_PORT_TYPE_IB)
705 return !MLX5_CAP_GEN(dev->mdev, ib_virt);
706 return 0;
707}
708
709enum {
710 MLX5_VPORT_ACCESS_METHOD_MAD,
711 MLX5_VPORT_ACCESS_METHOD_HCA,
712 MLX5_VPORT_ACCESS_METHOD_NIC,
713};
714
715static int mlx5_get_vport_access_method(struct ib_device *ibdev)
716{
717 if (mlx5_use_mad_ifc(to_mdev(ibdev)))
718 return MLX5_VPORT_ACCESS_METHOD_MAD;
719
720 if (mlx5_ib_port_link_layer(ibdev, 1) ==
721 IB_LINK_LAYER_ETHERNET)
722 return MLX5_VPORT_ACCESS_METHOD_NIC;
723
724 return MLX5_VPORT_ACCESS_METHOD_HCA;
725}
726
727static void get_atomic_caps(struct mlx5_ib_dev *dev,
728 u8 atomic_size_qp,
729 struct ib_device_attr *props)
730{
731 u8 tmp;
732 u8 atomic_operations = MLX5_CAP_ATOMIC(dev->mdev, atomic_operations);
733 u8 atomic_req_8B_endianness_mode =
734 MLX5_CAP_ATOMIC(dev->mdev, atomic_req_8B_endianness_mode);
735
736 /* Check if HW supports 8 bytes standard atomic operations and capable
737 * of host endianness respond
738 */
739 tmp = MLX5_ATOMIC_OPS_CMP_SWAP | MLX5_ATOMIC_OPS_FETCH_ADD;
740 if (((atomic_operations & tmp) == tmp) &&
741 (atomic_size_qp & MLX5_ATOMIC_SIZE_QP_8BYTES) &&
742 (atomic_req_8B_endianness_mode)) {
743 props->atomic_cap = IB_ATOMIC_HCA;
744 } else {
745 props->atomic_cap = IB_ATOMIC_NONE;
746 }
747}
748
749static void get_atomic_caps_qp(struct mlx5_ib_dev *dev,
750 struct ib_device_attr *props)
751{
752 u8 atomic_size_qp = MLX5_CAP_ATOMIC(dev->mdev, atomic_size_qp);
753
754 get_atomic_caps(dev, atomic_size_qp, props);
755}
756
757static int mlx5_query_system_image_guid(struct ib_device *ibdev,
758 __be64 *sys_image_guid)
759{
760 struct mlx5_ib_dev *dev = to_mdev(ibdev);
761 struct mlx5_core_dev *mdev = dev->mdev;
762 u64 tmp;
763 int err;
764
765 switch (mlx5_get_vport_access_method(ibdev)) {
766 case MLX5_VPORT_ACCESS_METHOD_MAD:
767 return mlx5_query_mad_ifc_system_image_guid(ibdev,
768 sys_image_guid);
769
770 case MLX5_VPORT_ACCESS_METHOD_HCA:
771 err = mlx5_query_hca_vport_system_image_guid(mdev, &tmp);
772 break;
773
774 case MLX5_VPORT_ACCESS_METHOD_NIC:
775 err = mlx5_query_nic_vport_system_image_guid(mdev, &tmp);
776 break;
777
778 default:
779 return -EINVAL;
780 }
781
782 if (!err)
783 *sys_image_guid = cpu_to_be64(tmp);
784
785 return err;
786
787}
788
789static int mlx5_query_max_pkeys(struct ib_device *ibdev,
790 u16 *max_pkeys)
791{
792 struct mlx5_ib_dev *dev = to_mdev(ibdev);
793 struct mlx5_core_dev *mdev = dev->mdev;
794
795 switch (mlx5_get_vport_access_method(ibdev)) {
796 case MLX5_VPORT_ACCESS_METHOD_MAD:
797 return mlx5_query_mad_ifc_max_pkeys(ibdev, max_pkeys);
798
799 case MLX5_VPORT_ACCESS_METHOD_HCA:
800 case MLX5_VPORT_ACCESS_METHOD_NIC:
801 *max_pkeys = mlx5_to_sw_pkey_sz(MLX5_CAP_GEN(mdev,
802 pkey_table_size));
803 return 0;
804
805 default:
806 return -EINVAL;
807 }
808}
809
810static int mlx5_query_vendor_id(struct ib_device *ibdev,
811 u32 *vendor_id)
812{
813 struct mlx5_ib_dev *dev = to_mdev(ibdev);
814
815 switch (mlx5_get_vport_access_method(ibdev)) {
816 case MLX5_VPORT_ACCESS_METHOD_MAD:
817 return mlx5_query_mad_ifc_vendor_id(ibdev, vendor_id);
818
819 case MLX5_VPORT_ACCESS_METHOD_HCA:
820 case MLX5_VPORT_ACCESS_METHOD_NIC:
821 return mlx5_core_query_vendor_id(dev->mdev, vendor_id);
822
823 default:
824 return -EINVAL;
825 }
826}
827
828static int mlx5_query_node_guid(struct mlx5_ib_dev *dev,
829 __be64 *node_guid)
830{
831 u64 tmp;
832 int err;
833
834 switch (mlx5_get_vport_access_method(&dev->ib_dev)) {
835 case MLX5_VPORT_ACCESS_METHOD_MAD:
836 return mlx5_query_mad_ifc_node_guid(dev, node_guid);
837
838 case MLX5_VPORT_ACCESS_METHOD_HCA:
839 err = mlx5_query_hca_vport_node_guid(dev->mdev, &tmp);
840 break;
841
842 case MLX5_VPORT_ACCESS_METHOD_NIC:
843 err = mlx5_query_nic_vport_node_guid(dev->mdev, &tmp);
844 break;
845
846 default:
847 return -EINVAL;
848 }
849
850 if (!err)
851 *node_guid = cpu_to_be64(tmp);
852
853 return err;
854}
855
856struct mlx5_reg_node_desc {
857 u8 desc[IB_DEVICE_NODE_DESC_MAX];
858};
859
860static int mlx5_query_node_desc(struct mlx5_ib_dev *dev, char *node_desc)
861{
862 struct mlx5_reg_node_desc in;
863
864 if (mlx5_use_mad_ifc(dev))
865 return mlx5_query_mad_ifc_node_desc(dev, node_desc);
866
867 memset(&in, 0, sizeof(in));
868
869 return mlx5_core_access_reg(dev->mdev, &in, sizeof(in), node_desc,
870 sizeof(struct mlx5_reg_node_desc),
871 MLX5_REG_NODE_DESC, 0, 0);
872}
873
874static void fill_esw_mgr_reg_c0(struct mlx5_core_dev *mdev,
875 struct mlx5_ib_query_device_resp *resp)
876{
877 struct mlx5_eswitch *esw = mdev->priv.eswitch;
878 u16 vport = mlx5_eswitch_manager_vport(mdev);
879
880 resp->reg_c0.value = mlx5_eswitch_get_vport_metadata_for_match(esw,
881 vport);
882 resp->reg_c0.mask = mlx5_eswitch_get_vport_metadata_mask();
883}
884
885static int mlx5_ib_query_device(struct ib_device *ibdev,
886 struct ib_device_attr *props,
887 struct ib_udata *uhw)
888{
889 size_t uhw_outlen = (uhw) ? uhw->outlen : 0;
890 struct mlx5_ib_dev *dev = to_mdev(ibdev);
891 struct mlx5_core_dev *mdev = dev->mdev;
892 int err = -ENOMEM;
893 int max_sq_desc;
894 int max_rq_sg;
895 int max_sq_sg;
896 u64 min_page_size = 1ull << MLX5_CAP_GEN(mdev, log_pg_sz);
897 bool raw_support = !mlx5_core_mp_enabled(mdev);
898 struct mlx5_ib_query_device_resp resp = {};
899 size_t resp_len;
900 u64 max_tso;
901
902 resp_len = sizeof(resp.comp_mask) + sizeof(resp.response_length);
903 if (uhw_outlen && uhw_outlen < resp_len)
904 return -EINVAL;
905
906 resp.response_length = resp_len;
907
908 if (uhw && uhw->inlen && !ib_is_udata_cleared(uhw, 0, uhw->inlen))
909 return -EINVAL;
910
911 memset(props, 0, sizeof(*props));
912 err = mlx5_query_system_image_guid(ibdev,
913 &props->sys_image_guid);
914 if (err)
915 return err;
916
917 props->max_pkeys = dev->pkey_table_len;
918
919 err = mlx5_query_vendor_id(ibdev, &props->vendor_id);
920 if (err)
921 return err;
922
923 props->fw_ver = ((u64)fw_rev_maj(dev->mdev) << 32) |
924 (fw_rev_min(dev->mdev) << 16) |
925 fw_rev_sub(dev->mdev);
926 props->device_cap_flags = IB_DEVICE_CHANGE_PHY_PORT |
927 IB_DEVICE_PORT_ACTIVE_EVENT |
928 IB_DEVICE_SYS_IMAGE_GUID |
929 IB_DEVICE_RC_RNR_NAK_GEN;
930
931 if (MLX5_CAP_GEN(mdev, pkv))
932 props->device_cap_flags |= IB_DEVICE_BAD_PKEY_CNTR;
933 if (MLX5_CAP_GEN(mdev, qkv))
934 props->device_cap_flags |= IB_DEVICE_BAD_QKEY_CNTR;
935 if (MLX5_CAP_GEN(mdev, apm))
936 props->device_cap_flags |= IB_DEVICE_AUTO_PATH_MIG;
937 if (MLX5_CAP_GEN(mdev, xrc))
938 props->device_cap_flags |= IB_DEVICE_XRC;
939 if (MLX5_CAP_GEN(mdev, imaicl)) {
940 props->device_cap_flags |= IB_DEVICE_MEM_WINDOW |
941 IB_DEVICE_MEM_WINDOW_TYPE_2B;
942 props->max_mw = 1 << MLX5_CAP_GEN(mdev, log_max_mkey);
943 /* We support 'Gappy' memory registration too */
944 props->kernel_cap_flags |= IBK_SG_GAPS_REG;
945 }
946 /* IB_WR_REG_MR always requires changing the entity size with UMR */
947 if (!MLX5_CAP_GEN(dev->mdev, umr_modify_entity_size_disabled))
948 props->device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS;
949 if (MLX5_CAP_GEN(mdev, sho)) {
950 props->kernel_cap_flags |= IBK_INTEGRITY_HANDOVER;
951 /* At this stage no support for signature handover */
952 props->sig_prot_cap = IB_PROT_T10DIF_TYPE_1 |
953 IB_PROT_T10DIF_TYPE_2 |
954 IB_PROT_T10DIF_TYPE_3;
955 props->sig_guard_cap = IB_GUARD_T10DIF_CRC |
956 IB_GUARD_T10DIF_CSUM;
957 }
958 if (MLX5_CAP_GEN(mdev, block_lb_mc))
959 props->kernel_cap_flags |= IBK_BLOCK_MULTICAST_LOOPBACK;
960
961 if (MLX5_CAP_GEN(dev->mdev, eth_net_offloads) && raw_support) {
962 if (MLX5_CAP_ETH(mdev, csum_cap)) {
963 /* Legacy bit to support old userspace libraries */
964 props->device_cap_flags |= IB_DEVICE_RAW_IP_CSUM;
965 props->raw_packet_caps |= IB_RAW_PACKET_CAP_IP_CSUM;
966 }
967
968 if (MLX5_CAP_ETH(dev->mdev, vlan_cap))
969 props->raw_packet_caps |=
970 IB_RAW_PACKET_CAP_CVLAN_STRIPPING;
971
972 if (offsetofend(typeof(resp), tso_caps) <= uhw_outlen) {
973 max_tso = MLX5_CAP_ETH(mdev, max_lso_cap);
974 if (max_tso) {
975 resp.tso_caps.max_tso = 1 << max_tso;
976 resp.tso_caps.supported_qpts |=
977 1 << IB_QPT_RAW_PACKET;
978 resp.response_length += sizeof(resp.tso_caps);
979 }
980 }
981
982 if (offsetofend(typeof(resp), rss_caps) <= uhw_outlen) {
983 resp.rss_caps.rx_hash_function =
984 MLX5_RX_HASH_FUNC_TOEPLITZ;
985 resp.rss_caps.rx_hash_fields_mask =
986 MLX5_RX_HASH_SRC_IPV4 |
987 MLX5_RX_HASH_DST_IPV4 |
988 MLX5_RX_HASH_SRC_IPV6 |
989 MLX5_RX_HASH_DST_IPV6 |
990 MLX5_RX_HASH_SRC_PORT_TCP |
991 MLX5_RX_HASH_DST_PORT_TCP |
992 MLX5_RX_HASH_SRC_PORT_UDP |
993 MLX5_RX_HASH_DST_PORT_UDP |
994 MLX5_RX_HASH_INNER;
995 resp.response_length += sizeof(resp.rss_caps);
996 }
997 } else {
998 if (offsetofend(typeof(resp), tso_caps) <= uhw_outlen)
999 resp.response_length += sizeof(resp.tso_caps);
1000 if (offsetofend(typeof(resp), rss_caps) <= uhw_outlen)
1001 resp.response_length += sizeof(resp.rss_caps);
1002 }
1003
1004 if (MLX5_CAP_GEN(mdev, ipoib_basic_offloads)) {
1005 props->device_cap_flags |= IB_DEVICE_UD_IP_CSUM;
1006 props->kernel_cap_flags |= IBK_UD_TSO;
1007 }
1008
1009 if (MLX5_CAP_GEN(dev->mdev, rq_delay_drop) &&
1010 MLX5_CAP_GEN(dev->mdev, general_notification_event) &&
1011 raw_support)
1012 props->raw_packet_caps |= IB_RAW_PACKET_CAP_DELAY_DROP;
1013
1014 if (MLX5_CAP_GEN(mdev, ipoib_enhanced_offloads) &&
1015 MLX5_CAP_IPOIB_ENHANCED(mdev, csum_cap))
1016 props->device_cap_flags |= IB_DEVICE_UD_IP_CSUM;
1017
1018 if (MLX5_CAP_GEN(dev->mdev, eth_net_offloads) &&
1019 MLX5_CAP_ETH(dev->mdev, scatter_fcs) &&
1020 raw_support) {
1021 /* Legacy bit to support old userspace libraries */
1022 props->device_cap_flags |= IB_DEVICE_RAW_SCATTER_FCS;
1023 props->raw_packet_caps |= IB_RAW_PACKET_CAP_SCATTER_FCS;
1024 }
1025
1026 if (MLX5_CAP_DEV_MEM(mdev, memic)) {
1027 props->max_dm_size =
1028 MLX5_CAP_DEV_MEM(mdev, max_memic_size);
1029 }
1030
1031 if (mlx5_get_flow_namespace(dev->mdev, MLX5_FLOW_NAMESPACE_BYPASS))
1032 props->device_cap_flags |= IB_DEVICE_MANAGED_FLOW_STEERING;
1033
1034 if (MLX5_CAP_GEN(mdev, end_pad))
1035 props->device_cap_flags |= IB_DEVICE_PCI_WRITE_END_PADDING;
1036
1037 props->vendor_part_id = mdev->pdev->device;
1038 props->hw_ver = mdev->pdev->revision;
1039
1040 props->max_mr_size = ~0ull;
1041 props->page_size_cap = ~(min_page_size - 1);
1042 props->max_qp = 1 << MLX5_CAP_GEN(mdev, log_max_qp);
1043 props->max_qp_wr = 1 << MLX5_CAP_GEN(mdev, log_max_qp_sz);
1044 max_rq_sg = MLX5_CAP_GEN(mdev, max_wqe_sz_rq) /
1045 sizeof(struct mlx5_wqe_data_seg);
1046 max_sq_desc = min_t(int, MLX5_CAP_GEN(mdev, max_wqe_sz_sq), 512);
1047 max_sq_sg = (max_sq_desc - sizeof(struct mlx5_wqe_ctrl_seg) -
1048 sizeof(struct mlx5_wqe_raddr_seg)) /
1049 sizeof(struct mlx5_wqe_data_seg);
1050 props->max_send_sge = max_sq_sg;
1051 props->max_recv_sge = max_rq_sg;
1052 props->max_sge_rd = MLX5_MAX_SGE_RD;
1053 props->max_cq = 1 << MLX5_CAP_GEN(mdev, log_max_cq);
1054 props->max_cqe = (1 << MLX5_CAP_GEN(mdev, log_max_cq_sz)) - 1;
1055 props->max_mr = 1 << MLX5_CAP_GEN(mdev, log_max_mkey);
1056 props->max_pd = 1 << MLX5_CAP_GEN(mdev, log_max_pd);
1057 props->max_qp_rd_atom = 1 << MLX5_CAP_GEN(mdev, log_max_ra_req_qp);
1058 props->max_qp_init_rd_atom = 1 << MLX5_CAP_GEN(mdev, log_max_ra_res_qp);
1059 props->max_srq = 1 << MLX5_CAP_GEN(mdev, log_max_srq);
1060 props->max_srq_wr = (1 << MLX5_CAP_GEN(mdev, log_max_srq_sz)) - 1;
1061 props->local_ca_ack_delay = MLX5_CAP_GEN(mdev, local_ca_ack_delay);
1062 props->max_res_rd_atom = props->max_qp_rd_atom * props->max_qp;
1063 props->max_srq_sge = max_rq_sg - 1;
1064 props->max_fast_reg_page_list_len =
1065 1 << MLX5_CAP_GEN(mdev, log_max_klm_list_size);
1066 props->max_pi_fast_reg_page_list_len =
1067 props->max_fast_reg_page_list_len / 2;
1068 props->max_sgl_rd =
1069 MLX5_CAP_GEN(mdev, max_sgl_for_optimized_performance);
1070 get_atomic_caps_qp(dev, props);
1071 props->masked_atomic_cap = IB_ATOMIC_NONE;
1072 props->max_mcast_grp = 1 << MLX5_CAP_GEN(mdev, log_max_mcg);
1073 props->max_mcast_qp_attach = MLX5_CAP_GEN(mdev, max_qp_mcg);
1074 props->max_total_mcast_qp_attach = props->max_mcast_qp_attach *
1075 props->max_mcast_grp;
1076 props->max_ah = INT_MAX;
1077 props->hca_core_clock = MLX5_CAP_GEN(mdev, device_frequency_khz);
1078 props->timestamp_mask = 0x7FFFFFFFFFFFFFFFULL;
1079
1080 if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING)) {
1081 if (dev->odp_caps.general_caps & IB_ODP_SUPPORT)
1082 props->kernel_cap_flags |= IBK_ON_DEMAND_PAGING;
1083 props->odp_caps = dev->odp_caps;
1084 if (!uhw) {
1085 /* ODP for kernel QPs is not implemented for receive
1086 * WQEs and SRQ WQEs
1087 */
1088 props->odp_caps.per_transport_caps.rc_odp_caps &=
1089 ~(IB_ODP_SUPPORT_READ |
1090 IB_ODP_SUPPORT_SRQ_RECV);
1091 props->odp_caps.per_transport_caps.uc_odp_caps &=
1092 ~(IB_ODP_SUPPORT_READ |
1093 IB_ODP_SUPPORT_SRQ_RECV);
1094 props->odp_caps.per_transport_caps.ud_odp_caps &=
1095 ~(IB_ODP_SUPPORT_READ |
1096 IB_ODP_SUPPORT_SRQ_RECV);
1097 props->odp_caps.per_transport_caps.xrc_odp_caps &=
1098 ~(IB_ODP_SUPPORT_READ |
1099 IB_ODP_SUPPORT_SRQ_RECV);
1100 }
1101 }
1102
1103 if (mlx5_core_is_vf(mdev))
1104 props->kernel_cap_flags |= IBK_VIRTUAL_FUNCTION;
1105
1106 if (mlx5_ib_port_link_layer(ibdev, 1) ==
1107 IB_LINK_LAYER_ETHERNET && raw_support) {
1108 props->rss_caps.max_rwq_indirection_tables =
1109 1 << MLX5_CAP_GEN(dev->mdev, log_max_rqt);
1110 props->rss_caps.max_rwq_indirection_table_size =
1111 1 << MLX5_CAP_GEN(dev->mdev, log_max_rqt_size);
1112 props->rss_caps.supported_qpts = 1 << IB_QPT_RAW_PACKET;
1113 props->max_wq_type_rq =
1114 1 << MLX5_CAP_GEN(dev->mdev, log_max_rq);
1115 }
1116
1117 if (MLX5_CAP_GEN(mdev, tag_matching)) {
1118 props->tm_caps.max_num_tags =
1119 (1 << MLX5_CAP_GEN(mdev, log_tag_matching_list_sz)) - 1;
1120 props->tm_caps.max_ops =
1121 1 << MLX5_CAP_GEN(mdev, log_max_qp_sz);
1122 props->tm_caps.max_sge = MLX5_TM_MAX_SGE;
1123 }
1124
1125 if (MLX5_CAP_GEN(mdev, tag_matching) &&
1126 MLX5_CAP_GEN(mdev, rndv_offload_rc)) {
1127 props->tm_caps.flags = IB_TM_CAP_RNDV_RC;
1128 props->tm_caps.max_rndv_hdr_size = MLX5_TM_MAX_RNDV_MSG_SIZE;
1129 }
1130
1131 if (MLX5_CAP_GEN(dev->mdev, cq_moderation)) {
1132 props->cq_caps.max_cq_moderation_count =
1133 MLX5_MAX_CQ_COUNT;
1134 props->cq_caps.max_cq_moderation_period =
1135 MLX5_MAX_CQ_PERIOD;
1136 }
1137
1138 if (offsetofend(typeof(resp), cqe_comp_caps) <= uhw_outlen) {
1139 resp.response_length += sizeof(resp.cqe_comp_caps);
1140
1141 if (MLX5_CAP_GEN(dev->mdev, cqe_compression)) {
1142 resp.cqe_comp_caps.max_num =
1143 MLX5_CAP_GEN(dev->mdev,
1144 cqe_compression_max_num);
1145
1146 resp.cqe_comp_caps.supported_format =
1147 MLX5_IB_CQE_RES_FORMAT_HASH |
1148 MLX5_IB_CQE_RES_FORMAT_CSUM;
1149
1150 if (MLX5_CAP_GEN(dev->mdev, mini_cqe_resp_stride_index))
1151 resp.cqe_comp_caps.supported_format |=
1152 MLX5_IB_CQE_RES_FORMAT_CSUM_STRIDX;
1153 }
1154 }
1155
1156 if (offsetofend(typeof(resp), packet_pacing_caps) <= uhw_outlen &&
1157 raw_support) {
1158 if (MLX5_CAP_QOS(mdev, packet_pacing) &&
1159 MLX5_CAP_GEN(mdev, qos)) {
1160 resp.packet_pacing_caps.qp_rate_limit_max =
1161 MLX5_CAP_QOS(mdev, packet_pacing_max_rate);
1162 resp.packet_pacing_caps.qp_rate_limit_min =
1163 MLX5_CAP_QOS(mdev, packet_pacing_min_rate);
1164 resp.packet_pacing_caps.supported_qpts |=
1165 1 << IB_QPT_RAW_PACKET;
1166 if (MLX5_CAP_QOS(mdev, packet_pacing_burst_bound) &&
1167 MLX5_CAP_QOS(mdev, packet_pacing_typical_size))
1168 resp.packet_pacing_caps.cap_flags |=
1169 MLX5_IB_PP_SUPPORT_BURST;
1170 }
1171 resp.response_length += sizeof(resp.packet_pacing_caps);
1172 }
1173
1174 if (offsetofend(typeof(resp), mlx5_ib_support_multi_pkt_send_wqes) <=
1175 uhw_outlen) {
1176 if (MLX5_CAP_ETH(mdev, multi_pkt_send_wqe))
1177 resp.mlx5_ib_support_multi_pkt_send_wqes =
1178 MLX5_IB_ALLOW_MPW;
1179
1180 if (MLX5_CAP_ETH(mdev, enhanced_multi_pkt_send_wqe))
1181 resp.mlx5_ib_support_multi_pkt_send_wqes |=
1182 MLX5_IB_SUPPORT_EMPW;
1183
1184 resp.response_length +=
1185 sizeof(resp.mlx5_ib_support_multi_pkt_send_wqes);
1186 }
1187
1188 if (offsetofend(typeof(resp), flags) <= uhw_outlen) {
1189 resp.response_length += sizeof(resp.flags);
1190
1191 if (MLX5_CAP_GEN(mdev, cqe_compression_128))
1192 resp.flags |=
1193 MLX5_IB_QUERY_DEV_RESP_FLAGS_CQE_128B_COMP;
1194
1195 if (MLX5_CAP_GEN(mdev, cqe_128_always))
1196 resp.flags |= MLX5_IB_QUERY_DEV_RESP_FLAGS_CQE_128B_PAD;
1197 if (MLX5_CAP_GEN(mdev, qp_packet_based))
1198 resp.flags |=
1199 MLX5_IB_QUERY_DEV_RESP_PACKET_BASED_CREDIT_MODE;
1200
1201 resp.flags |= MLX5_IB_QUERY_DEV_RESP_FLAGS_SCAT2CQE_DCT;
1202
1203 if (MLX5_CAP_GEN_2(mdev, dp_ordering_force) &&
1204 (MLX5_CAP_GEN(mdev, dp_ordering_ooo_all_xrc) ||
1205 MLX5_CAP_GEN(mdev, dp_ordering_ooo_all_dc) ||
1206 MLX5_CAP_GEN(mdev, dp_ordering_ooo_all_rc) ||
1207 MLX5_CAP_GEN(mdev, dp_ordering_ooo_all_ud) ||
1208 MLX5_CAP_GEN(mdev, dp_ordering_ooo_all_uc)))
1209 resp.flags |= MLX5_IB_QUERY_DEV_RESP_FLAGS_OOO_DP;
1210 }
1211
1212 if (offsetofend(typeof(resp), sw_parsing_caps) <= uhw_outlen) {
1213 resp.response_length += sizeof(resp.sw_parsing_caps);
1214 if (MLX5_CAP_ETH(mdev, swp)) {
1215 resp.sw_parsing_caps.sw_parsing_offloads |=
1216 MLX5_IB_SW_PARSING;
1217
1218 if (MLX5_CAP_ETH(mdev, swp_csum))
1219 resp.sw_parsing_caps.sw_parsing_offloads |=
1220 MLX5_IB_SW_PARSING_CSUM;
1221
1222 if (MLX5_CAP_ETH(mdev, swp_lso))
1223 resp.sw_parsing_caps.sw_parsing_offloads |=
1224 MLX5_IB_SW_PARSING_LSO;
1225
1226 if (resp.sw_parsing_caps.sw_parsing_offloads)
1227 resp.sw_parsing_caps.supported_qpts =
1228 BIT(IB_QPT_RAW_PACKET);
1229 }
1230 }
1231
1232 if (offsetofend(typeof(resp), striding_rq_caps) <= uhw_outlen &&
1233 raw_support) {
1234 resp.response_length += sizeof(resp.striding_rq_caps);
1235 if (MLX5_CAP_GEN(mdev, striding_rq)) {
1236 resp.striding_rq_caps.min_single_stride_log_num_of_bytes =
1237 MLX5_MIN_SINGLE_STRIDE_LOG_NUM_BYTES;
1238 resp.striding_rq_caps.max_single_stride_log_num_of_bytes =
1239 MLX5_MAX_SINGLE_STRIDE_LOG_NUM_BYTES;
1240 if (MLX5_CAP_GEN(dev->mdev, ext_stride_num_range))
1241 resp.striding_rq_caps
1242 .min_single_wqe_log_num_of_strides =
1243 MLX5_EXT_MIN_SINGLE_WQE_LOG_NUM_STRIDES;
1244 else
1245 resp.striding_rq_caps
1246 .min_single_wqe_log_num_of_strides =
1247 MLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES;
1248 resp.striding_rq_caps.max_single_wqe_log_num_of_strides =
1249 MLX5_MAX_SINGLE_WQE_LOG_NUM_STRIDES;
1250 resp.striding_rq_caps.supported_qpts =
1251 BIT(IB_QPT_RAW_PACKET);
1252 }
1253 }
1254
1255 if (offsetofend(typeof(resp), tunnel_offloads_caps) <= uhw_outlen) {
1256 resp.response_length += sizeof(resp.tunnel_offloads_caps);
1257 if (MLX5_CAP_ETH(mdev, tunnel_stateless_vxlan))
1258 resp.tunnel_offloads_caps |=
1259 MLX5_IB_TUNNELED_OFFLOADS_VXLAN;
1260 if (MLX5_CAP_ETH(mdev, tunnel_stateless_geneve_rx))
1261 resp.tunnel_offloads_caps |=
1262 MLX5_IB_TUNNELED_OFFLOADS_GENEVE;
1263 if (MLX5_CAP_ETH(mdev, tunnel_stateless_gre))
1264 resp.tunnel_offloads_caps |=
1265 MLX5_IB_TUNNELED_OFFLOADS_GRE;
1266 if (MLX5_CAP_ETH(mdev, tunnel_stateless_mpls_over_gre))
1267 resp.tunnel_offloads_caps |=
1268 MLX5_IB_TUNNELED_OFFLOADS_MPLS_GRE;
1269 if (MLX5_CAP_ETH(mdev, tunnel_stateless_mpls_over_udp))
1270 resp.tunnel_offloads_caps |=
1271 MLX5_IB_TUNNELED_OFFLOADS_MPLS_UDP;
1272 }
1273
1274 if (offsetofend(typeof(resp), dci_streams_caps) <= uhw_outlen) {
1275 resp.response_length += sizeof(resp.dci_streams_caps);
1276
1277 resp.dci_streams_caps.max_log_num_concurent =
1278 MLX5_CAP_GEN(mdev, log_max_dci_stream_channels);
1279
1280 resp.dci_streams_caps.max_log_num_errored =
1281 MLX5_CAP_GEN(mdev, log_max_dci_errored_streams);
1282 }
1283
1284 if (offsetofend(typeof(resp), reserved) <= uhw_outlen)
1285 resp.response_length += sizeof(resp.reserved);
1286
1287 if (offsetofend(typeof(resp), reg_c0) <= uhw_outlen) {
1288 struct mlx5_eswitch *esw = mdev->priv.eswitch;
1289
1290 resp.response_length += sizeof(resp.reg_c0);
1291
1292 if (mlx5_eswitch_mode(mdev) == MLX5_ESWITCH_OFFLOADS &&
1293 mlx5_eswitch_vport_match_metadata_enabled(esw))
1294 fill_esw_mgr_reg_c0(mdev, &resp);
1295 }
1296
1297 if (uhw_outlen) {
1298 err = ib_copy_to_udata(uhw, &resp, resp.response_length);
1299
1300 if (err)
1301 return err;
1302 }
1303
1304 return 0;
1305}
1306
1307static void translate_active_width(struct ib_device *ibdev, u16 active_width,
1308 u8 *ib_width)
1309{
1310 struct mlx5_ib_dev *dev = to_mdev(ibdev);
1311
1312 if (active_width & MLX5_PTYS_WIDTH_1X)
1313 *ib_width = IB_WIDTH_1X;
1314 else if (active_width & MLX5_PTYS_WIDTH_2X)
1315 *ib_width = IB_WIDTH_2X;
1316 else if (active_width & MLX5_PTYS_WIDTH_4X)
1317 *ib_width = IB_WIDTH_4X;
1318 else if (active_width & MLX5_PTYS_WIDTH_8X)
1319 *ib_width = IB_WIDTH_8X;
1320 else if (active_width & MLX5_PTYS_WIDTH_12X)
1321 *ib_width = IB_WIDTH_12X;
1322 else {
1323 mlx5_ib_dbg(dev, "Invalid active_width %d, setting width to default value: 4x\n",
1324 active_width);
1325 *ib_width = IB_WIDTH_4X;
1326 }
1327
1328 return;
1329}
1330
1331static int mlx5_mtu_to_ib_mtu(int mtu)
1332{
1333 switch (mtu) {
1334 case 256: return 1;
1335 case 512: return 2;
1336 case 1024: return 3;
1337 case 2048: return 4;
1338 case 4096: return 5;
1339 default:
1340 pr_warn("invalid mtu\n");
1341 return -1;
1342 }
1343}
1344
1345enum ib_max_vl_num {
1346 __IB_MAX_VL_0 = 1,
1347 __IB_MAX_VL_0_1 = 2,
1348 __IB_MAX_VL_0_3 = 3,
1349 __IB_MAX_VL_0_7 = 4,
1350 __IB_MAX_VL_0_14 = 5,
1351};
1352
1353enum mlx5_vl_hw_cap {
1354 MLX5_VL_HW_0 = 1,
1355 MLX5_VL_HW_0_1 = 2,
1356 MLX5_VL_HW_0_2 = 3,
1357 MLX5_VL_HW_0_3 = 4,
1358 MLX5_VL_HW_0_4 = 5,
1359 MLX5_VL_HW_0_5 = 6,
1360 MLX5_VL_HW_0_6 = 7,
1361 MLX5_VL_HW_0_7 = 8,
1362 MLX5_VL_HW_0_14 = 15
1363};
1364
1365static int translate_max_vl_num(struct ib_device *ibdev, u8 vl_hw_cap,
1366 u8 *max_vl_num)
1367{
1368 switch (vl_hw_cap) {
1369 case MLX5_VL_HW_0:
1370 *max_vl_num = __IB_MAX_VL_0;
1371 break;
1372 case MLX5_VL_HW_0_1:
1373 *max_vl_num = __IB_MAX_VL_0_1;
1374 break;
1375 case MLX5_VL_HW_0_3:
1376 *max_vl_num = __IB_MAX_VL_0_3;
1377 break;
1378 case MLX5_VL_HW_0_7:
1379 *max_vl_num = __IB_MAX_VL_0_7;
1380 break;
1381 case MLX5_VL_HW_0_14:
1382 *max_vl_num = __IB_MAX_VL_0_14;
1383 break;
1384
1385 default:
1386 return -EINVAL;
1387 }
1388
1389 return 0;
1390}
1391
1392static int mlx5_query_hca_port(struct ib_device *ibdev, u32 port,
1393 struct ib_port_attr *props)
1394{
1395 struct mlx5_ib_dev *dev = to_mdev(ibdev);
1396 struct mlx5_core_dev *mdev = dev->mdev;
1397 struct mlx5_hca_vport_context *rep;
1398 u8 vl_hw_cap, plane_index = 0;
1399 u16 max_mtu;
1400 u16 oper_mtu;
1401 int err;
1402 u16 ib_link_width_oper;
1403
1404 rep = kzalloc(sizeof(*rep), GFP_KERNEL);
1405 if (!rep) {
1406 err = -ENOMEM;
1407 goto out;
1408 }
1409
1410 /* props being zeroed by the caller, avoid zeroing it here */
1411
1412 if (ibdev->type == RDMA_DEVICE_TYPE_SMI) {
1413 plane_index = port;
1414 port = smi_to_native_portnum(dev, port);
1415 }
1416
1417 err = mlx5_query_hca_vport_context(mdev, 0, port, 0, rep);
1418 if (err)
1419 goto out;
1420
1421 props->lid = rep->lid;
1422 props->lmc = rep->lmc;
1423 props->sm_lid = rep->sm_lid;
1424 props->sm_sl = rep->sm_sl;
1425 props->state = rep->vport_state;
1426 props->phys_state = rep->port_physical_state;
1427
1428 props->port_cap_flags = rep->cap_mask1;
1429 if (dev->num_plane) {
1430 props->port_cap_flags |= IB_PORT_SM_DISABLED;
1431 props->port_cap_flags &= ~IB_PORT_SM;
1432 } else if (ibdev->type == RDMA_DEVICE_TYPE_SMI)
1433 props->port_cap_flags &= ~IB_PORT_CM_SUP;
1434
1435 props->gid_tbl_len = mlx5_get_gid_table_len(MLX5_CAP_GEN(mdev, gid_table_size));
1436 props->max_msg_sz = 1 << MLX5_CAP_GEN(mdev, log_max_msg);
1437 props->pkey_tbl_len = mlx5_to_sw_pkey_sz(MLX5_CAP_GEN(mdev, pkey_table_size));
1438 props->bad_pkey_cntr = rep->pkey_violation_counter;
1439 props->qkey_viol_cntr = rep->qkey_violation_counter;
1440 props->subnet_timeout = rep->subnet_timeout;
1441 props->init_type_reply = rep->init_type_reply;
1442
1443 if (props->port_cap_flags & IB_PORT_CAP_MASK2_SUP)
1444 props->port_cap_flags2 = rep->cap_mask2;
1445
1446 err = mlx5_query_ib_port_oper(mdev, &ib_link_width_oper,
1447 &props->active_speed, port, plane_index);
1448 if (err)
1449 goto out;
1450
1451 translate_active_width(ibdev, ib_link_width_oper, &props->active_width);
1452
1453 mlx5_query_port_max_mtu(mdev, &max_mtu, port);
1454
1455 props->max_mtu = mlx5_mtu_to_ib_mtu(max_mtu);
1456
1457 mlx5_query_port_oper_mtu(mdev, &oper_mtu, port);
1458
1459 props->active_mtu = mlx5_mtu_to_ib_mtu(oper_mtu);
1460
1461 err = mlx5_query_port_vl_hw_cap(mdev, &vl_hw_cap, port);
1462 if (err)
1463 goto out;
1464
1465 err = translate_max_vl_num(ibdev, vl_hw_cap,
1466 &props->max_vl_num);
1467out:
1468 kfree(rep);
1469 return err;
1470}
1471
1472int mlx5_ib_query_port(struct ib_device *ibdev, u32 port,
1473 struct ib_port_attr *props)
1474{
1475 unsigned int count;
1476 int ret;
1477
1478 switch (mlx5_get_vport_access_method(ibdev)) {
1479 case MLX5_VPORT_ACCESS_METHOD_MAD:
1480 ret = mlx5_query_mad_ifc_port(ibdev, port, props);
1481 break;
1482
1483 case MLX5_VPORT_ACCESS_METHOD_HCA:
1484 ret = mlx5_query_hca_port(ibdev, port, props);
1485 break;
1486
1487 case MLX5_VPORT_ACCESS_METHOD_NIC:
1488 ret = mlx5_query_port_roce(ibdev, port, props);
1489 break;
1490
1491 default:
1492 ret = -EINVAL;
1493 }
1494
1495 if (!ret && props) {
1496 struct mlx5_ib_dev *dev = to_mdev(ibdev);
1497 struct mlx5_core_dev *mdev;
1498 bool put_mdev = true;
1499
1500 mdev = mlx5_ib_get_native_port_mdev(dev, port, NULL);
1501 if (!mdev) {
1502 /* If the port isn't affiliated yet query the master.
1503 * The master and slave will have the same values.
1504 */
1505 mdev = dev->mdev;
1506 port = 1;
1507 put_mdev = false;
1508 }
1509 count = mlx5_core_reserved_gids_count(mdev);
1510 if (put_mdev)
1511 mlx5_ib_put_native_port_mdev(dev, port);
1512 props->gid_tbl_len -= count;
1513 }
1514 return ret;
1515}
1516
1517static int mlx5_ib_rep_query_port(struct ib_device *ibdev, u32 port,
1518 struct ib_port_attr *props)
1519{
1520 return mlx5_query_port_roce(ibdev, port, props);
1521}
1522
1523static int mlx5_ib_rep_query_pkey(struct ib_device *ibdev, u32 port, u16 index,
1524 u16 *pkey)
1525{
1526 /* Default special Pkey for representor device port as per the
1527 * IB specification 1.3 section 10.9.1.2.
1528 */
1529 *pkey = 0xffff;
1530 return 0;
1531}
1532
1533static int mlx5_ib_query_gid(struct ib_device *ibdev, u32 port, int index,
1534 union ib_gid *gid)
1535{
1536 struct mlx5_ib_dev *dev = to_mdev(ibdev);
1537 struct mlx5_core_dev *mdev = dev->mdev;
1538
1539 switch (mlx5_get_vport_access_method(ibdev)) {
1540 case MLX5_VPORT_ACCESS_METHOD_MAD:
1541 return mlx5_query_mad_ifc_gids(ibdev, port, index, gid);
1542
1543 case MLX5_VPORT_ACCESS_METHOD_HCA:
1544 return mlx5_query_hca_vport_gid(mdev, 0, port, 0, index, gid);
1545
1546 default:
1547 return -EINVAL;
1548 }
1549
1550}
1551
1552static int mlx5_query_hca_nic_pkey(struct ib_device *ibdev, u32 port,
1553 u16 index, u16 *pkey)
1554{
1555 struct mlx5_ib_dev *dev = to_mdev(ibdev);
1556 struct mlx5_core_dev *mdev;
1557 bool put_mdev = true;
1558 u32 mdev_port_num;
1559 int err;
1560
1561 mdev = mlx5_ib_get_native_port_mdev(dev, port, &mdev_port_num);
1562 if (!mdev) {
1563 /* The port isn't affiliated yet, get the PKey from the master
1564 * port. For RoCE the PKey tables will be the same.
1565 */
1566 put_mdev = false;
1567 mdev = dev->mdev;
1568 mdev_port_num = 1;
1569 }
1570
1571 err = mlx5_query_hca_vport_pkey(mdev, 0, mdev_port_num, 0,
1572 index, pkey);
1573 if (put_mdev)
1574 mlx5_ib_put_native_port_mdev(dev, port);
1575
1576 return err;
1577}
1578
1579static int mlx5_ib_query_pkey(struct ib_device *ibdev, u32 port, u16 index,
1580 u16 *pkey)
1581{
1582 switch (mlx5_get_vport_access_method(ibdev)) {
1583 case MLX5_VPORT_ACCESS_METHOD_MAD:
1584 return mlx5_query_mad_ifc_pkey(ibdev, port, index, pkey);
1585
1586 case MLX5_VPORT_ACCESS_METHOD_HCA:
1587 case MLX5_VPORT_ACCESS_METHOD_NIC:
1588 return mlx5_query_hca_nic_pkey(ibdev, port, index, pkey);
1589 default:
1590 return -EINVAL;
1591 }
1592}
1593
1594static int mlx5_ib_modify_device(struct ib_device *ibdev, int mask,
1595 struct ib_device_modify *props)
1596{
1597 struct mlx5_ib_dev *dev = to_mdev(ibdev);
1598 struct mlx5_reg_node_desc in;
1599 struct mlx5_reg_node_desc out;
1600 int err;
1601
1602 if (mask & ~IB_DEVICE_MODIFY_NODE_DESC)
1603 return -EOPNOTSUPP;
1604
1605 if (!(mask & IB_DEVICE_MODIFY_NODE_DESC))
1606 return 0;
1607
1608 /*
1609 * If possible, pass node desc to FW, so it can generate
1610 * a 144 trap. If cmd fails, just ignore.
1611 */
1612 memcpy(&in, props->node_desc, IB_DEVICE_NODE_DESC_MAX);
1613 err = mlx5_core_access_reg(dev->mdev, &in, sizeof(in), &out,
1614 sizeof(out), MLX5_REG_NODE_DESC, 0, 1);
1615 if (err)
1616 return err;
1617
1618 memcpy(ibdev->node_desc, props->node_desc, IB_DEVICE_NODE_DESC_MAX);
1619
1620 return err;
1621}
1622
1623static int set_port_caps_atomic(struct mlx5_ib_dev *dev, u32 port_num, u32 mask,
1624 u32 value)
1625{
1626 struct mlx5_hca_vport_context ctx = {};
1627 struct mlx5_core_dev *mdev;
1628 u32 mdev_port_num;
1629 int err;
1630
1631 mdev = mlx5_ib_get_native_port_mdev(dev, port_num, &mdev_port_num);
1632 if (!mdev)
1633 return -ENODEV;
1634
1635 err = mlx5_query_hca_vport_context(mdev, 0, mdev_port_num, 0, &ctx);
1636 if (err)
1637 goto out;
1638
1639 if (~ctx.cap_mask1_perm & mask) {
1640 mlx5_ib_warn(dev, "trying to change bitmask 0x%X but change supported 0x%X\n",
1641 mask, ctx.cap_mask1_perm);
1642 err = -EINVAL;
1643 goto out;
1644 }
1645
1646 ctx.cap_mask1 = value;
1647 ctx.cap_mask1_perm = mask;
1648 err = mlx5_core_modify_hca_vport_context(mdev, 0, mdev_port_num,
1649 0, &ctx);
1650
1651out:
1652 mlx5_ib_put_native_port_mdev(dev, port_num);
1653
1654 return err;
1655}
1656
1657static int mlx5_ib_modify_port(struct ib_device *ibdev, u32 port, int mask,
1658 struct ib_port_modify *props)
1659{
1660 struct mlx5_ib_dev *dev = to_mdev(ibdev);
1661 struct ib_port_attr attr;
1662 u32 tmp;
1663 int err;
1664 u32 change_mask;
1665 u32 value;
1666 bool is_ib = (mlx5_ib_port_link_layer(ibdev, port) ==
1667 IB_LINK_LAYER_INFINIBAND);
1668
1669 /* CM layer calls ib_modify_port() regardless of the link layer. For
1670 * Ethernet ports, qkey violation and Port capabilities are meaningless.
1671 */
1672 if (!is_ib)
1673 return 0;
1674
1675 if (MLX5_CAP_GEN(dev->mdev, ib_virt) && is_ib) {
1676 change_mask = props->clr_port_cap_mask | props->set_port_cap_mask;
1677 value = ~props->clr_port_cap_mask | props->set_port_cap_mask;
1678 return set_port_caps_atomic(dev, port, change_mask, value);
1679 }
1680
1681 mutex_lock(&dev->cap_mask_mutex);
1682
1683 err = ib_query_port(ibdev, port, &attr);
1684 if (err)
1685 goto out;
1686
1687 tmp = (attr.port_cap_flags | props->set_port_cap_mask) &
1688 ~props->clr_port_cap_mask;
1689
1690 err = mlx5_set_port_caps(dev->mdev, port, tmp);
1691
1692out:
1693 mutex_unlock(&dev->cap_mask_mutex);
1694 return err;
1695}
1696
1697static void print_lib_caps(struct mlx5_ib_dev *dev, u64 caps)
1698{
1699 mlx5_ib_dbg(dev, "MLX5_LIB_CAP_4K_UAR = %s\n",
1700 caps & MLX5_LIB_CAP_4K_UAR ? "y" : "n");
1701}
1702
1703static u16 calc_dynamic_bfregs(int uars_per_sys_page)
1704{
1705 /* Large page with non 4k uar support might limit the dynamic size */
1706 if (uars_per_sys_page == 1 && PAGE_SIZE > 4096)
1707 return MLX5_MIN_DYN_BFREGS;
1708
1709 return MLX5_MAX_DYN_BFREGS;
1710}
1711
1712static int calc_total_bfregs(struct mlx5_ib_dev *dev, bool lib_uar_4k,
1713 struct mlx5_ib_alloc_ucontext_req_v2 *req,
1714 struct mlx5_bfreg_info *bfregi)
1715{
1716 int uars_per_sys_page;
1717 int bfregs_per_sys_page;
1718 int ref_bfregs = req->total_num_bfregs;
1719
1720 if (req->total_num_bfregs == 0)
1721 return -EINVAL;
1722
1723 BUILD_BUG_ON(MLX5_MAX_BFREGS % MLX5_NON_FP_BFREGS_IN_PAGE);
1724 BUILD_BUG_ON(MLX5_MAX_BFREGS < MLX5_NON_FP_BFREGS_IN_PAGE);
1725
1726 if (req->total_num_bfregs > MLX5_MAX_BFREGS)
1727 return -ENOMEM;
1728
1729 uars_per_sys_page = get_uars_per_sys_page(dev, lib_uar_4k);
1730 bfregs_per_sys_page = uars_per_sys_page * MLX5_NON_FP_BFREGS_PER_UAR;
1731 /* This holds the required static allocation asked by the user */
1732 req->total_num_bfregs = ALIGN(req->total_num_bfregs, bfregs_per_sys_page);
1733 if (req->num_low_latency_bfregs > req->total_num_bfregs - 1)
1734 return -EINVAL;
1735
1736 bfregi->num_static_sys_pages = req->total_num_bfregs / bfregs_per_sys_page;
1737 bfregi->num_dyn_bfregs = ALIGN(calc_dynamic_bfregs(uars_per_sys_page), bfregs_per_sys_page);
1738 bfregi->total_num_bfregs = req->total_num_bfregs + bfregi->num_dyn_bfregs;
1739 bfregi->num_sys_pages = bfregi->total_num_bfregs / bfregs_per_sys_page;
1740
1741 mlx5_ib_dbg(dev, "uar_4k: fw support %s, lib support %s, user requested %d bfregs, allocated %d, total bfregs %d, using %d sys pages\n",
1742 MLX5_CAP_GEN(dev->mdev, uar_4k) ? "yes" : "no",
1743 lib_uar_4k ? "yes" : "no", ref_bfregs,
1744 req->total_num_bfregs, bfregi->total_num_bfregs,
1745 bfregi->num_sys_pages);
1746
1747 return 0;
1748}
1749
1750static int allocate_uars(struct mlx5_ib_dev *dev, struct mlx5_ib_ucontext *context)
1751{
1752 struct mlx5_bfreg_info *bfregi;
1753 int err;
1754 int i;
1755
1756 bfregi = &context->bfregi;
1757 for (i = 0; i < bfregi->num_static_sys_pages; i++) {
1758 err = mlx5_cmd_uar_alloc(dev->mdev, &bfregi->sys_pages[i],
1759 context->devx_uid);
1760 if (err)
1761 goto error;
1762
1763 mlx5_ib_dbg(dev, "allocated uar %d\n", bfregi->sys_pages[i]);
1764 }
1765
1766 for (i = bfregi->num_static_sys_pages; i < bfregi->num_sys_pages; i++)
1767 bfregi->sys_pages[i] = MLX5_IB_INVALID_UAR_INDEX;
1768
1769 return 0;
1770
1771error:
1772 for (--i; i >= 0; i--)
1773 if (mlx5_cmd_uar_dealloc(dev->mdev, bfregi->sys_pages[i],
1774 context->devx_uid))
1775 mlx5_ib_warn(dev, "failed to free uar %d\n", i);
1776
1777 return err;
1778}
1779
1780static void deallocate_uars(struct mlx5_ib_dev *dev,
1781 struct mlx5_ib_ucontext *context)
1782{
1783 struct mlx5_bfreg_info *bfregi;
1784 int i;
1785
1786 bfregi = &context->bfregi;
1787 for (i = 0; i < bfregi->num_sys_pages; i++)
1788 if (i < bfregi->num_static_sys_pages ||
1789 bfregi->sys_pages[i] != MLX5_IB_INVALID_UAR_INDEX)
1790 mlx5_cmd_uar_dealloc(dev->mdev, bfregi->sys_pages[i],
1791 context->devx_uid);
1792}
1793
1794static int mlx5_ib_enable_lb_mp(struct mlx5_core_dev *master,
1795 struct mlx5_core_dev *slave)
1796{
1797 int err;
1798
1799 err = mlx5_nic_vport_update_local_lb(master, true);
1800 if (err)
1801 return err;
1802
1803 err = mlx5_nic_vport_update_local_lb(slave, true);
1804 if (err)
1805 goto out;
1806
1807 return 0;
1808
1809out:
1810 mlx5_nic_vport_update_local_lb(master, false);
1811 return err;
1812}
1813
1814static void mlx5_ib_disable_lb_mp(struct mlx5_core_dev *master,
1815 struct mlx5_core_dev *slave)
1816{
1817 mlx5_nic_vport_update_local_lb(slave, false);
1818 mlx5_nic_vport_update_local_lb(master, false);
1819}
1820
1821int mlx5_ib_enable_lb(struct mlx5_ib_dev *dev, bool td, bool qp)
1822{
1823 int err = 0;
1824
1825 mutex_lock(&dev->lb.mutex);
1826 if (td)
1827 dev->lb.user_td++;
1828 if (qp)
1829 dev->lb.qps++;
1830
1831 if (dev->lb.user_td == 2 ||
1832 dev->lb.qps == 1) {
1833 if (!dev->lb.enabled) {
1834 err = mlx5_nic_vport_update_local_lb(dev->mdev, true);
1835 dev->lb.enabled = true;
1836 }
1837 }
1838
1839 mutex_unlock(&dev->lb.mutex);
1840
1841 return err;
1842}
1843
1844void mlx5_ib_disable_lb(struct mlx5_ib_dev *dev, bool td, bool qp)
1845{
1846 mutex_lock(&dev->lb.mutex);
1847 if (td)
1848 dev->lb.user_td--;
1849 if (qp)
1850 dev->lb.qps--;
1851
1852 if (dev->lb.user_td == 1 &&
1853 dev->lb.qps == 0) {
1854 if (dev->lb.enabled) {
1855 mlx5_nic_vport_update_local_lb(dev->mdev, false);
1856 dev->lb.enabled = false;
1857 }
1858 }
1859
1860 mutex_unlock(&dev->lb.mutex);
1861}
1862
1863static int mlx5_ib_alloc_transport_domain(struct mlx5_ib_dev *dev, u32 *tdn,
1864 u16 uid)
1865{
1866 int err;
1867
1868 if (!MLX5_CAP_GEN(dev->mdev, log_max_transport_domain))
1869 return 0;
1870
1871 err = mlx5_cmd_alloc_transport_domain(dev->mdev, tdn, uid);
1872 if (err)
1873 return err;
1874
1875 if ((MLX5_CAP_GEN(dev->mdev, port_type) != MLX5_CAP_PORT_TYPE_ETH) ||
1876 (!MLX5_CAP_GEN(dev->mdev, disable_local_lb_uc) &&
1877 !MLX5_CAP_GEN(dev->mdev, disable_local_lb_mc)))
1878 return err;
1879
1880 return mlx5_ib_enable_lb(dev, true, false);
1881}
1882
1883static void mlx5_ib_dealloc_transport_domain(struct mlx5_ib_dev *dev, u32 tdn,
1884 u16 uid)
1885{
1886 if (!MLX5_CAP_GEN(dev->mdev, log_max_transport_domain))
1887 return;
1888
1889 mlx5_cmd_dealloc_transport_domain(dev->mdev, tdn, uid);
1890
1891 if ((MLX5_CAP_GEN(dev->mdev, port_type) != MLX5_CAP_PORT_TYPE_ETH) ||
1892 (!MLX5_CAP_GEN(dev->mdev, disable_local_lb_uc) &&
1893 !MLX5_CAP_GEN(dev->mdev, disable_local_lb_mc)))
1894 return;
1895
1896 mlx5_ib_disable_lb(dev, true, false);
1897}
1898
1899static int set_ucontext_resp(struct ib_ucontext *uctx,
1900 struct mlx5_ib_alloc_ucontext_resp *resp)
1901{
1902 struct ib_device *ibdev = uctx->device;
1903 struct mlx5_ib_dev *dev = to_mdev(ibdev);
1904 struct mlx5_ib_ucontext *context = to_mucontext(uctx);
1905 struct mlx5_bfreg_info *bfregi = &context->bfregi;
1906
1907 if (MLX5_CAP_GEN(dev->mdev, dump_fill_mkey)) {
1908 resp->dump_fill_mkey = dev->mkeys.dump_fill_mkey;
1909 resp->comp_mask |=
1910 MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_DUMP_FILL_MKEY;
1911 }
1912
1913 resp->qp_tab_size = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp);
1914 if (mlx5_wc_support_get(dev->mdev))
1915 resp->bf_reg_size = 1 << MLX5_CAP_GEN(dev->mdev,
1916 log_bf_reg_size);
1917 resp->cache_line_size = cache_line_size();
1918 resp->max_sq_desc_sz = MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq);
1919 resp->max_rq_desc_sz = MLX5_CAP_GEN(dev->mdev, max_wqe_sz_rq);
1920 resp->max_send_wqebb = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz);
1921 resp->max_recv_wr = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz);
1922 resp->max_srq_recv_wr = 1 << MLX5_CAP_GEN(dev->mdev, log_max_srq_sz);
1923 resp->cqe_version = context->cqe_version;
1924 resp->log_uar_size = MLX5_CAP_GEN(dev->mdev, uar_4k) ?
1925 MLX5_ADAPTER_PAGE_SHIFT : PAGE_SHIFT;
1926 resp->num_uars_per_page = MLX5_CAP_GEN(dev->mdev, uar_4k) ?
1927 MLX5_CAP_GEN(dev->mdev,
1928 num_of_uars_per_page) : 1;
1929 resp->tot_bfregs = bfregi->lib_uar_dyn ? 0 :
1930 bfregi->total_num_bfregs - bfregi->num_dyn_bfregs;
1931 resp->num_ports = dev->num_ports;
1932 resp->cmds_supp_uhw |= MLX5_USER_CMDS_SUPP_UHW_QUERY_DEVICE |
1933 MLX5_USER_CMDS_SUPP_UHW_CREATE_AH;
1934
1935 if (mlx5_ib_port_link_layer(ibdev, 1) == IB_LINK_LAYER_ETHERNET) {
1936 mlx5_query_min_inline(dev->mdev, &resp->eth_min_inline);
1937 resp->eth_min_inline++;
1938 }
1939
1940 if (dev->mdev->clock_info)
1941 resp->clock_info_versions = BIT(MLX5_IB_CLOCK_INFO_V1);
1942
1943 /*
1944 * We don't want to expose information from the PCI bar that is located
1945 * after 4096 bytes, so if the arch only supports larger pages, let's
1946 * pretend we don't support reading the HCA's core clock. This is also
1947 * forced by mmap function.
1948 */
1949 if (PAGE_SIZE <= 4096) {
1950 resp->comp_mask |=
1951 MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_CORE_CLOCK_OFFSET;
1952 resp->hca_core_clock_offset =
1953 offsetof(struct mlx5_init_seg,
1954 internal_timer_h) % PAGE_SIZE;
1955 }
1956
1957 if (MLX5_CAP_GEN(dev->mdev, ece_support))
1958 resp->comp_mask |= MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_ECE;
1959
1960 if (rt_supported(MLX5_CAP_GEN(dev->mdev, sq_ts_format)) &&
1961 rt_supported(MLX5_CAP_GEN(dev->mdev, rq_ts_format)) &&
1962 rt_supported(MLX5_CAP_ROCE(dev->mdev, qp_ts_format)))
1963 resp->comp_mask |=
1964 MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_REAL_TIME_TS;
1965
1966 resp->num_dyn_bfregs = bfregi->num_dyn_bfregs;
1967
1968 if (MLX5_CAP_GEN(dev->mdev, drain_sigerr))
1969 resp->comp_mask |= MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_SQD2RTS;
1970
1971 resp->comp_mask |=
1972 MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_MKEY_UPDATE_TAG;
1973
1974 return 0;
1975}
1976
1977static bool uctx_rdma_ctrl_is_enabled(u64 enabled_caps)
1978{
1979 return UCAP_ENABLED(enabled_caps, RDMA_UCAP_MLX5_CTRL_LOCAL) ||
1980 UCAP_ENABLED(enabled_caps, RDMA_UCAP_MLX5_CTRL_OTHER_VHCA);
1981}
1982
1983static int mlx5_ib_alloc_ucontext(struct ib_ucontext *uctx,
1984 struct ib_udata *udata)
1985{
1986 struct ib_device *ibdev = uctx->device;
1987 struct mlx5_ib_dev *dev = to_mdev(ibdev);
1988 struct mlx5_ib_alloc_ucontext_req_v2 req = {};
1989 struct mlx5_ib_alloc_ucontext_resp resp = {};
1990 struct mlx5_ib_ucontext *context = to_mucontext(uctx);
1991 struct mlx5_bfreg_info *bfregi;
1992 int ver;
1993 int err;
1994 size_t min_req_v2 = offsetof(struct mlx5_ib_alloc_ucontext_req_v2,
1995 max_cqe_version);
1996 bool lib_uar_4k;
1997 bool lib_uar_dyn;
1998
1999 if (!dev->ib_active)
2000 return -EAGAIN;
2001
2002 if (udata->inlen == sizeof(struct mlx5_ib_alloc_ucontext_req))
2003 ver = 0;
2004 else if (udata->inlen >= min_req_v2)
2005 ver = 2;
2006 else
2007 return -EINVAL;
2008
2009 err = ib_copy_from_udata(&req, udata, min(udata->inlen, sizeof(req)));
2010 if (err)
2011 return err;
2012
2013 if (req.flags & ~MLX5_IB_ALLOC_UCTX_DEVX)
2014 return -EOPNOTSUPP;
2015
2016 if (req.comp_mask || req.reserved0 || req.reserved1 || req.reserved2)
2017 return -EOPNOTSUPP;
2018
2019 req.total_num_bfregs = ALIGN(req.total_num_bfregs,
2020 MLX5_NON_FP_BFREGS_PER_UAR);
2021 if (req.num_low_latency_bfregs > req.total_num_bfregs - 1)
2022 return -EINVAL;
2023
2024 if (req.flags & MLX5_IB_ALLOC_UCTX_DEVX) {
2025 err = mlx5_ib_devx_create(dev, true, uctx->enabled_caps);
2026 if (err < 0)
2027 goto out_ctx;
2028 context->devx_uid = err;
2029
2030 if (uctx_rdma_ctrl_is_enabled(uctx->enabled_caps)) {
2031 err = mlx5_cmd_add_privileged_uid(dev->mdev,
2032 context->devx_uid);
2033 if (err)
2034 goto out_devx;
2035 }
2036 }
2037
2038 lib_uar_4k = req.lib_caps & MLX5_LIB_CAP_4K_UAR;
2039 lib_uar_dyn = req.lib_caps & MLX5_LIB_CAP_DYN_UAR;
2040 bfregi = &context->bfregi;
2041
2042 if (lib_uar_dyn) {
2043 bfregi->lib_uar_dyn = lib_uar_dyn;
2044 goto uar_done;
2045 }
2046
2047 /* updates req->total_num_bfregs */
2048 err = calc_total_bfregs(dev, lib_uar_4k, &req, bfregi);
2049 if (err)
2050 goto out_ucap;
2051
2052 mutex_init(&bfregi->lock);
2053 bfregi->lib_uar_4k = lib_uar_4k;
2054 bfregi->count = kcalloc(bfregi->total_num_bfregs, sizeof(*bfregi->count),
2055 GFP_KERNEL);
2056 if (!bfregi->count) {
2057 err = -ENOMEM;
2058 goto out_ucap;
2059 }
2060
2061 bfregi->sys_pages = kcalloc(bfregi->num_sys_pages,
2062 sizeof(*bfregi->sys_pages),
2063 GFP_KERNEL);
2064 if (!bfregi->sys_pages) {
2065 err = -ENOMEM;
2066 goto out_count;
2067 }
2068
2069 err = allocate_uars(dev, context);
2070 if (err)
2071 goto out_sys_pages;
2072
2073uar_done:
2074 err = mlx5_ib_alloc_transport_domain(dev, &context->tdn,
2075 context->devx_uid);
2076 if (err)
2077 goto out_uars;
2078
2079 INIT_LIST_HEAD(&context->db_page_list);
2080 mutex_init(&context->db_page_mutex);
2081
2082 context->cqe_version = min_t(__u8,
2083 (__u8)MLX5_CAP_GEN(dev->mdev, cqe_version),
2084 req.max_cqe_version);
2085
2086 err = set_ucontext_resp(uctx, &resp);
2087 if (err)
2088 goto out_mdev;
2089
2090 resp.response_length = min(udata->outlen, sizeof(resp));
2091 err = ib_copy_to_udata(udata, &resp, resp.response_length);
2092 if (err)
2093 goto out_mdev;
2094
2095 bfregi->ver = ver;
2096 bfregi->num_low_latency_bfregs = req.num_low_latency_bfregs;
2097 context->lib_caps = req.lib_caps;
2098 print_lib_caps(dev, context->lib_caps);
2099
2100 if (mlx5_ib_lag_should_assign_affinity(dev)) {
2101 u32 port = mlx5_core_native_port_num(dev->mdev) - 1;
2102
2103 atomic_set(&context->tx_port_affinity,
2104 atomic_add_return(
2105 1, &dev->port[port].roce.tx_port_affinity));
2106 }
2107
2108 return 0;
2109
2110out_mdev:
2111 mlx5_ib_dealloc_transport_domain(dev, context->tdn, context->devx_uid);
2112
2113out_uars:
2114 deallocate_uars(dev, context);
2115
2116out_sys_pages:
2117 kfree(bfregi->sys_pages);
2118
2119out_count:
2120 kfree(bfregi->count);
2121
2122out_ucap:
2123 if (req.flags & MLX5_IB_ALLOC_UCTX_DEVX &&
2124 uctx_rdma_ctrl_is_enabled(uctx->enabled_caps))
2125 mlx5_cmd_remove_privileged_uid(dev->mdev, context->devx_uid);
2126
2127out_devx:
2128 if (req.flags & MLX5_IB_ALLOC_UCTX_DEVX)
2129 mlx5_ib_devx_destroy(dev, context->devx_uid);
2130
2131out_ctx:
2132 return err;
2133}
2134
2135static int mlx5_ib_query_ucontext(struct ib_ucontext *ibcontext,
2136 struct uverbs_attr_bundle *attrs)
2137{
2138 struct mlx5_ib_alloc_ucontext_resp uctx_resp = {};
2139 int ret;
2140
2141 ret = set_ucontext_resp(ibcontext, &uctx_resp);
2142 if (ret)
2143 return ret;
2144
2145 uctx_resp.response_length =
2146 min_t(size_t,
2147 uverbs_attr_get_len(attrs,
2148 MLX5_IB_ATTR_QUERY_CONTEXT_RESP_UCTX),
2149 sizeof(uctx_resp));
2150
2151 ret = uverbs_copy_to_struct_or_zero(attrs,
2152 MLX5_IB_ATTR_QUERY_CONTEXT_RESP_UCTX,
2153 &uctx_resp,
2154 sizeof(uctx_resp));
2155 return ret;
2156}
2157
2158static void mlx5_ib_dealloc_ucontext(struct ib_ucontext *ibcontext)
2159{
2160 struct mlx5_ib_ucontext *context = to_mucontext(ibcontext);
2161 struct mlx5_ib_dev *dev = to_mdev(ibcontext->device);
2162 struct mlx5_bfreg_info *bfregi;
2163
2164 bfregi = &context->bfregi;
2165 mlx5_ib_dealloc_transport_domain(dev, context->tdn, context->devx_uid);
2166
2167 deallocate_uars(dev, context);
2168 kfree(bfregi->sys_pages);
2169 kfree(bfregi->count);
2170
2171 if (context->devx_uid) {
2172 if (uctx_rdma_ctrl_is_enabled(ibcontext->enabled_caps))
2173 mlx5_cmd_remove_privileged_uid(dev->mdev,
2174 context->devx_uid);
2175 mlx5_ib_devx_destroy(dev, context->devx_uid);
2176 }
2177}
2178
2179static phys_addr_t uar_index2pfn(struct mlx5_ib_dev *dev,
2180 int uar_idx)
2181{
2182 int fw_uars_per_page;
2183
2184 fw_uars_per_page = MLX5_CAP_GEN(dev->mdev, uar_4k) ? MLX5_UARS_IN_PAGE : 1;
2185
2186 return (dev->mdev->bar_addr >> PAGE_SHIFT) + uar_idx / fw_uars_per_page;
2187}
2188
2189static u64 uar_index2paddress(struct mlx5_ib_dev *dev,
2190 int uar_idx)
2191{
2192 unsigned int fw_uars_per_page;
2193
2194 fw_uars_per_page = MLX5_CAP_GEN(dev->mdev, uar_4k) ?
2195 MLX5_UARS_IN_PAGE : 1;
2196
2197 return (dev->mdev->bar_addr + (uar_idx / fw_uars_per_page) * PAGE_SIZE);
2198}
2199
2200static int get_command(unsigned long offset)
2201{
2202 return (offset >> MLX5_IB_MMAP_CMD_SHIFT) & MLX5_IB_MMAP_CMD_MASK;
2203}
2204
2205static int get_arg(unsigned long offset)
2206{
2207 return offset & ((1 << MLX5_IB_MMAP_CMD_SHIFT) - 1);
2208}
2209
2210static int get_index(unsigned long offset)
2211{
2212 return get_arg(offset);
2213}
2214
2215/* Index resides in an extra byte to enable larger values than 255 */
2216static int get_extended_index(unsigned long offset)
2217{
2218 return get_arg(offset) | ((offset >> 16) & 0xff) << 8;
2219}
2220
2221
2222static void mlx5_ib_disassociate_ucontext(struct ib_ucontext *ibcontext)
2223{
2224}
2225
2226static inline char *mmap_cmd2str(enum mlx5_ib_mmap_cmd cmd)
2227{
2228 switch (cmd) {
2229 case MLX5_IB_MMAP_WC_PAGE:
2230 return "WC";
2231 case MLX5_IB_MMAP_REGULAR_PAGE:
2232 return "best effort WC";
2233 case MLX5_IB_MMAP_NC_PAGE:
2234 return "NC";
2235 case MLX5_IB_MMAP_DEVICE_MEM:
2236 return "Device Memory";
2237 default:
2238 return "Unknown";
2239 }
2240}
2241
2242static int mlx5_ib_mmap_clock_info_page(struct mlx5_ib_dev *dev,
2243 struct vm_area_struct *vma,
2244 struct mlx5_ib_ucontext *context)
2245{
2246 if ((vma->vm_end - vma->vm_start != PAGE_SIZE) ||
2247 !(vma->vm_flags & VM_SHARED))
2248 return -EINVAL;
2249
2250 if (get_index(vma->vm_pgoff) != MLX5_IB_CLOCK_INFO_V1)
2251 return -EOPNOTSUPP;
2252
2253 if (vma->vm_flags & (VM_WRITE | VM_EXEC))
2254 return -EPERM;
2255 vm_flags_clear(vma, VM_MAYWRITE);
2256
2257 if (!dev->mdev->clock_info)
2258 return -EOPNOTSUPP;
2259
2260 return vm_insert_page(vma, vma->vm_start,
2261 virt_to_page(dev->mdev->clock_info));
2262}
2263
2264static void mlx5_ib_mmap_free(struct rdma_user_mmap_entry *entry)
2265{
2266 struct mlx5_user_mmap_entry *mentry = to_mmmap(entry);
2267 struct mlx5_ib_dev *dev = to_mdev(entry->ucontext->device);
2268 struct mlx5_var_table *var_table = &dev->var_table;
2269 struct mlx5_ib_ucontext *context = to_mucontext(entry->ucontext);
2270
2271 switch (mentry->mmap_flag) {
2272 case MLX5_IB_MMAP_TYPE_MEMIC:
2273 case MLX5_IB_MMAP_TYPE_MEMIC_OP:
2274 mlx5_ib_dm_mmap_free(dev, mentry);
2275 break;
2276 case MLX5_IB_MMAP_TYPE_VAR:
2277 mutex_lock(&var_table->bitmap_lock);
2278 clear_bit(mentry->page_idx, var_table->bitmap);
2279 mutex_unlock(&var_table->bitmap_lock);
2280 kfree(mentry);
2281 break;
2282 case MLX5_IB_MMAP_TYPE_UAR_WC:
2283 case MLX5_IB_MMAP_TYPE_UAR_NC:
2284 mlx5_cmd_uar_dealloc(dev->mdev, mentry->page_idx,
2285 context->devx_uid);
2286 kfree(mentry);
2287 break;
2288 default:
2289 WARN_ON(true);
2290 }
2291}
2292
2293static int uar_mmap(struct mlx5_ib_dev *dev, enum mlx5_ib_mmap_cmd cmd,
2294 struct vm_area_struct *vma,
2295 struct mlx5_ib_ucontext *context)
2296{
2297 struct mlx5_bfreg_info *bfregi = &context->bfregi;
2298 int err;
2299 unsigned long idx;
2300 phys_addr_t pfn;
2301 pgprot_t prot;
2302 u32 bfreg_dyn_idx = 0;
2303 u32 uar_index;
2304 int dyn_uar = (cmd == MLX5_IB_MMAP_ALLOC_WC);
2305 int max_valid_idx = dyn_uar ? bfregi->num_sys_pages :
2306 bfregi->num_static_sys_pages;
2307
2308 if (bfregi->lib_uar_dyn)
2309 return -EINVAL;
2310
2311 if (vma->vm_end - vma->vm_start != PAGE_SIZE)
2312 return -EINVAL;
2313
2314 if (dyn_uar)
2315 idx = get_extended_index(vma->vm_pgoff) + bfregi->num_static_sys_pages;
2316 else
2317 idx = get_index(vma->vm_pgoff);
2318
2319 if (idx >= max_valid_idx) {
2320 mlx5_ib_warn(dev, "invalid uar index %lu, max=%d\n",
2321 idx, max_valid_idx);
2322 return -EINVAL;
2323 }
2324
2325 switch (cmd) {
2326 case MLX5_IB_MMAP_WC_PAGE:
2327 case MLX5_IB_MMAP_ALLOC_WC:
2328 case MLX5_IB_MMAP_REGULAR_PAGE:
2329 /* For MLX5_IB_MMAP_REGULAR_PAGE do the best effort to get WC */
2330 prot = pgprot_writecombine(vma->vm_page_prot);
2331 break;
2332 case MLX5_IB_MMAP_NC_PAGE:
2333 prot = pgprot_noncached(vma->vm_page_prot);
2334 break;
2335 default:
2336 return -EINVAL;
2337 }
2338
2339 if (dyn_uar) {
2340 int uars_per_page;
2341
2342 uars_per_page = get_uars_per_sys_page(dev, bfregi->lib_uar_4k);
2343 bfreg_dyn_idx = idx * (uars_per_page * MLX5_NON_FP_BFREGS_PER_UAR);
2344 if (bfreg_dyn_idx >= bfregi->total_num_bfregs) {
2345 mlx5_ib_warn(dev, "invalid bfreg_dyn_idx %u, max=%u\n",
2346 bfreg_dyn_idx, bfregi->total_num_bfregs);
2347 return -EINVAL;
2348 }
2349
2350 mutex_lock(&bfregi->lock);
2351 /* Fail if uar already allocated, first bfreg index of each
2352 * page holds its count.
2353 */
2354 if (bfregi->count[bfreg_dyn_idx]) {
2355 mlx5_ib_warn(dev, "wrong offset, idx %lu is busy, bfregn=%u\n", idx, bfreg_dyn_idx);
2356 mutex_unlock(&bfregi->lock);
2357 return -EINVAL;
2358 }
2359
2360 bfregi->count[bfreg_dyn_idx]++;
2361 mutex_unlock(&bfregi->lock);
2362
2363 err = mlx5_cmd_uar_alloc(dev->mdev, &uar_index,
2364 context->devx_uid);
2365 if (err) {
2366 mlx5_ib_warn(dev, "UAR alloc failed\n");
2367 goto free_bfreg;
2368 }
2369 } else {
2370 uar_index = bfregi->sys_pages[idx];
2371 }
2372
2373 pfn = uar_index2pfn(dev, uar_index);
2374 mlx5_ib_dbg(dev, "uar idx 0x%lx, pfn %pa\n", idx, &pfn);
2375
2376 err = rdma_user_mmap_io(&context->ibucontext, vma, pfn, PAGE_SIZE,
2377 prot, NULL);
2378 if (err) {
2379 mlx5_ib_err(dev,
2380 "rdma_user_mmap_io failed with error=%d, mmap_cmd=%s\n",
2381 err, mmap_cmd2str(cmd));
2382 goto err;
2383 }
2384
2385 if (dyn_uar)
2386 bfregi->sys_pages[idx] = uar_index;
2387 return 0;
2388
2389err:
2390 if (!dyn_uar)
2391 return err;
2392
2393 mlx5_cmd_uar_dealloc(dev->mdev, idx, context->devx_uid);
2394
2395free_bfreg:
2396 mlx5_ib_free_bfreg(dev, bfregi, bfreg_dyn_idx);
2397
2398 return err;
2399}
2400
2401static unsigned long mlx5_vma_to_pgoff(struct vm_area_struct *vma)
2402{
2403 unsigned long idx;
2404 u8 command;
2405
2406 command = get_command(vma->vm_pgoff);
2407 idx = get_extended_index(vma->vm_pgoff);
2408
2409 return (command << 16 | idx);
2410}
2411
2412static int mlx5_ib_mmap_offset(struct mlx5_ib_dev *dev,
2413 struct vm_area_struct *vma,
2414 struct ib_ucontext *ucontext)
2415{
2416 struct mlx5_user_mmap_entry *mentry;
2417 struct rdma_user_mmap_entry *entry;
2418 unsigned long pgoff;
2419 pgprot_t prot;
2420 phys_addr_t pfn;
2421 int ret;
2422
2423 pgoff = mlx5_vma_to_pgoff(vma);
2424 entry = rdma_user_mmap_entry_get_pgoff(ucontext, pgoff);
2425 if (!entry)
2426 return -EINVAL;
2427
2428 mentry = to_mmmap(entry);
2429 pfn = (mentry->address >> PAGE_SHIFT);
2430 if (mentry->mmap_flag == MLX5_IB_MMAP_TYPE_VAR ||
2431 mentry->mmap_flag == MLX5_IB_MMAP_TYPE_UAR_NC)
2432 prot = pgprot_noncached(vma->vm_page_prot);
2433 else
2434 prot = pgprot_writecombine(vma->vm_page_prot);
2435 ret = rdma_user_mmap_io(ucontext, vma, pfn,
2436 entry->npages * PAGE_SIZE,
2437 prot,
2438 entry);
2439 rdma_user_mmap_entry_put(&mentry->rdma_entry);
2440 return ret;
2441}
2442
2443static u64 mlx5_entry_to_mmap_offset(struct mlx5_user_mmap_entry *entry)
2444{
2445 u64 cmd = (entry->rdma_entry.start_pgoff >> 16) & 0xFFFF;
2446 u64 index = entry->rdma_entry.start_pgoff & 0xFFFF;
2447
2448 return (((index >> 8) << 16) | (cmd << MLX5_IB_MMAP_CMD_SHIFT) |
2449 (index & 0xFF)) << PAGE_SHIFT;
2450}
2451
2452static int mlx5_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma)
2453{
2454 struct mlx5_ib_ucontext *context = to_mucontext(ibcontext);
2455 struct mlx5_ib_dev *dev = to_mdev(ibcontext->device);
2456 unsigned long command;
2457 phys_addr_t pfn;
2458
2459 command = get_command(vma->vm_pgoff);
2460 switch (command) {
2461 case MLX5_IB_MMAP_WC_PAGE:
2462 case MLX5_IB_MMAP_ALLOC_WC:
2463 if (!mlx5_wc_support_get(dev->mdev))
2464 return -EPERM;
2465 fallthrough;
2466 case MLX5_IB_MMAP_NC_PAGE:
2467 case MLX5_IB_MMAP_REGULAR_PAGE:
2468 return uar_mmap(dev, command, vma, context);
2469
2470 case MLX5_IB_MMAP_GET_CONTIGUOUS_PAGES:
2471 return -ENOSYS;
2472
2473 case MLX5_IB_MMAP_CORE_CLOCK:
2474 if (vma->vm_end - vma->vm_start != PAGE_SIZE)
2475 return -EINVAL;
2476
2477 if (vma->vm_flags & VM_WRITE)
2478 return -EPERM;
2479 vm_flags_clear(vma, VM_MAYWRITE);
2480
2481 /* Don't expose to user-space information it shouldn't have */
2482 if (PAGE_SIZE > 4096)
2483 return -EOPNOTSUPP;
2484
2485 pfn = (dev->mdev->iseg_base +
2486 offsetof(struct mlx5_init_seg, internal_timer_h)) >>
2487 PAGE_SHIFT;
2488 return rdma_user_mmap_io(&context->ibucontext, vma, pfn,
2489 PAGE_SIZE,
2490 pgprot_noncached(vma->vm_page_prot),
2491 NULL);
2492 case MLX5_IB_MMAP_CLOCK_INFO:
2493 return mlx5_ib_mmap_clock_info_page(dev, vma, context);
2494
2495 default:
2496 return mlx5_ib_mmap_offset(dev, vma, ibcontext);
2497 }
2498
2499 return 0;
2500}
2501
2502static int mlx5_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
2503{
2504 struct mlx5_ib_pd *pd = to_mpd(ibpd);
2505 struct ib_device *ibdev = ibpd->device;
2506 struct mlx5_ib_alloc_pd_resp resp;
2507 int err;
2508 u32 out[MLX5_ST_SZ_DW(alloc_pd_out)] = {};
2509 u32 in[MLX5_ST_SZ_DW(alloc_pd_in)] = {};
2510 u16 uid = 0;
2511 struct mlx5_ib_ucontext *context = rdma_udata_to_drv_context(
2512 udata, struct mlx5_ib_ucontext, ibucontext);
2513
2514 uid = context ? context->devx_uid : 0;
2515 MLX5_SET(alloc_pd_in, in, opcode, MLX5_CMD_OP_ALLOC_PD);
2516 MLX5_SET(alloc_pd_in, in, uid, uid);
2517 err = mlx5_cmd_exec_inout(to_mdev(ibdev)->mdev, alloc_pd, in, out);
2518 if (err)
2519 return err;
2520
2521 pd->pdn = MLX5_GET(alloc_pd_out, out, pd);
2522 pd->uid = uid;
2523 if (udata) {
2524 resp.pdn = pd->pdn;
2525 if (ib_copy_to_udata(udata, &resp, sizeof(resp))) {
2526 mlx5_cmd_dealloc_pd(to_mdev(ibdev)->mdev, pd->pdn, uid);
2527 return -EFAULT;
2528 }
2529 }
2530
2531 return 0;
2532}
2533
2534static int mlx5_ib_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata)
2535{
2536 struct mlx5_ib_dev *mdev = to_mdev(pd->device);
2537 struct mlx5_ib_pd *mpd = to_mpd(pd);
2538
2539 return mlx5_cmd_dealloc_pd(mdev->mdev, mpd->pdn, mpd->uid);
2540}
2541
2542static int mlx5_ib_mcg_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
2543{
2544 struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
2545 struct mlx5_ib_qp *mqp = to_mqp(ibqp);
2546 int err;
2547 u16 uid;
2548
2549 uid = ibqp->pd ?
2550 to_mpd(ibqp->pd)->uid : 0;
2551
2552 if (mqp->flags & IB_QP_CREATE_SOURCE_QPN) {
2553 mlx5_ib_dbg(dev, "Attaching a multi cast group to underlay QP is not supported\n");
2554 return -EOPNOTSUPP;
2555 }
2556
2557 err = mlx5_cmd_attach_mcg(dev->mdev, gid, ibqp->qp_num, uid);
2558 if (err)
2559 mlx5_ib_warn(dev, "failed attaching QPN 0x%x, MGID %pI6\n",
2560 ibqp->qp_num, gid->raw);
2561
2562 return err;
2563}
2564
2565static int mlx5_ib_mcg_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
2566{
2567 struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
2568 int err;
2569 u16 uid;
2570
2571 uid = ibqp->pd ?
2572 to_mpd(ibqp->pd)->uid : 0;
2573 err = mlx5_cmd_detach_mcg(dev->mdev, gid, ibqp->qp_num, uid);
2574 if (err)
2575 mlx5_ib_warn(dev, "failed detaching QPN 0x%x, MGID %pI6\n",
2576 ibqp->qp_num, gid->raw);
2577
2578 return err;
2579}
2580
2581static int init_node_data(struct mlx5_ib_dev *dev)
2582{
2583 int err;
2584
2585 err = mlx5_query_node_desc(dev, dev->ib_dev.node_desc);
2586 if (err)
2587 return err;
2588
2589 dev->mdev->rev_id = dev->mdev->pdev->revision;
2590
2591 return mlx5_query_node_guid(dev, &dev->ib_dev.node_guid);
2592}
2593
2594static ssize_t fw_pages_show(struct device *device,
2595 struct device_attribute *attr, char *buf)
2596{
2597 struct mlx5_ib_dev *dev =
2598 rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev);
2599
2600 return sysfs_emit(buf, "%d\n", dev->mdev->priv.fw_pages);
2601}
2602static DEVICE_ATTR_RO(fw_pages);
2603
2604static ssize_t reg_pages_show(struct device *device,
2605 struct device_attribute *attr, char *buf)
2606{
2607 struct mlx5_ib_dev *dev =
2608 rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev);
2609
2610 return sysfs_emit(buf, "%d\n", atomic_read(&dev->mdev->priv.reg_pages));
2611}
2612static DEVICE_ATTR_RO(reg_pages);
2613
2614static ssize_t hca_type_show(struct device *device,
2615 struct device_attribute *attr, char *buf)
2616{
2617 struct mlx5_ib_dev *dev =
2618 rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev);
2619
2620 return sysfs_emit(buf, "MT%d\n", dev->mdev->pdev->device);
2621}
2622static DEVICE_ATTR_RO(hca_type);
2623
2624static ssize_t hw_rev_show(struct device *device,
2625 struct device_attribute *attr, char *buf)
2626{
2627 struct mlx5_ib_dev *dev =
2628 rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev);
2629
2630 return sysfs_emit(buf, "%x\n", dev->mdev->rev_id);
2631}
2632static DEVICE_ATTR_RO(hw_rev);
2633
2634static ssize_t board_id_show(struct device *device,
2635 struct device_attribute *attr, char *buf)
2636{
2637 struct mlx5_ib_dev *dev =
2638 rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev);
2639
2640 return sysfs_emit(buf, "%.*s\n", MLX5_BOARD_ID_LEN,
2641 dev->mdev->board_id);
2642}
2643static DEVICE_ATTR_RO(board_id);
2644
2645static struct attribute *mlx5_class_attributes[] = {
2646 &dev_attr_hw_rev.attr,
2647 &dev_attr_hca_type.attr,
2648 &dev_attr_board_id.attr,
2649 &dev_attr_fw_pages.attr,
2650 &dev_attr_reg_pages.attr,
2651 NULL,
2652};
2653
2654static const struct attribute_group mlx5_attr_group = {
2655 .attrs = mlx5_class_attributes,
2656};
2657
2658static void pkey_change_handler(struct work_struct *work)
2659{
2660 struct mlx5_ib_port_resources *ports =
2661 container_of(work, struct mlx5_ib_port_resources,
2662 pkey_change_work);
2663
2664 if (!ports->gsi)
2665 /*
2666 * We got this event before device was fully configured
2667 * and MAD registration code wasn't called/finished yet.
2668 */
2669 return;
2670
2671 mlx5_ib_gsi_pkey_change(ports->gsi);
2672}
2673
2674static void mlx5_ib_handle_internal_error(struct mlx5_ib_dev *ibdev)
2675{
2676 struct mlx5_ib_qp *mqp;
2677 struct mlx5_ib_cq *send_mcq, *recv_mcq;
2678 struct mlx5_core_cq *mcq;
2679 struct list_head cq_armed_list;
2680 unsigned long flags_qp;
2681 unsigned long flags_cq;
2682 unsigned long flags;
2683
2684 INIT_LIST_HEAD(&cq_armed_list);
2685
2686 /* Go over qp list reside on that ibdev, sync with create/destroy qp.*/
2687 spin_lock_irqsave(&ibdev->reset_flow_resource_lock, flags);
2688 list_for_each_entry(mqp, &ibdev->qp_list, qps_list) {
2689 spin_lock_irqsave(&mqp->sq.lock, flags_qp);
2690 if (mqp->sq.tail != mqp->sq.head) {
2691 send_mcq = to_mcq(mqp->ibqp.send_cq);
2692 spin_lock_irqsave(&send_mcq->lock, flags_cq);
2693 if (send_mcq->mcq.comp &&
2694 mqp->ibqp.send_cq->comp_handler) {
2695 if (!send_mcq->mcq.reset_notify_added) {
2696 send_mcq->mcq.reset_notify_added = 1;
2697 list_add_tail(&send_mcq->mcq.reset_notify,
2698 &cq_armed_list);
2699 }
2700 }
2701 spin_unlock_irqrestore(&send_mcq->lock, flags_cq);
2702 }
2703 spin_unlock_irqrestore(&mqp->sq.lock, flags_qp);
2704 spin_lock_irqsave(&mqp->rq.lock, flags_qp);
2705 /* no handling is needed for SRQ */
2706 if (!mqp->ibqp.srq) {
2707 if (mqp->rq.tail != mqp->rq.head) {
2708 recv_mcq = to_mcq(mqp->ibqp.recv_cq);
2709 spin_lock_irqsave(&recv_mcq->lock, flags_cq);
2710 if (recv_mcq->mcq.comp &&
2711 mqp->ibqp.recv_cq->comp_handler) {
2712 if (!recv_mcq->mcq.reset_notify_added) {
2713 recv_mcq->mcq.reset_notify_added = 1;
2714 list_add_tail(&recv_mcq->mcq.reset_notify,
2715 &cq_armed_list);
2716 }
2717 }
2718 spin_unlock_irqrestore(&recv_mcq->lock,
2719 flags_cq);
2720 }
2721 }
2722 spin_unlock_irqrestore(&mqp->rq.lock, flags_qp);
2723 }
2724 /*At that point all inflight post send were put to be executed as of we
2725 * lock/unlock above locks Now need to arm all involved CQs.
2726 */
2727 list_for_each_entry(mcq, &cq_armed_list, reset_notify) {
2728 mcq->comp(mcq, NULL);
2729 }
2730 spin_unlock_irqrestore(&ibdev->reset_flow_resource_lock, flags);
2731}
2732
2733static void delay_drop_handler(struct work_struct *work)
2734{
2735 int err;
2736 struct mlx5_ib_delay_drop *delay_drop =
2737 container_of(work, struct mlx5_ib_delay_drop,
2738 delay_drop_work);
2739
2740 atomic_inc(&delay_drop->events_cnt);
2741
2742 mutex_lock(&delay_drop->lock);
2743 err = mlx5_core_set_delay_drop(delay_drop->dev, delay_drop->timeout);
2744 if (err) {
2745 mlx5_ib_warn(delay_drop->dev, "Failed to set delay drop, timeout=%u\n",
2746 delay_drop->timeout);
2747 delay_drop->activate = false;
2748 }
2749 mutex_unlock(&delay_drop->lock);
2750}
2751
2752static void handle_general_event(struct mlx5_ib_dev *ibdev, struct mlx5_eqe *eqe,
2753 struct ib_event *ibev)
2754{
2755 u32 port = (eqe->data.port.port >> 4) & 0xf;
2756
2757 switch (eqe->sub_type) {
2758 case MLX5_GENERAL_SUBTYPE_DELAY_DROP_TIMEOUT:
2759 if (mlx5_ib_port_link_layer(&ibdev->ib_dev, port) ==
2760 IB_LINK_LAYER_ETHERNET)
2761 schedule_work(&ibdev->delay_drop.delay_drop_work);
2762 break;
2763 default: /* do nothing */
2764 return;
2765 }
2766}
2767
2768static int handle_port_change(struct mlx5_ib_dev *ibdev, struct mlx5_eqe *eqe,
2769 struct ib_event *ibev)
2770{
2771 u32 port = (eqe->data.port.port >> 4) & 0xf;
2772
2773 ibev->element.port_num = port;
2774
2775 switch (eqe->sub_type) {
2776 case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE:
2777 case MLX5_PORT_CHANGE_SUBTYPE_DOWN:
2778 case MLX5_PORT_CHANGE_SUBTYPE_INITIALIZED:
2779 /* In RoCE, port up/down events are handled in
2780 * mlx5_netdev_event().
2781 */
2782 if (mlx5_ib_port_link_layer(&ibdev->ib_dev, port) ==
2783 IB_LINK_LAYER_ETHERNET)
2784 return -EINVAL;
2785
2786 ibev->event = (eqe->sub_type == MLX5_PORT_CHANGE_SUBTYPE_ACTIVE) ?
2787 IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR;
2788 break;
2789
2790 case MLX5_PORT_CHANGE_SUBTYPE_LID:
2791 ibev->event = IB_EVENT_LID_CHANGE;
2792 break;
2793
2794 case MLX5_PORT_CHANGE_SUBTYPE_PKEY:
2795 ibev->event = IB_EVENT_PKEY_CHANGE;
2796 schedule_work(&ibdev->devr.ports[port - 1].pkey_change_work);
2797 break;
2798
2799 case MLX5_PORT_CHANGE_SUBTYPE_GUID:
2800 ibev->event = IB_EVENT_GID_CHANGE;
2801 break;
2802
2803 case MLX5_PORT_CHANGE_SUBTYPE_CLIENT_REREG:
2804 ibev->event = IB_EVENT_CLIENT_REREGISTER;
2805 break;
2806 default:
2807 return -EINVAL;
2808 }
2809
2810 return 0;
2811}
2812
2813static void mlx5_ib_handle_event(struct work_struct *_work)
2814{
2815 struct mlx5_ib_event_work *work =
2816 container_of(_work, struct mlx5_ib_event_work, work);
2817 struct mlx5_ib_dev *ibdev;
2818 struct ib_event ibev;
2819 bool fatal = false;
2820
2821 if (work->is_slave) {
2822 ibdev = mlx5_ib_get_ibdev_from_mpi(work->mpi);
2823 if (!ibdev)
2824 goto out;
2825 } else {
2826 ibdev = work->dev;
2827 }
2828
2829 switch (work->event) {
2830 case MLX5_DEV_EVENT_SYS_ERROR:
2831 ibev.event = IB_EVENT_DEVICE_FATAL;
2832 mlx5_ib_handle_internal_error(ibdev);
2833 ibev.element.port_num = (u8)(unsigned long)work->param;
2834 fatal = true;
2835 break;
2836 case MLX5_EVENT_TYPE_PORT_CHANGE:
2837 if (handle_port_change(ibdev, work->param, &ibev))
2838 goto out;
2839 break;
2840 case MLX5_EVENT_TYPE_GENERAL_EVENT:
2841 handle_general_event(ibdev, work->param, &ibev);
2842 fallthrough;
2843 default:
2844 goto out;
2845 }
2846
2847 ibev.device = &ibdev->ib_dev;
2848
2849 if (!rdma_is_port_valid(&ibdev->ib_dev, ibev.element.port_num)) {
2850 mlx5_ib_warn(ibdev, "warning: event on port %d\n", ibev.element.port_num);
2851 goto out;
2852 }
2853
2854 if (ibdev->ib_active)
2855 ib_dispatch_event(&ibev);
2856
2857 if (fatal)
2858 ibdev->ib_active = false;
2859out:
2860 kfree(work);
2861}
2862
2863static int mlx5_ib_event(struct notifier_block *nb,
2864 unsigned long event, void *param)
2865{
2866 struct mlx5_ib_event_work *work;
2867
2868 work = kmalloc(sizeof(*work), GFP_ATOMIC);
2869 if (!work)
2870 return NOTIFY_DONE;
2871
2872 INIT_WORK(&work->work, mlx5_ib_handle_event);
2873 work->dev = container_of(nb, struct mlx5_ib_dev, mdev_events);
2874 work->is_slave = false;
2875 work->param = param;
2876 work->event = event;
2877
2878 queue_work(mlx5_ib_event_wq, &work->work);
2879
2880 return NOTIFY_OK;
2881}
2882
2883static int mlx5_ib_event_slave_port(struct notifier_block *nb,
2884 unsigned long event, void *param)
2885{
2886 struct mlx5_ib_event_work *work;
2887
2888 work = kmalloc(sizeof(*work), GFP_ATOMIC);
2889 if (!work)
2890 return NOTIFY_DONE;
2891
2892 INIT_WORK(&work->work, mlx5_ib_handle_event);
2893 work->mpi = container_of(nb, struct mlx5_ib_multiport_info, mdev_events);
2894 work->is_slave = true;
2895 work->param = param;
2896 work->event = event;
2897 queue_work(mlx5_ib_event_wq, &work->work);
2898
2899 return NOTIFY_OK;
2900}
2901
2902static int mlx5_ib_get_plane_num(struct mlx5_core_dev *mdev, u8 *num_plane)
2903{
2904 struct mlx5_hca_vport_context vport_ctx;
2905 int err;
2906
2907 *num_plane = 0;
2908 if (!MLX5_CAP_GEN(mdev, ib_virt) || !MLX5_CAP_GEN_2(mdev, multiplane))
2909 return 0;
2910
2911 err = mlx5_query_hca_vport_context(mdev, 0, 1, 0, &vport_ctx);
2912 if (err)
2913 return err;
2914
2915 *num_plane = vport_ctx.num_plane;
2916 return 0;
2917}
2918
2919static int set_has_smi_cap(struct mlx5_ib_dev *dev)
2920{
2921 struct mlx5_hca_vport_context vport_ctx;
2922 int err;
2923 int port;
2924
2925 if (MLX5_CAP_GEN(dev->mdev, port_type) != MLX5_CAP_PORT_TYPE_IB)
2926 return 0;
2927
2928 for (port = 1; port <= dev->num_ports; port++) {
2929 if (dev->num_plane) {
2930 dev->port_caps[port - 1].has_smi = false;
2931 continue;
2932 } else if (!MLX5_CAP_GEN(dev->mdev, ib_virt) ||
2933 dev->ib_dev.type == RDMA_DEVICE_TYPE_SMI) {
2934 dev->port_caps[port - 1].has_smi = true;
2935 continue;
2936 }
2937
2938 err = mlx5_query_hca_vport_context(dev->mdev, 0, port, 0,
2939 &vport_ctx);
2940 if (err) {
2941 mlx5_ib_err(dev, "query_hca_vport_context for port=%d failed %d\n",
2942 port, err);
2943 return err;
2944 }
2945 dev->port_caps[port - 1].has_smi = vport_ctx.has_smi;
2946 }
2947
2948 return 0;
2949}
2950
2951static void get_ext_port_caps(struct mlx5_ib_dev *dev)
2952{
2953 unsigned int port;
2954
2955 rdma_for_each_port (&dev->ib_dev, port)
2956 mlx5_query_ext_port_caps(dev, port);
2957}
2958
2959static u8 mlx5_get_umr_fence(u8 umr_fence_cap)
2960{
2961 switch (umr_fence_cap) {
2962 case MLX5_CAP_UMR_FENCE_NONE:
2963 return MLX5_FENCE_MODE_NONE;
2964 case MLX5_CAP_UMR_FENCE_SMALL:
2965 return MLX5_FENCE_MODE_INITIATOR_SMALL;
2966 default:
2967 return MLX5_FENCE_MODE_STRONG_ORDERING;
2968 }
2969}
2970
2971int mlx5_ib_dev_res_cq_init(struct mlx5_ib_dev *dev)
2972{
2973 struct mlx5_ib_resources *devr = &dev->devr;
2974 struct ib_cq_init_attr cq_attr = {.cqe = 1};
2975 struct ib_device *ibdev;
2976 struct ib_pd *pd;
2977 struct ib_cq *cq;
2978 int ret = 0;
2979
2980
2981 /*
2982 * devr->c0 is set once, never changed until device unload.
2983 * Avoid taking the mutex if initialization is already done.
2984 */
2985 if (devr->c0)
2986 return 0;
2987
2988 mutex_lock(&devr->cq_lock);
2989 if (devr->c0)
2990 goto unlock;
2991
2992 ibdev = &dev->ib_dev;
2993 pd = ib_alloc_pd(ibdev, 0);
2994 if (IS_ERR(pd)) {
2995 ret = PTR_ERR(pd);
2996 mlx5_ib_err(dev, "Couldn't allocate PD for res init, err=%d\n", ret);
2997 goto unlock;
2998 }
2999
3000 cq = ib_create_cq(ibdev, NULL, NULL, NULL, &cq_attr);
3001 if (IS_ERR(cq)) {
3002 ret = PTR_ERR(cq);
3003 mlx5_ib_err(dev, "Couldn't create CQ for res init, err=%d\n", ret);
3004 ib_dealloc_pd(pd);
3005 goto unlock;
3006 }
3007
3008 devr->p0 = pd;
3009 devr->c0 = cq;
3010
3011unlock:
3012 mutex_unlock(&devr->cq_lock);
3013 return ret;
3014}
3015
3016int mlx5_ib_dev_res_srq_init(struct mlx5_ib_dev *dev)
3017{
3018 struct mlx5_ib_resources *devr = &dev->devr;
3019 struct ib_srq_init_attr attr;
3020 struct ib_srq *s0, *s1;
3021 int ret = 0;
3022
3023 /*
3024 * devr->s1 is set once, never changed until device unload.
3025 * Avoid taking the mutex if initialization is already done.
3026 */
3027 if (devr->s1)
3028 return 0;
3029
3030 mutex_lock(&devr->srq_lock);
3031 if (devr->s1)
3032 goto unlock;
3033
3034 ret = mlx5_ib_dev_res_cq_init(dev);
3035 if (ret)
3036 goto unlock;
3037
3038 memset(&attr, 0, sizeof(attr));
3039 attr.attr.max_sge = 1;
3040 attr.attr.max_wr = 1;
3041 attr.srq_type = IB_SRQT_XRC;
3042 attr.ext.cq = devr->c0;
3043
3044 s0 = ib_create_srq(devr->p0, &attr);
3045 if (IS_ERR(s0)) {
3046 ret = PTR_ERR(s0);
3047 mlx5_ib_err(dev, "Couldn't create SRQ 0 for res init, err=%d\n", ret);
3048 goto unlock;
3049 }
3050
3051 memset(&attr, 0, sizeof(attr));
3052 attr.attr.max_sge = 1;
3053 attr.attr.max_wr = 1;
3054 attr.srq_type = IB_SRQT_BASIC;
3055
3056 s1 = ib_create_srq(devr->p0, &attr);
3057 if (IS_ERR(s1)) {
3058 ret = PTR_ERR(s1);
3059 mlx5_ib_err(dev, "Couldn't create SRQ 1 for res init, err=%d\n", ret);
3060 ib_destroy_srq(s0);
3061 }
3062
3063 devr->s0 = s0;
3064 devr->s1 = s1;
3065
3066unlock:
3067 mutex_unlock(&devr->srq_lock);
3068 return ret;
3069}
3070
3071static int mlx5_ib_dev_res_init(struct mlx5_ib_dev *dev)
3072{
3073 struct mlx5_ib_resources *devr = &dev->devr;
3074 int ret;
3075
3076 if (!MLX5_CAP_GEN(dev->mdev, xrc))
3077 return -EOPNOTSUPP;
3078
3079 ret = mlx5_cmd_xrcd_alloc(dev->mdev, &devr->xrcdn0, 0);
3080 if (ret)
3081 return ret;
3082
3083 ret = mlx5_cmd_xrcd_alloc(dev->mdev, &devr->xrcdn1, 0);
3084 if (ret) {
3085 mlx5_cmd_xrcd_dealloc(dev->mdev, devr->xrcdn0, 0);
3086 return ret;
3087 }
3088
3089 mutex_init(&devr->cq_lock);
3090 mutex_init(&devr->srq_lock);
3091
3092 return 0;
3093}
3094
3095static void mlx5_ib_dev_res_cleanup(struct mlx5_ib_dev *dev)
3096{
3097 struct mlx5_ib_resources *devr = &dev->devr;
3098
3099 /* After s0/s1 init, they are not unset during the device lifetime. */
3100 if (devr->s1) {
3101 ib_destroy_srq(devr->s1);
3102 ib_destroy_srq(devr->s0);
3103 }
3104 mlx5_cmd_xrcd_dealloc(dev->mdev, devr->xrcdn1, 0);
3105 mlx5_cmd_xrcd_dealloc(dev->mdev, devr->xrcdn0, 0);
3106 /* After p0/c0 init, they are not unset during the device lifetime. */
3107 if (devr->c0) {
3108 ib_destroy_cq(devr->c0);
3109 ib_dealloc_pd(devr->p0);
3110 }
3111 mutex_destroy(&devr->cq_lock);
3112 mutex_destroy(&devr->srq_lock);
3113}
3114
3115static int
3116mlx5_ib_create_data_direct_resources(struct mlx5_ib_dev *dev)
3117{
3118 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
3119 struct mlx5_core_dev *mdev = dev->mdev;
3120 void *mkc;
3121 u32 mkey;
3122 u32 pdn;
3123 u32 *in;
3124 int err;
3125
3126 err = mlx5_core_alloc_pd(mdev, &pdn);
3127 if (err)
3128 return err;
3129
3130 in = kvzalloc(inlen, GFP_KERNEL);
3131 if (!in) {
3132 err = -ENOMEM;
3133 goto err;
3134 }
3135
3136 MLX5_SET(create_mkey_in, in, data_direct, 1);
3137 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
3138 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_PA);
3139 MLX5_SET(mkc, mkc, lw, 1);
3140 MLX5_SET(mkc, mkc, lr, 1);
3141 MLX5_SET(mkc, mkc, rw, 1);
3142 MLX5_SET(mkc, mkc, rr, 1);
3143 MLX5_SET(mkc, mkc, a, 1);
3144 MLX5_SET(mkc, mkc, pd, pdn);
3145 MLX5_SET(mkc, mkc, length64, 1);
3146 MLX5_SET(mkc, mkc, qpn, 0xffffff);
3147 err = mlx5_core_create_mkey(mdev, &mkey, in, inlen);
3148 kvfree(in);
3149 if (err)
3150 goto err;
3151
3152 dev->ddr.mkey = mkey;
3153 dev->ddr.pdn = pdn;
3154 return 0;
3155
3156err:
3157 mlx5_core_dealloc_pd(mdev, pdn);
3158 return err;
3159}
3160
3161static void
3162mlx5_ib_free_data_direct_resources(struct mlx5_ib_dev *dev)
3163{
3164 mlx5_core_destroy_mkey(dev->mdev, dev->ddr.mkey);
3165 mlx5_core_dealloc_pd(dev->mdev, dev->ddr.pdn);
3166}
3167
3168static u32 get_core_cap_flags(struct ib_device *ibdev,
3169 struct mlx5_hca_vport_context *rep)
3170{
3171 struct mlx5_ib_dev *dev = to_mdev(ibdev);
3172 enum rdma_link_layer ll = mlx5_ib_port_link_layer(ibdev, 1);
3173 u8 l3_type_cap = MLX5_CAP_ROCE(dev->mdev, l3_type);
3174 u8 roce_version_cap = MLX5_CAP_ROCE(dev->mdev, roce_version);
3175 bool raw_support = !mlx5_core_mp_enabled(dev->mdev);
3176 u32 ret = 0;
3177
3178 if (rep->grh_required)
3179 ret |= RDMA_CORE_CAP_IB_GRH_REQUIRED;
3180
3181 if (dev->num_plane)
3182 return ret | RDMA_CORE_CAP_PROT_IB | RDMA_CORE_CAP_IB_MAD |
3183 RDMA_CORE_CAP_IB_CM | RDMA_CORE_CAP_IB_SA |
3184 RDMA_CORE_CAP_AF_IB;
3185 else if (ibdev->type == RDMA_DEVICE_TYPE_SMI)
3186 return ret | RDMA_CORE_CAP_IB_MAD | RDMA_CORE_CAP_IB_SMI;
3187
3188 if (ll == IB_LINK_LAYER_INFINIBAND)
3189 return ret | RDMA_CORE_PORT_IBA_IB;
3190
3191 if (raw_support)
3192 ret |= RDMA_CORE_PORT_RAW_PACKET;
3193
3194 if (!(l3_type_cap & MLX5_ROCE_L3_TYPE_IPV4_CAP))
3195 return ret;
3196
3197 if (!(l3_type_cap & MLX5_ROCE_L3_TYPE_IPV6_CAP))
3198 return ret;
3199
3200 if (roce_version_cap & MLX5_ROCE_VERSION_1_CAP)
3201 ret |= RDMA_CORE_PORT_IBA_ROCE;
3202
3203 if (roce_version_cap & MLX5_ROCE_VERSION_2_CAP)
3204 ret |= RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
3205
3206 return ret;
3207}
3208
3209static int mlx5_port_immutable(struct ib_device *ibdev, u32 port_num,
3210 struct ib_port_immutable *immutable)
3211{
3212 struct ib_port_attr attr;
3213 struct mlx5_ib_dev *dev = to_mdev(ibdev);
3214 enum rdma_link_layer ll = mlx5_ib_port_link_layer(ibdev, port_num);
3215 struct mlx5_hca_vport_context rep = {0};
3216 int err;
3217
3218 err = ib_query_port(ibdev, port_num, &attr);
3219 if (err)
3220 return err;
3221
3222 if (ll == IB_LINK_LAYER_INFINIBAND) {
3223 if (ibdev->type == RDMA_DEVICE_TYPE_SMI)
3224 port_num = smi_to_native_portnum(dev, port_num);
3225
3226 err = mlx5_query_hca_vport_context(dev->mdev, 0, port_num, 0,
3227 &rep);
3228 if (err)
3229 return err;
3230 }
3231
3232 immutable->pkey_tbl_len = attr.pkey_tbl_len;
3233 immutable->gid_tbl_len = attr.gid_tbl_len;
3234 immutable->core_cap_flags = get_core_cap_flags(ibdev, &rep);
3235 immutable->max_mad_size = IB_MGMT_MAD_SIZE;
3236
3237 return 0;
3238}
3239
3240static int mlx5_port_rep_immutable(struct ib_device *ibdev, u32 port_num,
3241 struct ib_port_immutable *immutable)
3242{
3243 struct ib_port_attr attr;
3244 int err;
3245
3246 immutable->core_cap_flags = RDMA_CORE_PORT_RAW_PACKET;
3247
3248 err = ib_query_port(ibdev, port_num, &attr);
3249 if (err)
3250 return err;
3251
3252 immutable->pkey_tbl_len = attr.pkey_tbl_len;
3253 immutable->gid_tbl_len = attr.gid_tbl_len;
3254 immutable->core_cap_flags = RDMA_CORE_PORT_RAW_PACKET;
3255
3256 return 0;
3257}
3258
3259static void get_dev_fw_str(struct ib_device *ibdev, char *str)
3260{
3261 struct mlx5_ib_dev *dev =
3262 container_of(ibdev, struct mlx5_ib_dev, ib_dev);
3263 snprintf(str, IB_FW_VERSION_NAME_MAX, "%d.%d.%04d",
3264 fw_rev_maj(dev->mdev), fw_rev_min(dev->mdev),
3265 fw_rev_sub(dev->mdev));
3266}
3267
3268static int lag_event(struct notifier_block *nb, unsigned long event, void *data)
3269{
3270 struct mlx5_ib_dev *dev = container_of(nb, struct mlx5_ib_dev,
3271 lag_events);
3272 struct mlx5_core_dev *mdev = dev->mdev;
3273 struct ib_device *ibdev = &dev->ib_dev;
3274 struct net_device *old_ndev = NULL;
3275 struct mlx5_ib_port *port;
3276 struct net_device *ndev;
3277 u32 portnum = 0;
3278 int ret = 0;
3279 int i;
3280
3281 switch (event) {
3282 case MLX5_DRIVER_EVENT_ACTIVE_BACKUP_LAG_CHANGE_LOWERSTATE:
3283 ndev = data;
3284 if (ndev) {
3285 if (!mlx5_lag_is_roce(mdev)) {
3286 // sriov lag
3287 for (i = 0; i < dev->num_ports; i++) {
3288 port = &dev->port[i];
3289 if (port->rep && port->rep->vport ==
3290 MLX5_VPORT_UPLINK) {
3291 portnum = i;
3292 break;
3293 }
3294 }
3295 }
3296 old_ndev = ib_device_get_netdev(ibdev, portnum + 1);
3297 ret = ib_device_set_netdev(ibdev, ndev, portnum + 1);
3298 if (ret)
3299 goto out;
3300
3301 if (old_ndev)
3302 roce_del_all_netdev_gids(ibdev, portnum + 1,
3303 old_ndev);
3304 rdma_roce_rescan_port(ibdev, portnum + 1);
3305 }
3306 break;
3307 default:
3308 return NOTIFY_DONE;
3309 }
3310
3311out:
3312 dev_put(old_ndev);
3313 return notifier_from_errno(ret);
3314}
3315
3316static void mlx5e_lag_event_register(struct mlx5_ib_dev *dev)
3317{
3318 dev->lag_events.notifier_call = lag_event;
3319 blocking_notifier_chain_register(&dev->mdev->priv.lag_nh,
3320 &dev->lag_events);
3321}
3322
3323static void mlx5e_lag_event_unregister(struct mlx5_ib_dev *dev)
3324{
3325 blocking_notifier_chain_unregister(&dev->mdev->priv.lag_nh,
3326 &dev->lag_events);
3327}
3328
3329static int mlx5_eth_lag_init(struct mlx5_ib_dev *dev)
3330{
3331 struct mlx5_core_dev *mdev = dev->mdev;
3332 struct mlx5_flow_namespace *ns = mlx5_get_flow_namespace(mdev,
3333 MLX5_FLOW_NAMESPACE_LAG);
3334 struct mlx5_flow_table *ft;
3335 int err;
3336
3337 if (!ns || !mlx5_lag_is_active(mdev))
3338 return 0;
3339
3340 err = mlx5_cmd_create_vport_lag(mdev);
3341 if (err)
3342 return err;
3343
3344 ft = mlx5_create_lag_demux_flow_table(ns, 0, 0);
3345 if (IS_ERR(ft)) {
3346 err = PTR_ERR(ft);
3347 goto err_destroy_vport_lag;
3348 }
3349
3350 mlx5e_lag_event_register(dev);
3351 dev->flow_db->lag_demux_ft = ft;
3352 dev->lag_ports = mlx5_lag_get_num_ports(mdev);
3353 dev->lag_active = true;
3354 return 0;
3355
3356err_destroy_vport_lag:
3357 mlx5_cmd_destroy_vport_lag(mdev);
3358 return err;
3359}
3360
3361static void mlx5_eth_lag_cleanup(struct mlx5_ib_dev *dev)
3362{
3363 struct mlx5_core_dev *mdev = dev->mdev;
3364
3365 if (dev->lag_active) {
3366 dev->lag_active = false;
3367
3368 mlx5e_lag_event_unregister(dev);
3369 mlx5_destroy_flow_table(dev->flow_db->lag_demux_ft);
3370 dev->flow_db->lag_demux_ft = NULL;
3371
3372 mlx5_cmd_destroy_vport_lag(mdev);
3373 }
3374}
3375
3376static void mlx5_netdev_notifier_register(struct mlx5_roce *roce,
3377 struct net_device *netdev)
3378{
3379 int err;
3380
3381 if (roce->tracking_netdev)
3382 return;
3383 roce->tracking_netdev = netdev;
3384 roce->nb.notifier_call = mlx5_netdev_event;
3385 err = register_netdevice_notifier_dev_net(netdev, &roce->nb, &roce->nn);
3386 WARN_ON(err);
3387}
3388
3389static void mlx5_netdev_notifier_unregister(struct mlx5_roce *roce)
3390{
3391 if (!roce->tracking_netdev)
3392 return;
3393 unregister_netdevice_notifier_dev_net(roce->tracking_netdev, &roce->nb,
3394 &roce->nn);
3395 roce->tracking_netdev = NULL;
3396}
3397
3398static int mlx5e_mdev_notifier_event(struct notifier_block *nb,
3399 unsigned long event, void *data)
3400{
3401 struct mlx5_roce *roce = container_of(nb, struct mlx5_roce, mdev_nb);
3402 struct net_device *netdev = data;
3403
3404 switch (event) {
3405 case MLX5_DRIVER_EVENT_UPLINK_NETDEV:
3406 if (netdev)
3407 mlx5_netdev_notifier_register(roce, netdev);
3408 else
3409 mlx5_netdev_notifier_unregister(roce);
3410 break;
3411 default:
3412 return NOTIFY_DONE;
3413 }
3414
3415 return NOTIFY_OK;
3416}
3417
3418static void mlx5_mdev_netdev_track(struct mlx5_ib_dev *dev, u32 port_num)
3419{
3420 struct mlx5_roce *roce = &dev->port[port_num].roce;
3421
3422 roce->mdev_nb.notifier_call = mlx5e_mdev_notifier_event;
3423 mlx5_blocking_notifier_register(dev->mdev, &roce->mdev_nb);
3424 mlx5_core_uplink_netdev_event_replay(dev->mdev);
3425}
3426
3427static void mlx5_mdev_netdev_untrack(struct mlx5_ib_dev *dev, u32 port_num)
3428{
3429 struct mlx5_roce *roce = &dev->port[port_num].roce;
3430
3431 mlx5_blocking_notifier_unregister(dev->mdev, &roce->mdev_nb);
3432 mlx5_netdev_notifier_unregister(roce);
3433}
3434
3435static int mlx5_enable_eth(struct mlx5_ib_dev *dev)
3436{
3437 int err;
3438
3439 if (!dev->is_rep && dev->profile != &raw_eth_profile) {
3440 err = mlx5_nic_vport_enable_roce(dev->mdev);
3441 if (err)
3442 return err;
3443 }
3444
3445 err = mlx5_eth_lag_init(dev);
3446 if (err)
3447 goto err_disable_roce;
3448
3449 return 0;
3450
3451err_disable_roce:
3452 if (!dev->is_rep && dev->profile != &raw_eth_profile)
3453 mlx5_nic_vport_disable_roce(dev->mdev);
3454
3455 return err;
3456}
3457
3458static void mlx5_disable_eth(struct mlx5_ib_dev *dev)
3459{
3460 mlx5_eth_lag_cleanup(dev);
3461 if (!dev->is_rep && dev->profile != &raw_eth_profile)
3462 mlx5_nic_vport_disable_roce(dev->mdev);
3463}
3464
3465static int mlx5_ib_rn_get_params(struct ib_device *device, u32 port_num,
3466 enum rdma_netdev_t type,
3467 struct rdma_netdev_alloc_params *params)
3468{
3469 if (type != RDMA_NETDEV_IPOIB)
3470 return -EOPNOTSUPP;
3471
3472 return mlx5_rdma_rn_get_params(to_mdev(device)->mdev, device, params);
3473}
3474
3475static ssize_t delay_drop_timeout_read(struct file *filp, char __user *buf,
3476 size_t count, loff_t *pos)
3477{
3478 struct mlx5_ib_delay_drop *delay_drop = filp->private_data;
3479 char lbuf[20];
3480 int len;
3481
3482 len = snprintf(lbuf, sizeof(lbuf), "%u\n", delay_drop->timeout);
3483 return simple_read_from_buffer(buf, count, pos, lbuf, len);
3484}
3485
3486static ssize_t delay_drop_timeout_write(struct file *filp, const char __user *buf,
3487 size_t count, loff_t *pos)
3488{
3489 struct mlx5_ib_delay_drop *delay_drop = filp->private_data;
3490 u32 timeout;
3491 u32 var;
3492
3493 if (kstrtouint_from_user(buf, count, 0, &var))
3494 return -EFAULT;
3495
3496 timeout = min_t(u32, roundup(var, 100), MLX5_MAX_DELAY_DROP_TIMEOUT_MS *
3497 1000);
3498 if (timeout != var)
3499 mlx5_ib_dbg(delay_drop->dev, "Round delay drop timeout to %u usec\n",
3500 timeout);
3501
3502 delay_drop->timeout = timeout;
3503
3504 return count;
3505}
3506
3507static const struct file_operations fops_delay_drop_timeout = {
3508 .owner = THIS_MODULE,
3509 .open = simple_open,
3510 .write = delay_drop_timeout_write,
3511 .read = delay_drop_timeout_read,
3512};
3513
3514static void mlx5_ib_unbind_slave_port(struct mlx5_ib_dev *ibdev,
3515 struct mlx5_ib_multiport_info *mpi)
3516{
3517 u32 port_num = mlx5_core_native_port_num(mpi->mdev) - 1;
3518 struct mlx5_ib_port *port = &ibdev->port[port_num];
3519 int comps;
3520 int err;
3521 int i;
3522
3523 lockdep_assert_held(&mlx5_ib_multiport_mutex);
3524
3525 mlx5_ib_disable_lb_mp(ibdev->mdev, mpi->mdev);
3526
3527 mlx5_core_mp_event_replay(ibdev->mdev,
3528 MLX5_DRIVER_EVENT_AFFILIATION_REMOVED,
3529 NULL);
3530 mlx5_core_mp_event_replay(mpi->mdev,
3531 MLX5_DRIVER_EVENT_AFFILIATION_REMOVED,
3532 NULL);
3533
3534 mlx5_ib_cleanup_cong_debugfs(ibdev, port_num);
3535
3536 spin_lock(&port->mp.mpi_lock);
3537 if (!mpi->ibdev) {
3538 spin_unlock(&port->mp.mpi_lock);
3539 return;
3540 }
3541
3542 mpi->ibdev = NULL;
3543
3544 spin_unlock(&port->mp.mpi_lock);
3545 if (mpi->mdev_events.notifier_call)
3546 mlx5_notifier_unregister(mpi->mdev, &mpi->mdev_events);
3547 mpi->mdev_events.notifier_call = NULL;
3548 mlx5_mdev_netdev_untrack(ibdev, port_num);
3549 spin_lock(&port->mp.mpi_lock);
3550
3551 comps = mpi->mdev_refcnt;
3552 if (comps) {
3553 mpi->unaffiliate = true;
3554 init_completion(&mpi->unref_comp);
3555 spin_unlock(&port->mp.mpi_lock);
3556
3557 for (i = 0; i < comps; i++)
3558 wait_for_completion(&mpi->unref_comp);
3559
3560 spin_lock(&port->mp.mpi_lock);
3561 mpi->unaffiliate = false;
3562 }
3563
3564 port->mp.mpi = NULL;
3565
3566 spin_unlock(&port->mp.mpi_lock);
3567
3568 err = mlx5_nic_vport_unaffiliate_multiport(mpi->mdev);
3569
3570 mlx5_ib_dbg(ibdev, "unaffiliated port %u\n", port_num + 1);
3571 /* Log an error, still needed to cleanup the pointers and add
3572 * it back to the list.
3573 */
3574 if (err)
3575 mlx5_ib_err(ibdev, "Failed to unaffiliate port %u\n",
3576 port_num + 1);
3577
3578 ibdev->port[port_num].roce.last_port_state = IB_PORT_DOWN;
3579}
3580
3581static bool mlx5_ib_bind_slave_port(struct mlx5_ib_dev *ibdev,
3582 struct mlx5_ib_multiport_info *mpi)
3583{
3584 u32 port_num = mlx5_core_native_port_num(mpi->mdev) - 1;
3585 u64 key;
3586 int err;
3587
3588 lockdep_assert_held(&mlx5_ib_multiport_mutex);
3589
3590 spin_lock(&ibdev->port[port_num].mp.mpi_lock);
3591 if (ibdev->port[port_num].mp.mpi) {
3592 mlx5_ib_dbg(ibdev, "port %u already affiliated.\n",
3593 port_num + 1);
3594 spin_unlock(&ibdev->port[port_num].mp.mpi_lock);
3595 return false;
3596 }
3597
3598 ibdev->port[port_num].mp.mpi = mpi;
3599 mpi->ibdev = ibdev;
3600 mpi->mdev_events.notifier_call = NULL;
3601 spin_unlock(&ibdev->port[port_num].mp.mpi_lock);
3602
3603 err = mlx5_nic_vport_affiliate_multiport(ibdev->mdev, mpi->mdev);
3604 if (err)
3605 goto unbind;
3606
3607 mlx5_mdev_netdev_track(ibdev, port_num);
3608
3609 mpi->mdev_events.notifier_call = mlx5_ib_event_slave_port;
3610 mlx5_notifier_register(mpi->mdev, &mpi->mdev_events);
3611
3612 mlx5_ib_init_cong_debugfs(ibdev, port_num);
3613
3614 key = mpi->mdev->priv.adev_idx;
3615 mlx5_core_mp_event_replay(mpi->mdev,
3616 MLX5_DRIVER_EVENT_AFFILIATION_DONE,
3617 &key);
3618 mlx5_core_mp_event_replay(ibdev->mdev,
3619 MLX5_DRIVER_EVENT_AFFILIATION_DONE,
3620 &key);
3621
3622 err = mlx5_ib_enable_lb_mp(ibdev->mdev, mpi->mdev);
3623 if (err)
3624 goto unbind;
3625
3626 return true;
3627
3628unbind:
3629 mlx5_ib_unbind_slave_port(ibdev, mpi);
3630 return false;
3631}
3632
3633static int mlx5_ib_data_direct_init(struct mlx5_ib_dev *dev)
3634{
3635 char vuid[MLX5_ST_SZ_BYTES(array1024_auto) + 1] = {};
3636 int ret;
3637
3638 if (!MLX5_CAP_GEN(dev->mdev, data_direct) ||
3639 !MLX5_CAP_GEN_2(dev->mdev, query_vuid))
3640 return 0;
3641
3642 ret = mlx5_cmd_query_vuid(dev->mdev, true, vuid);
3643 if (ret)
3644 return ret;
3645
3646 ret = mlx5_ib_create_data_direct_resources(dev);
3647 if (ret)
3648 return ret;
3649
3650 INIT_LIST_HEAD(&dev->data_direct_mr_list);
3651 ret = mlx5_data_direct_ib_reg(dev, vuid);
3652 if (ret)
3653 mlx5_ib_free_data_direct_resources(dev);
3654
3655 return ret;
3656}
3657
3658static void mlx5_ib_data_direct_cleanup(struct mlx5_ib_dev *dev)
3659{
3660 if (!MLX5_CAP_GEN(dev->mdev, data_direct) ||
3661 !MLX5_CAP_GEN_2(dev->mdev, query_vuid))
3662 return;
3663
3664 mlx5_data_direct_ib_unreg(dev);
3665 mlx5_ib_free_data_direct_resources(dev);
3666}
3667
3668static int mlx5_ib_init_multiport_master(struct mlx5_ib_dev *dev)
3669{
3670 u32 port_num = mlx5_core_native_port_num(dev->mdev) - 1;
3671 enum rdma_link_layer ll = mlx5_ib_port_link_layer(&dev->ib_dev,
3672 port_num + 1);
3673 struct mlx5_ib_multiport_info *mpi;
3674 int err;
3675 u32 i;
3676
3677 if (!mlx5_core_is_mp_master(dev->mdev) || ll != IB_LINK_LAYER_ETHERNET)
3678 return 0;
3679
3680 err = mlx5_query_nic_vport_system_image_guid(dev->mdev,
3681 &dev->sys_image_guid);
3682 if (err)
3683 return err;
3684
3685 err = mlx5_nic_vport_enable_roce(dev->mdev);
3686 if (err)
3687 return err;
3688
3689 mutex_lock(&mlx5_ib_multiport_mutex);
3690 for (i = 0; i < dev->num_ports; i++) {
3691 bool bound = false;
3692
3693 /* build a stub multiport info struct for the native port. */
3694 if (i == port_num) {
3695 mpi = kzalloc(sizeof(*mpi), GFP_KERNEL);
3696 if (!mpi) {
3697 mutex_unlock(&mlx5_ib_multiport_mutex);
3698 mlx5_nic_vport_disable_roce(dev->mdev);
3699 return -ENOMEM;
3700 }
3701
3702 mpi->is_master = true;
3703 mpi->mdev = dev->mdev;
3704 mpi->sys_image_guid = dev->sys_image_guid;
3705 dev->port[i].mp.mpi = mpi;
3706 mpi->ibdev = dev;
3707 mpi = NULL;
3708 continue;
3709 }
3710
3711 list_for_each_entry(mpi, &mlx5_ib_unaffiliated_port_list,
3712 list) {
3713 if (dev->sys_image_guid == mpi->sys_image_guid &&
3714 (mlx5_core_native_port_num(mpi->mdev) - 1) == i &&
3715 mlx5_core_same_coredev_type(dev->mdev, mpi->mdev)) {
3716 bound = mlx5_ib_bind_slave_port(dev, mpi);
3717 }
3718
3719 if (bound) {
3720 dev_dbg(mpi->mdev->device,
3721 "removing port from unaffiliated list.\n");
3722 mlx5_ib_dbg(dev, "port %d bound\n", i + 1);
3723 list_del(&mpi->list);
3724 break;
3725 }
3726 }
3727 if (!bound)
3728 mlx5_ib_dbg(dev, "no free port found for port %d\n",
3729 i + 1);
3730 }
3731
3732 list_add_tail(&dev->ib_dev_list, &mlx5_ib_dev_list);
3733 mutex_unlock(&mlx5_ib_multiport_mutex);
3734 return err;
3735}
3736
3737static void mlx5_ib_cleanup_multiport_master(struct mlx5_ib_dev *dev)
3738{
3739 u32 port_num = mlx5_core_native_port_num(dev->mdev) - 1;
3740 enum rdma_link_layer ll = mlx5_ib_port_link_layer(&dev->ib_dev,
3741 port_num + 1);
3742 u32 i;
3743
3744 if (!mlx5_core_is_mp_master(dev->mdev) || ll != IB_LINK_LAYER_ETHERNET)
3745 return;
3746
3747 mutex_lock(&mlx5_ib_multiport_mutex);
3748 for (i = 0; i < dev->num_ports; i++) {
3749 if (dev->port[i].mp.mpi) {
3750 /* Destroy the native port stub */
3751 if (i == port_num) {
3752 kfree(dev->port[i].mp.mpi);
3753 dev->port[i].mp.mpi = NULL;
3754 } else {
3755 mlx5_ib_dbg(dev, "unbinding port_num: %u\n",
3756 i + 1);
3757 list_add_tail(&dev->port[i].mp.mpi->list,
3758 &mlx5_ib_unaffiliated_port_list);
3759 mlx5_ib_unbind_slave_port(dev,
3760 dev->port[i].mp.mpi);
3761 }
3762 }
3763 }
3764
3765 mlx5_ib_dbg(dev, "removing from devlist\n");
3766 list_del(&dev->ib_dev_list);
3767 mutex_unlock(&mlx5_ib_multiport_mutex);
3768
3769 mlx5_nic_vport_disable_roce(dev->mdev);
3770}
3771
3772static int mmap_obj_cleanup(struct ib_uobject *uobject,
3773 enum rdma_remove_reason why,
3774 struct uverbs_attr_bundle *attrs)
3775{
3776 struct mlx5_user_mmap_entry *obj = uobject->object;
3777
3778 rdma_user_mmap_entry_remove(&obj->rdma_entry);
3779 return 0;
3780}
3781
3782static int mlx5_rdma_user_mmap_entry_insert(struct mlx5_ib_ucontext *c,
3783 struct mlx5_user_mmap_entry *entry,
3784 size_t length)
3785{
3786 return rdma_user_mmap_entry_insert_range(
3787 &c->ibucontext, &entry->rdma_entry, length,
3788 (MLX5_IB_MMAP_OFFSET_START << 16),
3789 ((MLX5_IB_MMAP_OFFSET_END << 16) + (1UL << 16) - 1));
3790}
3791
3792static struct mlx5_user_mmap_entry *
3793alloc_var_entry(struct mlx5_ib_ucontext *c)
3794{
3795 struct mlx5_user_mmap_entry *entry;
3796 struct mlx5_var_table *var_table;
3797 u32 page_idx;
3798 int err;
3799
3800 var_table = &to_mdev(c->ibucontext.device)->var_table;
3801 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
3802 if (!entry)
3803 return ERR_PTR(-ENOMEM);
3804
3805 mutex_lock(&var_table->bitmap_lock);
3806 page_idx = find_first_zero_bit(var_table->bitmap,
3807 var_table->num_var_hw_entries);
3808 if (page_idx >= var_table->num_var_hw_entries) {
3809 err = -ENOSPC;
3810 mutex_unlock(&var_table->bitmap_lock);
3811 goto end;
3812 }
3813
3814 set_bit(page_idx, var_table->bitmap);
3815 mutex_unlock(&var_table->bitmap_lock);
3816
3817 entry->address = var_table->hw_start_addr +
3818 (page_idx * var_table->stride_size);
3819 entry->page_idx = page_idx;
3820 entry->mmap_flag = MLX5_IB_MMAP_TYPE_VAR;
3821
3822 err = mlx5_rdma_user_mmap_entry_insert(c, entry,
3823 var_table->stride_size);
3824 if (err)
3825 goto err_insert;
3826
3827 return entry;
3828
3829err_insert:
3830 mutex_lock(&var_table->bitmap_lock);
3831 clear_bit(page_idx, var_table->bitmap);
3832 mutex_unlock(&var_table->bitmap_lock);
3833end:
3834 kfree(entry);
3835 return ERR_PTR(err);
3836}
3837
3838static int UVERBS_HANDLER(MLX5_IB_METHOD_VAR_OBJ_ALLOC)(
3839 struct uverbs_attr_bundle *attrs)
3840{
3841 struct ib_uobject *uobj = uverbs_attr_get_uobject(
3842 attrs, MLX5_IB_ATTR_VAR_OBJ_ALLOC_HANDLE);
3843 struct mlx5_ib_ucontext *c;
3844 struct mlx5_user_mmap_entry *entry;
3845 u64 mmap_offset;
3846 u32 length;
3847 int err;
3848
3849 c = to_mucontext(ib_uverbs_get_ucontext(attrs));
3850 if (IS_ERR(c))
3851 return PTR_ERR(c);
3852
3853 entry = alloc_var_entry(c);
3854 if (IS_ERR(entry))
3855 return PTR_ERR(entry);
3856
3857 mmap_offset = mlx5_entry_to_mmap_offset(entry);
3858 length = entry->rdma_entry.npages * PAGE_SIZE;
3859 uobj->object = entry;
3860 uverbs_finalize_uobj_create(attrs, MLX5_IB_ATTR_VAR_OBJ_ALLOC_HANDLE);
3861
3862 err = uverbs_copy_to(attrs, MLX5_IB_ATTR_VAR_OBJ_ALLOC_MMAP_OFFSET,
3863 &mmap_offset, sizeof(mmap_offset));
3864 if (err)
3865 return err;
3866
3867 err = uverbs_copy_to(attrs, MLX5_IB_ATTR_VAR_OBJ_ALLOC_PAGE_ID,
3868 &entry->page_idx, sizeof(entry->page_idx));
3869 if (err)
3870 return err;
3871
3872 err = uverbs_copy_to(attrs, MLX5_IB_ATTR_VAR_OBJ_ALLOC_MMAP_LENGTH,
3873 &length, sizeof(length));
3874 return err;
3875}
3876
3877DECLARE_UVERBS_NAMED_METHOD(
3878 MLX5_IB_METHOD_VAR_OBJ_ALLOC,
3879 UVERBS_ATTR_IDR(MLX5_IB_ATTR_VAR_OBJ_ALLOC_HANDLE,
3880 MLX5_IB_OBJECT_VAR,
3881 UVERBS_ACCESS_NEW,
3882 UA_MANDATORY),
3883 UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_VAR_OBJ_ALLOC_PAGE_ID,
3884 UVERBS_ATTR_TYPE(u32),
3885 UA_MANDATORY),
3886 UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_VAR_OBJ_ALLOC_MMAP_LENGTH,
3887 UVERBS_ATTR_TYPE(u32),
3888 UA_MANDATORY),
3889 UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_VAR_OBJ_ALLOC_MMAP_OFFSET,
3890 UVERBS_ATTR_TYPE(u64),
3891 UA_MANDATORY));
3892
3893DECLARE_UVERBS_NAMED_METHOD_DESTROY(
3894 MLX5_IB_METHOD_VAR_OBJ_DESTROY,
3895 UVERBS_ATTR_IDR(MLX5_IB_ATTR_VAR_OBJ_DESTROY_HANDLE,
3896 MLX5_IB_OBJECT_VAR,
3897 UVERBS_ACCESS_DESTROY,
3898 UA_MANDATORY));
3899
3900DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_VAR,
3901 UVERBS_TYPE_ALLOC_IDR(mmap_obj_cleanup),
3902 &UVERBS_METHOD(MLX5_IB_METHOD_VAR_OBJ_ALLOC),
3903 &UVERBS_METHOD(MLX5_IB_METHOD_VAR_OBJ_DESTROY));
3904
3905static bool var_is_supported(struct ib_device *device)
3906{
3907 struct mlx5_ib_dev *dev = to_mdev(device);
3908
3909 return (MLX5_CAP_GEN_64(dev->mdev, general_obj_types) &
3910 MLX5_GENERAL_OBJ_TYPES_CAP_VIRTIO_NET_Q);
3911}
3912
3913static struct mlx5_user_mmap_entry *
3914alloc_uar_entry(struct mlx5_ib_ucontext *c,
3915 enum mlx5_ib_uapi_uar_alloc_type alloc_type)
3916{
3917 struct mlx5_user_mmap_entry *entry;
3918 struct mlx5_ib_dev *dev;
3919 u32 uar_index;
3920 int err;
3921
3922 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
3923 if (!entry)
3924 return ERR_PTR(-ENOMEM);
3925
3926 dev = to_mdev(c->ibucontext.device);
3927 err = mlx5_cmd_uar_alloc(dev->mdev, &uar_index, c->devx_uid);
3928 if (err)
3929 goto end;
3930
3931 entry->page_idx = uar_index;
3932 entry->address = uar_index2paddress(dev, uar_index);
3933 if (alloc_type == MLX5_IB_UAPI_UAR_ALLOC_TYPE_BF)
3934 entry->mmap_flag = MLX5_IB_MMAP_TYPE_UAR_WC;
3935 else
3936 entry->mmap_flag = MLX5_IB_MMAP_TYPE_UAR_NC;
3937
3938 err = mlx5_rdma_user_mmap_entry_insert(c, entry, PAGE_SIZE);
3939 if (err)
3940 goto err_insert;
3941
3942 return entry;
3943
3944err_insert:
3945 mlx5_cmd_uar_dealloc(dev->mdev, uar_index, c->devx_uid);
3946end:
3947 kfree(entry);
3948 return ERR_PTR(err);
3949}
3950
3951static int UVERBS_HANDLER(MLX5_IB_METHOD_UAR_OBJ_ALLOC)(
3952 struct uverbs_attr_bundle *attrs)
3953{
3954 struct ib_uobject *uobj = uverbs_attr_get_uobject(
3955 attrs, MLX5_IB_ATTR_UAR_OBJ_ALLOC_HANDLE);
3956 enum mlx5_ib_uapi_uar_alloc_type alloc_type;
3957 struct mlx5_ib_ucontext *c;
3958 struct mlx5_user_mmap_entry *entry;
3959 u64 mmap_offset;
3960 u32 length;
3961 int err;
3962
3963 c = to_mucontext(ib_uverbs_get_ucontext(attrs));
3964 if (IS_ERR(c))
3965 return PTR_ERR(c);
3966
3967 err = uverbs_get_const(&alloc_type, attrs,
3968 MLX5_IB_ATTR_UAR_OBJ_ALLOC_TYPE);
3969 if (err)
3970 return err;
3971
3972 if (alloc_type != MLX5_IB_UAPI_UAR_ALLOC_TYPE_BF &&
3973 alloc_type != MLX5_IB_UAPI_UAR_ALLOC_TYPE_NC)
3974 return -EOPNOTSUPP;
3975
3976 if (!mlx5_wc_support_get(to_mdev(c->ibucontext.device)->mdev) &&
3977 alloc_type == MLX5_IB_UAPI_UAR_ALLOC_TYPE_BF)
3978 return -EOPNOTSUPP;
3979
3980 entry = alloc_uar_entry(c, alloc_type);
3981 if (IS_ERR(entry))
3982 return PTR_ERR(entry);
3983
3984 mmap_offset = mlx5_entry_to_mmap_offset(entry);
3985 length = entry->rdma_entry.npages * PAGE_SIZE;
3986 uobj->object = entry;
3987 uverbs_finalize_uobj_create(attrs, MLX5_IB_ATTR_UAR_OBJ_ALLOC_HANDLE);
3988
3989 err = uverbs_copy_to(attrs, MLX5_IB_ATTR_UAR_OBJ_ALLOC_MMAP_OFFSET,
3990 &mmap_offset, sizeof(mmap_offset));
3991 if (err)
3992 return err;
3993
3994 err = uverbs_copy_to(attrs, MLX5_IB_ATTR_UAR_OBJ_ALLOC_PAGE_ID,
3995 &entry->page_idx, sizeof(entry->page_idx));
3996 if (err)
3997 return err;
3998
3999 err = uverbs_copy_to(attrs, MLX5_IB_ATTR_UAR_OBJ_ALLOC_MMAP_LENGTH,
4000 &length, sizeof(length));
4001 return err;
4002}
4003
4004DECLARE_UVERBS_NAMED_METHOD(
4005 MLX5_IB_METHOD_UAR_OBJ_ALLOC,
4006 UVERBS_ATTR_IDR(MLX5_IB_ATTR_UAR_OBJ_ALLOC_HANDLE,
4007 MLX5_IB_OBJECT_UAR,
4008 UVERBS_ACCESS_NEW,
4009 UA_MANDATORY),
4010 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_UAR_OBJ_ALLOC_TYPE,
4011 enum mlx5_ib_uapi_uar_alloc_type,
4012 UA_MANDATORY),
4013 UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_UAR_OBJ_ALLOC_PAGE_ID,
4014 UVERBS_ATTR_TYPE(u32),
4015 UA_MANDATORY),
4016 UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_UAR_OBJ_ALLOC_MMAP_LENGTH,
4017 UVERBS_ATTR_TYPE(u32),
4018 UA_MANDATORY),
4019 UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_UAR_OBJ_ALLOC_MMAP_OFFSET,
4020 UVERBS_ATTR_TYPE(u64),
4021 UA_MANDATORY));
4022
4023DECLARE_UVERBS_NAMED_METHOD_DESTROY(
4024 MLX5_IB_METHOD_UAR_OBJ_DESTROY,
4025 UVERBS_ATTR_IDR(MLX5_IB_ATTR_UAR_OBJ_DESTROY_HANDLE,
4026 MLX5_IB_OBJECT_UAR,
4027 UVERBS_ACCESS_DESTROY,
4028 UA_MANDATORY));
4029
4030DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_UAR,
4031 UVERBS_TYPE_ALLOC_IDR(mmap_obj_cleanup),
4032 &UVERBS_METHOD(MLX5_IB_METHOD_UAR_OBJ_ALLOC),
4033 &UVERBS_METHOD(MLX5_IB_METHOD_UAR_OBJ_DESTROY));
4034
4035ADD_UVERBS_ATTRIBUTES_SIMPLE(
4036 mlx5_ib_query_context,
4037 UVERBS_OBJECT_DEVICE,
4038 UVERBS_METHOD_QUERY_CONTEXT,
4039 UVERBS_ATTR_PTR_OUT(
4040 MLX5_IB_ATTR_QUERY_CONTEXT_RESP_UCTX,
4041 UVERBS_ATTR_STRUCT(struct mlx5_ib_alloc_ucontext_resp,
4042 dump_fill_mkey),
4043 UA_MANDATORY));
4044
4045ADD_UVERBS_ATTRIBUTES_SIMPLE(
4046 mlx5_ib_reg_dmabuf_mr,
4047 UVERBS_OBJECT_MR,
4048 UVERBS_METHOD_REG_DMABUF_MR,
4049 UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_REG_DMABUF_MR_ACCESS_FLAGS,
4050 enum mlx5_ib_uapi_reg_dmabuf_flags,
4051 UA_OPTIONAL));
4052
4053static const struct uapi_definition mlx5_ib_defs[] = {
4054 UAPI_DEF_CHAIN(mlx5_ib_devx_defs),
4055 UAPI_DEF_CHAIN(mlx5_ib_flow_defs),
4056 UAPI_DEF_CHAIN(mlx5_ib_qos_defs),
4057 UAPI_DEF_CHAIN(mlx5_ib_std_types_defs),
4058 UAPI_DEF_CHAIN(mlx5_ib_dm_defs),
4059 UAPI_DEF_CHAIN(mlx5_ib_create_cq_defs),
4060
4061 UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_DEVICE, &mlx5_ib_query_context),
4062 UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_MR, &mlx5_ib_reg_dmabuf_mr),
4063 UAPI_DEF_CHAIN_OBJ_TREE_NAMED(MLX5_IB_OBJECT_VAR,
4064 UAPI_DEF_IS_OBJ_SUPPORTED(var_is_supported)),
4065 UAPI_DEF_CHAIN_OBJ_TREE_NAMED(MLX5_IB_OBJECT_UAR),
4066 {}
4067};
4068
4069static void mlx5_ib_stage_init_cleanup(struct mlx5_ib_dev *dev)
4070{
4071 mlx5_ib_data_direct_cleanup(dev);
4072 mlx5_ib_cleanup_multiport_master(dev);
4073 WARN_ON(!xa_empty(&dev->odp_mkeys));
4074 mutex_destroy(&dev->cap_mask_mutex);
4075 WARN_ON(!xa_empty(&dev->sig_mrs));
4076 WARN_ON(!bitmap_empty(dev->dm.memic_alloc_pages, MLX5_MAX_MEMIC_PAGES));
4077 mlx5r_macsec_dealloc_gids(dev);
4078}
4079
4080static int mlx5_ib_stage_init_init(struct mlx5_ib_dev *dev)
4081{
4082 struct mlx5_core_dev *mdev = dev->mdev;
4083 int err, i;
4084
4085 dev->ib_dev.node_type = RDMA_NODE_IB_CA;
4086 dev->ib_dev.local_dma_lkey = 0 /* not supported for now */;
4087 dev->ib_dev.dev.parent = mdev->device;
4088 dev->ib_dev.lag_flags = RDMA_LAG_FLAGS_HASH_ALL_SLAVES;
4089
4090 for (i = 0; i < dev->num_ports; i++) {
4091 spin_lock_init(&dev->port[i].mp.mpi_lock);
4092 dev->port[i].roce.dev = dev;
4093 dev->port[i].roce.native_port_num = i + 1;
4094 dev->port[i].roce.last_port_state = IB_PORT_DOWN;
4095 }
4096
4097 err = mlx5r_cmd_query_special_mkeys(dev);
4098 if (err)
4099 return err;
4100
4101 err = mlx5r_macsec_init_gids_and_devlist(dev);
4102 if (err)
4103 return err;
4104
4105 err = mlx5_ib_init_multiport_master(dev);
4106 if (err)
4107 goto err;
4108
4109 err = set_has_smi_cap(dev);
4110 if (err)
4111 goto err_mp;
4112
4113 err = mlx5_query_max_pkeys(&dev->ib_dev, &dev->pkey_table_len);
4114 if (err)
4115 goto err_mp;
4116
4117 if (mlx5_use_mad_ifc(dev))
4118 get_ext_port_caps(dev);
4119
4120 dev->ib_dev.num_comp_vectors = mlx5_comp_vectors_max(mdev);
4121
4122 mutex_init(&dev->cap_mask_mutex);
4123 mutex_init(&dev->data_direct_lock);
4124 INIT_LIST_HEAD(&dev->qp_list);
4125 spin_lock_init(&dev->reset_flow_resource_lock);
4126 xa_init(&dev->odp_mkeys);
4127 xa_init(&dev->sig_mrs);
4128 atomic_set(&dev->mkey_var, 0);
4129
4130 spin_lock_init(&dev->dm.lock);
4131 dev->dm.dev = mdev;
4132 err = mlx5_ib_data_direct_init(dev);
4133 if (err)
4134 goto err_mp;
4135
4136 return 0;
4137err_mp:
4138 mlx5_ib_cleanup_multiport_master(dev);
4139err:
4140 mlx5r_macsec_dealloc_gids(dev);
4141 return err;
4142}
4143
4144static struct ib_device *mlx5_ib_add_sub_dev(struct ib_device *parent,
4145 enum rdma_nl_dev_type type,
4146 const char *name);
4147static void mlx5_ib_del_sub_dev(struct ib_device *sub_dev);
4148
4149static const struct ib_device_ops mlx5_ib_dev_ops = {
4150 .owner = THIS_MODULE,
4151 .driver_id = RDMA_DRIVER_MLX5,
4152 .uverbs_abi_ver = MLX5_IB_UVERBS_ABI_VERSION,
4153
4154 .add_gid = mlx5_ib_add_gid,
4155 .add_sub_dev = mlx5_ib_add_sub_dev,
4156 .alloc_mr = mlx5_ib_alloc_mr,
4157 .alloc_mr_integrity = mlx5_ib_alloc_mr_integrity,
4158 .alloc_pd = mlx5_ib_alloc_pd,
4159 .alloc_ucontext = mlx5_ib_alloc_ucontext,
4160 .attach_mcast = mlx5_ib_mcg_attach,
4161 .check_mr_status = mlx5_ib_check_mr_status,
4162 .create_ah = mlx5_ib_create_ah,
4163 .create_cq = mlx5_ib_create_cq,
4164 .create_qp = mlx5_ib_create_qp,
4165 .create_srq = mlx5_ib_create_srq,
4166 .create_user_ah = mlx5_ib_create_ah,
4167 .dealloc_pd = mlx5_ib_dealloc_pd,
4168 .dealloc_ucontext = mlx5_ib_dealloc_ucontext,
4169 .del_gid = mlx5_ib_del_gid,
4170 .del_sub_dev = mlx5_ib_del_sub_dev,
4171 .dereg_mr = mlx5_ib_dereg_mr,
4172 .destroy_ah = mlx5_ib_destroy_ah,
4173 .destroy_cq = mlx5_ib_destroy_cq,
4174 .destroy_qp = mlx5_ib_destroy_qp,
4175 .destroy_srq = mlx5_ib_destroy_srq,
4176 .detach_mcast = mlx5_ib_mcg_detach,
4177 .disassociate_ucontext = mlx5_ib_disassociate_ucontext,
4178 .drain_rq = mlx5_ib_drain_rq,
4179 .drain_sq = mlx5_ib_drain_sq,
4180 .device_group = &mlx5_attr_group,
4181 .get_dev_fw_str = get_dev_fw_str,
4182 .get_dma_mr = mlx5_ib_get_dma_mr,
4183 .get_link_layer = mlx5_ib_port_link_layer,
4184 .map_mr_sg = mlx5_ib_map_mr_sg,
4185 .map_mr_sg_pi = mlx5_ib_map_mr_sg_pi,
4186 .mmap = mlx5_ib_mmap,
4187 .mmap_free = mlx5_ib_mmap_free,
4188 .modify_cq = mlx5_ib_modify_cq,
4189 .modify_device = mlx5_ib_modify_device,
4190 .modify_port = mlx5_ib_modify_port,
4191 .modify_qp = mlx5_ib_modify_qp,
4192 .modify_srq = mlx5_ib_modify_srq,
4193 .poll_cq = mlx5_ib_poll_cq,
4194 .post_recv = mlx5_ib_post_recv_nodrain,
4195 .post_send = mlx5_ib_post_send_nodrain,
4196 .post_srq_recv = mlx5_ib_post_srq_recv,
4197 .process_mad = mlx5_ib_process_mad,
4198 .query_ah = mlx5_ib_query_ah,
4199 .query_device = mlx5_ib_query_device,
4200 .query_gid = mlx5_ib_query_gid,
4201 .query_pkey = mlx5_ib_query_pkey,
4202 .query_qp = mlx5_ib_query_qp,
4203 .query_srq = mlx5_ib_query_srq,
4204 .query_ucontext = mlx5_ib_query_ucontext,
4205 .reg_user_mr = mlx5_ib_reg_user_mr,
4206 .reg_user_mr_dmabuf = mlx5_ib_reg_user_mr_dmabuf,
4207 .req_notify_cq = mlx5_ib_arm_cq,
4208 .rereg_user_mr = mlx5_ib_rereg_user_mr,
4209 .resize_cq = mlx5_ib_resize_cq,
4210 .ufile_hw_cleanup = mlx5_ib_ufile_hw_cleanup,
4211
4212 INIT_RDMA_OBJ_SIZE(ib_ah, mlx5_ib_ah, ibah),
4213 INIT_RDMA_OBJ_SIZE(ib_counters, mlx5_ib_mcounters, ibcntrs),
4214 INIT_RDMA_OBJ_SIZE(ib_cq, mlx5_ib_cq, ibcq),
4215 INIT_RDMA_OBJ_SIZE(ib_pd, mlx5_ib_pd, ibpd),
4216 INIT_RDMA_OBJ_SIZE(ib_qp, mlx5_ib_qp, ibqp),
4217 INIT_RDMA_OBJ_SIZE(ib_srq, mlx5_ib_srq, ibsrq),
4218 INIT_RDMA_OBJ_SIZE(ib_ucontext, mlx5_ib_ucontext, ibucontext),
4219};
4220
4221static const struct ib_device_ops mlx5_ib_dev_ipoib_enhanced_ops = {
4222 .rdma_netdev_get_params = mlx5_ib_rn_get_params,
4223};
4224
4225static const struct ib_device_ops mlx5_ib_dev_sriov_ops = {
4226 .get_vf_config = mlx5_ib_get_vf_config,
4227 .get_vf_guid = mlx5_ib_get_vf_guid,
4228 .get_vf_stats = mlx5_ib_get_vf_stats,
4229 .set_vf_guid = mlx5_ib_set_vf_guid,
4230 .set_vf_link_state = mlx5_ib_set_vf_link_state,
4231};
4232
4233static const struct ib_device_ops mlx5_ib_dev_mw_ops = {
4234 .alloc_mw = mlx5_ib_alloc_mw,
4235 .dealloc_mw = mlx5_ib_dealloc_mw,
4236
4237 INIT_RDMA_OBJ_SIZE(ib_mw, mlx5_ib_mw, ibmw),
4238};
4239
4240static const struct ib_device_ops mlx5_ib_dev_xrc_ops = {
4241 .alloc_xrcd = mlx5_ib_alloc_xrcd,
4242 .dealloc_xrcd = mlx5_ib_dealloc_xrcd,
4243
4244 INIT_RDMA_OBJ_SIZE(ib_xrcd, mlx5_ib_xrcd, ibxrcd),
4245};
4246
4247static int mlx5_ib_init_var_table(struct mlx5_ib_dev *dev)
4248{
4249 struct mlx5_core_dev *mdev = dev->mdev;
4250 struct mlx5_var_table *var_table = &dev->var_table;
4251 u8 log_doorbell_bar_size;
4252 u8 log_doorbell_stride;
4253 u64 bar_size;
4254
4255 log_doorbell_bar_size = MLX5_CAP_DEV_VDPA_EMULATION(mdev,
4256 log_doorbell_bar_size);
4257 log_doorbell_stride = MLX5_CAP_DEV_VDPA_EMULATION(mdev,
4258 log_doorbell_stride);
4259 var_table->hw_start_addr = dev->mdev->bar_addr +
4260 MLX5_CAP64_DEV_VDPA_EMULATION(mdev,
4261 doorbell_bar_offset);
4262 bar_size = (1ULL << log_doorbell_bar_size) * 4096;
4263 var_table->stride_size = 1ULL << log_doorbell_stride;
4264 var_table->num_var_hw_entries = div_u64(bar_size,
4265 var_table->stride_size);
4266 mutex_init(&var_table->bitmap_lock);
4267 var_table->bitmap = bitmap_zalloc(var_table->num_var_hw_entries,
4268 GFP_KERNEL);
4269 return (var_table->bitmap) ? 0 : -ENOMEM;
4270}
4271
4272static void mlx5_ib_cleanup_ucaps(struct mlx5_ib_dev *dev)
4273{
4274 if (MLX5_CAP_GEN(dev->mdev, uctx_cap) & MLX5_UCTX_CAP_RDMA_CTRL)
4275 ib_remove_ucap(RDMA_UCAP_MLX5_CTRL_LOCAL);
4276
4277 if (MLX5_CAP_GEN(dev->mdev, uctx_cap) &
4278 MLX5_UCTX_CAP_RDMA_CTRL_OTHER_VHCA)
4279 ib_remove_ucap(RDMA_UCAP_MLX5_CTRL_OTHER_VHCA);
4280}
4281
4282static int mlx5_ib_init_ucaps(struct mlx5_ib_dev *dev)
4283{
4284 int ret;
4285
4286 if (MLX5_CAP_GEN(dev->mdev, uctx_cap) & MLX5_UCTX_CAP_RDMA_CTRL) {
4287 ret = ib_create_ucap(RDMA_UCAP_MLX5_CTRL_LOCAL);
4288 if (ret)
4289 return ret;
4290 }
4291
4292 if (MLX5_CAP_GEN(dev->mdev, uctx_cap) &
4293 MLX5_UCTX_CAP_RDMA_CTRL_OTHER_VHCA) {
4294 ret = ib_create_ucap(RDMA_UCAP_MLX5_CTRL_OTHER_VHCA);
4295 if (ret)
4296 goto remove_local;
4297 }
4298
4299 return 0;
4300
4301remove_local:
4302 if (MLX5_CAP_GEN(dev->mdev, uctx_cap) & MLX5_UCTX_CAP_RDMA_CTRL)
4303 ib_remove_ucap(RDMA_UCAP_MLX5_CTRL_LOCAL);
4304 return ret;
4305}
4306
4307static void mlx5_ib_stage_caps_cleanup(struct mlx5_ib_dev *dev)
4308{
4309 if (MLX5_CAP_GEN_2_64(dev->mdev, general_obj_types_127_64) &
4310 MLX5_HCA_CAP_2_GENERAL_OBJECT_TYPES_RDMA_CTRL)
4311 mlx5_ib_cleanup_ucaps(dev);
4312
4313 bitmap_free(dev->var_table.bitmap);
4314}
4315
4316static int mlx5_ib_stage_caps_init(struct mlx5_ib_dev *dev)
4317{
4318 struct mlx5_core_dev *mdev = dev->mdev;
4319 int err;
4320
4321 if (MLX5_CAP_GEN(mdev, ipoib_enhanced_offloads) &&
4322 IS_ENABLED(CONFIG_MLX5_CORE_IPOIB))
4323 ib_set_device_ops(&dev->ib_dev,
4324 &mlx5_ib_dev_ipoib_enhanced_ops);
4325
4326 if (mlx5_core_is_pf(mdev))
4327 ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_sriov_ops);
4328
4329 dev->umr_fence = mlx5_get_umr_fence(MLX5_CAP_GEN(mdev, umr_fence));
4330
4331 if (MLX5_CAP_GEN(mdev, imaicl))
4332 ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_mw_ops);
4333
4334 if (MLX5_CAP_GEN(mdev, xrc))
4335 ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_xrc_ops);
4336
4337 if (MLX5_CAP_DEV_MEM(mdev, memic) ||
4338 MLX5_CAP_GEN_64(dev->mdev, general_obj_types) &
4339 MLX5_GENERAL_OBJ_TYPES_CAP_SW_ICM)
4340 ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_dm_ops);
4341
4342 ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_ops);
4343
4344 if (IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS))
4345 dev->ib_dev.driver_def = mlx5_ib_defs;
4346
4347 err = init_node_data(dev);
4348 if (err)
4349 return err;
4350
4351 if ((MLX5_CAP_GEN(dev->mdev, port_type) == MLX5_CAP_PORT_TYPE_ETH) &&
4352 (MLX5_CAP_GEN(dev->mdev, disable_local_lb_uc) ||
4353 MLX5_CAP_GEN(dev->mdev, disable_local_lb_mc)))
4354 mutex_init(&dev->lb.mutex);
4355
4356 if (MLX5_CAP_GEN_64(dev->mdev, general_obj_types) &
4357 MLX5_GENERAL_OBJ_TYPES_CAP_VIRTIO_NET_Q) {
4358 err = mlx5_ib_init_var_table(dev);
4359 if (err)
4360 return err;
4361 }
4362
4363 if (MLX5_CAP_GEN_2_64(dev->mdev, general_obj_types_127_64) &
4364 MLX5_HCA_CAP_2_GENERAL_OBJECT_TYPES_RDMA_CTRL) {
4365 err = mlx5_ib_init_ucaps(dev);
4366 if (err)
4367 return err;
4368 }
4369
4370 dev->ib_dev.use_cq_dim = true;
4371
4372 return 0;
4373}
4374
4375static const struct ib_device_ops mlx5_ib_dev_port_ops = {
4376 .get_port_immutable = mlx5_port_immutable,
4377 .query_port = mlx5_ib_query_port,
4378};
4379
4380static int mlx5_ib_stage_non_default_cb(struct mlx5_ib_dev *dev)
4381{
4382 ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_port_ops);
4383 return 0;
4384}
4385
4386static const struct ib_device_ops mlx5_ib_dev_port_rep_ops = {
4387 .get_port_immutable = mlx5_port_rep_immutable,
4388 .query_port = mlx5_ib_rep_query_port,
4389 .query_pkey = mlx5_ib_rep_query_pkey,
4390};
4391
4392static int mlx5_ib_stage_raw_eth_non_default_cb(struct mlx5_ib_dev *dev)
4393{
4394 ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_port_rep_ops);
4395 return 0;
4396}
4397
4398static const struct ib_device_ops mlx5_ib_dev_common_roce_ops = {
4399 .create_rwq_ind_table = mlx5_ib_create_rwq_ind_table,
4400 .create_wq = mlx5_ib_create_wq,
4401 .destroy_rwq_ind_table = mlx5_ib_destroy_rwq_ind_table,
4402 .destroy_wq = mlx5_ib_destroy_wq,
4403 .modify_wq = mlx5_ib_modify_wq,
4404
4405 INIT_RDMA_OBJ_SIZE(ib_rwq_ind_table, mlx5_ib_rwq_ind_table,
4406 ib_rwq_ind_tbl),
4407};
4408
4409static int mlx5_ib_roce_init(struct mlx5_ib_dev *dev)
4410{
4411 struct mlx5_core_dev *mdev = dev->mdev;
4412 enum rdma_link_layer ll;
4413 int port_type_cap;
4414 u32 port_num = 0;
4415 int err;
4416
4417 port_type_cap = MLX5_CAP_GEN(mdev, port_type);
4418 ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap);
4419
4420 if (ll == IB_LINK_LAYER_ETHERNET) {
4421 ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_common_roce_ops);
4422
4423 port_num = mlx5_core_native_port_num(dev->mdev) - 1;
4424
4425 /* Register only for native ports */
4426 mlx5_mdev_netdev_track(dev, port_num);
4427
4428 err = mlx5_enable_eth(dev);
4429 if (err)
4430 goto cleanup;
4431 }
4432
4433 return 0;
4434cleanup:
4435 mlx5_mdev_netdev_untrack(dev, port_num);
4436 return err;
4437}
4438
4439static void mlx5_ib_roce_cleanup(struct mlx5_ib_dev *dev)
4440{
4441 struct mlx5_core_dev *mdev = dev->mdev;
4442 enum rdma_link_layer ll;
4443 int port_type_cap;
4444 u32 port_num;
4445
4446 port_type_cap = MLX5_CAP_GEN(mdev, port_type);
4447 ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap);
4448
4449 if (ll == IB_LINK_LAYER_ETHERNET) {
4450 mlx5_disable_eth(dev);
4451
4452 port_num = mlx5_core_native_port_num(dev->mdev) - 1;
4453 mlx5_mdev_netdev_untrack(dev, port_num);
4454 }
4455}
4456
4457static int mlx5_ib_stage_cong_debugfs_init(struct mlx5_ib_dev *dev)
4458{
4459 mlx5_ib_init_cong_debugfs(dev,
4460 mlx5_core_native_port_num(dev->mdev) - 1);
4461 return 0;
4462}
4463
4464static void mlx5_ib_stage_cong_debugfs_cleanup(struct mlx5_ib_dev *dev)
4465{
4466 mlx5_ib_cleanup_cong_debugfs(dev,
4467 mlx5_core_native_port_num(dev->mdev) - 1);
4468}
4469
4470static int mlx5_ib_stage_bfrag_init(struct mlx5_ib_dev *dev)
4471{
4472 int err;
4473
4474 err = mlx5_alloc_bfreg(dev->mdev, &dev->bfreg, false, false);
4475 if (err)
4476 return err;
4477
4478 err = mlx5_alloc_bfreg(dev->mdev, &dev->fp_bfreg, false, true);
4479 if (err)
4480 mlx5_free_bfreg(dev->mdev, &dev->bfreg);
4481
4482 return err;
4483}
4484
4485static void mlx5_ib_stage_bfrag_cleanup(struct mlx5_ib_dev *dev)
4486{
4487 mlx5_free_bfreg(dev->mdev, &dev->fp_bfreg);
4488 mlx5_free_bfreg(dev->mdev, &dev->bfreg);
4489}
4490
4491static int mlx5_ib_stage_ib_reg_init(struct mlx5_ib_dev *dev)
4492{
4493 const char *name;
4494
4495 if (dev->sub_dev_name) {
4496 name = dev->sub_dev_name;
4497 ib_mark_name_assigned_by_user(&dev->ib_dev);
4498 } else if (!mlx5_lag_is_active(dev->mdev))
4499 name = "mlx5_%d";
4500 else
4501 name = "mlx5_bond_%d";
4502 return ib_register_device(&dev->ib_dev, name, &dev->mdev->pdev->dev);
4503}
4504
4505static void mlx5_ib_stage_pre_ib_reg_umr_cleanup(struct mlx5_ib_dev *dev)
4506{
4507 mlx5_mkey_cache_cleanup(dev);
4508 mlx5r_umr_resource_cleanup(dev);
4509 mlx5r_umr_cleanup(dev);
4510}
4511
4512static void mlx5_ib_stage_ib_reg_cleanup(struct mlx5_ib_dev *dev)
4513{
4514 ib_unregister_device(&dev->ib_dev);
4515}
4516
4517static int mlx5_ib_stage_post_ib_reg_umr_init(struct mlx5_ib_dev *dev)
4518{
4519 int ret;
4520
4521 ret = mlx5r_umr_init(dev);
4522 if (ret)
4523 return ret;
4524
4525 ret = mlx5_mkey_cache_init(dev);
4526 if (ret)
4527 mlx5_ib_warn(dev, "mr cache init failed %d\n", ret);
4528 return ret;
4529}
4530
4531static int mlx5_ib_stage_delay_drop_init(struct mlx5_ib_dev *dev)
4532{
4533 struct dentry *root;
4534
4535 if (!(dev->ib_dev.attrs.raw_packet_caps & IB_RAW_PACKET_CAP_DELAY_DROP))
4536 return 0;
4537
4538 mutex_init(&dev->delay_drop.lock);
4539 dev->delay_drop.dev = dev;
4540 dev->delay_drop.activate = false;
4541 dev->delay_drop.timeout = MLX5_MAX_DELAY_DROP_TIMEOUT_MS * 1000;
4542 INIT_WORK(&dev->delay_drop.delay_drop_work, delay_drop_handler);
4543 atomic_set(&dev->delay_drop.rqs_cnt, 0);
4544 atomic_set(&dev->delay_drop.events_cnt, 0);
4545
4546 if (!mlx5_debugfs_root)
4547 return 0;
4548
4549 root = debugfs_create_dir("delay_drop", mlx5_debugfs_get_dev_root(dev->mdev));
4550 dev->delay_drop.dir_debugfs = root;
4551
4552 debugfs_create_atomic_t("num_timeout_events", 0400, root,
4553 &dev->delay_drop.events_cnt);
4554 debugfs_create_atomic_t("num_rqs", 0400, root,
4555 &dev->delay_drop.rqs_cnt);
4556 debugfs_create_file("timeout", 0600, root, &dev->delay_drop,
4557 &fops_delay_drop_timeout);
4558 return 0;
4559}
4560
4561static void mlx5_ib_stage_delay_drop_cleanup(struct mlx5_ib_dev *dev)
4562{
4563 if (!(dev->ib_dev.attrs.raw_packet_caps & IB_RAW_PACKET_CAP_DELAY_DROP))
4564 return;
4565
4566 cancel_work_sync(&dev->delay_drop.delay_drop_work);
4567 if (!dev->delay_drop.dir_debugfs)
4568 return;
4569
4570 debugfs_remove_recursive(dev->delay_drop.dir_debugfs);
4571 dev->delay_drop.dir_debugfs = NULL;
4572}
4573
4574static int mlx5_ib_stage_dev_notifier_init(struct mlx5_ib_dev *dev)
4575{
4576 struct mlx5_ib_resources *devr = &dev->devr;
4577 int port;
4578
4579 for (port = 0; port < ARRAY_SIZE(devr->ports); ++port)
4580 INIT_WORK(&devr->ports[port].pkey_change_work,
4581 pkey_change_handler);
4582
4583 dev->mdev_events.notifier_call = mlx5_ib_event;
4584 mlx5_notifier_register(dev->mdev, &dev->mdev_events);
4585
4586 mlx5r_macsec_event_register(dev);
4587
4588 return 0;
4589}
4590
4591static void mlx5_ib_stage_dev_notifier_cleanup(struct mlx5_ib_dev *dev)
4592{
4593 struct mlx5_ib_resources *devr = &dev->devr;
4594 int port;
4595
4596 mlx5r_macsec_event_unregister(dev);
4597 mlx5_notifier_unregister(dev->mdev, &dev->mdev_events);
4598
4599 for (port = 0; port < ARRAY_SIZE(devr->ports); ++port)
4600 cancel_work_sync(&devr->ports[port].pkey_change_work);
4601}
4602
4603void mlx5_ib_data_direct_bind(struct mlx5_ib_dev *ibdev,
4604 struct mlx5_data_direct_dev *dev)
4605{
4606 mutex_lock(&ibdev->data_direct_lock);
4607 ibdev->data_direct_dev = dev;
4608 mutex_unlock(&ibdev->data_direct_lock);
4609}
4610
4611void mlx5_ib_data_direct_unbind(struct mlx5_ib_dev *ibdev)
4612{
4613 mutex_lock(&ibdev->data_direct_lock);
4614 mlx5_ib_revoke_data_direct_mrs(ibdev);
4615 ibdev->data_direct_dev = NULL;
4616 mutex_unlock(&ibdev->data_direct_lock);
4617}
4618
4619void __mlx5_ib_remove(struct mlx5_ib_dev *dev,
4620 const struct mlx5_ib_profile *profile,
4621 int stage)
4622{
4623 dev->ib_active = false;
4624
4625 /* Number of stages to cleanup */
4626 while (stage) {
4627 stage--;
4628 if (profile->stage[stage].cleanup)
4629 profile->stage[stage].cleanup(dev);
4630 }
4631
4632 kfree(dev->port);
4633 ib_dealloc_device(&dev->ib_dev);
4634}
4635
4636int __mlx5_ib_add(struct mlx5_ib_dev *dev,
4637 const struct mlx5_ib_profile *profile)
4638{
4639 int err;
4640 int i;
4641
4642 dev->profile = profile;
4643
4644 for (i = 0; i < MLX5_IB_STAGE_MAX; i++) {
4645 if (profile->stage[i].init) {
4646 err = profile->stage[i].init(dev);
4647 if (err)
4648 goto err_out;
4649 }
4650 }
4651
4652 dev->ib_active = true;
4653 return 0;
4654
4655err_out:
4656 /* Clean up stages which were initialized */
4657 while (i) {
4658 i--;
4659 if (profile->stage[i].cleanup)
4660 profile->stage[i].cleanup(dev);
4661 }
4662 return -ENOMEM;
4663}
4664
4665static const struct mlx5_ib_profile pf_profile = {
4666 STAGE_CREATE(MLX5_IB_STAGE_INIT,
4667 mlx5_ib_stage_init_init,
4668 mlx5_ib_stage_init_cleanup),
4669 STAGE_CREATE(MLX5_IB_STAGE_FS,
4670 mlx5_ib_fs_init,
4671 mlx5_ib_fs_cleanup),
4672 STAGE_CREATE(MLX5_IB_STAGE_CAPS,
4673 mlx5_ib_stage_caps_init,
4674 mlx5_ib_stage_caps_cleanup),
4675 STAGE_CREATE(MLX5_IB_STAGE_NON_DEFAULT_CB,
4676 mlx5_ib_stage_non_default_cb,
4677 NULL),
4678 STAGE_CREATE(MLX5_IB_STAGE_ROCE,
4679 mlx5_ib_roce_init,
4680 mlx5_ib_roce_cleanup),
4681 STAGE_CREATE(MLX5_IB_STAGE_QP,
4682 mlx5_init_qp_table,
4683 mlx5_cleanup_qp_table),
4684 STAGE_CREATE(MLX5_IB_STAGE_SRQ,
4685 mlx5_init_srq_table,
4686 mlx5_cleanup_srq_table),
4687 STAGE_CREATE(MLX5_IB_STAGE_DEVICE_RESOURCES,
4688 mlx5_ib_dev_res_init,
4689 mlx5_ib_dev_res_cleanup),
4690 STAGE_CREATE(MLX5_IB_STAGE_ODP,
4691 mlx5_ib_odp_init_one,
4692 mlx5_ib_odp_cleanup_one),
4693 STAGE_CREATE(MLX5_IB_STAGE_COUNTERS,
4694 mlx5_ib_counters_init,
4695 mlx5_ib_counters_cleanup),
4696 STAGE_CREATE(MLX5_IB_STAGE_CONG_DEBUGFS,
4697 mlx5_ib_stage_cong_debugfs_init,
4698 mlx5_ib_stage_cong_debugfs_cleanup),
4699 STAGE_CREATE(MLX5_IB_STAGE_BFREG,
4700 mlx5_ib_stage_bfrag_init,
4701 mlx5_ib_stage_bfrag_cleanup),
4702 STAGE_CREATE(MLX5_IB_STAGE_PRE_IB_REG_UMR,
4703 NULL,
4704 mlx5_ib_stage_pre_ib_reg_umr_cleanup),
4705 STAGE_CREATE(MLX5_IB_STAGE_WHITELIST_UID,
4706 mlx5_ib_devx_init,
4707 mlx5_ib_devx_cleanup),
4708 STAGE_CREATE(MLX5_IB_STAGE_IB_REG,
4709 mlx5_ib_stage_ib_reg_init,
4710 mlx5_ib_stage_ib_reg_cleanup),
4711 STAGE_CREATE(MLX5_IB_STAGE_DEVICE_NOTIFIER,
4712 mlx5_ib_stage_dev_notifier_init,
4713 mlx5_ib_stage_dev_notifier_cleanup),
4714 STAGE_CREATE(MLX5_IB_STAGE_POST_IB_REG_UMR,
4715 mlx5_ib_stage_post_ib_reg_umr_init,
4716 NULL),
4717 STAGE_CREATE(MLX5_IB_STAGE_DELAY_DROP,
4718 mlx5_ib_stage_delay_drop_init,
4719 mlx5_ib_stage_delay_drop_cleanup),
4720 STAGE_CREATE(MLX5_IB_STAGE_RESTRACK,
4721 mlx5_ib_restrack_init,
4722 NULL),
4723};
4724
4725const struct mlx5_ib_profile raw_eth_profile = {
4726 STAGE_CREATE(MLX5_IB_STAGE_INIT,
4727 mlx5_ib_stage_init_init,
4728 mlx5_ib_stage_init_cleanup),
4729 STAGE_CREATE(MLX5_IB_STAGE_FS,
4730 mlx5_ib_fs_init,
4731 mlx5_ib_fs_cleanup),
4732 STAGE_CREATE(MLX5_IB_STAGE_CAPS,
4733 mlx5_ib_stage_caps_init,
4734 mlx5_ib_stage_caps_cleanup),
4735 STAGE_CREATE(MLX5_IB_STAGE_NON_DEFAULT_CB,
4736 mlx5_ib_stage_raw_eth_non_default_cb,
4737 NULL),
4738 STAGE_CREATE(MLX5_IB_STAGE_ROCE,
4739 mlx5_ib_roce_init,
4740 mlx5_ib_roce_cleanup),
4741 STAGE_CREATE(MLX5_IB_STAGE_QP,
4742 mlx5_init_qp_table,
4743 mlx5_cleanup_qp_table),
4744 STAGE_CREATE(MLX5_IB_STAGE_SRQ,
4745 mlx5_init_srq_table,
4746 mlx5_cleanup_srq_table),
4747 STAGE_CREATE(MLX5_IB_STAGE_DEVICE_RESOURCES,
4748 mlx5_ib_dev_res_init,
4749 mlx5_ib_dev_res_cleanup),
4750 STAGE_CREATE(MLX5_IB_STAGE_COUNTERS,
4751 mlx5_ib_counters_init,
4752 mlx5_ib_counters_cleanup),
4753 STAGE_CREATE(MLX5_IB_STAGE_CONG_DEBUGFS,
4754 mlx5_ib_stage_cong_debugfs_init,
4755 mlx5_ib_stage_cong_debugfs_cleanup),
4756 STAGE_CREATE(MLX5_IB_STAGE_BFREG,
4757 mlx5_ib_stage_bfrag_init,
4758 mlx5_ib_stage_bfrag_cleanup),
4759 STAGE_CREATE(MLX5_IB_STAGE_PRE_IB_REG_UMR,
4760 NULL,
4761 mlx5_ib_stage_pre_ib_reg_umr_cleanup),
4762 STAGE_CREATE(MLX5_IB_STAGE_WHITELIST_UID,
4763 mlx5_ib_devx_init,
4764 mlx5_ib_devx_cleanup),
4765 STAGE_CREATE(MLX5_IB_STAGE_IB_REG,
4766 mlx5_ib_stage_ib_reg_init,
4767 mlx5_ib_stage_ib_reg_cleanup),
4768 STAGE_CREATE(MLX5_IB_STAGE_DEVICE_NOTIFIER,
4769 mlx5_ib_stage_dev_notifier_init,
4770 mlx5_ib_stage_dev_notifier_cleanup),
4771 STAGE_CREATE(MLX5_IB_STAGE_POST_IB_REG_UMR,
4772 mlx5_ib_stage_post_ib_reg_umr_init,
4773 NULL),
4774 STAGE_CREATE(MLX5_IB_STAGE_DELAY_DROP,
4775 mlx5_ib_stage_delay_drop_init,
4776 mlx5_ib_stage_delay_drop_cleanup),
4777 STAGE_CREATE(MLX5_IB_STAGE_RESTRACK,
4778 mlx5_ib_restrack_init,
4779 NULL),
4780};
4781
4782static const struct mlx5_ib_profile plane_profile = {
4783 STAGE_CREATE(MLX5_IB_STAGE_INIT,
4784 mlx5_ib_stage_init_init,
4785 mlx5_ib_stage_init_cleanup),
4786 STAGE_CREATE(MLX5_IB_STAGE_CAPS,
4787 mlx5_ib_stage_caps_init,
4788 mlx5_ib_stage_caps_cleanup),
4789 STAGE_CREATE(MLX5_IB_STAGE_NON_DEFAULT_CB,
4790 mlx5_ib_stage_non_default_cb,
4791 NULL),
4792 STAGE_CREATE(MLX5_IB_STAGE_QP,
4793 mlx5_init_qp_table,
4794 mlx5_cleanup_qp_table),
4795 STAGE_CREATE(MLX5_IB_STAGE_SRQ,
4796 mlx5_init_srq_table,
4797 mlx5_cleanup_srq_table),
4798 STAGE_CREATE(MLX5_IB_STAGE_DEVICE_RESOURCES,
4799 mlx5_ib_dev_res_init,
4800 mlx5_ib_dev_res_cleanup),
4801 STAGE_CREATE(MLX5_IB_STAGE_BFREG,
4802 mlx5_ib_stage_bfrag_init,
4803 mlx5_ib_stage_bfrag_cleanup),
4804 STAGE_CREATE(MLX5_IB_STAGE_IB_REG,
4805 mlx5_ib_stage_ib_reg_init,
4806 mlx5_ib_stage_ib_reg_cleanup),
4807};
4808
4809static struct ib_device *mlx5_ib_add_sub_dev(struct ib_device *parent,
4810 enum rdma_nl_dev_type type,
4811 const char *name)
4812{
4813 struct mlx5_ib_dev *mparent = to_mdev(parent), *mplane;
4814 enum rdma_link_layer ll;
4815 int ret;
4816
4817 if (mparent->smi_dev)
4818 return ERR_PTR(-EEXIST);
4819
4820 ll = mlx5_port_type_cap_to_rdma_ll(MLX5_CAP_GEN(mparent->mdev,
4821 port_type));
4822 if (type != RDMA_DEVICE_TYPE_SMI || !mparent->num_plane ||
4823 ll != IB_LINK_LAYER_INFINIBAND ||
4824 !MLX5_CAP_GEN_2(mparent->mdev, multiplane_qp_ud))
4825 return ERR_PTR(-EOPNOTSUPP);
4826
4827 mplane = ib_alloc_device(mlx5_ib_dev, ib_dev);
4828 if (!mplane)
4829 return ERR_PTR(-ENOMEM);
4830
4831 mplane->port = kcalloc(mparent->num_plane * mparent->num_ports,
4832 sizeof(*mplane->port), GFP_KERNEL);
4833 if (!mplane->port) {
4834 ret = -ENOMEM;
4835 goto fail_kcalloc;
4836 }
4837
4838 mplane->ib_dev.type = type;
4839 mplane->mdev = mparent->mdev;
4840 mplane->num_ports = mparent->num_plane;
4841 mplane->sub_dev_name = name;
4842 mplane->ib_dev.phys_port_cnt = mplane->num_ports;
4843
4844 ret = __mlx5_ib_add(mplane, &plane_profile);
4845 if (ret)
4846 goto fail_ib_add;
4847
4848 mparent->smi_dev = mplane;
4849 return &mplane->ib_dev;
4850
4851fail_ib_add:
4852 kfree(mplane->port);
4853fail_kcalloc:
4854 ib_dealloc_device(&mplane->ib_dev);
4855 return ERR_PTR(ret);
4856}
4857
4858static void mlx5_ib_del_sub_dev(struct ib_device *sub_dev)
4859{
4860 struct mlx5_ib_dev *mdev = to_mdev(sub_dev);
4861
4862 to_mdev(sub_dev->parent)->smi_dev = NULL;
4863 __mlx5_ib_remove(mdev, mdev->profile, MLX5_IB_STAGE_MAX);
4864}
4865
4866static int mlx5r_mp_probe(struct auxiliary_device *adev,
4867 const struct auxiliary_device_id *id)
4868{
4869 struct mlx5_adev *idev = container_of(adev, struct mlx5_adev, adev);
4870 struct mlx5_core_dev *mdev = idev->mdev;
4871 struct mlx5_ib_multiport_info *mpi;
4872 struct mlx5_ib_dev *dev;
4873 bool bound = false;
4874 int err;
4875
4876 mpi = kzalloc(sizeof(*mpi), GFP_KERNEL);
4877 if (!mpi)
4878 return -ENOMEM;
4879
4880 mpi->mdev = mdev;
4881 err = mlx5_query_nic_vport_system_image_guid(mdev,
4882 &mpi->sys_image_guid);
4883 if (err) {
4884 kfree(mpi);
4885 return err;
4886 }
4887
4888 mutex_lock(&mlx5_ib_multiport_mutex);
4889 list_for_each_entry(dev, &mlx5_ib_dev_list, ib_dev_list) {
4890 if (dev->sys_image_guid == mpi->sys_image_guid &&
4891 mlx5_core_same_coredev_type(dev->mdev, mpi->mdev))
4892 bound = mlx5_ib_bind_slave_port(dev, mpi);
4893
4894 if (bound) {
4895 rdma_roce_rescan_device(&dev->ib_dev);
4896 mpi->ibdev->ib_active = true;
4897 break;
4898 }
4899 }
4900
4901 if (!bound) {
4902 list_add_tail(&mpi->list, &mlx5_ib_unaffiliated_port_list);
4903 dev_dbg(mdev->device,
4904 "no suitable IB device found to bind to, added to unaffiliated list.\n");
4905 }
4906 mutex_unlock(&mlx5_ib_multiport_mutex);
4907
4908 auxiliary_set_drvdata(adev, mpi);
4909 return 0;
4910}
4911
4912static void mlx5r_mp_remove(struct auxiliary_device *adev)
4913{
4914 struct mlx5_ib_multiport_info *mpi;
4915
4916 mpi = auxiliary_get_drvdata(adev);
4917 mutex_lock(&mlx5_ib_multiport_mutex);
4918 if (mpi->ibdev)
4919 mlx5_ib_unbind_slave_port(mpi->ibdev, mpi);
4920 else
4921 list_del(&mpi->list);
4922 mutex_unlock(&mlx5_ib_multiport_mutex);
4923 kfree(mpi);
4924}
4925
4926static int mlx5r_probe(struct auxiliary_device *adev,
4927 const struct auxiliary_device_id *id)
4928{
4929 struct mlx5_adev *idev = container_of(adev, struct mlx5_adev, adev);
4930 struct mlx5_core_dev *mdev = idev->mdev;
4931 const struct mlx5_ib_profile *profile;
4932 int port_type_cap, num_ports, ret;
4933 enum rdma_link_layer ll;
4934 struct mlx5_ib_dev *dev;
4935
4936 port_type_cap = MLX5_CAP_GEN(mdev, port_type);
4937 ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap);
4938
4939 num_ports = max(MLX5_CAP_GEN(mdev, num_ports),
4940 MLX5_CAP_GEN(mdev, num_vhca_ports));
4941 dev = ib_alloc_device(mlx5_ib_dev, ib_dev);
4942 if (!dev)
4943 return -ENOMEM;
4944
4945 if (ll == IB_LINK_LAYER_INFINIBAND) {
4946 ret = mlx5_ib_get_plane_num(mdev, &dev->num_plane);
4947 if (ret)
4948 goto fail;
4949 }
4950
4951 dev->port = kcalloc(num_ports, sizeof(*dev->port),
4952 GFP_KERNEL);
4953 if (!dev->port) {
4954 ret = -ENOMEM;
4955 goto fail;
4956 }
4957
4958 dev->mdev = mdev;
4959 dev->num_ports = num_ports;
4960 dev->ib_dev.phys_port_cnt = num_ports;
4961
4962 if (ll == IB_LINK_LAYER_ETHERNET && !mlx5_get_roce_state(mdev))
4963 profile = &raw_eth_profile;
4964 else
4965 profile = &pf_profile;
4966
4967 ret = __mlx5_ib_add(dev, profile);
4968 if (ret)
4969 goto fail_ib_add;
4970
4971 auxiliary_set_drvdata(adev, dev);
4972 return 0;
4973
4974fail_ib_add:
4975 kfree(dev->port);
4976fail:
4977 ib_dealloc_device(&dev->ib_dev);
4978 return ret;
4979}
4980
4981static void mlx5r_remove(struct auxiliary_device *adev)
4982{
4983 struct mlx5_ib_dev *dev;
4984
4985 dev = auxiliary_get_drvdata(adev);
4986 __mlx5_ib_remove(dev, dev->profile, MLX5_IB_STAGE_MAX);
4987}
4988
4989static const struct auxiliary_device_id mlx5r_mp_id_table[] = {
4990 { .name = MLX5_ADEV_NAME ".multiport", },
4991 {},
4992};
4993
4994static const struct auxiliary_device_id mlx5r_id_table[] = {
4995 { .name = MLX5_ADEV_NAME ".rdma", },
4996 {},
4997};
4998
4999MODULE_DEVICE_TABLE(auxiliary, mlx5r_mp_id_table);
5000MODULE_DEVICE_TABLE(auxiliary, mlx5r_id_table);
5001
5002static struct auxiliary_driver mlx5r_mp_driver = {
5003 .name = "multiport",
5004 .probe = mlx5r_mp_probe,
5005 .remove = mlx5r_mp_remove,
5006 .id_table = mlx5r_mp_id_table,
5007};
5008
5009static struct auxiliary_driver mlx5r_driver = {
5010 .name = "rdma",
5011 .probe = mlx5r_probe,
5012 .remove = mlx5r_remove,
5013 .id_table = mlx5r_id_table,
5014};
5015
5016static int __init mlx5_ib_init(void)
5017{
5018 int ret;
5019
5020 xlt_emergency_page = (void *)__get_free_page(GFP_KERNEL);
5021 if (!xlt_emergency_page)
5022 return -ENOMEM;
5023
5024 mlx5_ib_event_wq = alloc_ordered_workqueue("mlx5_ib_event_wq", 0);
5025 if (!mlx5_ib_event_wq) {
5026 free_page((unsigned long)xlt_emergency_page);
5027 return -ENOMEM;
5028 }
5029
5030 ret = mlx5_ib_qp_event_init();
5031 if (ret)
5032 goto qp_event_err;
5033
5034 mlx5_ib_odp_init();
5035 ret = mlx5r_rep_init();
5036 if (ret)
5037 goto rep_err;
5038 ret = mlx5_data_direct_driver_register();
5039 if (ret)
5040 goto dd_err;
5041 ret = auxiliary_driver_register(&mlx5r_mp_driver);
5042 if (ret)
5043 goto mp_err;
5044 ret = auxiliary_driver_register(&mlx5r_driver);
5045 if (ret)
5046 goto drv_err;
5047
5048 return 0;
5049
5050drv_err:
5051 auxiliary_driver_unregister(&mlx5r_mp_driver);
5052mp_err:
5053 mlx5_data_direct_driver_unregister();
5054dd_err:
5055 mlx5r_rep_cleanup();
5056rep_err:
5057 mlx5_ib_qp_event_cleanup();
5058qp_event_err:
5059 destroy_workqueue(mlx5_ib_event_wq);
5060 free_page((unsigned long)xlt_emergency_page);
5061 return ret;
5062}
5063
5064static void __exit mlx5_ib_cleanup(void)
5065{
5066 mlx5_data_direct_driver_unregister();
5067 auxiliary_driver_unregister(&mlx5r_driver);
5068 auxiliary_driver_unregister(&mlx5r_mp_driver);
5069 mlx5r_rep_cleanup();
5070
5071 mlx5_ib_qp_event_cleanup();
5072 destroy_workqueue(mlx5_ib_event_wq);
5073 free_page((unsigned long)xlt_emergency_page);
5074}
5075
5076module_init(mlx5_ib_init);
5077module_exit(mlx5_ib_cleanup);
Linux 6.x block layer and io_uring tree(s)