projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Linux 6.16-rc6
[linux-2.6-block.git] / drivers / infiniband / hw / mlx5 / main.c
CommitLineData
b572ebe6 1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
e126ba97 2/*
b572ebe6 3 * Copyright (c) 2013-2020, Mellanox Technologies inc. All rights reserved.
90da7dc8 4 * Copyright (c) 2020, Intel Corporation. All rights reserved.
e126ba97
EC		 5 */
6
fe248c3a 7#include <linux/debugfs.h>
adec640e 8#include <linux/highmem.h>
e126ba97
EC		 9#include <linux/module.h>
10#include <linux/init.h>
11#include <linux/errno.h>
12#include <linux/pci.h>
13#include <linux/dma-mapping.h>
14#include <linux/slab.h>
24da0016 15#include <linux/bitmap.h>
e126ba97 16#include <linux/sched.h>
6e84f315 17#include <linux/sched/mm.h>
0881e7bd 18#include <linux/sched/task.h>
7c2344c3 19#include <linux/delay.h>
e126ba97 20#include <rdma/ib_user_verbs.h>
3f89a643 21#include <rdma/ib_addr.h>
2811ba51 22#include <rdma/ib_cache.h>
ada68c31 23#include <linux/mlx5/port.h>
1b5daf11 24#include <linux/mlx5/vport.h>
72c7fe90 25#include <linux/mlx5/fs.h>
cecae747 26#include <linux/mlx5/eswitch.h>
0d293714 27#include <linux/mlx5/driver.h>
7c2344c3 28#include <linux/list.h>
e126ba97 29#include <rdma/ib_smi.h>
ca7ef7ad 30#include <rdma/ib_umem_odp.h>
cfc1a89e 31#include <rdma/lag.h>
038d2ef8
MG		 32#include <linux/in.h>
33#include <linux/etherdevice.h>
e126ba97 34#include "mlx5_ib.h"
fc385b7a 35#include "ib_rep.h"
e1f24a79 36#include "cmd.h"
d8b7515e 37#include "devx.h"
831df883 38#include "dm.h"
f7c4ffda 39#include "fs.h"
f3da6577 40#include "srq.h"
333fbaa0 41#include "qp.h"
029e88fd 42#include "wr.h"
b572ebe6 43#include "restrack.h"
64825827 44#include "counters.h"
04876c12 45#include "umr.h"
8c84660b 46#include <rdma/uverbs_std_types.h>
2904bb37 47#include <rdma/uverbs_ioctl.h>
c6475a0b
AY		 48#include <rdma/mlx5_user_ioctl_verbs.h>
49#include <rdma/mlx5_user_ioctl_cmds.h>
cf7174e8 50#include <rdma/ib_ucaps.h>
758ce14a 51#include "macsec.h"
6910e366 52#include "data_direct.h"
8c84660b
MB		 53
54#define UVERBS_MODULE_NAME mlx5_ib
55#include <rdma/uverbs_named_ioctl.h>
e126ba97 56
e126ba97 57MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>");
daeee976 58MODULE_DESCRIPTION("Mellanox 5th generation network adapters (ConnectX series) IB driver");
e126ba97 59MODULE_LICENSE("Dual BSD/GPL");
e126ba97 60
d69a24e0
DJ		 61struct mlx5_ib_event_work {
62 struct work_struct work;
df097a27
SM		 63 union {
64 struct mlx5_ib_dev *dev;
65 struct mlx5_ib_multiport_info *mpi;
66 };
67 bool is_slave;
134e9349 68 unsigned int event;
df097a27 69 void *param;
d69a24e0
DJ		 70};
71
da7525d2
EBE		 72enum {
73 MLX5_ATOMIC_SIZE_QP_8BYTES = 1 << 3,
74};
75
d69a24e0 76static struct workqueue_struct *mlx5_ib_event_wq;
32f69e4b
DJ		 77static LIST_HEAD(mlx5_ib_unaffiliated_port_list);
78static LIST_HEAD(mlx5_ib_dev_list);
79/*
80 * This mutex should be held when accessing either of the above lists
81 */
82static DEFINE_MUTEX(mlx5_ib_multiport_mutex);
83
84struct mlx5_ib_dev *mlx5_ib_get_ibdev_from_mpi(struct mlx5_ib_multiport_info *mpi)
85{
86 struct mlx5_ib_dev *dev;
87
88 mutex_lock(&mlx5_ib_multiport_mutex);
89 dev = mpi->ibdev;
90 mutex_unlock(&mlx5_ib_multiport_mutex);
91 return dev;
92}
93
1b5daf11 94static enum rdma_link_layer
ebd61f68 95mlx5_port_type_cap_to_rdma_ll(int port_type_cap)
1b5daf11 96{
ebd61f68 97 switch (port_type_cap) {
1b5daf11
MD		 98 case MLX5_CAP_PORT_TYPE_IB:
99 return IB_LINK_LAYER_INFINIBAND;
100 case MLX5_CAP_PORT_TYPE_ETH:
101 return IB_LINK_LAYER_ETHERNET;
102 default:
103 return IB_LINK_LAYER_UNSPECIFIED;
104 }
105}
106
ebd61f68 107static enum rdma_link_layer
1fb7f897 108mlx5_ib_port_link_layer(struct ib_device *device, u32 port_num)
ebd61f68
AS		 109{
110 struct mlx5_ib_dev *dev = to_mdev(device);
111 int port_type_cap = MLX5_CAP_GEN(dev->mdev, port_type);
112
113 return mlx5_port_type_cap_to_rdma_ll(port_type_cap);
114}
115
fd65f1b8 116static int get_port_state(struct ib_device *ibdev,
1fb7f897 117 u32 port_num,
fd65f1b8
MS		 118 enum ib_port_state *state)
119{
120 struct ib_port_attr attr;
121 int ret;
122
123 memset(&attr, 0, sizeof(attr));
3023a1e9 124 ret = ibdev->ops.query_port(ibdev, port_num, &attr);
fd65f1b8
MS		 125 if (!ret)
126 *state = attr.state;
127 return ret;
128}
129
35b0aa67
MB		 130static struct mlx5_roce *mlx5_get_rep_roce(struct mlx5_ib_dev *dev,
131 struct net_device *ndev,
c446d9da 132 struct net_device *upper,
1fb7f897 133 u32 *port_num)
35b0aa67 134{
35b0aa67
MB		 135 struct net_device *rep_ndev;
136 struct mlx5_ib_port *port;
137 int i;
138
139 for (i = 0; i < dev->num_ports; i++) {
140 port = &dev->port[i];
141 if (!port->rep)
142 continue;
143
c446d9da
MB		 144 if (upper == ndev && port->rep->vport == MLX5_VPORT_UPLINK) {
145 *port_num = i + 1;
146 return &port->roce;
147 }
148
149 if (upper && port->rep->vport == MLX5_VPORT_UPLINK)
150 continue;
8d159eb2
CM		 151 rep_ndev = ib_device_get_netdev(&dev->ib_dev, i + 1);
152 if (rep_ndev && rep_ndev == ndev) {
153 dev_put(rep_ndev);
35b0aa67
MB		 154 *port_num = i + 1;
155 return &port->roce;
156 }
8d159eb2
CM		 157
158 dev_put(rep_ndev);
159 }
160
161 return NULL;
162}
163
164static bool mlx5_netdev_send_event(struct mlx5_ib_dev *dev,
165 struct net_device *ndev,
166 struct net_device *upper,
167 struct net_device *ib_ndev)
168{
169 if (!dev->ib_active)
170 return false;
171
172 /* Event is about our upper device */
173 if (upper == ndev)
174 return true;
175
176 /* RDMA device is not in lag and not in switchdev */
177 if (!dev->is_rep && !upper && ndev == ib_ndev)
178 return true;
179
180 /* RDMA devie is in switchdev */
181 if (dev->is_rep && ndev == ib_ndev)
182 return true;
183
184 return false;
185}
186
187static struct net_device *mlx5_ib_get_rep_uplink_netdev(struct mlx5_ib_dev *ibdev)
188{
189 struct mlx5_ib_port *port;
190 int i;
191
192 for (i = 0; i < ibdev->num_ports; i++) {
193 port = &ibdev->port[i];
194 if (port->rep && port->rep->vport == MLX5_VPORT_UPLINK) {
195 return ib_device_get_netdev(&ibdev->ib_dev, i + 1);
196 }
35b0aa67
MB		 197 }
198
199 return NULL;
200}
201
fc24fc5e
AS		 202static int mlx5_netdev_event(struct notifier_block *this,
203 unsigned long event, void *ptr)
204{
7fd8aefb 205 struct mlx5_roce *roce = container_of(this, struct mlx5_roce, nb);
fc24fc5e 206 struct net_device *ndev = netdev_notifier_info_to_dev(ptr);
1fb7f897 207 u32 port_num = roce->native_port_num;
8d159eb2 208 struct net_device *ib_ndev = NULL;
7fd8aefb
DJ		 209 struct mlx5_core_dev *mdev;
210 struct mlx5_ib_dev *ibdev;
211
212 ibdev = roce->dev;
32f69e4b
DJ		 213 mdev = mlx5_ib_get_native_port_mdev(ibdev, port_num, NULL);
214 if (!mdev)
215 return NOTIFY_DONE;
fc24fc5e 216
5ec8c83e
AH		 217 switch (event) {
218 case NETDEV_REGISTER:
35b0aa67
MB		 219 /* Should already be registered during the load */
220 if (ibdev->is_rep)
221 break;
8d159eb2
CM		 222
223 ib_ndev = ib_device_get_netdev(&ibdev->ib_dev, port_num);
224 /* Exit if already registered */
225 if (ib_ndev)
226 goto put_ndev;
227
dce45af5 228 if (ndev->dev.parent == mdev->device)
8d159eb2 229 ib_device_set_netdev(&ibdev->ib_dev, ndev, port_num);
5ec8c83e 230 break;
fc24fc5e 231
842a9c83 232 case NETDEV_UNREGISTER:
35b0aa67 233 /* In case of reps, ib device goes away before the netdevs */
8d159eb2
CM		 234 if (ibdev->is_rep)
235 break;
236 ib_ndev = ib_device_get_netdev(&ibdev->ib_dev, port_num);
237 if (ib_ndev == ndev)
238 ib_device_set_netdev(&ibdev->ib_dev, NULL, port_num);
239 goto put_ndev;
842a9c83 240
fd65f1b8 241 case NETDEV_CHANGE:
5ec8c83e 242 case NETDEV_UP:
88621dfe 243 case NETDEV_DOWN: {
88621dfe
AH		 244 struct net_device *upper = NULL;
245
220043b0
PH		 246 if (!netif_is_lag_master(ndev) && !netif_is_lag_port(ndev) &&
247 !mlx5_core_mp_enabled(mdev))
37901377
YL		 248 return NOTIFY_DONE;
249
8d159eb2 250 if (mlx5_lag_is_roce(mdev) || mlx5_lag_is_sriov(mdev)) {
303ee44a
MB		 251 struct net_device *lag_ndev;
252
8d159eb2
CM		 253 if(mlx5_lag_is_roce(mdev))
254 lag_ndev = ib_device_get_netdev(&ibdev->ib_dev, 1);
255 else /* sriov lag */
256 lag_ndev = mlx5_ib_get_rep_uplink_netdev(ibdev);
257
303ee44a
MB		 258 if (lag_ndev) {
259 upper = netdev_master_upper_dev_get(lag_ndev);
260 dev_put(lag_ndev);
261 } else {
262 goto done;
263 }
88621dfe
AH		 264 }
265
35b0aa67 266 if (ibdev->is_rep)
c446d9da 267 roce = mlx5_get_rep_roce(ibdev, ndev, upper, &port_num);
35b0aa67
MB		 268 if (!roce)
269 return NOTIFY_DONE;
8d159eb2
CM		 270
271 ib_ndev = ib_device_get_netdev(&ibdev->ib_dev, port_num);
272
273 if (mlx5_netdev_send_event(ibdev, ndev, upper, ib_ndev)) {
626bc02d 274 struct ib_event ibev = { };
fd65f1b8 275 enum ib_port_state port_state;
5ec8c83e 276
7fd8aefb
DJ		 277 if (get_port_state(&ibdev->ib_dev, port_num,
278 &port_state))
8d159eb2 279 goto put_ndev;
fd65f1b8 280
7fd8aefb 281 if (roce->last_port_state == port_state)
8d159eb2 282 goto put_ndev;
fd65f1b8 283
7fd8aefb 284 roce->last_port_state = port_state;
5ec8c83e 285 ibev.device = &ibdev->ib_dev;
fd65f1b8
MS		 286 if (port_state == IB_PORT_DOWN)
287 ibev.event = IB_EVENT_PORT_ERR;
288 else if (port_state == IB_PORT_ACTIVE)
289 ibev.event = IB_EVENT_PORT_ACTIVE;
290 else
8d159eb2 291 goto put_ndev;
fd65f1b8 292
7fd8aefb 293 ibev.element.port_num = port_num;
5ec8c83e
AH		 294 ib_dispatch_event(&ibev);
295 }
296 break;
88621dfe 297 }
fc24fc5e 298
5ec8c83e
AH		 299 default:
300 break;
301 }
8d159eb2
CM		 302put_ndev:
303 dev_put(ib_ndev);
7fd8aefb 304done:
32f69e4b 305 mlx5_ib_put_native_port_mdev(ibdev, port_num);
fc24fc5e
AS		 306 return NOTIFY_DONE;
307}
308
32f69e4b 309struct mlx5_core_dev *mlx5_ib_get_native_port_mdev(struct mlx5_ib_dev *ibdev,
1fb7f897
MB		 310 u32 ib_port_num,
311 u32 *native_port_num)
32f69e4b
DJ		 312{
313 enum rdma_link_layer ll = mlx5_ib_port_link_layer(&ibdev->ib_dev,
314 ib_port_num);
315 struct mlx5_core_dev *mdev = NULL;
316 struct mlx5_ib_multiport_info *mpi;
317 struct mlx5_ib_port *port;
318
026a4259
MZ		 319 if (ibdev->ib_dev.type == RDMA_DEVICE_TYPE_SMI) {
320 if (native_port_num)
321 *native_port_num = smi_to_native_portnum(ibdev,
322 ib_port_num);
323 return ibdev->mdev;
324
325 }
326
210b1f78
MB		 327 if (!mlx5_core_mp_enabled(ibdev->mdev) ||
328 ll != IB_LINK_LAYER_ETHERNET) {
329 if (native_port_num)
330 *native_port_num = ib_port_num;
331 return ibdev->mdev;
332 }
333
32f69e4b
DJ		 334 if (native_port_num)
335 *native_port_num = 1;
336
32f69e4b 337 port = &ibdev->port[ib_port_num - 1];
32f69e4b
DJ		 338 spin_lock(&port->mp.mpi_lock);
339 mpi = ibdev->port[ib_port_num - 1].mp.mpi;
340 if (mpi && !mpi->unaffiliate) {
341 mdev = mpi->mdev;
342 /* If it's the master no need to refcount, it'll exist
343 * as long as the ib_dev exists.
344 */
345 if (!mpi->is_master)
346 mpi->mdev_refcnt++;
347 }
348 spin_unlock(&port->mp.mpi_lock);
349
350 return mdev;
351}
352
1fb7f897 353void mlx5_ib_put_native_port_mdev(struct mlx5_ib_dev *ibdev, u32 port_num)
32f69e4b
DJ		 354{
355 enum rdma_link_layer ll = mlx5_ib_port_link_layer(&ibdev->ib_dev,
356 port_num);
357 struct mlx5_ib_multiport_info *mpi;
358 struct mlx5_ib_port *port;
359
360 if (!mlx5_core_mp_enabled(ibdev->mdev) || ll != IB_LINK_LAYER_ETHERNET)
361 return;
362
363 port = &ibdev->port[port_num - 1];
364
365 spin_lock(&port->mp.mpi_lock);
366 mpi = ibdev->port[port_num - 1].mp.mpi;
367 if (mpi->is_master)
368 goto out;
369
370 mpi->mdev_refcnt--;
371 if (mpi->unaffiliate)
372 complete(&mpi->unref_comp);
373out:
374 spin_unlock(&port->mp.mpi_lock);
375}
376
639bf441
AL		 377static int translate_eth_legacy_proto_oper(u32 eth_proto_oper,
378 u16 *active_speed, u8 *active_width)
f1b65df5
NO		 379{
380 switch (eth_proto_oper) {
381 case MLX5E_PROT_MASK(MLX5E_1000BASE_CX_SGMII):
382 case MLX5E_PROT_MASK(MLX5E_1000BASE_KX):
383 case MLX5E_PROT_MASK(MLX5E_100BASE_TX):
384 case MLX5E_PROT_MASK(MLX5E_1000BASE_T):
385 *active_width = IB_WIDTH_1X;
386 *active_speed = IB_SPEED_SDR;
387 break;
388 case MLX5E_PROT_MASK(MLX5E_10GBASE_T):
389 case MLX5E_PROT_MASK(MLX5E_10GBASE_CX4):
390 case MLX5E_PROT_MASK(MLX5E_10GBASE_KX4):
391 case MLX5E_PROT_MASK(MLX5E_10GBASE_KR):
392 case MLX5E_PROT_MASK(MLX5E_10GBASE_CR):
393 case MLX5E_PROT_MASK(MLX5E_10GBASE_SR):
394 case MLX5E_PROT_MASK(MLX5E_10GBASE_ER):
395 *active_width = IB_WIDTH_1X;
396 *active_speed = IB_SPEED_QDR;
397 break;
398 case MLX5E_PROT_MASK(MLX5E_25GBASE_CR):
399 case MLX5E_PROT_MASK(MLX5E_25GBASE_KR):
400 case MLX5E_PROT_MASK(MLX5E_25GBASE_SR):
401 *active_width = IB_WIDTH_1X;
402 *active_speed = IB_SPEED_EDR;
403 break;
404 case MLX5E_PROT_MASK(MLX5E_40GBASE_CR4):
405 case MLX5E_PROT_MASK(MLX5E_40GBASE_KR4):
406 case MLX5E_PROT_MASK(MLX5E_40GBASE_SR4):
407 case MLX5E_PROT_MASK(MLX5E_40GBASE_LR4):
408 *active_width = IB_WIDTH_4X;
409 *active_speed = IB_SPEED_QDR;
410 break;
411 case MLX5E_PROT_MASK(MLX5E_50GBASE_CR2):
412 case MLX5E_PROT_MASK(MLX5E_50GBASE_KR2):
413 case MLX5E_PROT_MASK(MLX5E_50GBASE_SR2):
414 *active_width = IB_WIDTH_1X;
415 *active_speed = IB_SPEED_HDR;
416 break;
417 case MLX5E_PROT_MASK(MLX5E_56GBASE_R4):
418 *active_width = IB_WIDTH_4X;
419 *active_speed = IB_SPEED_FDR;
420 break;
421 case MLX5E_PROT_MASK(MLX5E_100GBASE_CR4):
422 case MLX5E_PROT_MASK(MLX5E_100GBASE_SR4):
423 case MLX5E_PROT_MASK(MLX5E_100GBASE_KR4):
424 case MLX5E_PROT_MASK(MLX5E_100GBASE_LR4):
425 *active_width = IB_WIDTH_4X;
426 *active_speed = IB_SPEED_EDR;
427 break;
428 default:
429 return -EINVAL;
430 }
431
432 return 0;
433}
434
639bf441 435static int translate_eth_ext_proto_oper(u32 eth_proto_oper, u16 *active_speed,
08e8676f
AL		 436 u8 *active_width)
437{
438 switch (eth_proto_oper) {
439 case MLX5E_PROT_MASK(MLX5E_SGMII_100M):
440 case MLX5E_PROT_MASK(MLX5E_1000BASE_X_SGMII):
441 *active_width = IB_WIDTH_1X;
442 *active_speed = IB_SPEED_SDR;
443 break;
444 case MLX5E_PROT_MASK(MLX5E_5GBASE_R):
445 *active_width = IB_WIDTH_1X;
446 *active_speed = IB_SPEED_DDR;
447 break;
448 case MLX5E_PROT_MASK(MLX5E_10GBASE_XFI_XAUI_1):
449 *active_width = IB_WIDTH_1X;
450 *active_speed = IB_SPEED_QDR;
451 break;
452 case MLX5E_PROT_MASK(MLX5E_40GBASE_XLAUI_4_XLPPI_4):
453 *active_width = IB_WIDTH_4X;
454 *active_speed = IB_SPEED_QDR;
455 break;
456 case MLX5E_PROT_MASK(MLX5E_25GAUI_1_25GBASE_CR_KR):
457 *active_width = IB_WIDTH_1X;
458 *active_speed = IB_SPEED_EDR;
459 break;
460 case MLX5E_PROT_MASK(MLX5E_50GAUI_2_LAUI_2_50GBASE_CR2_KR2):
cd272875
AL		 461 *active_width = IB_WIDTH_2X;
462 *active_speed = IB_SPEED_EDR;
463 break;
08e8676f
AL		 464 case MLX5E_PROT_MASK(MLX5E_50GAUI_1_LAUI_1_50GBASE_CR_KR):
465 *active_width = IB_WIDTH_1X;
466 *active_speed = IB_SPEED_HDR;
467 break;
cd272875
AL		 468 case MLX5E_PROT_MASK(MLX5E_CAUI_4_100GBASE_CR4_KR4):
469 *active_width = IB_WIDTH_4X;
470 *active_speed = IB_SPEED_EDR;
471 break;
08e8676f
AL		 472 case MLX5E_PROT_MASK(MLX5E_100GAUI_2_100GBASE_CR2_KR2):
473 *active_width = IB_WIDTH_2X;
474 *active_speed = IB_SPEED_HDR;
475 break;
f946e45f
ML		 476 case MLX5E_PROT_MASK(MLX5E_100GAUI_1_100GBASE_CR_KR):
477 *active_width = IB_WIDTH_1X;
478 *active_speed = IB_SPEED_NDR;
479 break;
08e8676f
AL		 480 case MLX5E_PROT_MASK(MLX5E_200GAUI_4_200GBASE_CR4_KR4):
481 *active_width = IB_WIDTH_4X;
482 *active_speed = IB_SPEED_HDR;
483 break;
f946e45f
ML		 484 case MLX5E_PROT_MASK(MLX5E_200GAUI_2_200GBASE_CR2_KR2):
485 *active_width = IB_WIDTH_2X;
486 *active_speed = IB_SPEED_NDR;
487 break;
d00d16bc
PH		 488 case MLX5E_PROT_MASK(MLX5E_200GAUI_1_200GBASE_CR1_KR1):
489 *active_width = IB_WIDTH_1X;
490 *active_speed = IB_SPEED_XDR;
491 break;
b28ad324 492 case MLX5E_PROT_MASK(MLX5E_400GAUI_8_400GBASE_CR8):
88c9483f
MS		 493 *active_width = IB_WIDTH_8X;
494 *active_speed = IB_SPEED_HDR;
495 break;
f946e45f
ML		 496 case MLX5E_PROT_MASK(MLX5E_400GAUI_4_400GBASE_CR4_KR4):
497 *active_width = IB_WIDTH_4X;
498 *active_speed = IB_SPEED_NDR;
499 break;
d00d16bc
PH		 500 case MLX5E_PROT_MASK(MLX5E_400GAUI_2_400GBASE_CR2_KR2):
501 *active_width = IB_WIDTH_2X;
502 *active_speed = IB_SPEED_XDR;
503 break;
948f0bf5
PH		 504 case MLX5E_PROT_MASK(MLX5E_800GAUI_8_800GBASE_CR8_KR8):
505 *active_width = IB_WIDTH_8X;
506 *active_speed = IB_SPEED_NDR;
507 break;
d00d16bc
PH		 508 case MLX5E_PROT_MASK(MLX5E_800GAUI_4_800GBASE_CR4_KR4):
509 *active_width = IB_WIDTH_4X;
510 *active_speed = IB_SPEED_XDR;
511 break;
08e8676f
AL		 512 default:
513 return -EINVAL;
514 }
515
516 return 0;
517}
518
639bf441 519static int translate_eth_proto_oper(u32 eth_proto_oper, u16 *active_speed,
08e8676f
AL		 520 u8 *active_width, bool ext)
521{
522 return ext ?
523 translate_eth_ext_proto_oper(eth_proto_oper, active_speed,
524 active_width) :
525 translate_eth_legacy_proto_oper(eth_proto_oper, active_speed,
526 active_width);
527}
528
1fb7f897 529static int mlx5_query_port_roce(struct ib_device *device, u32 port_num,
095b0927 530 struct ib_port_attr *props)
3f89a643
AS		 531{
532 struct mlx5_ib_dev *dev = to_mdev(device);
bc4e12ff 533 u32 out[MLX5_ST_SZ_DW(ptys_reg)] = {0};
da005f9f 534 struct mlx5_core_dev *mdev;
88621dfe 535 struct net_device *ndev, *upper;
3f89a643 536 enum ib_mtu ndev_ib_mtu;
b3cbd6f0 537 bool put_mdev = true;
f1b65df5 538 u32 eth_prot_oper;
1fb7f897 539 u32 mdev_port_num;
08e8676f 540 bool ext;
095b0927 541 int err;
3f89a643 542
b3cbd6f0
DJ		 543 mdev = mlx5_ib_get_native_port_mdev(dev, port_num, &mdev_port_num);
544 if (!mdev) {
545 /* This means the port isn't affiliated yet. Get the
546 * info for the master port instead.
547 */
548 put_mdev = false;
549 mdev = dev->mdev;
550 mdev_port_num = 1;
551 port_num = 1;
552 }
553
f1b65df5
NO		 554 /* Possible bad flows are checked before filling out props so in case
555 * of an error it will still be zeroed out.
26628e2d 556 * Use native port in case of reps
50f22fd8 557 */
26628e2d
MB		 558 if (dev->is_rep)
559 err = mlx5_query_port_ptys(mdev, out, sizeof(out), MLX5_PTYS_EN,
3b43399b 560 1, 0);
26628e2d
MB		 561 else
562 err = mlx5_query_port_ptys(mdev, out, sizeof(out), MLX5_PTYS_EN,
3b43399b 563 mdev_port_num, 0);
095b0927 564 if (err)
b3cbd6f0 565 goto out;
530c8632 566 ext = !!MLX5_GET_ETH_PROTO(ptys_reg, out, true, eth_proto_capability);
08e8676f 567 eth_prot_oper = MLX5_GET_ETH_PROTO(ptys_reg, out, ext, eth_proto_oper);
f1b65df5 568
7672ed33
HL		 569 props->active_width = IB_WIDTH_4X;
570 props->active_speed = IB_SPEED_QDR;
571
f1b65df5 572 translate_eth_proto_oper(eth_prot_oper, &props->active_speed,
08e8676f 573 &props->active_width, ext);
3f89a643 574
7852546f 575 if (!dev->is_rep && dev->mdev->roce.roce_en) {
7a58779e 576 u16 qkey_viol_cntr;
3f89a643 577
7a58779e
PP		 578 props->port_cap_flags |= IB_PORT_CM_SUP;
579 props->ip_gids = true;
580 props->gid_tbl_len = MLX5_CAP_ROCE(dev->mdev,
581 roce_address_table_size);
582 mlx5_query_nic_vport_qkey_viol_cntr(mdev, &qkey_viol_cntr);
583 props->qkey_viol_cntr = qkey_viol_cntr;
584 }
3f89a643
AS		 585 props->max_mtu = IB_MTU_4096;
586 props->max_msg_sz = 1 << MLX5_CAP_GEN(dev->mdev, log_max_msg);
587 props->pkey_tbl_len = 1;
588 props->state = IB_PORT_DOWN;
72a7720f 589 props->phys_state = IB_PORT_PHYS_STATE_DISABLED;
3f89a643 590
b3cbd6f0
DJ		 591 /* If this is a stub query for an unaffiliated port stop here */
592 if (!put_mdev)
593 goto out;
594
8d159eb2 595 ndev = ib_device_get_netdev(device, port_num);
3f89a643 596 if (!ndev)
b3cbd6f0 597 goto out;
3f89a643 598
3ed7f9e2 599 if (mlx5_lag_is_roce(mdev) || mlx5_lag_is_sriov(mdev)) {
88621dfe
AH		 600 rcu_read_lock();
601 upper = netdev_master_upper_dev_get_rcu(ndev);
602 if (upper) {
603 dev_put(ndev);
604 ndev = upper;
605 dev_hold(ndev);
606 }
607 rcu_read_unlock();
608 }
609
3f89a643
AS		 610 if (netif_running(ndev) && netif_carrier_ok(ndev)) {
611 props->state = IB_PORT_ACTIVE;
72a7720f 612 props->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
3f89a643
AS		 613 }
614
615 ndev_ib_mtu = iboe_get_mtu(ndev->mtu);
616
617 dev_put(ndev);
618
619 props->active_mtu = min(props->max_mtu, ndev_ib_mtu);
b3cbd6f0
DJ		 620out:
621 if (put_mdev)
622 mlx5_ib_put_native_port_mdev(dev, port_num);
623 return err;
3f89a643
AS		 624}
625
758ce14a
PH		 626int set_roce_addr(struct mlx5_ib_dev *dev, u32 port_num,
627 unsigned int index, const union ib_gid *gid,
628 const struct ib_gid_attr *attr)
3cca2606 629{
dedbc2d3 630 enum ib_gid_type gid_type;
a70c0739 631 u16 vlan_id = 0xffff;
095b0927
IT		 632 u8 roce_version = 0;
633 u8 roce_l3_type = 0;
095b0927 634 u8 mac[ETH_ALEN];
a70c0739 635 int ret;
095b0927 636
dedbc2d3 637 gid_type = attr->gid_type;
095b0927 638 if (gid) {
a70c0739
PP		 639 ret = rdma_read_gid_l2_fields(attr, &vlan_id, &mac[0]);
640 if (ret)
641 return ret;
3cca2606
AS		 642 }
643
095b0927 644 switch (gid_type) {
1c15b4f2 645 case IB_GID_TYPE_ROCE:
095b0927 646 roce_version = MLX5_ROCE_VERSION_1;
3cca2606
AS		 647 break;
648 case IB_GID_TYPE_ROCE_UDP_ENCAP:
095b0927 649 roce_version = MLX5_ROCE_VERSION_2;
dedbc2d3 650 if (gid && ipv6_addr_v4mapped((void *)gid))
095b0927
IT		 651 roce_l3_type = MLX5_ROCE_L3_TYPE_IPV4;
652 else
653 roce_l3_type = MLX5_ROCE_L3_TYPE_IPV6;
3cca2606
AS		 654 break;
655
656 default:
095b0927 657 mlx5_ib_warn(dev, "Unexpected GID type %u\n", gid_type);
3cca2606
AS		 658 }
659
095b0927 660 return mlx5_core_roce_gid_set(dev->mdev, index, roce_version,
cf34e1fe 661 roce_l3_type, gid->raw, mac,
a70c0739 662 vlan_id < VLAN_CFI_MASK, vlan_id,
cf34e1fe 663 port_num);
3cca2606
AS		 664}
665
f4df9a7c 666static int mlx5_ib_add_gid(const struct ib_gid_attr *attr,
3cca2606
AS		 667 __always_unused void **context)
668{
758ce14a
PH		 669 int ret;
670
671 ret = mlx5r_add_gid_macsec_operations(attr);
672 if (ret)
673 return ret;
674
414448d2 675 return set_roce_addr(to_mdev(attr->device), attr->port_num,
f4df9a7c 676 attr->index, &attr->gid, attr);
3cca2606
AS		 677}
678
414448d2
PP		 679static int mlx5_ib_del_gid(const struct ib_gid_attr *attr,
680 __always_unused void **context)
3cca2606 681{
758ce14a
PH		 682 int ret;
683
684 ret = set_roce_addr(to_mdev(attr->device), attr->port_num,
685 attr->index, NULL, attr);
686 if (ret)
687 return ret;
688
689 mlx5r_del_gid_macsec_operations(attr);
690 return 0;
3cca2606
AS		 691}
692
5ac55dfc
MZ		 693__be16 mlx5_get_roce_udp_sport_min(const struct mlx5_ib_dev *dev,
694 const struct ib_gid_attr *attr)
2811ba51 695{
47ec3866 696 if (attr->gid_type != IB_GID_TYPE_ROCE_UDP_ENCAP)
2811ba51
AS		 697 return 0;
698
699 return cpu_to_be16(MLX5_CAP_ROCE(dev->mdev, r_roce_min_src_udp_port));
700}
701
1b5daf11
MD		 702static int mlx5_use_mad_ifc(struct mlx5_ib_dev *dev)
703{
7fae6655
NO		 704 if (MLX5_CAP_GEN(dev->mdev, port_type) == MLX5_CAP_PORT_TYPE_IB)
705 return !MLX5_CAP_GEN(dev->mdev, ib_virt);
706 return 0;
1b5daf11
MD		 707}
708
709enum {
710 MLX5_VPORT_ACCESS_METHOD_MAD,
711 MLX5_VPORT_ACCESS_METHOD_HCA,
712 MLX5_VPORT_ACCESS_METHOD_NIC,
713};
714
715static int mlx5_get_vport_access_method(struct ib_device *ibdev)
716{
717 if (mlx5_use_mad_ifc(to_mdev(ibdev)))
718 return MLX5_VPORT_ACCESS_METHOD_MAD;
719
ebd61f68 720 if (mlx5_ib_port_link_layer(ibdev, 1) ==
1b5daf11
MD		 721 IB_LINK_LAYER_ETHERNET)
722 return MLX5_VPORT_ACCESS_METHOD_NIC;
723
724 return MLX5_VPORT_ACCESS_METHOD_HCA;
725}
726
da7525d2 727static void get_atomic_caps(struct mlx5_ib_dev *dev,
776a3906 728 u8 atomic_size_qp,
da7525d2
EBE		 729 struct ib_device_attr *props)
730{
731 u8 tmp;
732 u8 atomic_operations = MLX5_CAP_ATOMIC(dev->mdev, atomic_operations);
da7525d2 733 u8 atomic_req_8B_endianness_mode =
bd10838a 734 MLX5_CAP_ATOMIC(dev->mdev, atomic_req_8B_endianness_mode);
da7525d2
EBE		 735
736 /* Check if HW supports 8 bytes standard atomic operations and capable
737 * of host endianness respond
738 */
739 tmp = MLX5_ATOMIC_OPS_CMP_SWAP | MLX5_ATOMIC_OPS_FETCH_ADD;
740 if (((atomic_operations & tmp) == tmp) &&
741 (atomic_size_qp & MLX5_ATOMIC_SIZE_QP_8BYTES) &&
742 (atomic_req_8B_endianness_mode)) {
743 props->atomic_cap = IB_ATOMIC_HCA;
744 } else {
745 props->atomic_cap = IB_ATOMIC_NONE;
746 }
747}
748
776a3906
MS		 749static void get_atomic_caps_qp(struct mlx5_ib_dev *dev,
750 struct ib_device_attr *props)
751{
752 u8 atomic_size_qp = MLX5_CAP_ATOMIC(dev->mdev, atomic_size_qp);
753
754 get_atomic_caps(dev, atomic_size_qp, props);
755}
756
1b5daf11
MD		 757static int mlx5_query_system_image_guid(struct ib_device *ibdev,
758 __be64 *sys_image_guid)
759{
760 struct mlx5_ib_dev *dev = to_mdev(ibdev);
761 struct mlx5_core_dev *mdev = dev->mdev;
762 u64 tmp;
763 int err;
764
765 switch (mlx5_get_vport_access_method(ibdev)) {
766 case MLX5_VPORT_ACCESS_METHOD_MAD:
767 return mlx5_query_mad_ifc_system_image_guid(ibdev,
768 sys_image_guid);
769
770 case MLX5_VPORT_ACCESS_METHOD_HCA:
771 err = mlx5_query_hca_vport_system_image_guid(mdev, &tmp);
3f89a643
AS		 772 break;
773
774 case MLX5_VPORT_ACCESS_METHOD_NIC:
775 err = mlx5_query_nic_vport_system_image_guid(mdev, &tmp);
776 break;
1b5daf11
MD		 777
778 default:
779 return -EINVAL;
780 }
3f89a643
AS		 781
782 if (!err)
783 *sys_image_guid = cpu_to_be64(tmp);
784
785 return err;
786
1b5daf11
MD		 787}
788
789static int mlx5_query_max_pkeys(struct ib_device *ibdev,
790 u16 *max_pkeys)
791{
792 struct mlx5_ib_dev *dev = to_mdev(ibdev);
793 struct mlx5_core_dev *mdev = dev->mdev;
794
795 switch (mlx5_get_vport_access_method(ibdev)) {
796 case MLX5_VPORT_ACCESS_METHOD_MAD:
797 return mlx5_query_mad_ifc_max_pkeys(ibdev, max_pkeys);
798
799 case MLX5_VPORT_ACCESS_METHOD_HCA:
800 case MLX5_VPORT_ACCESS_METHOD_NIC:
801 *max_pkeys = mlx5_to_sw_pkey_sz(MLX5_CAP_GEN(mdev,
802 pkey_table_size));
803 return 0;
804
805 default:
806 return -EINVAL;
807 }
808}
809
810static int mlx5_query_vendor_id(struct ib_device *ibdev,
811 u32 *vendor_id)
812{
813 struct mlx5_ib_dev *dev = to_mdev(ibdev);
814
815 switch (mlx5_get_vport_access_method(ibdev)) {
816 case MLX5_VPORT_ACCESS_METHOD_MAD:
817 return mlx5_query_mad_ifc_vendor_id(ibdev, vendor_id);
818
819 case MLX5_VPORT_ACCESS_METHOD_HCA:
820 case MLX5_VPORT_ACCESS_METHOD_NIC:
821 return mlx5_core_query_vendor_id(dev->mdev, vendor_id);
822
823 default:
824 return -EINVAL;
825 }
826}
827
828static int mlx5_query_node_guid(struct mlx5_ib_dev *dev,
829 __be64 *node_guid)
830{
831 u64 tmp;
832 int err;
833
834 switch (mlx5_get_vport_access_method(&dev->ib_dev)) {
835 case MLX5_VPORT_ACCESS_METHOD_MAD:
836 return mlx5_query_mad_ifc_node_guid(dev, node_guid);
837
838 case MLX5_VPORT_ACCESS_METHOD_HCA:
839 err = mlx5_query_hca_vport_node_guid(dev->mdev, &tmp);
3f89a643
AS		 840 break;
841
842 case MLX5_VPORT_ACCESS_METHOD_NIC:
843 err = mlx5_query_nic_vport_node_guid(dev->mdev, &tmp);
844 break;
1b5daf11
MD		 845
846 default:
847 return -EINVAL;
848 }
3f89a643
AS		 849
850 if (!err)
851 *node_guid = cpu_to_be64(tmp);
852
853 return err;
1b5daf11
MD		 854}
855
856struct mlx5_reg_node_desc {
bd99fdea 857 u8 desc[IB_DEVICE_NODE_DESC_MAX];
1b5daf11
MD		 858};
859
860static int mlx5_query_node_desc(struct mlx5_ib_dev *dev, char *node_desc)
861{
862 struct mlx5_reg_node_desc in;
863
864 if (mlx5_use_mad_ifc(dev))
865 return mlx5_query_mad_ifc_node_desc(dev, node_desc);
866
867 memset(&in, 0, sizeof(in));
868
869 return mlx5_core_access_reg(dev->mdev, &in, sizeof(in), node_desc,
870 sizeof(struct mlx5_reg_node_desc),
871 MLX5_REG_NODE_DESC, 0, 0);
872}
873
d727d27d
MB		 874static void fill_esw_mgr_reg_c0(struct mlx5_core_dev *mdev,
875 struct mlx5_ib_query_device_resp *resp)
876{
877 struct mlx5_eswitch *esw = mdev->priv.eswitch;
878 u16 vport = mlx5_eswitch_manager_vport(mdev);
879
880 resp->reg_c0.value = mlx5_eswitch_get_vport_metadata_for_match(esw,
881 vport);
882 resp->reg_c0.mask = mlx5_eswitch_get_vport_metadata_mask();
883}
884
e126ba97 885static int mlx5_ib_query_device(struct ib_device *ibdev,
2528e33e
MB		 886 struct ib_device_attr *props,
887 struct ib_udata *uhw)
e126ba97 888{
48357091 889 size_t uhw_outlen = (uhw) ? uhw->outlen : 0;
e126ba97 890 struct mlx5_ib_dev *dev = to_mdev(ibdev);
938fe83c 891 struct mlx5_core_dev *mdev = dev->mdev;
e126ba97 892 int err = -ENOMEM;
288c01b7 893 int max_sq_desc;
e126ba97
EC		 894 int max_rq_sg;
895 int max_sq_sg;
e0238a6a 896 u64 min_page_size = 1ull << MLX5_CAP_GEN(mdev, log_pg_sz);
85c7c014 897 bool raw_support = !mlx5_core_mp_enabled(mdev);
402ca536
BW		 898 struct mlx5_ib_query_device_resp resp = {};
899 size_t resp_len;
900 u64 max_tso;
e126ba97 901
402ca536 902 resp_len = sizeof(resp.comp_mask) + sizeof(resp.response_length);
48357091 903 if (uhw_outlen && uhw_outlen < resp_len)
402ca536 904 return -EINVAL;
6f26b2ac
EA		 905
906 resp.response_length = resp_len;
402ca536 907
48357091 908 if (uhw && uhw->inlen && !ib_is_udata_cleared(uhw, 0, uhw->inlen))
2528e33e
MB		 909 return -EINVAL;
910
1b5daf11
MD		 911 memset(props, 0, sizeof(*props));
912 err = mlx5_query_system_image_guid(ibdev,
913 &props->sys_image_guid);
914 if (err)
915 return err;
e126ba97 916
2019d70e 917 props->max_pkeys = dev->pkey_table_len;
e126ba97 918
1b5daf11
MD		 919 err = mlx5_query_vendor_id(ibdev, &props->vendor_id);
920 if (err)
921 return err;
e126ba97 922
9603b61d
JM		 923 props->fw_ver = ((u64)fw_rev_maj(dev->mdev) << 32) |
924 (fw_rev_min(dev->mdev) << 16) |
925 fw_rev_sub(dev->mdev);
e126ba97
EC		 926 props->device_cap_flags = IB_DEVICE_CHANGE_PHY_PORT |
927 IB_DEVICE_PORT_ACTIVE_EVENT |
928 IB_DEVICE_SYS_IMAGE_GUID |
1a4c3a3d 929 IB_DEVICE_RC_RNR_NAK_GEN;
938fe83c
SM		 930
931 if (MLX5_CAP_GEN(mdev, pkv))
e126ba97 932 props->device_cap_flags |= IB_DEVICE_BAD_PKEY_CNTR;
938fe83c 933 if (MLX5_CAP_GEN(mdev, qkv))
e126ba97 934 props->device_cap_flags |= IB_DEVICE_BAD_QKEY_CNTR;
938fe83c 935 if (MLX5_CAP_GEN(mdev, apm))
e126ba97 936 props->device_cap_flags |= IB_DEVICE_AUTO_PATH_MIG;
938fe83c 937 if (MLX5_CAP_GEN(mdev, xrc))
e126ba97 938 props->device_cap_flags |= IB_DEVICE_XRC;
d2370e0a
MB		 939 if (MLX5_CAP_GEN(mdev, imaicl)) {
940 props->device_cap_flags |= IB_DEVICE_MEM_WINDOW |
941 IB_DEVICE_MEM_WINDOW_TYPE_2B;
942 props->max_mw = 1 << MLX5_CAP_GEN(mdev, log_max_mkey);
b005d316 943 /* We support 'Gappy' memory registration too */
e945c653 944 props->kernel_cap_flags |= IBK_SG_GAPS_REG;
d2370e0a 945 }
0ec52f01
JG		 946 /* IB_WR_REG_MR always requires changing the entity size with UMR */
947 if (!MLX5_CAP_GEN(dev->mdev, umr_modify_entity_size_disabled))
948 props->device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS;
938fe83c 949 if (MLX5_CAP_GEN(mdev, sho)) {
e945c653 950 props->kernel_cap_flags |= IBK_INTEGRITY_HANDOVER;
2dea9094
SG		 951 /* At this stage no support for signature handover */
952 props->sig_prot_cap = IB_PROT_T10DIF_TYPE_1 |
953 IB_PROT_T10DIF_TYPE_2 |
954 IB_PROT_T10DIF_TYPE_3;
955 props->sig_guard_cap = IB_GUARD_T10DIF_CRC |
956 IB_GUARD_T10DIF_CSUM;
957 }
938fe83c 958 if (MLX5_CAP_GEN(mdev, block_lb_mc))
e945c653 959 props->kernel_cap_flags |= IBK_BLOCK_MULTICAST_LOOPBACK;
e126ba97 960
85c7c014 961 if (MLX5_CAP_GEN(dev->mdev, eth_net_offloads) && raw_support) {
e8161334
NO		 962 if (MLX5_CAP_ETH(mdev, csum_cap)) {
963 /* Legacy bit to support old userspace libraries */
88115fe7 964 props->device_cap_flags |= IB_DEVICE_RAW_IP_CSUM;
e8161334
NO		 965 props->raw_packet_caps |= IB_RAW_PACKET_CAP_IP_CSUM;
966 }
967
968 if (MLX5_CAP_ETH(dev->mdev, vlan_cap))
969 props->raw_packet_caps |=
970 IB_RAW_PACKET_CAP_CVLAN_STRIPPING;
88115fe7 971
a762d460 972 if (offsetofend(typeof(resp), tso_caps) <= uhw_outlen) {
402ca536
BW		 973 max_tso = MLX5_CAP_ETH(mdev, max_lso_cap);
974 if (max_tso) {
975 resp.tso_caps.max_tso = 1 << max_tso;
976 resp.tso_caps.supported_qpts |=
977 1 << IB_QPT_RAW_PACKET;
978 resp.response_length += sizeof(resp.tso_caps);
979 }
980 }
31f69a82 981
a762d460 982 if (offsetofend(typeof(resp), rss_caps) <= uhw_outlen) {
31f69a82
YH		 983 resp.rss_caps.rx_hash_function =
984 MLX5_RX_HASH_FUNC_TOEPLITZ;
985 resp.rss_caps.rx_hash_fields_mask =
986 MLX5_RX_HASH_SRC_IPV4 |
987 MLX5_RX_HASH_DST_IPV4 |
988 MLX5_RX_HASH_SRC_IPV6 |
989 MLX5_RX_HASH_DST_IPV6 |
990 MLX5_RX_HASH_SRC_PORT_TCP |
991 MLX5_RX_HASH_DST_PORT_TCP |
992 MLX5_RX_HASH_SRC_PORT_UDP |
4e2b53a5
MG		 993 MLX5_RX_HASH_DST_PORT_UDP |
994 MLX5_RX_HASH_INNER;
31f69a82
YH		 995 resp.response_length += sizeof(resp.rss_caps);
996 }
997 } else {
a762d460 998 if (offsetofend(typeof(resp), tso_caps) <= uhw_outlen)
31f69a82 999 resp.response_length += sizeof(resp.tso_caps);
a762d460 1000 if (offsetofend(typeof(resp), rss_caps) <= uhw_outlen)
31f69a82 1001 resp.response_length += sizeof(resp.rss_caps);
402ca536
BW		 1002 }
1003
f0313965
ES		 1004 if (MLX5_CAP_GEN(mdev, ipoib_basic_offloads)) {
1005 props->device_cap_flags |= IB_DEVICE_UD_IP_CSUM;
e945c653 1006 props->kernel_cap_flags |= IBK_UD_TSO;
f0313965
ES		 1007 }
1008
03404e8a 1009 if (MLX5_CAP_GEN(dev->mdev, rq_delay_drop) &&
85c7c014
DJ		 1010 MLX5_CAP_GEN(dev->mdev, general_notification_event) &&
1011 raw_support)
03404e8a
MG		 1012 props->raw_packet_caps |= IB_RAW_PACKET_CAP_DELAY_DROP;
1013
1d54f890
YH		 1014 if (MLX5_CAP_GEN(mdev, ipoib_enhanced_offloads) &&
1015 MLX5_CAP_IPOIB_ENHANCED(mdev, csum_cap))
1016 props->device_cap_flags |= IB_DEVICE_UD_IP_CSUM;
1017
cff5a0f3 1018 if (MLX5_CAP_GEN(dev->mdev, eth_net_offloads) &&
85c7c014
DJ		 1019 MLX5_CAP_ETH(dev->mdev, scatter_fcs) &&
1020 raw_support) {
e8161334 1021 /* Legacy bit to support old userspace libraries */
cff5a0f3 1022 props->device_cap_flags |= IB_DEVICE_RAW_SCATTER_FCS;
e8161334
NO		 1023 props->raw_packet_caps |= IB_RAW_PACKET_CAP_SCATTER_FCS;
1024 }
cff5a0f3 1025
24da0016
AL		 1026 if (MLX5_CAP_DEV_MEM(mdev, memic)) {
1027 props->max_dm_size =
1028 MLX5_CAP_DEV_MEM(mdev, max_memic_size);
1029 }
1030
da6d6ba3
MG		 1031 if (mlx5_get_flow_namespace(dev->mdev, MLX5_FLOW_NAMESPACE_BYPASS))
1032 props->device_cap_flags |= IB_DEVICE_MANAGED_FLOW_STEERING;
1033
b1383aa6
NO		 1034 if (MLX5_CAP_GEN(mdev, end_pad))
1035 props->device_cap_flags |= IB_DEVICE_PCI_WRITE_END_PADDING;
1036
1b5daf11
MD		 1037 props->vendor_part_id = mdev->pdev->device;
1038 props->hw_ver = mdev->pdev->revision;
e126ba97
EC		 1039
1040 props->max_mr_size = ~0ull;
e0238a6a 1041 props->page_size_cap = ~(min_page_size - 1);
938fe83c
SM		 1042 props->max_qp = 1 << MLX5_CAP_GEN(mdev, log_max_qp);
1043 props->max_qp_wr = 1 << MLX5_CAP_GEN(mdev, log_max_qp_sz);
1044 max_rq_sg = MLX5_CAP_GEN(mdev, max_wqe_sz_rq) /
1045 sizeof(struct mlx5_wqe_data_seg);
288c01b7
EC		 1046 max_sq_desc = min_t(int, MLX5_CAP_GEN(mdev, max_wqe_sz_sq), 512);
1047 max_sq_sg = (max_sq_desc - sizeof(struct mlx5_wqe_ctrl_seg) -
1048 sizeof(struct mlx5_wqe_raddr_seg)) /
1049 sizeof(struct mlx5_wqe_data_seg);
33023fb8
SW		 1050 props->max_send_sge = max_sq_sg;
1051 props->max_recv_sge = max_rq_sg;
986ef95e 1052 props->max_sge_rd = MLX5_MAX_SGE_RD;
938fe83c 1053 props->max_cq = 1 << MLX5_CAP_GEN(mdev, log_max_cq);
9f177686 1054 props->max_cqe = (1 << MLX5_CAP_GEN(mdev, log_max_cq_sz)) - 1;
938fe83c
SM		 1055 props->max_mr = 1 << MLX5_CAP_GEN(mdev, log_max_mkey);
1056 props->max_pd = 1 << MLX5_CAP_GEN(mdev, log_max_pd);
1057 props->max_qp_rd_atom = 1 << MLX5_CAP_GEN(mdev, log_max_ra_req_qp);
1058 props->max_qp_init_rd_atom = 1 << MLX5_CAP_GEN(mdev, log_max_ra_res_qp);
1059 props->max_srq = 1 << MLX5_CAP_GEN(mdev, log_max_srq);
1060 props->max_srq_wr = (1 << MLX5_CAP_GEN(mdev, log_max_srq_sz)) - 1;
1061 props->local_ca_ack_delay = MLX5_CAP_GEN(mdev, local_ca_ack_delay);
e126ba97 1062 props->max_res_rd_atom = props->max_qp_rd_atom * props->max_qp;
e126ba97 1063 props->max_srq_sge = max_rq_sg - 1;
911f4331
SG		 1064 props->max_fast_reg_page_list_len =
1065 1 << MLX5_CAP_GEN(mdev, log_max_klm_list_size);
62e3c379
MG		 1066 props->max_pi_fast_reg_page_list_len =
1067 props->max_fast_reg_page_list_len / 2;
36609056
YF		 1068 props->max_sgl_rd =
1069 MLX5_CAP_GEN(mdev, max_sgl_for_optimized_performance);
776a3906 1070 get_atomic_caps_qp(dev, props);
81bea28f 1071 props->masked_atomic_cap = IB_ATOMIC_NONE;
938fe83c
SM		 1072 props->max_mcast_grp = 1 << MLX5_CAP_GEN(mdev, log_max_mcg);
1073 props->max_mcast_qp_attach = MLX5_CAP_GEN(mdev, max_qp_mcg);
e126ba97
EC		 1074 props->max_total_mcast_qp_attach = props->max_mcast_qp_attach *
1075 props->max_mcast_grp;
86695a65 1076 props->max_ah = INT_MAX;
7c60bcbb
MB		 1077 props->hca_core_clock = MLX5_CAP_GEN(mdev, device_frequency_khz);
1078 props->timestamp_mask = 0x7FFFFFFFFFFFFFFFULL;
e126ba97 1079
e502b8b0 1080 if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING)) {
00815752 1081 if (dev->odp_caps.general_caps & IB_ODP_SUPPORT)
e945c653 1082 props->kernel_cap_flags |= IBK_ON_DEMAND_PAGING;
e502b8b0 1083 props->odp_caps = dev->odp_caps;
a73a8955
MS		 1084 if (!uhw) {
1085 /* ODP for kernel QPs is not implemented for receive
1086 * WQEs and SRQ WQEs
1087 */
1088 props->odp_caps.per_transport_caps.rc_odp_caps &=
1089 ~(IB_ODP_SUPPORT_READ |
1090 IB_ODP_SUPPORT_SRQ_RECV);
1091 props->odp_caps.per_transport_caps.uc_odp_caps &=
1092 ~(IB_ODP_SUPPORT_READ |
1093 IB_ODP_SUPPORT_SRQ_RECV);
1094 props->odp_caps.per_transport_caps.ud_odp_caps &=
1095 ~(IB_ODP_SUPPORT_READ |
1096 IB_ODP_SUPPORT_SRQ_RECV);
1097 props->odp_caps.per_transport_caps.xrc_odp_caps &=
1098 ~(IB_ODP_SUPPORT_READ |
1099 IB_ODP_SUPPORT_SRQ_RECV);
1100 }
e502b8b0 1101 }
8cdd312c 1102
e53a9d26 1103 if (mlx5_core_is_vf(mdev))
e945c653 1104 props->kernel_cap_flags |= IBK_VIRTUAL_FUNCTION;
eff901d3 1105
31f69a82 1106 if (mlx5_ib_port_link_layer(ibdev, 1) ==
85c7c014 1107 IB_LINK_LAYER_ETHERNET && raw_support) {
31f69a82
YH		 1108 props->rss_caps.max_rwq_indirection_tables =
1109 1 << MLX5_CAP_GEN(dev->mdev, log_max_rqt);
1110 props->rss_caps.max_rwq_indirection_table_size =
1111 1 << MLX5_CAP_GEN(dev->mdev, log_max_rqt_size);
1112 props->rss_caps.supported_qpts = 1 << IB_QPT_RAW_PACKET;
1113 props->max_wq_type_rq =
1114 1 << MLX5_CAP_GEN(dev->mdev, log_max_rq);
1115 }
1116
eb761894 1117 if (MLX5_CAP_GEN(mdev, tag_matching)) {
78b1beb0 1118 props->tm_caps.max_num_tags =
eb761894 1119 (1 << MLX5_CAP_GEN(mdev, log_tag_matching_list_sz)) - 1;
78b1beb0 1120 props->tm_caps.max_ops =
eb761894 1121 1 << MLX5_CAP_GEN(mdev, log_max_qp_sz);
78b1beb0 1122 props->tm_caps.max_sge = MLX5_TM_MAX_SGE;
eb761894
AK		 1123 }
1124
89705e92
DG		 1125 if (MLX5_CAP_GEN(mdev, tag_matching) &&
1126 MLX5_CAP_GEN(mdev, rndv_offload_rc)) {
1127 props->tm_caps.flags = IB_TM_CAP_RNDV_RC;
1128 props->tm_caps.max_rndv_hdr_size = MLX5_TM_MAX_RNDV_MSG_SIZE;
1129 }
1130
87ab3f52
YC		 1131 if (MLX5_CAP_GEN(dev->mdev, cq_moderation)) {
1132 props->cq_caps.max_cq_moderation_count =
1133 MLX5_MAX_CQ_COUNT;
1134 props->cq_caps.max_cq_moderation_period =
1135 MLX5_MAX_CQ_PERIOD;
1136 }
1137
a762d460 1138 if (offsetofend(typeof(resp), cqe_comp_caps) <= uhw_outlen) {
7e43a2a5 1139 resp.response_length += sizeof(resp.cqe_comp_caps);
572f46bf
YC		 1140
1141 if (MLX5_CAP_GEN(dev->mdev, cqe_compression)) {
1142 resp.cqe_comp_caps.max_num =
1143 MLX5_CAP_GEN(dev->mdev,
1144 cqe_compression_max_num);
1145
1146 resp.cqe_comp_caps.supported_format =
1147 MLX5_IB_CQE_RES_FORMAT_HASH |
1148 MLX5_IB_CQE_RES_FORMAT_CSUM;
6f1006a4
YC		 1149
1150 if (MLX5_CAP_GEN(dev->mdev, mini_cqe_resp_stride_index))
1151 resp.cqe_comp_caps.supported_format |=
1152 MLX5_IB_CQE_RES_FORMAT_CSUM_STRIDX;
572f46bf 1153 }
7e43a2a5
BW		 1154 }
1155
a762d460 1156 if (offsetofend(typeof(resp), packet_pacing_caps) <= uhw_outlen &&
85c7c014 1157 raw_support) {
d949167d
BW		 1158 if (MLX5_CAP_QOS(mdev, packet_pacing) &&
1159 MLX5_CAP_GEN(mdev, qos)) {
1160 resp.packet_pacing_caps.qp_rate_limit_max =
1161 MLX5_CAP_QOS(mdev, packet_pacing_max_rate);
1162 resp.packet_pacing_caps.qp_rate_limit_min =
1163 MLX5_CAP_QOS(mdev, packet_pacing_min_rate);
1164 resp.packet_pacing_caps.supported_qpts |=
1165 1 << IB_QPT_RAW_PACKET;
61147f39
BW		 1166 if (MLX5_CAP_QOS(mdev, packet_pacing_burst_bound) &&
1167 MLX5_CAP_QOS(mdev, packet_pacing_typical_size))
1168 resp.packet_pacing_caps.cap_flags |=
1169 MLX5_IB_PP_SUPPORT_BURST;
d949167d
BW		 1170 }
1171 resp.response_length += sizeof(resp.packet_pacing_caps);
1172 }
1173
a762d460
LR		 1174 if (offsetofend(typeof(resp), mlx5_ib_support_multi_pkt_send_wqes) <=
1175 uhw_outlen) {
795b609c
BW		 1176 if (MLX5_CAP_ETH(mdev, multi_pkt_send_wqe))
1177 resp.mlx5_ib_support_multi_pkt_send_wqes =
1178 MLX5_IB_ALLOW_MPW;
050da902
BW		 1179
1180 if (MLX5_CAP_ETH(mdev, enhanced_multi_pkt_send_wqe))
1181 resp.mlx5_ib_support_multi_pkt_send_wqes |=
1182 MLX5_IB_SUPPORT_EMPW;
1183
9f885201
LR		 1184 resp.response_length +=
1185 sizeof(resp.mlx5_ib_support_multi_pkt_send_wqes);
1186 }
1187
a762d460 1188 if (offsetofend(typeof(resp), flags) <= uhw_outlen) {
de57f2ad 1189 resp.response_length += sizeof(resp.flags);
7a0c8f42 1190
de57f2ad
GL		 1191 if (MLX5_CAP_GEN(mdev, cqe_compression_128))
1192 resp.flags |=
1193 MLX5_IB_QUERY_DEV_RESP_FLAGS_CQE_128B_COMP;
7a0c8f42
GL		 1194
1195 if (MLX5_CAP_GEN(mdev, cqe_128_always))
1196 resp.flags |= MLX5_IB_QUERY_DEV_RESP_FLAGS_CQE_128B_PAD;
7e11b911
DG		 1197 if (MLX5_CAP_GEN(mdev, qp_packet_based))
1198 resp.flags |=
1199 MLX5_IB_QUERY_DEV_RESP_PACKET_BASED_CREDIT_MODE;
7249c8ea
GL		 1200
1201 resp.flags |= MLX5_IB_QUERY_DEV_RESP_FLAGS_SCAT2CQE_DCT;
8b36f7c3
ES		 1202
1203 if (MLX5_CAP_GEN_2(mdev, dp_ordering_force) &&
1204 (MLX5_CAP_GEN(mdev, dp_ordering_ooo_all_xrc) ||
1205 MLX5_CAP_GEN(mdev, dp_ordering_ooo_all_dc) ||
1206 MLX5_CAP_GEN(mdev, dp_ordering_ooo_all_rc) ||
1207 MLX5_CAP_GEN(mdev, dp_ordering_ooo_all_ud) ||
1208 MLX5_CAP_GEN(mdev, dp_ordering_ooo_all_uc)))
1209 resp.flags |= MLX5_IB_QUERY_DEV_RESP_FLAGS_OOO_DP;
de57f2ad 1210 }
9f885201 1211
a762d460 1212 if (offsetofend(typeof(resp), sw_parsing_caps) <= uhw_outlen) {
96dc3fc5
NO		 1213 resp.response_length += sizeof(resp.sw_parsing_caps);
1214 if (MLX5_CAP_ETH(mdev, swp)) {
1215 resp.sw_parsing_caps.sw_parsing_offloads |=
1216 MLX5_IB_SW_PARSING;
1217
1218 if (MLX5_CAP_ETH(mdev, swp_csum))
1219 resp.sw_parsing_caps.sw_parsing_offloads |=
1220 MLX5_IB_SW_PARSING_CSUM;
1221
1222 if (MLX5_CAP_ETH(mdev, swp_lso))
1223 resp.sw_parsing_caps.sw_parsing_offloads |=
1224 MLX5_IB_SW_PARSING_LSO;
1225
1226 if (resp.sw_parsing_caps.sw_parsing_offloads)
1227 resp.sw_parsing_caps.supported_qpts =
1228 BIT(IB_QPT_RAW_PACKET);
1229 }
1230 }
1231
a762d460 1232 if (offsetofend(typeof(resp), striding_rq_caps) <= uhw_outlen &&
85c7c014 1233 raw_support) {
b4f34597
NO		 1234 resp.response_length += sizeof(resp.striding_rq_caps);
1235 if (MLX5_CAP_GEN(mdev, striding_rq)) {
1236 resp.striding_rq_caps.min_single_stride_log_num_of_bytes =
1237 MLX5_MIN_SINGLE_STRIDE_LOG_NUM_BYTES;
1238 resp.striding_rq_caps.max_single_stride_log_num_of_bytes =
1239 MLX5_MAX_SINGLE_STRIDE_LOG_NUM_BYTES;
c16339b6
MZ		 1240 if (MLX5_CAP_GEN(dev->mdev, ext_stride_num_range))
1241 resp.striding_rq_caps
1242 .min_single_wqe_log_num_of_strides =
1243 MLX5_EXT_MIN_SINGLE_WQE_LOG_NUM_STRIDES;
1244 else
1245 resp.striding_rq_caps
1246 .min_single_wqe_log_num_of_strides =
1247 MLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES;
b4f34597
NO		 1248 resp.striding_rq_caps.max_single_wqe_log_num_of_strides =
1249 MLX5_MAX_SINGLE_WQE_LOG_NUM_STRIDES;
1250 resp.striding_rq_caps.supported_qpts =
1251 BIT(IB_QPT_RAW_PACKET);
1252 }
1253 }
1254
a762d460 1255 if (offsetofend(typeof(resp), tunnel_offloads_caps) <= uhw_outlen) {
f95ef6cb
MG		 1256 resp.response_length += sizeof(resp.tunnel_offloads_caps);
1257 if (MLX5_CAP_ETH(mdev, tunnel_stateless_vxlan))
1258 resp.tunnel_offloads_caps |=
1259 MLX5_IB_TUNNELED_OFFLOADS_VXLAN;
1260 if (MLX5_CAP_ETH(mdev, tunnel_stateless_geneve_rx))
1261 resp.tunnel_offloads_caps |=
1262 MLX5_IB_TUNNELED_OFFLOADS_GENEVE;
1263 if (MLX5_CAP_ETH(mdev, tunnel_stateless_gre))
1264 resp.tunnel_offloads_caps |=
1265 MLX5_IB_TUNNELED_OFFLOADS_GRE;
41e684ef 1266 if (MLX5_CAP_ETH(mdev, tunnel_stateless_mpls_over_gre))
e818e255
AL		 1267 resp.tunnel_offloads_caps |=
1268 MLX5_IB_TUNNELED_OFFLOADS_MPLS_GRE;
41e684ef 1269 if (MLX5_CAP_ETH(mdev, tunnel_stateless_mpls_over_udp))
e818e255
AL		 1270 resp.tunnel_offloads_caps |=
1271 MLX5_IB_TUNNELED_OFFLOADS_MPLS_UDP;
f95ef6cb
MG		 1272 }
1273
11656f59
LN		 1274 if (offsetofend(typeof(resp), dci_streams_caps) <= uhw_outlen) {
1275 resp.response_length += sizeof(resp.dci_streams_caps);
1276
1277 resp.dci_streams_caps.max_log_num_concurent =
1278 MLX5_CAP_GEN(mdev, log_max_dci_stream_channels);
1279
1280 resp.dci_streams_caps.max_log_num_errored =
1281 MLX5_CAP_GEN(mdev, log_max_dci_errored_streams);
1282 }
1283
d727d27d
MB		 1284 if (offsetofend(typeof(resp), reserved) <= uhw_outlen)
1285 resp.response_length += sizeof(resp.reserved);
1286
1287 if (offsetofend(typeof(resp), reg_c0) <= uhw_outlen) {
1288 struct mlx5_eswitch *esw = mdev->priv.eswitch;
1289
1290 resp.response_length += sizeof(resp.reg_c0);
1291
1292 if (mlx5_eswitch_mode(mdev) == MLX5_ESWITCH_OFFLOADS &&
1293 mlx5_eswitch_vport_match_metadata_enabled(esw))
1294 fill_esw_mgr_reg_c0(mdev, &resp);
1295 }
1296
48357091 1297 if (uhw_outlen) {
402ca536
BW		 1298 err = ib_copy_to_udata(uhw, &resp, resp.response_length);
1299
1300 if (err)
1301 return err;
1302 }
1303
1b5daf11 1304 return 0;
e126ba97
EC		 1305}
1306
639bf441
AL		 1307static void translate_active_width(struct ib_device *ibdev, u16 active_width,
1308 u8 *ib_width)
e126ba97
EC		 1309{
1310 struct mlx5_ib_dev *dev = to_mdev(ibdev);
1b5daf11 1311
e27014bd 1312 if (active_width & MLX5_PTYS_WIDTH_1X)
1b5daf11 1313 *ib_width = IB_WIDTH_1X;
e27014bd 1314 else if (active_width & MLX5_PTYS_WIDTH_2X)
d764970b 1315 *ib_width = IB_WIDTH_2X;
e27014bd 1316 else if (active_width & MLX5_PTYS_WIDTH_4X)
1b5daf11 1317 *ib_width = IB_WIDTH_4X;
e27014bd 1318 else if (active_width & MLX5_PTYS_WIDTH_8X)
1b5daf11 1319 *ib_width = IB_WIDTH_8X;
e27014bd 1320 else if (active_width & MLX5_PTYS_WIDTH_12X)
1b5daf11 1321 *ib_width = IB_WIDTH_12X;
db7a691a
MG		 1322 else {
1323 mlx5_ib_dbg(dev, "Invalid active_width %d, setting width to default value: 4x\n",
e27014bd 1324 active_width);
db7a691a 1325 *ib_width = IB_WIDTH_4X;
e126ba97
EC		 1326 }
1327
db7a691a 1328 return;
1b5daf11 1329}
e126ba97 1330
1b5daf11
MD		 1331static int mlx5_mtu_to_ib_mtu(int mtu)
1332{
1333 switch (mtu) {
1334 case 256: return 1;
1335 case 512: return 2;
1336 case 1024: return 3;
1337 case 2048: return 4;
1338 case 4096: return 5;
1339 default:
1340 pr_warn("invalid mtu\n");
1341 return -1;
e126ba97 1342 }
1b5daf11 1343}
e126ba97 1344
1b5daf11
MD		 1345enum ib_max_vl_num {
1346 __IB_MAX_VL_0 = 1,
1347 __IB_MAX_VL_0_1 = 2,
1348 __IB_MAX_VL_0_3 = 3,
1349 __IB_MAX_VL_0_7 = 4,
1350 __IB_MAX_VL_0_14 = 5,
1351};
e126ba97 1352
1b5daf11
MD		 1353enum mlx5_vl_hw_cap {
1354 MLX5_VL_HW_0 = 1,
1355 MLX5_VL_HW_0_1 = 2,
1356 MLX5_VL_HW_0_2 = 3,
1357 MLX5_VL_HW_0_3 = 4,
1358 MLX5_VL_HW_0_4 = 5,
1359 MLX5_VL_HW_0_5 = 6,
1360 MLX5_VL_HW_0_6 = 7,
1361 MLX5_VL_HW_0_7 = 8,
1362 MLX5_VL_HW_0_14 = 15
1363};
e126ba97 1364
1b5daf11
MD		 1365static int translate_max_vl_num(struct ib_device *ibdev, u8 vl_hw_cap,
1366 u8 *max_vl_num)
1367{
1368 switch (vl_hw_cap) {
1369 case MLX5_VL_HW_0:
1370 *max_vl_num = __IB_MAX_VL_0;
1371 break;
1372 case MLX5_VL_HW_0_1:
1373 *max_vl_num = __IB_MAX_VL_0_1;
1374 break;
1375 case MLX5_VL_HW_0_3:
1376 *max_vl_num = __IB_MAX_VL_0_3;
1377 break;
1378 case MLX5_VL_HW_0_7:
1379 *max_vl_num = __IB_MAX_VL_0_7;
1380 break;
1381 case MLX5_VL_HW_0_14:
1382 *max_vl_num = __IB_MAX_VL_0_14;
1383 break;
e126ba97 1384
1b5daf11
MD		 1385 default:
1386 return -EINVAL;
e126ba97 1387 }
e126ba97 1388
1b5daf11 1389 return 0;
e126ba97
EC		 1390}
1391
1fb7f897 1392static int mlx5_query_hca_port(struct ib_device *ibdev, u32 port,
1b5daf11 1393 struct ib_port_attr *props)
e126ba97 1394{
1b5daf11
MD		 1395 struct mlx5_ib_dev *dev = to_mdev(ibdev);
1396 struct mlx5_core_dev *mdev = dev->mdev;
1397 struct mlx5_hca_vport_context *rep;
3b43399b 1398 u8 vl_hw_cap, plane_index = 0;
046339ea
SM		 1399 u16 max_mtu;
1400 u16 oper_mtu;
1b5daf11 1401 int err;
639bf441 1402 u16 ib_link_width_oper;
e126ba97 1403
1b5daf11
MD		 1404 rep = kzalloc(sizeof(*rep), GFP_KERNEL);
1405 if (!rep) {
1406 err = -ENOMEM;
e126ba97 1407 goto out;
e126ba97 1408 }
e126ba97 1409
c4550c63 1410 /* props being zeroed by the caller, avoid zeroing it here */
e126ba97 1411
3b43399b
MZ		 1412 if (ibdev->type == RDMA_DEVICE_TYPE_SMI) {
1413 plane_index = port;
026a4259 1414 port = smi_to_native_portnum(dev, port);
3b43399b 1415 }
026a4259 1416
1b5daf11 1417 err = mlx5_query_hca_vport_context(mdev, 0, port, 0, rep);
e126ba97
EC		 1418 if (err)
1419 goto out;
1420
1b5daf11
MD		 1421 props->lid = rep->lid;
1422 props->lmc = rep->lmc;
1423 props->sm_lid = rep->sm_lid;
1424 props->sm_sl = rep->sm_sl;
1425 props->state = rep->vport_state;
1426 props->phys_state = rep->port_physical_state;
2a5db20f
MZ		 1427
1428 props->port_cap_flags = rep->cap_mask1;
1429 if (dev->num_plane) {
1430 props->port_cap_flags |= IB_PORT_SM_DISABLED;
1431 props->port_cap_flags &= ~IB_PORT_SM;
026a4259
MZ		 1432 } else if (ibdev->type == RDMA_DEVICE_TYPE_SMI)
1433 props->port_cap_flags &= ~IB_PORT_CM_SUP;
2a5db20f 1434
1b5daf11
MD		 1435 props->gid_tbl_len = mlx5_get_gid_table_len(MLX5_CAP_GEN(mdev, gid_table_size));
1436 props->max_msg_sz = 1 << MLX5_CAP_GEN(mdev, log_max_msg);
1437 props->pkey_tbl_len = mlx5_to_sw_pkey_sz(MLX5_CAP_GEN(mdev, pkey_table_size));
1438 props->bad_pkey_cntr = rep->pkey_violation_counter;
1439 props->qkey_viol_cntr = rep->qkey_violation_counter;
1440 props->subnet_timeout = rep->subnet_timeout;
1441 props->init_type_reply = rep->init_type_reply;
e126ba97 1442
4106a758
MG		 1443 if (props->port_cap_flags & IB_PORT_CAP_MASK2_SUP)
1444 props->port_cap_flags2 = rep->cap_mask2;
1445
639bf441 1446 err = mlx5_query_ib_port_oper(mdev, &ib_link_width_oper,
3b43399b 1447 &props->active_speed, port, plane_index);
1b5daf11 1448 if (err)
e126ba97 1449 goto out;
e126ba97 1450
db7a691a
MG		 1451 translate_active_width(ibdev, ib_link_width_oper, &props->active_width);
1452
facc9699 1453 mlx5_query_port_max_mtu(mdev, &max_mtu, port);
e126ba97 1454
1b5daf11 1455 props->max_mtu = mlx5_mtu_to_ib_mtu(max_mtu);
e126ba97 1456
facc9699 1457 mlx5_query_port_oper_mtu(mdev, &oper_mtu, port);
e126ba97 1458
1b5daf11 1459 props->active_mtu = mlx5_mtu_to_ib_mtu(oper_mtu);
e126ba97 1460
1b5daf11
MD		 1461 err = mlx5_query_port_vl_hw_cap(mdev, &vl_hw_cap, port);
1462 if (err)
1463 goto out;
e126ba97 1464
1b5daf11
MD		 1465 err = translate_max_vl_num(ibdev, vl_hw_cap,
1466 &props->max_vl_num);
e126ba97 1467out:
1b5daf11 1468 kfree(rep);
e126ba97
EC		 1469 return err;
1470}
1471
1fb7f897 1472int mlx5_ib_query_port(struct ib_device *ibdev, u32 port,
1b5daf11 1473 struct ib_port_attr *props)
e126ba97 1474{
095b0927
IT		 1475 unsigned int count;
1476 int ret;
1477
1b5daf11
MD		 1478 switch (mlx5_get_vport_access_method(ibdev)) {
1479 case MLX5_VPORT_ACCESS_METHOD_MAD:
095b0927
IT		 1480 ret = mlx5_query_mad_ifc_port(ibdev, port, props);
1481 break;
e126ba97 1482
1b5daf11 1483 case MLX5_VPORT_ACCESS_METHOD_HCA:
095b0927
IT		 1484 ret = mlx5_query_hca_port(ibdev, port, props);
1485 break;
e126ba97 1486
3f89a643 1487 case MLX5_VPORT_ACCESS_METHOD_NIC:
095b0927
IT		 1488 ret = mlx5_query_port_roce(ibdev, port, props);
1489 break;
3f89a643 1490
1b5daf11 1491 default:
095b0927
IT		 1492 ret = -EINVAL;
1493 }
1494
1495 if (!ret && props) {
b3cbd6f0
DJ		 1496 struct mlx5_ib_dev *dev = to_mdev(ibdev);
1497 struct mlx5_core_dev *mdev;
1498 bool put_mdev = true;
1499
1500 mdev = mlx5_ib_get_native_port_mdev(dev, port, NULL);
1501 if (!mdev) {
1502 /* If the port isn't affiliated yet query the master.
1503 * The master and slave will have the same values.
1504 */
1505 mdev = dev->mdev;
1506 port = 1;
1507 put_mdev = false;
1508 }
1509 count = mlx5_core_reserved_gids_count(mdev);
1510 if (put_mdev)
1511 mlx5_ib_put_native_port_mdev(dev, port);
095b0927 1512 props->gid_tbl_len -= count;
1b5daf11 1513 }
095b0927 1514 return ret;
1b5daf11 1515}
e126ba97 1516
1fb7f897 1517static int mlx5_ib_rep_query_port(struct ib_device *ibdev, u32 port,
8e6efa3a
MB		 1518 struct ib_port_attr *props)
1519{
7a58779e 1520 return mlx5_query_port_roce(ibdev, port, props);
8e6efa3a
MB		 1521}
1522
1fb7f897 1523static int mlx5_ib_rep_query_pkey(struct ib_device *ibdev, u32 port, u16 index,
d6fd59e1
PP		 1524 u16 *pkey)
1525{
1526 /* Default special Pkey for representor device port as per the
1527 * IB specification 1.3 section 10.9.1.2.
1528 */
1529 *pkey = 0xffff;
1530 return 0;
1531}
1532
1fb7f897 1533static int mlx5_ib_query_gid(struct ib_device *ibdev, u32 port, int index,
1b5daf11
MD		 1534 union ib_gid *gid)
1535{
1536 struct mlx5_ib_dev *dev = to_mdev(ibdev);
1537 struct mlx5_core_dev *mdev = dev->mdev;
e126ba97 1538
1b5daf11
MD		 1539 switch (mlx5_get_vport_access_method(ibdev)) {
1540 case MLX5_VPORT_ACCESS_METHOD_MAD:
1541 return mlx5_query_mad_ifc_gids(ibdev, port, index, gid);
e126ba97 1542
1b5daf11
MD		 1543 case MLX5_VPORT_ACCESS_METHOD_HCA:
1544 return mlx5_query_hca_vport_gid(mdev, 0, port, 0, index, gid);
1545
1546 default:
1547 return -EINVAL;
1548 }
e126ba97 1549
e126ba97
EC		 1550}
1551
1fb7f897 1552static int mlx5_query_hca_nic_pkey(struct ib_device *ibdev, u32 port,
b3cbd6f0 1553 u16 index, u16 *pkey)
1b5daf11
MD		 1554{
1555 struct mlx5_ib_dev *dev = to_mdev(ibdev);
b3cbd6f0
DJ		 1556 struct mlx5_core_dev *mdev;
1557 bool put_mdev = true;
1fb7f897 1558 u32 mdev_port_num;
b3cbd6f0 1559 int err;
1b5daf11 1560
b3cbd6f0
DJ		 1561 mdev = mlx5_ib_get_native_port_mdev(dev, port, &mdev_port_num);
1562 if (!mdev) {
1563 /* The port isn't affiliated yet, get the PKey from the master
1564 * port. For RoCE the PKey tables will be the same.
1565 */
1566 put_mdev = false;
1567 mdev = dev->mdev;
1568 mdev_port_num = 1;
1569 }
1570
1571 err = mlx5_query_hca_vport_pkey(mdev, 0, mdev_port_num, 0,
1572 index, pkey);
1573 if (put_mdev)
1574 mlx5_ib_put_native_port_mdev(dev, port);
1575
1576 return err;
1577}
1578
1fb7f897 1579static int mlx5_ib_query_pkey(struct ib_device *ibdev, u32 port, u16 index,
b3cbd6f0
DJ		 1580 u16 *pkey)
1581{
1b5daf11
MD		 1582 switch (mlx5_get_vport_access_method(ibdev)) {
1583 case MLX5_VPORT_ACCESS_METHOD_MAD:
1584 return mlx5_query_mad_ifc_pkey(ibdev, port, index, pkey);
1585
1586 case MLX5_VPORT_ACCESS_METHOD_HCA:
1587 case MLX5_VPORT_ACCESS_METHOD_NIC:
b3cbd6f0 1588 return mlx5_query_hca_nic_pkey(ibdev, port, index, pkey);
1b5daf11
MD		 1589 default:
1590 return -EINVAL;
1591 }
1592}
e126ba97
EC		 1593
1594static int mlx5_ib_modify_device(struct ib_device *ibdev, int mask,
1595 struct ib_device_modify *props)
1596{
1597 struct mlx5_ib_dev *dev = to_mdev(ibdev);
1598 struct mlx5_reg_node_desc in;
1599 struct mlx5_reg_node_desc out;
1600 int err;
1601
1602 if (mask & ~IB_DEVICE_MODIFY_NODE_DESC)
1603 return -EOPNOTSUPP;
1604
1605 if (!(mask & IB_DEVICE_MODIFY_NODE_DESC))
1606 return 0;
1607
1608 /*
1609 * If possible, pass node desc to FW, so it can generate
1610 * a 144 trap. If cmd fails, just ignore.
1611 */
bd99fdea 1612 memcpy(&in, props->node_desc, IB_DEVICE_NODE_DESC_MAX);
9603b61d 1613 err = mlx5_core_access_reg(dev->mdev, &in, sizeof(in), &out,
e126ba97
EC		 1614 sizeof(out), MLX5_REG_NODE_DESC, 0, 1);
1615 if (err)
1616 return err;
1617
bd99fdea 1618 memcpy(ibdev->node_desc, props->node_desc, IB_DEVICE_NODE_DESC_MAX);
e126ba97
EC		 1619
1620 return err;
1621}
1622
1fb7f897 1623static int set_port_caps_atomic(struct mlx5_ib_dev *dev, u32 port_num, u32 mask,
cdbe33d0
EC		 1624 u32 value)
1625{
1626 struct mlx5_hca_vport_context ctx = {};
b3cbd6f0 1627 struct mlx5_core_dev *mdev;
1fb7f897 1628 u32 mdev_port_num;
cdbe33d0
EC		 1629 int err;
1630
b3cbd6f0
DJ		 1631 mdev = mlx5_ib_get_native_port_mdev(dev, port_num, &mdev_port_num);
1632 if (!mdev)
1633 return -ENODEV;
1634
1635 err = mlx5_query_hca_vport_context(mdev, 0, mdev_port_num, 0, &ctx);
cdbe33d0 1636 if (err)
b3cbd6f0 1637 goto out;
cdbe33d0
EC		 1638
1639 if (~ctx.cap_mask1_perm & mask) {
1640 mlx5_ib_warn(dev, "trying to change bitmask 0x%X but change supported 0x%X\n",
1641 mask, ctx.cap_mask1_perm);
b3cbd6f0
DJ		 1642 err = -EINVAL;
1643 goto out;
cdbe33d0
EC		 1644 }
1645
1646 ctx.cap_mask1 = value;
1647 ctx.cap_mask1_perm = mask;
b3cbd6f0
DJ		 1648 err = mlx5_core_modify_hca_vport_context(mdev, 0, mdev_port_num,
1649 0, &ctx);
1650
1651out:
1652 mlx5_ib_put_native_port_mdev(dev, port_num);
cdbe33d0
EC		 1653
1654 return err;
1655}
1656
1fb7f897 1657static int mlx5_ib_modify_port(struct ib_device *ibdev, u32 port, int mask,
e126ba97
EC		 1658 struct ib_port_modify *props)
1659{
1660 struct mlx5_ib_dev *dev = to_mdev(ibdev);
1661 struct ib_port_attr attr;
1662 u32 tmp;
1663 int err;
cdbe33d0
EC		 1664 u32 change_mask;
1665 u32 value;
1666 bool is_ib = (mlx5_ib_port_link_layer(ibdev, port) ==
1667 IB_LINK_LAYER_INFINIBAND);
1668
ec255879
MD		 1669 /* CM layer calls ib_modify_port() regardless of the link layer. For
1670 * Ethernet ports, qkey violation and Port capabilities are meaningless.
1671 */
1672 if (!is_ib)
1673 return 0;
1674
cdbe33d0
EC		 1675 if (MLX5_CAP_GEN(dev->mdev, ib_virt) && is_ib) {
1676 change_mask = props->clr_port_cap_mask | props->set_port_cap_mask;
1677 value = ~props->clr_port_cap_mask | props->set_port_cap_mask;
1678 return set_port_caps_atomic(dev, port, change_mask, value);
1679 }
e126ba97
EC		 1680
1681 mutex_lock(&dev->cap_mask_mutex);
1682
c4550c63 1683 err = ib_query_port(ibdev, port, &attr);
e126ba97
EC		 1684 if (err)
1685 goto out;
1686
1687 tmp = (attr.port_cap_flags | props->set_port_cap_mask) &
1688 ~props->clr_port_cap_mask;
1689
9603b61d 1690 err = mlx5_set_port_caps(dev->mdev, port, tmp);
e126ba97
EC		 1691
1692out:
1693 mutex_unlock(&dev->cap_mask_mutex);
1694 return err;
1695}
1696
30aa60b3
EC		 1697static void print_lib_caps(struct mlx5_ib_dev *dev, u64 caps)
1698{
1699 mlx5_ib_dbg(dev, "MLX5_LIB_CAP_4K_UAR = %s\n",
1700 caps & MLX5_LIB_CAP_4K_UAR ? "y" : "n");
1701}
1702
31a78a5a
YH		 1703static u16 calc_dynamic_bfregs(int uars_per_sys_page)
1704{
1705 /* Large page with non 4k uar support might limit the dynamic size */
1706 if (uars_per_sys_page == 1 && PAGE_SIZE > 4096)
1707 return MLX5_MIN_DYN_BFREGS;
1708
1709 return MLX5_MAX_DYN_BFREGS;
1710}
1711
b037c29a
EC		 1712static int calc_total_bfregs(struct mlx5_ib_dev *dev, bool lib_uar_4k,
1713 struct mlx5_ib_alloc_ucontext_req_v2 *req,
31a78a5a 1714 struct mlx5_bfreg_info *bfregi)
b037c29a
EC		 1715{
1716 int uars_per_sys_page;
1717 int bfregs_per_sys_page;
1718 int ref_bfregs = req->total_num_bfregs;
1719
1720 if (req->total_num_bfregs == 0)
1721 return -EINVAL;
1722
1723 BUILD_BUG_ON(MLX5_MAX_BFREGS % MLX5_NON_FP_BFREGS_IN_PAGE);
1724 BUILD_BUG_ON(MLX5_MAX_BFREGS < MLX5_NON_FP_BFREGS_IN_PAGE);
1725
1726 if (req->total_num_bfregs > MLX5_MAX_BFREGS)
1727 return -ENOMEM;
1728
1729 uars_per_sys_page = get_uars_per_sys_page(dev, lib_uar_4k);
1730 bfregs_per_sys_page = uars_per_sys_page * MLX5_NON_FP_BFREGS_PER_UAR;
31a78a5a 1731 /* This holds the required static allocation asked by the user */
b037c29a 1732 req->total_num_bfregs = ALIGN(req->total_num_bfregs, bfregs_per_sys_page);
b037c29a
EC		 1733 if (req->num_low_latency_bfregs > req->total_num_bfregs - 1)
1734 return -EINVAL;
1735
31a78a5a
YH		 1736 bfregi->num_static_sys_pages = req->total_num_bfregs / bfregs_per_sys_page;
1737 bfregi->num_dyn_bfregs = ALIGN(calc_dynamic_bfregs(uars_per_sys_page), bfregs_per_sys_page);
1738 bfregi->total_num_bfregs = req->total_num_bfregs + bfregi->num_dyn_bfregs;
1739 bfregi->num_sys_pages = bfregi->total_num_bfregs / bfregs_per_sys_page;
1740
1741 mlx5_ib_dbg(dev, "uar_4k: fw support %s, lib support %s, user requested %d bfregs, allocated %d, total bfregs %d, using %d sys pages\n",
b037c29a
EC		 1742 MLX5_CAP_GEN(dev->mdev, uar_4k) ? "yes" : "no",
1743 lib_uar_4k ? "yes" : "no", ref_bfregs,
31a78a5a
YH		 1744 req->total_num_bfregs, bfregi->total_num_bfregs,
1745 bfregi->num_sys_pages);
b037c29a
EC		 1746
1747 return 0;
1748}
1749
1750static int allocate_uars(struct mlx5_ib_dev *dev, struct mlx5_ib_ucontext *context)
1751{
1752 struct mlx5_bfreg_info *bfregi;
1753 int err;
1754 int i;
1755
1756 bfregi = &context->bfregi;
31a78a5a 1757 for (i = 0; i < bfregi->num_static_sys_pages; i++) {
d2c8a155
ML		 1758 err = mlx5_cmd_uar_alloc(dev->mdev, &bfregi->sys_pages[i],
1759 context->devx_uid);
b037c29a
EC		 1760 if (err)
1761 goto error;
1762
1763 mlx5_ib_dbg(dev, "allocated uar %d\n", bfregi->sys_pages[i]);
1764 }
4ed131d0
YH		 1765
1766 for (i = bfregi->num_static_sys_pages; i < bfregi->num_sys_pages; i++)
1767 bfregi->sys_pages[i] = MLX5_IB_INVALID_UAR_INDEX;
1768
b037c29a
EC		 1769 return 0;
1770
1771error:
1772 for (--i; i >= 0; i--)
d2c8a155
ML		 1773 if (mlx5_cmd_uar_dealloc(dev->mdev, bfregi->sys_pages[i],
1774 context->devx_uid))
b037c29a
EC		 1775 mlx5_ib_warn(dev, "failed to free uar %d\n", i);
1776
1777 return err;
1778}
1779
15177999
LR		 1780static void deallocate_uars(struct mlx5_ib_dev *dev,
1781 struct mlx5_ib_ucontext *context)
b037c29a
EC		 1782{
1783 struct mlx5_bfreg_info *bfregi;
b037c29a
EC		 1784 int i;
1785
1786 bfregi = &context->bfregi;
15177999 1787 for (i = 0; i < bfregi->num_sys_pages; i++)
4ed131d0 1788 if (i < bfregi->num_static_sys_pages ||
15177999 1789 bfregi->sys_pages[i] != MLX5_IB_INVALID_UAR_INDEX)
d2c8a155
ML		 1790 mlx5_cmd_uar_dealloc(dev->mdev, bfregi->sys_pages[i],
1791 context->devx_uid);
b037c29a
EC		 1792}
1793
a9a9e689
PH		 1794static int mlx5_ib_enable_lb_mp(struct mlx5_core_dev *master,
1795 struct mlx5_core_dev *slave)
1796{
1797 int err;
1798
1799 err = mlx5_nic_vport_update_local_lb(master, true);
1800 if (err)
1801 return err;
1802
1803 err = mlx5_nic_vport_update_local_lb(slave, true);
1804 if (err)
1805 goto out;
1806
1807 return 0;
1808
1809out:
1810 mlx5_nic_vport_update_local_lb(master, false);
1811 return err;
1812}
1813
1814static void mlx5_ib_disable_lb_mp(struct mlx5_core_dev *master,
1815 struct mlx5_core_dev *slave)
1816{
1817 mlx5_nic_vport_update_local_lb(slave, false);
1818 mlx5_nic_vport_update_local_lb(master, false);
1819}
1820
0042f9e4 1821int mlx5_ib_enable_lb(struct mlx5_ib_dev *dev, bool td, bool qp)
a560f1d9
MB		 1822{
1823 int err = 0;
1824
1825 mutex_lock(&dev->lb.mutex);
0042f9e4
MB		 1826 if (td)
1827 dev->lb.user_td++;
1828 if (qp)
1829 dev->lb.qps++;
1830
1831 if (dev->lb.user_td == 2 ||
1832 dev->lb.qps == 1) {
1833 if (!dev->lb.enabled) {
1834 err = mlx5_nic_vport_update_local_lb(dev->mdev, true);
1835 dev->lb.enabled = true;
1836 }
1837 }
a560f1d9
MB		 1838
1839 mutex_unlock(&dev->lb.mutex);
1840
1841 return err;
1842}
1843
0042f9e4 1844void mlx5_ib_disable_lb(struct mlx5_ib_dev *dev, bool td, bool qp)
a560f1d9
MB		 1845{
1846 mutex_lock(&dev->lb.mutex);
0042f9e4
MB		 1847 if (td)
1848 dev->lb.user_td--;
1849 if (qp)
1850 dev->lb.qps--;
1851
1852 if (dev->lb.user_td == 1 &&
1853 dev->lb.qps == 0) {
1854 if (dev->lb.enabled) {
1855 mlx5_nic_vport_update_local_lb(dev->mdev, false);
1856 dev->lb.enabled = false;
1857 }
1858 }
a560f1d9
MB		 1859
1860 mutex_unlock(&dev->lb.mutex);
1861}
1862
d2d19121
YH		 1863static int mlx5_ib_alloc_transport_domain(struct mlx5_ib_dev *dev, u32 *tdn,
1864 u16 uid)
c85023e1
HN		 1865{
1866 int err;
1867
cfdeb893
LR		 1868 if (!MLX5_CAP_GEN(dev->mdev, log_max_transport_domain))
1869 return 0;
1870
d2d19121 1871 err = mlx5_cmd_alloc_transport_domain(dev->mdev, tdn, uid);
c85023e1
HN		 1872 if (err)
1873 return err;
1874
1875 if ((MLX5_CAP_GEN(dev->mdev, port_type) != MLX5_CAP_PORT_TYPE_ETH) ||
8978cc92
EBE		 1876 (!MLX5_CAP_GEN(dev->mdev, disable_local_lb_uc) &&
1877 !MLX5_CAP_GEN(dev->mdev, disable_local_lb_mc)))
c85023e1
HN		 1878 return err;
1879
0042f9e4 1880 return mlx5_ib_enable_lb(dev, true, false);
c85023e1
HN		 1881}
1882
d2d19121
YH		 1883static void mlx5_ib_dealloc_transport_domain(struct mlx5_ib_dev *dev, u32 tdn,
1884 u16 uid)
c85023e1 1885{
cfdeb893
LR		 1886 if (!MLX5_CAP_GEN(dev->mdev, log_max_transport_domain))
1887 return;
1888
d2d19121 1889 mlx5_cmd_dealloc_transport_domain(dev->mdev, tdn, uid);
c85023e1
HN		 1890
1891 if ((MLX5_CAP_GEN(dev->mdev, port_type) != MLX5_CAP_PORT_TYPE_ETH) ||
8978cc92
EBE		 1892 (!MLX5_CAP_GEN(dev->mdev, disable_local_lb_uc) &&
1893 !MLX5_CAP_GEN(dev->mdev, disable_local_lb_mc)))
c85023e1
HN		 1894 return;
1895
0042f9e4 1896 mlx5_ib_disable_lb(dev, true, false);
c85023e1
HN		 1897}
1898
45ec21c9
YH		 1899static int set_ucontext_resp(struct ib_ucontext *uctx,
1900 struct mlx5_ib_alloc_ucontext_resp *resp)
1901{
1902 struct ib_device *ibdev = uctx->device;
1903 struct mlx5_ib_dev *dev = to_mdev(ibdev);
1904 struct mlx5_ib_ucontext *context = to_mucontext(uctx);
1905 struct mlx5_bfreg_info *bfregi = &context->bfregi;
45ec21c9
YH		 1906
1907 if (MLX5_CAP_GEN(dev->mdev, dump_fill_mkey)) {
594cac11 1908 resp->dump_fill_mkey = dev->mkeys.dump_fill_mkey;
45ec21c9
YH		 1909 resp->comp_mask |=
1910 MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_DUMP_FILL_MKEY;
1911 }
1912
1913 resp->qp_tab_size = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp);
d98995b4 1914 if (mlx5_wc_support_get(dev->mdev))
45ec21c9
YH		 1915 resp->bf_reg_size = 1 << MLX5_CAP_GEN(dev->mdev,
1916 log_bf_reg_size);
1917 resp->cache_line_size = cache_line_size();
1918 resp->max_sq_desc_sz = MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq);
1919 resp->max_rq_desc_sz = MLX5_CAP_GEN(dev->mdev, max_wqe_sz_rq);
1920 resp->max_send_wqebb = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz);
1921 resp->max_recv_wr = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz);
1922 resp->max_srq_recv_wr = 1 << MLX5_CAP_GEN(dev->mdev, log_max_srq_sz);
1923 resp->cqe_version = context->cqe_version;
1924 resp->log_uar_size = MLX5_CAP_GEN(dev->mdev, uar_4k) ?
1925 MLX5_ADAPTER_PAGE_SHIFT : PAGE_SHIFT;
1926 resp->num_uars_per_page = MLX5_CAP_GEN(dev->mdev, uar_4k) ?
1927 MLX5_CAP_GEN(dev->mdev,
1928 num_of_uars_per_page) : 1;
45ec21c9
YH		 1929 resp->tot_bfregs = bfregi->lib_uar_dyn ? 0 :
1930 bfregi->total_num_bfregs - bfregi->num_dyn_bfregs;
1931 resp->num_ports = dev->num_ports;
1932 resp->cmds_supp_uhw |= MLX5_USER_CMDS_SUPP_UHW_QUERY_DEVICE |
1933 MLX5_USER_CMDS_SUPP_UHW_CREATE_AH;
1934
1935 if (mlx5_ib_port_link_layer(ibdev, 1) == IB_LINK_LAYER_ETHERNET) {
1936 mlx5_query_min_inline(dev->mdev, &resp->eth_min_inline);
1937 resp->eth_min_inline++;
1938 }
1939
1940 if (dev->mdev->clock_info)
1941 resp->clock_info_versions = BIT(MLX5_IB_CLOCK_INFO_V1);
1942
1943 /*
1944 * We don't want to expose information from the PCI bar that is located
1945 * after 4096 bytes, so if the arch only supports larger pages, let's
1946 * pretend we don't support reading the HCA's core clock. This is also
1947 * forced by mmap function.
1948 */
1949 if (PAGE_SIZE <= 4096) {
1950 resp->comp_mask |=
1951 MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_CORE_CLOCK_OFFSET;
1952 resp->hca_core_clock_offset =
1953 offsetof(struct mlx5_init_seg,
1954 internal_timer_h) % PAGE_SIZE;
1955 }
1956
1957 if (MLX5_CAP_GEN(dev->mdev, ece_support))
1958 resp->comp_mask |= MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_ECE;
1959
33652951
AL		 1960 if (rt_supported(MLX5_CAP_GEN(dev->mdev, sq_ts_format)) &&
1961 rt_supported(MLX5_CAP_GEN(dev->mdev, rq_ts_format)) &&
1962 rt_supported(MLX5_CAP_ROCE(dev->mdev, qp_ts_format)))
1963 resp->comp_mask |=
1964 MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_REAL_TIME_TS;
1965
45ec21c9 1966 resp->num_dyn_bfregs = bfregi->num_dyn_bfregs;
c906b86e
SG		 1967
1968 if (MLX5_CAP_GEN(dev->mdev, drain_sigerr))
1969 resp->comp_mask |= MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_SQD2RTS;
1970
13ad1125
AL		 1971 resp->comp_mask |=
1972 MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_MKEY_UPDATE_TAG;
1973
45ec21c9
YH		 1974 return 0;
1975}
1976
17ade536
CM		 1977static bool uctx_rdma_ctrl_is_enabled(u64 enabled_caps)
1978{
1979 return UCAP_ENABLED(enabled_caps, RDMA_UCAP_MLX5_CTRL_LOCAL) ||
1980 UCAP_ENABLED(enabled_caps, RDMA_UCAP_MLX5_CTRL_OTHER_VHCA);
1981}
1982
a2a074ef
LR		 1983static int mlx5_ib_alloc_ucontext(struct ib_ucontext *uctx,
1984 struct ib_udata *udata)
e126ba97 1985{
a2a074ef 1986 struct ib_device *ibdev = uctx->device;
e126ba97 1987 struct mlx5_ib_dev *dev = to_mdev(ibdev);
b368d7cb
MB		 1988 struct mlx5_ib_alloc_ucontext_req_v2 req = {};
1989 struct mlx5_ib_alloc_ucontext_resp resp = {};
a2a074ef 1990 struct mlx5_ib_ucontext *context = to_mucontext(uctx);
2f5ff264 1991 struct mlx5_bfreg_info *bfregi;
78c0f98c 1992 int ver;
e126ba97 1993 int err;
a168a41c
MD		 1994 size_t min_req_v2 = offsetof(struct mlx5_ib_alloc_ucontext_req_v2,
1995 max_cqe_version);
b037c29a 1996 bool lib_uar_4k;
0a2fd01c 1997 bool lib_uar_dyn;
e126ba97
EC		 1998
1999 if (!dev->ib_active)
a2a074ef 2000 return -EAGAIN;
e126ba97 2001
e093111d 2002 if (udata->inlen == sizeof(struct mlx5_ib_alloc_ucontext_req))
78c0f98c 2003 ver = 0;
e093111d 2004 else if (udata->inlen >= min_req_v2)
78c0f98c
EC		 2005 ver = 2;
2006 else
a2a074ef 2007 return -EINVAL;
78c0f98c 2008
e093111d 2009 err = ib_copy_from_udata(&req, udata, min(udata->inlen, sizeof(req)));
e126ba97 2010 if (err)
a2a074ef 2011 return err;
e126ba97 2012
a8b92ca1 2013 if (req.flags & ~MLX5_IB_ALLOC_UCTX_DEVX)
a2a074ef 2014 return -EOPNOTSUPP;
78c0f98c 2015
f72300c5 2016 if (req.comp_mask || req.reserved0 || req.reserved1 || req.reserved2)
a2a074ef 2017 return -EOPNOTSUPP;
b368d7cb 2018
2f5ff264
EC		 2019 req.total_num_bfregs = ALIGN(req.total_num_bfregs,
2020 MLX5_NON_FP_BFREGS_PER_UAR);
2021 if (req.num_low_latency_bfregs > req.total_num_bfregs - 1)
a2a074ef 2022 return -EINVAL;
e126ba97 2023
d2c8a155 2024 if (req.flags & MLX5_IB_ALLOC_UCTX_DEVX) {
17ade536 2025 err = mlx5_ib_devx_create(dev, true, uctx->enabled_caps);
d2c8a155
ML		 2026 if (err < 0)
2027 goto out_ctx;
2028 context->devx_uid = err;
17ade536
CM		 2029
2030 if (uctx_rdma_ctrl_is_enabled(uctx->enabled_caps)) {
2031 err = mlx5_cmd_add_privileged_uid(dev->mdev,
2032 context->devx_uid);
2033 if (err)
2034 goto out_devx;
2035 }
d2c8a155
ML		 2036 }
2037
30aa60b3 2038 lib_uar_4k = req.lib_caps & MLX5_LIB_CAP_4K_UAR;
0a2fd01c 2039 lib_uar_dyn = req.lib_caps & MLX5_LIB_CAP_DYN_UAR;
2f5ff264 2040 bfregi = &context->bfregi;
b037c29a 2041
0a2fd01c
YH		 2042 if (lib_uar_dyn) {
2043 bfregi->lib_uar_dyn = lib_uar_dyn;
2044 goto uar_done;
2045 }
2046
b037c29a 2047 /* updates req->total_num_bfregs */
31a78a5a 2048 err = calc_total_bfregs(dev, lib_uar_4k, &req, bfregi);
b037c29a 2049 if (err)
17ade536 2050 goto out_ucap;
e126ba97 2051
b037c29a
EC		 2052 mutex_init(&bfregi->lock);
2053 bfregi->lib_uar_4k = lib_uar_4k;
31a78a5a 2054 bfregi->count = kcalloc(bfregi->total_num_bfregs, sizeof(*bfregi->count),
e126ba97 2055 GFP_KERNEL);
b037c29a 2056 if (!bfregi->count) {
e126ba97 2057 err = -ENOMEM;
17ade536 2058 goto out_ucap;
e126ba97
EC		 2059 }
2060
b037c29a
EC		 2061 bfregi->sys_pages = kcalloc(bfregi->num_sys_pages,
2062 sizeof(*bfregi->sys_pages),
2063 GFP_KERNEL);
2064 if (!bfregi->sys_pages) {
e126ba97 2065 err = -ENOMEM;
b037c29a 2066 goto out_count;
e126ba97
EC		 2067 }
2068
b037c29a
EC		 2069 err = allocate_uars(dev, context);
2070 if (err)
2071 goto out_sys_pages;
e126ba97 2072
0a2fd01c 2073uar_done:
d2d19121
YH		 2074 err = mlx5_ib_alloc_transport_domain(dev, &context->tdn,
2075 context->devx_uid);
2076 if (err)
d2c8a155 2077 goto out_uars;
d2d19121 2078
e126ba97
EC		 2079 INIT_LIST_HEAD(&context->db_page_list);
2080 mutex_init(&context->db_page_mutex);
2081
45ec21c9
YH		 2082 context->cqe_version = min_t(__u8,
2083 (__u8)MLX5_CAP_GEN(dev->mdev, cqe_version),
2084 req.max_cqe_version);
25bb36e7 2085
45ec21c9
YH		 2086 err = set_ucontext_resp(uctx, &resp);
2087 if (err)
2088 goto out_mdev;
5f62a521 2089
45ec21c9 2090 resp.response_length = min(udata->outlen, sizeof(resp));
b368d7cb 2091 err = ib_copy_to_udata(udata, &resp, resp.response_length);
e126ba97 2092 if (err)
a8b92ca1 2093 goto out_mdev;
e126ba97 2094
2f5ff264
EC		 2095 bfregi->ver = ver;
2096 bfregi->num_low_latency_bfregs = req.num_low_latency_bfregs;
30aa60b3
EC		 2097 context->lib_caps = req.lib_caps;
2098 print_lib_caps(dev, context->lib_caps);
f72300c5 2099
802dcc7f 2100 if (mlx5_ib_lag_should_assign_affinity(dev)) {
1fb7f897 2101 u32 port = mlx5_core_native_port_num(dev->mdev) - 1;
c6a21c38
MD		 2102
2103 atomic_set(&context->tx_port_affinity,
2104 atomic_add_return(
95579e78 2105 1, &dev->port[port].roce.tx_port_affinity));
c6a21c38
MD		 2106 }
2107
a2a074ef 2108 return 0;
e126ba97 2109
a8b92ca1 2110out_mdev:
d2d19121 2111 mlx5_ib_dealloc_transport_domain(dev, context->tdn, context->devx_uid);
146d2f1a 2112
e126ba97 2113out_uars:
b037c29a 2114 deallocate_uars(dev, context);
e126ba97 2115
b037c29a
EC		 2116out_sys_pages:
2117 kfree(bfregi->sys_pages);
e126ba97 2118
b037c29a
EC		 2119out_count:
2120 kfree(bfregi->count);
e126ba97 2121
17ade536
CM		 2122out_ucap:
2123 if (req.flags & MLX5_IB_ALLOC_UCTX_DEVX &&
2124 uctx_rdma_ctrl_is_enabled(uctx->enabled_caps))
2125 mlx5_cmd_remove_privileged_uid(dev->mdev, context->devx_uid);
2126
d2c8a155
ML		 2127out_devx:
2128 if (req.flags & MLX5_IB_ALLOC_UCTX_DEVX)
2129 mlx5_ib_devx_destroy(dev, context->devx_uid);
2130
e126ba97 2131out_ctx:
a2a074ef 2132 return err;
e126ba97
EC		 2133}
2134
0fb556b2
YH		 2135static int mlx5_ib_query_ucontext(struct ib_ucontext *ibcontext,
2136 struct uverbs_attr_bundle *attrs)
2137{
2138 struct mlx5_ib_alloc_ucontext_resp uctx_resp = {};
2139 int ret;
2140
2141 ret = set_ucontext_resp(ibcontext, &uctx_resp);
2142 if (ret)
2143 return ret;
2144
2145 uctx_resp.response_length =
2146 min_t(size_t,
2147 uverbs_attr_get_len(attrs,
2148 MLX5_IB_ATTR_QUERY_CONTEXT_RESP_UCTX),
2149 sizeof(uctx_resp));
2150
2151 ret = uverbs_copy_to_struct_or_zero(attrs,
2152 MLX5_IB_ATTR_QUERY_CONTEXT_RESP_UCTX,
2153 &uctx_resp,
2154 sizeof(uctx_resp));
2155 return ret;
2156}
2157
a2a074ef 2158static void mlx5_ib_dealloc_ucontext(struct ib_ucontext *ibcontext)
e126ba97
EC		 2159{
2160 struct mlx5_ib_ucontext *context = to_mucontext(ibcontext);
2161 struct mlx5_ib_dev *dev = to_mdev(ibcontext->device);
b037c29a 2162 struct mlx5_bfreg_info *bfregi;
e126ba97 2163
b037c29a 2164 bfregi = &context->bfregi;
d2d19121
YH		 2165 mlx5_ib_dealloc_transport_domain(dev, context->tdn, context->devx_uid);
2166
b037c29a
EC		 2167 deallocate_uars(dev, context);
2168 kfree(bfregi->sys_pages);
2f5ff264 2169 kfree(bfregi->count);
d2c8a155 2170
17ade536
CM		 2171 if (context->devx_uid) {
2172 if (uctx_rdma_ctrl_is_enabled(ibcontext->enabled_caps))
2173 mlx5_cmd_remove_privileged_uid(dev->mdev,
2174 context->devx_uid);
d2c8a155 2175 mlx5_ib_devx_destroy(dev, context->devx_uid);
17ade536 2176 }
e126ba97
EC		 2177}
2178
b037c29a 2179static phys_addr_t uar_index2pfn(struct mlx5_ib_dev *dev,
4ed131d0 2180 int uar_idx)
e126ba97 2181{
b037c29a
EC		 2182 int fw_uars_per_page;
2183
2184 fw_uars_per_page = MLX5_CAP_GEN(dev->mdev, uar_4k) ? MLX5_UARS_IN_PAGE : 1;
2185
aa8106f1 2186 return (dev->mdev->bar_addr >> PAGE_SHIFT) + uar_idx / fw_uars_per_page;
e126ba97
EC		 2187}
2188
342ee59d
YH		 2189static u64 uar_index2paddress(struct mlx5_ib_dev *dev,
2190 int uar_idx)
2191{
2192 unsigned int fw_uars_per_page;
2193
2194 fw_uars_per_page = MLX5_CAP_GEN(dev->mdev, uar_4k) ?
2195 MLX5_UARS_IN_PAGE : 1;
2196
2197 return (dev->mdev->bar_addr + (uar_idx / fw_uars_per_page) * PAGE_SIZE);
2198}
2199
e126ba97
EC		 2200static int get_command(unsigned long offset)
2201{
2202 return (offset >> MLX5_IB_MMAP_CMD_SHIFT) & MLX5_IB_MMAP_CMD_MASK;
2203}
2204
2205static int get_arg(unsigned long offset)
2206{
2207 return offset & ((1 << MLX5_IB_MMAP_CMD_SHIFT) - 1);
2208}
2209
2210static int get_index(unsigned long offset)
2211{
2212 return get_arg(offset);
2213}
2214
4ed131d0
YH		 2215/* Index resides in an extra byte to enable larger values than 255 */
2216static int get_extended_index(unsigned long offset)
2217{
2218 return get_arg(offset) | ((offset >> 16) & 0xff) << 8;
2219}
2220
7c2344c3
MG		 2221
2222static void mlx5_ib_disassociate_ucontext(struct ib_ucontext *ibcontext)
2223{
7c2344c3
MG		 2224}
2225
37aa5c36
GL		 2226static inline char *mmap_cmd2str(enum mlx5_ib_mmap_cmd cmd)
2227{
2228 switch (cmd) {
2229 case MLX5_IB_MMAP_WC_PAGE:
2230 return "WC";
2231 case MLX5_IB_MMAP_REGULAR_PAGE:
2232 return "best effort WC";
2233 case MLX5_IB_MMAP_NC_PAGE:
2234 return "NC";
24da0016
AL		 2235 case MLX5_IB_MMAP_DEVICE_MEM:
2236 return "Device Memory";
37aa5c36 2237 default:
dab994bc 2238 return "Unknown";
37aa5c36
GL		 2239 }
2240}
2241
5c99eaec
FD		 2242static int mlx5_ib_mmap_clock_info_page(struct mlx5_ib_dev *dev,
2243 struct vm_area_struct *vma,
2244 struct mlx5_ib_ucontext *context)
2245{
4eb6ab13
JG		 2246 if ((vma->vm_end - vma->vm_start != PAGE_SIZE) ||
2247 !(vma->vm_flags & VM_SHARED))
5c99eaec
FD		 2248 return -EINVAL;
2249
2250 if (get_index(vma->vm_pgoff) != MLX5_IB_CLOCK_INFO_V1)
2251 return -EOPNOTSUPP;
2252
4eb6ab13 2253 if (vma->vm_flags & (VM_WRITE | VM_EXEC))
5c99eaec 2254 return -EPERM;
1c71222e 2255 vm_flags_clear(vma, VM_MAYWRITE);
5c99eaec 2256
ddcdc368 2257 if (!dev->mdev->clock_info)
5c99eaec
FD		 2258 return -EOPNOTSUPP;
2259
4eb6ab13
JG		 2260 return vm_insert_page(vma, vma->vm_start,
2261 virt_to_page(dev->mdev->clock_info));
5c99eaec
FD		 2262}
2263
dc2316eb
YH		 2264static void mlx5_ib_mmap_free(struct rdma_user_mmap_entry *entry)
2265{
2266 struct mlx5_user_mmap_entry *mentry = to_mmmap(entry);
2267 struct mlx5_ib_dev *dev = to_mdev(entry->ucontext->device);
7be76bef 2268 struct mlx5_var_table *var_table = &dev->var_table;
d2c8a155 2269 struct mlx5_ib_ucontext *context = to_mucontext(entry->ucontext);
dc2316eb
YH		 2270
2271 switch (mentry->mmap_flag) {
2272 case MLX5_IB_MMAP_TYPE_MEMIC:
cea85fa5
MG		 2273 case MLX5_IB_MMAP_TYPE_MEMIC_OP:
2274 mlx5_ib_dm_mmap_free(dev, mentry);
dc2316eb 2275 break;
7be76bef
YH		 2276 case MLX5_IB_MMAP_TYPE_VAR:
2277 mutex_lock(&var_table->bitmap_lock);
2278 clear_bit(mentry->page_idx, var_table->bitmap);
2279 mutex_unlock(&var_table->bitmap_lock);
2280 kfree(mentry);
2281 break;
342ee59d
YH		 2282 case MLX5_IB_MMAP_TYPE_UAR_WC:
2283 case MLX5_IB_MMAP_TYPE_UAR_NC:
d2c8a155
ML		 2284 mlx5_cmd_uar_dealloc(dev->mdev, mentry->page_idx,
2285 context->devx_uid);
342ee59d
YH		 2286 kfree(mentry);
2287 break;
dc2316eb
YH		 2288 default:
2289 WARN_ON(true);
2290 }
2291}
2292
37aa5c36 2293static int uar_mmap(struct mlx5_ib_dev *dev, enum mlx5_ib_mmap_cmd cmd,
7c2344c3
MG		 2294 struct vm_area_struct *vma,
2295 struct mlx5_ib_ucontext *context)
37aa5c36 2296{
2f5ff264 2297 struct mlx5_bfreg_info *bfregi = &context->bfregi;
37aa5c36
GL		 2298 int err;
2299 unsigned long idx;
aa09ea6e 2300 phys_addr_t pfn;
37aa5c36 2301 pgprot_t prot;
4ed131d0
YH		 2302 u32 bfreg_dyn_idx = 0;
2303 u32 uar_index;
2304 int dyn_uar = (cmd == MLX5_IB_MMAP_ALLOC_WC);
2305 int max_valid_idx = dyn_uar ? bfregi->num_sys_pages :
2306 bfregi->num_static_sys_pages;
b037c29a 2307
0a2fd01c
YH		 2308 if (bfregi->lib_uar_dyn)
2309 return -EINVAL;
2310
b037c29a
EC		 2311 if (vma->vm_end - vma->vm_start != PAGE_SIZE)
2312 return -EINVAL;
2313
4ed131d0
YH		 2314 if (dyn_uar)
2315 idx = get_extended_index(vma->vm_pgoff) + bfregi->num_static_sys_pages;
2316 else
2317 idx = get_index(vma->vm_pgoff);
2318
2319 if (idx >= max_valid_idx) {
2320 mlx5_ib_warn(dev, "invalid uar index %lu, max=%d\n",
2321 idx, max_valid_idx);
b037c29a
EC		 2322 return -EINVAL;
2323 }
37aa5c36
GL		 2324
2325 switch (cmd) {
2326 case MLX5_IB_MMAP_WC_PAGE:
4ed131d0 2327 case MLX5_IB_MMAP_ALLOC_WC:
37aa5c36
GL		 2328 case MLX5_IB_MMAP_REGULAR_PAGE:
2329 /* For MLX5_IB_MMAP_REGULAR_PAGE do the best effort to get WC */
2330 prot = pgprot_writecombine(vma->vm_page_prot);
2331 break;
2332 case MLX5_IB_MMAP_NC_PAGE:
2333 prot = pgprot_noncached(vma->vm_page_prot);
2334 break;
2335 default:
2336 return -EINVAL;
2337 }
2338
4ed131d0
YH		 2339 if (dyn_uar) {
2340 int uars_per_page;
2341
2342 uars_per_page = get_uars_per_sys_page(dev, bfregi->lib_uar_4k);
2343 bfreg_dyn_idx = idx * (uars_per_page * MLX5_NON_FP_BFREGS_PER_UAR);
2344 if (bfreg_dyn_idx >= bfregi->total_num_bfregs) {
2345 mlx5_ib_warn(dev, "invalid bfreg_dyn_idx %u, max=%u\n",
2346 bfreg_dyn_idx, bfregi->total_num_bfregs);
2347 return -EINVAL;
2348 }
2349
2350 mutex_lock(&bfregi->lock);
2351 /* Fail if uar already allocated, first bfreg index of each
2352 * page holds its count.
2353 */
2354 if (bfregi->count[bfreg_dyn_idx]) {
2355 mlx5_ib_warn(dev, "wrong offset, idx %lu is busy, bfregn=%u\n", idx, bfreg_dyn_idx);
2356 mutex_unlock(&bfregi->lock);
2357 return -EINVAL;
2358 }
2359
2360 bfregi->count[bfreg_dyn_idx]++;
2361 mutex_unlock(&bfregi->lock);
2362
d2c8a155
ML		 2363 err = mlx5_cmd_uar_alloc(dev->mdev, &uar_index,
2364 context->devx_uid);
4ed131d0
YH		 2365 if (err) {
2366 mlx5_ib_warn(dev, "UAR alloc failed\n");
2367 goto free_bfreg;
2368 }
2369 } else {
2370 uar_index = bfregi->sys_pages[idx];
2371 }
2372
2373 pfn = uar_index2pfn(dev, uar_index);
37aa5c36
GL		 2374 mlx5_ib_dbg(dev, "uar idx 0x%lx, pfn %pa\n", idx, &pfn);
2375
e2cd1d1a 2376 err = rdma_user_mmap_io(&context->ibucontext, vma, pfn, PAGE_SIZE,
c043ff2c 2377 prot, NULL);
37aa5c36 2378 if (err) {
8f062287 2379 mlx5_ib_err(dev,
e2cd1d1a 2380 "rdma_user_mmap_io failed with error=%d, mmap_cmd=%s\n",
8f062287 2381 err, mmap_cmd2str(cmd));
4ed131d0 2382 goto err;
37aa5c36
GL		 2383 }
2384
4ed131d0
YH		 2385 if (dyn_uar)
2386 bfregi->sys_pages[idx] = uar_index;
2387 return 0;
2388
2389err:
2390 if (!dyn_uar)
2391 return err;
2392
d2c8a155 2393 mlx5_cmd_uar_dealloc(dev->mdev, idx, context->devx_uid);
4ed131d0
YH		 2394
2395free_bfreg:
2396 mlx5_ib_free_bfreg(dev, bfregi, bfreg_dyn_idx);
2397
2398 return err;
37aa5c36
GL		 2399}
2400
dc2316eb
YH		 2401static unsigned long mlx5_vma_to_pgoff(struct vm_area_struct *vma)
2402{
2403 unsigned long idx;
2404 u8 command;
2405
2406 command = get_command(vma->vm_pgoff);
2407 idx = get_extended_index(vma->vm_pgoff);
2408
2409 return (command << 16 | idx);
2410}
2411
2412static int mlx5_ib_mmap_offset(struct mlx5_ib_dev *dev,
2413 struct vm_area_struct *vma,
2414 struct ib_ucontext *ucontext)
24da0016 2415{
dc2316eb
YH		 2416 struct mlx5_user_mmap_entry *mentry;
2417 struct rdma_user_mmap_entry *entry;
2418 unsigned long pgoff;
2419 pgprot_t prot;
24da0016 2420 phys_addr_t pfn;
dc2316eb 2421 int ret;
24da0016 2422
dc2316eb
YH		 2423 pgoff = mlx5_vma_to_pgoff(vma);
2424 entry = rdma_user_mmap_entry_get_pgoff(ucontext, pgoff);
2425 if (!entry)
24da0016
AL		 2426 return -EINVAL;
2427
dc2316eb
YH		 2428 mentry = to_mmmap(entry);
2429 pfn = (mentry->address >> PAGE_SHIFT);
342ee59d
YH		 2430 if (mentry->mmap_flag == MLX5_IB_MMAP_TYPE_VAR ||
2431 mentry->mmap_flag == MLX5_IB_MMAP_TYPE_UAR_NC)
3f59b6c3
YH		 2432 prot = pgprot_noncached(vma->vm_page_prot);
2433 else
2434 prot = pgprot_writecombine(vma->vm_page_prot);
dc2316eb
YH		 2435 ret = rdma_user_mmap_io(ucontext, vma, pfn,
2436 entry->npages * PAGE_SIZE,
2437 prot,
2438 entry);
2439 rdma_user_mmap_entry_put(&mentry->rdma_entry);
2440 return ret;
24da0016
AL		 2441}
2442
7be76bef
YH		 2443static u64 mlx5_entry_to_mmap_offset(struct mlx5_user_mmap_entry *entry)
2444{
9b6d3bbc
LR		 2445 u64 cmd = (entry->rdma_entry.start_pgoff >> 16) & 0xFFFF;
2446 u64 index = entry->rdma_entry.start_pgoff & 0xFFFF;
7be76bef
YH		 2447
2448 return (((index >> 8) << 16) | (cmd << MLX5_IB_MMAP_CMD_SHIFT) |
2449 (index & 0xFF)) << PAGE_SHIFT;
2450}
2451
e126ba97
EC		 2452static int mlx5_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma)
2453{
2454 struct mlx5_ib_ucontext *context = to_mucontext(ibcontext);
2455 struct mlx5_ib_dev *dev = to_mdev(ibcontext->device);
e126ba97 2456 unsigned long command;
e126ba97
EC		 2457 phys_addr_t pfn;
2458
2459 command = get_command(vma->vm_pgoff);
2460 switch (command) {
37aa5c36 2461 case MLX5_IB_MMAP_WC_PAGE:
1f3db161 2462 case MLX5_IB_MMAP_ALLOC_WC:
d98995b4 2463 if (!mlx5_wc_support_get(dev->mdev))
1f3db161
YH		 2464 return -EPERM;
2465 fallthrough;
37aa5c36 2466 case MLX5_IB_MMAP_NC_PAGE:
e126ba97 2467 case MLX5_IB_MMAP_REGULAR_PAGE:
7c2344c3 2468 return uar_mmap(dev, command, vma, context);
e126ba97
EC		 2469
2470 case MLX5_IB_MMAP_GET_CONTIGUOUS_PAGES:
2471 return -ENOSYS;
2472
d69e3bcf 2473 case MLX5_IB_MMAP_CORE_CLOCK:
d69e3bcf
MB		 2474 if (vma->vm_end - vma->vm_start != PAGE_SIZE)
2475 return -EINVAL;
2476
6cbac1e4 2477 if (vma->vm_flags & VM_WRITE)
d69e3bcf 2478 return -EPERM;
1c71222e 2479 vm_flags_clear(vma, VM_MAYWRITE);
d69e3bcf
MB		 2480
2481 /* Don't expose to user-space information it shouldn't have */
2482 if (PAGE_SIZE > 4096)
2483 return -EOPNOTSUPP;
2484
d69e3bcf
MB		 2485 pfn = (dev->mdev->iseg_base +
2486 offsetof(struct mlx5_init_seg, internal_timer_h)) >>
2487 PAGE_SHIFT;
d5e560d3
JG		 2488 return rdma_user_mmap_io(&context->ibucontext, vma, pfn,
2489 PAGE_SIZE,
c043ff2c
MK		 2490 pgprot_noncached(vma->vm_page_prot),
2491 NULL);
5c99eaec
FD		 2492 case MLX5_IB_MMAP_CLOCK_INFO:
2493 return mlx5_ib_mmap_clock_info_page(dev, vma, context);
d69e3bcf 2494
e126ba97 2495 default:
dc2316eb 2496 return mlx5_ib_mmap_offset(dev, vma, ibcontext);
e126ba97
EC		 2497 }
2498
2499 return 0;
2500}
2501
ff23dfa1 2502static int mlx5_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
e126ba97 2503{
21a428a0
LR		 2504 struct mlx5_ib_pd *pd = to_mpd(ibpd);
2505 struct ib_device *ibdev = ibpd->device;
e126ba97 2506 struct mlx5_ib_alloc_pd_resp resp;
e126ba97 2507 int err;
a1069c1c 2508 u32 out[MLX5_ST_SZ_DW(alloc_pd_out)] = {};
31578def 2509 u32 in[MLX5_ST_SZ_DW(alloc_pd_in)] = {};
a1069c1c 2510 u16 uid = 0;
ff23dfa1
SR		 2511 struct mlx5_ib_ucontext *context = rdma_udata_to_drv_context(
2512 udata, struct mlx5_ib_ucontext, ibucontext);
e126ba97 2513
ff23dfa1 2514 uid = context ? context->devx_uid : 0;
a1069c1c
YH		 2515 MLX5_SET(alloc_pd_in, in, opcode, MLX5_CMD_OP_ALLOC_PD);
2516 MLX5_SET(alloc_pd_in, in, uid, uid);
31578def 2517 err = mlx5_cmd_exec_inout(to_mdev(ibdev)->mdev, alloc_pd, in, out);
21a428a0
LR		 2518 if (err)
2519 return err;
e126ba97 2520
a1069c1c
YH		 2521 pd->pdn = MLX5_GET(alloc_pd_out, out, pd);
2522 pd->uid = uid;
ff23dfa1 2523 if (udata) {
e126ba97
EC		 2524 resp.pdn = pd->pdn;
2525 if (ib_copy_to_udata(udata, &resp, sizeof(resp))) {
a1069c1c 2526 mlx5_cmd_dealloc_pd(to_mdev(ibdev)->mdev, pd->pdn, uid);
21a428a0 2527 return -EFAULT;
e126ba97 2528 }
e126ba97
EC		 2529 }
2530
21a428a0 2531 return 0;
e126ba97
EC		 2532}
2533
91a7c58f 2534static int mlx5_ib_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata)
e126ba97
EC		 2535{
2536 struct mlx5_ib_dev *mdev = to_mdev(pd->device);
2537 struct mlx5_ib_pd *mpd = to_mpd(pd);
2538
91a7c58f 2539 return mlx5_cmd_dealloc_pd(mdev->mdev, mpd->pdn, mpd->uid);
e126ba97
EC		 2540}
2541
2542static int mlx5_ib_mcg_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
038d2ef8 2543{
e126ba97 2544 struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
81e30880 2545 struct mlx5_ib_qp *mqp = to_mqp(ibqp);
e126ba97 2546 int err;
539ec982 2547 u16 uid;
466fa6d2 2548
539ec982
YH		 2549 uid = ibqp->pd ?
2550 to_mpd(ibqp->pd)->uid : 0;
038d2ef8 2551
2be08c30 2552 if (mqp->flags & IB_QP_CREATE_SOURCE_QPN) {
81e30880
YH		 2553 mlx5_ib_dbg(dev, "Attaching a multi cast group to underlay QP is not supported\n");
2554 return -EOPNOTSUPP;
2555 }
6113cc44 2556
539ec982 2557 err = mlx5_cmd_attach_mcg(dev->mdev, gid, ibqp->qp_num, uid);
e126ba97
EC		 2558 if (err)
2559 mlx5_ib_warn(dev, "failed attaching QPN 0x%x, MGID %pI6\n",
2560 ibqp->qp_num, gid->raw);
6113cc44 2561
6113cc44 2562 return err;
038d2ef8
MG		 2563}
2564
e126ba97 2565static int mlx5_ib_mcg_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
2d1e697e 2566{
e126ba97
EC		 2567 struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
2568 int err;
539ec982 2569 u16 uid;
2d1e697e 2570
539ec982
YH		 2571 uid = ibqp->pd ?
2572 to_mpd(ibqp->pd)->uid : 0;
2573 err = mlx5_cmd_detach_mcg(dev->mdev, gid, ibqp->qp_num, uid);
e126ba97
EC		 2574 if (err)
2575 mlx5_ib_warn(dev, "failed detaching QPN 0x%x, MGID %pI6\n",
2576 ibqp->qp_num, gid->raw);
2577
2578 return err;
ca0d4753
MG		 2579}
2580
e126ba97 2581static int init_node_data(struct mlx5_ib_dev *dev)
71c6e863 2582{
1b5daf11 2583 int err;
71c6e863 2584
1b5daf11 2585 err = mlx5_query_node_desc(dev, dev->ib_dev.node_desc);
e126ba97 2586 if (err)
1b5daf11 2587 return err;
71c6e863 2588
1b5daf11 2589 dev->mdev->rev_id = dev->mdev->pdev->revision;
71c6e863 2590
1b5daf11 2591 return mlx5_query_node_guid(dev, &dev->ib_dev.node_guid);
802c2125
AY		 2592}
2593
508a523f
PP		 2594static ssize_t fw_pages_show(struct device *device,
2595 struct device_attribute *attr, char *buf)
e126ba97
EC		 2596{
2597 struct mlx5_ib_dev *dev =
54747231 2598 rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev);
2d1e697e 2599
1c7fd726 2600 return sysfs_emit(buf, "%d\n", dev->mdev->priv.fw_pages);
e126ba97 2601}
508a523f 2602static DEVICE_ATTR_RO(fw_pages);
466fa6d2 2603
508a523f 2604static ssize_t reg_pages_show(struct device *device,
e126ba97
EC		 2605 struct device_attribute *attr, char *buf)
2606{
2607 struct mlx5_ib_dev *dev =
54747231 2608 rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev);
038d2ef8 2609
1c7fd726 2610 return sysfs_emit(buf, "%d\n", atomic_read(&dev->mdev->priv.reg_pages));
e126ba97 2611}
508a523f 2612static DEVICE_ATTR_RO(reg_pages);
038d2ef8 2613
508a523f
PP		 2614static ssize_t hca_type_show(struct device *device,
2615 struct device_attribute *attr, char *buf)
e126ba97
EC		 2616{
2617 struct mlx5_ib_dev *dev =
54747231 2618 rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev);
038d2ef8 2619
1c7fd726 2620 return sysfs_emit(buf, "MT%d\n", dev->mdev->pdev->device);
e126ba97 2621}
508a523f 2622static DEVICE_ATTR_RO(hca_type);
026bae0c 2623
508a523f
PP		 2624static ssize_t hw_rev_show(struct device *device,
2625 struct device_attribute *attr, char *buf)
e126ba97
EC		 2626{
2627 struct mlx5_ib_dev *dev =
54747231 2628 rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev);
026bae0c 2629
1c7fd726 2630 return sysfs_emit(buf, "%x\n", dev->mdev->rev_id);
e126ba97 2631}
508a523f 2632static DEVICE_ATTR_RO(hw_rev);
466fa6d2 2633
508a523f
PP		 2634static ssize_t board_id_show(struct device *device,
2635 struct device_attribute *attr, char *buf)
e126ba97
EC		 2636{
2637 struct mlx5_ib_dev *dev =
54747231 2638 rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev);
2d1e697e 2639
1c7fd726
JP		 2640 return sysfs_emit(buf, "%.*s\n", MLX5_BOARD_ID_LEN,
2641 dev->mdev->board_id);
e126ba97 2642}
508a523f 2643static DEVICE_ATTR_RO(board_id);
038d2ef8 2644
508a523f
PP		 2645static struct attribute *mlx5_class_attributes[] = {
2646 &dev_attr_hw_rev.attr,
2647 &dev_attr_hca_type.attr,
2648 &dev_attr_board_id.attr,
2649 &dev_attr_fw_pages.attr,
2650 &dev_attr_reg_pages.attr,
2651 NULL,
2652};
038d2ef8 2653
508a523f
PP		 2654static const struct attribute_group mlx5_attr_group = {
2655 .attrs = mlx5_class_attributes,
e126ba97 2656};
038d2ef8 2657
7722f47e
HE		 2658static void pkey_change_handler(struct work_struct *work)
2659{
2660 struct mlx5_ib_port_resources *ports =
2661 container_of(work, struct mlx5_ib_port_resources,
2662 pkey_change_work);
038d2ef8 2663
8c9e7f03
LR		 2664 if (!ports->gsi)
2665 /*
2666 * We got this event before device was fully configured
2667 * and MAD registration code wasn't called/finished yet.
2668 */
2669 return;
2670
7722f47e 2671 mlx5_ib_gsi_pkey_change(ports->gsi);
7722f47e 2672}
038d2ef8 2673
89ea94a7
MG		 2674static void mlx5_ib_handle_internal_error(struct mlx5_ib_dev *ibdev)
2675{
2676 struct mlx5_ib_qp *mqp;
2677 struct mlx5_ib_cq *send_mcq, *recv_mcq;
2678 struct mlx5_core_cq *mcq;
2679 struct list_head cq_armed_list;
2680 unsigned long flags_qp;
2681 unsigned long flags_cq;
2682 unsigned long flags;
038d2ef8 2683
89ea94a7 2684 INIT_LIST_HEAD(&cq_armed_list);
da2f22ae 2685
89ea94a7
MG		 2686 /* Go over qp list reside on that ibdev, sync with create/destroy qp.*/
2687 spin_lock_irqsave(&ibdev->reset_flow_resource_lock, flags);
2688 list_for_each_entry(mqp, &ibdev->qp_list, qps_list) {
2689 spin_lock_irqsave(&mqp->sq.lock, flags_qp);
2690 if (mqp->sq.tail != mqp->sq.head) {
2691 send_mcq = to_mcq(mqp->ibqp.send_cq);
2692 spin_lock_irqsave(&send_mcq->lock, flags_cq);
2693 if (send_mcq->mcq.comp &&
2694 mqp->ibqp.send_cq->comp_handler) {
2695 if (!send_mcq->mcq.reset_notify_added) {
2696 send_mcq->mcq.reset_notify_added = 1;
2697 list_add_tail(&send_mcq->mcq.reset_notify,
2698 &cq_armed_list);
2699 }
71c6e863 2700 }
89ea94a7 2701 spin_unlock_irqrestore(&send_mcq->lock, flags_cq);
71c6e863 2702 }
89ea94a7
MG		 2703 spin_unlock_irqrestore(&mqp->sq.lock, flags_qp);
2704 spin_lock_irqsave(&mqp->rq.lock, flags_qp);
2705 /* no handling is needed for SRQ */
2706 if (!mqp->ibqp.srq) {
2707 if (mqp->rq.tail != mqp->rq.head) {
2708 recv_mcq = to_mcq(mqp->ibqp.recv_cq);
2709 spin_lock_irqsave(&recv_mcq->lock, flags_cq);
2710 if (recv_mcq->mcq.comp &&
2711 mqp->ibqp.recv_cq->comp_handler) {
2712 if (!recv_mcq->mcq.reset_notify_added) {
2713 recv_mcq->mcq.reset_notify_added = 1;
2714 list_add_tail(&recv_mcq->mcq.reset_notify,
2715 &cq_armed_list);
2716 }
2717 }
2718 spin_unlock_irqrestore(&recv_mcq->lock,
2719 flags_cq);
2720 }
2721 }
2722 spin_unlock_irqrestore(&mqp->rq.lock, flags_qp);
81e30880 2723 }
89ea94a7
MG		 2724 /*At that point all inflight post send were put to be executed as of we
2725 * lock/unlock above locks Now need to arm all involved CQs.
2726 */
2727 list_for_each_entry(mcq, &cq_armed_list, reset_notify) {
4e0e2ea1 2728 mcq->comp(mcq, NULL);
81e30880 2729 }
89ea94a7 2730 spin_unlock_irqrestore(&ibdev->reset_flow_resource_lock, flags);
038d2ef8
MG		 2731}
2732
03404e8a 2733static void delay_drop_handler(struct work_struct *work)
802c2125 2734{
03404e8a
MG		 2735 int err;
2736 struct mlx5_ib_delay_drop *delay_drop =
2737 container_of(work, struct mlx5_ib_delay_drop,
2738 delay_drop_work);
0f750966 2739
fe248c3a 2740 atomic_inc(&delay_drop->events_cnt);
19cc7524 2741
03404e8a 2742 mutex_lock(&delay_drop->lock);
333fbaa0 2743 err = mlx5_core_set_delay_drop(delay_drop->dev, delay_drop->timeout);
03404e8a
MG		 2744 if (err) {
2745 mlx5_ib_warn(delay_drop->dev, "Failed to set delay drop, timeout=%u\n",
2746 delay_drop->timeout);
2747 delay_drop->activate = false;
0f750966 2748 }
03404e8a 2749 mutex_unlock(&delay_drop->lock);
0f750966
AL		 2750}
2751
09e574fa
SM		 2752static void handle_general_event(struct mlx5_ib_dev *ibdev, struct mlx5_eqe *eqe,
2753 struct ib_event *ibev)
0f750966 2754{
1fb7f897 2755 u32 port = (eqe->data.port.port >> 4) & 0xf;
038d2ef8 2756
09e574fa
SM		 2757 switch (eqe->sub_type) {
2758 case MLX5_GENERAL_SUBTYPE_DELAY_DROP_TIMEOUT:
6cfdc7e4
AL		 2759 if (mlx5_ib_port_link_layer(&ibdev->ib_dev, port) ==
2760 IB_LINK_LAYER_ETHERNET)
2761 schedule_work(&ibdev->delay_drop.delay_drop_work);
09e574fa
SM		 2762 break;
2763 default: /* do nothing */
2764 return;
038d2ef8
MG		 2765 }
2766}
2767
134e9349
SM		 2768static int handle_port_change(struct mlx5_ib_dev *ibdev, struct mlx5_eqe *eqe,
2769 struct ib_event *ibev)
3b3233fb 2770{
1fb7f897 2771 u32 port = (eqe->data.port.port >> 4) & 0xf;
3b3233fb 2772
134e9349 2773 ibev->element.port_num = port;
3b3233fb 2774
134e9349
SM		 2775 switch (eqe->sub_type) {
2776 case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE:
2777 case MLX5_PORT_CHANGE_SUBTYPE_DOWN:
2778 case MLX5_PORT_CHANGE_SUBTYPE_INITIALIZED:
2779 /* In RoCE, port up/down events are handled in
2780 * mlx5_netdev_event().
2781 */
2782 if (mlx5_ib_port_link_layer(&ibdev->ib_dev, port) ==
2783 IB_LINK_LAYER_ETHERNET)
2784 return -EINVAL;
2785
2786 ibev->event = (eqe->sub_type == MLX5_PORT_CHANGE_SUBTYPE_ACTIVE) ?
2787 IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR;
2788 break;
038d2ef8 2789
134e9349
SM		 2790 case MLX5_PORT_CHANGE_SUBTYPE_LID:
2791 ibev->event = IB_EVENT_LID_CHANGE;
2792 break;
038d2ef8 2793
134e9349
SM		 2794 case MLX5_PORT_CHANGE_SUBTYPE_PKEY:
2795 ibev->event = IB_EVENT_PKEY_CHANGE;
2796 schedule_work(&ibdev->devr.ports[port - 1].pkey_change_work);
2797 break;
038d2ef8 2798
134e9349
SM		 2799 case MLX5_PORT_CHANGE_SUBTYPE_GUID:
2800 ibev->event = IB_EVENT_GID_CHANGE;
2801 break;
038d2ef8 2802
134e9349
SM		 2803 case MLX5_PORT_CHANGE_SUBTYPE_CLIENT_REREG:
2804 ibev->event = IB_EVENT_CLIENT_REREGISTER;
2805 break;
2806 default:
2807 return -EINVAL;
2808 }
038d2ef8
MG		 2809
2810 return 0;
2811}
2812
d69a24e0 2813static void mlx5_ib_handle_event(struct work_struct *_work)
d4be3f44 2814{
d69a24e0
DJ		 2815 struct mlx5_ib_event_work *work =
2816 container_of(_work, struct mlx5_ib_event_work, work);
2817 struct mlx5_ib_dev *ibdev;
e126ba97 2818 struct ib_event ibev;
dbaaff2a 2819 bool fatal = false;
d4be3f44 2820
df097a27
SM		 2821 if (work->is_slave) {
2822 ibdev = mlx5_ib_get_ibdev_from_mpi(work->mpi);
d69a24e0
DJ		 2823 if (!ibdev)
2824 goto out;
2825 } else {
df097a27 2826 ibdev = work->dev;
038d2ef8
MG		 2827 }
2828
d69a24e0 2829 switch (work->event) {
e126ba97 2830 case MLX5_DEV_EVENT_SYS_ERROR:
e126ba97 2831 ibev.event = IB_EVENT_DEVICE_FATAL;
89ea94a7 2832 mlx5_ib_handle_internal_error(ibdev);
134e9349 2833 ibev.element.port_num = (u8)(unsigned long)work->param;
dbaaff2a 2834 fatal = true;
e126ba97 2835 break;
134e9349
SM		 2836 case MLX5_EVENT_TYPE_PORT_CHANGE:
2837 if (handle_port_change(ibdev, work->param, &ibev))
d69a24e0 2838 goto out;
e126ba97 2839 break;
09e574fa
SM		 2840 case MLX5_EVENT_TYPE_GENERAL_EVENT:
2841 handle_general_event(ibdev, work->param, &ibev);
df561f66 2842 fallthrough;
bdc37924 2843 default:
03404e8a 2844 goto out;
e126ba97 2845 }
038d2ef8 2846
134e9349 2847 ibev.device = &ibdev->ib_dev;
a550ddfc 2848
134e9349
SM		 2849 if (!rdma_is_port_valid(&ibdev->ib_dev, ibev.element.port_num)) {
2850 mlx5_ib_warn(ibdev, "warning: event on port %d\n", ibev.element.port_num);
03404e8a 2851 goto out;
a550ddfc 2852 }
a550ddfc 2853
e126ba97
EC		 2854 if (ibdev->ib_active)
2855 ib_dispatch_event(&ibev);
5e95af5f 2856
dbaaff2a
EC		 2857 if (fatal)
2858 ibdev->ib_active = false;
03404e8a 2859out:
d69a24e0 2860 kfree(work);
5e95af5f
RS		 2861}
2862
df097a27
SM		 2863static int mlx5_ib_event(struct notifier_block *nb,
2864 unsigned long event, void *param)
3b3233fb 2865{
d69a24e0 2866 struct mlx5_ib_event_work *work;
3b3233fb 2867
d69a24e0 2868 work = kmalloc(sizeof(*work), GFP_ATOMIC);
10bea9c8 2869 if (!work)
df097a27 2870 return NOTIFY_DONE;
3b3233fb 2871
10bea9c8 2872 INIT_WORK(&work->work, mlx5_ib_handle_event);
df097a27
SM		 2873 work->dev = container_of(nb, struct mlx5_ib_dev, mdev_events);
2874 work->is_slave = false;
10bea9c8 2875 work->param = param;
10bea9c8 2876 work->event = event;
3b3233fb 2877
10bea9c8 2878 queue_work(mlx5_ib_event_wq, &work->work);
3b3233fb 2879
df097a27 2880 return NOTIFY_OK;
3b3233fb
RS		 2881}
2882
df097a27
SM		 2883static int mlx5_ib_event_slave_port(struct notifier_block *nb,
2884 unsigned long event, void *param)
3b3233fb 2885{
df097a27 2886 struct mlx5_ib_event_work *work;
7c34ec19 2887
df097a27
SM		 2888 work = kmalloc(sizeof(*work), GFP_ATOMIC);
2889 if (!work)
2890 return NOTIFY_DONE;
9ef9c640 2891
df097a27
SM		 2892 INIT_WORK(&work->work, mlx5_ib_handle_event);
2893 work->mpi = container_of(nb, struct mlx5_ib_multiport_info, mdev_events);
2894 work->is_slave = true;
2895 work->param = param;
2896 work->event = event;
2897 queue_work(mlx5_ib_event_wq, &work->work);
2898
2899 return NOTIFY_OK;
9ef9c640
AH		 2900}
2901
2a5db20f
MZ		 2902static int mlx5_ib_get_plane_num(struct mlx5_core_dev *mdev, u8 *num_plane)
2903{
2904 struct mlx5_hca_vport_context vport_ctx;
2905 int err;
2906
2907 *num_plane = 0;
45d339fe 2908 if (!MLX5_CAP_GEN(mdev, ib_virt) || !MLX5_CAP_GEN_2(mdev, multiplane))
2a5db20f
MZ		 2909 return 0;
2910
2911 err = mlx5_query_hca_vport_context(mdev, 0, 1, 0, &vport_ctx);
2912 if (err)
2913 return err;
2914
2915 *num_plane = vport_ctx.num_plane;
2916 return 0;
2917}
2918
c43f1112 2919static int set_has_smi_cap(struct mlx5_ib_dev *dev)
d012f5d6 2920{
c43f1112 2921 struct mlx5_hca_vport_context vport_ctx;
d012f5d6 2922 int err;
c43f1112 2923 int port;
d012f5d6 2924
4b83c3ca
MB		 2925 if (MLX5_CAP_GEN(dev->mdev, port_type) != MLX5_CAP_PORT_TYPE_IB)
2926 return 0;
2927
2928 for (port = 1; port <= dev->num_ports; port++) {
2a5db20f
MZ		 2929 if (dev->num_plane) {
2930 dev->port_caps[port - 1].has_smi = false;
2931 continue;
026a4259
MZ		 2932 } else if (!MLX5_CAP_GEN(dev->mdev, ib_virt) ||
2933 dev->ib_dev.type == RDMA_DEVICE_TYPE_SMI) {
4b83c3ca
MB		 2934 dev->port_caps[port - 1].has_smi = true;
2935 continue;
c43f1112 2936 }
2a5db20f 2937
4b83c3ca
MB		 2938 err = mlx5_query_hca_vport_context(dev->mdev, 0, port, 0,
2939 &vport_ctx);
2940 if (err) {
2941 mlx5_ib_err(dev, "query_hca_vport_context for port=%d failed %d\n",
2942 port, err);
2943 return err;
2944 }
2945 dev->port_caps[port - 1].has_smi = vport_ctx.has_smi;
d012f5d6 2946 }
4b83c3ca 2947
d012f5d6
OG		 2948 return 0;
2949}
2950
e126ba97 2951static void get_ext_port_caps(struct mlx5_ib_dev *dev)
5ec8c83e 2952{
13179652 2953 unsigned int port;
e126ba97 2954
13179652 2955 rdma_for_each_port (&dev->ib_dev, port)
e126ba97 2956 mlx5_query_ext_port_caps(dev, port);
5ec8c83e
AH		 2957}
2958
6e8484c5
MG		 2959static u8 mlx5_get_umr_fence(u8 umr_fence_cap)
2960{
2961 switch (umr_fence_cap) {
2962 case MLX5_CAP_UMR_FENCE_NONE:
2963 return MLX5_FENCE_MODE_NONE;
2964 case MLX5_CAP_UMR_FENCE_SMALL:
2965 return MLX5_FENCE_MODE_INITIATOR_SMALL;
2966 default:
2967 return MLX5_FENCE_MODE_STRONG_ORDERING;
2968 }
2969}
7c16f477 2970
5895e70f 2971int mlx5_ib_dev_res_cq_init(struct mlx5_ib_dev *dev)
e126ba97 2972{
1e2b5a90 2973 struct mlx5_ib_resources *devr = &dev->devr;
bcf4c1ea 2974 struct ib_cq_init_attr cq_attr = {.cqe = 1};
5895e70f
JL		 2975 struct ib_device *ibdev;
2976 struct ib_pd *pd;
2977 struct ib_cq *cq;
e126ba97 2978 int ret = 0;
7c16f477 2979
7c16f477 2980
5895e70f
JL		 2981 /*
2982 * devr->c0 is set once, never changed until device unload.
2983 * Avoid taking the mutex if initialization is already done.
2984 */
2985 if (devr->c0)
2986 return 0;
7c16f477 2987
5895e70f
JL		 2988 mutex_lock(&devr->cq_lock);
2989 if (devr->c0)
2990 goto unlock;
d7fab916 2991
5895e70f
JL		 2992 ibdev = &dev->ib_dev;
2993 pd = ib_alloc_pd(ibdev, 0);
2994 if (IS_ERR(pd)) {
2995 ret = PTR_ERR(pd);
2996 mlx5_ib_err(dev, "Couldn't allocate PD for res init, err=%d\n", ret);
2997 goto unlock;
e126ba97 2998 }
9f876f3d 2999
5895e70f
JL		 3000 cq = ib_create_cq(ibdev, NULL, NULL, NULL, &cq_attr);
3001 if (IS_ERR(cq)) {
3002 ret = PTR_ERR(cq);
3003 mlx5_ib_err(dev, "Couldn't create CQ for res init, err=%d\n", ret);
3004 ib_dealloc_pd(pd);
3005 goto unlock;
3006 }
0837e86a 3007
5895e70f
JL		 3008 devr->p0 = pd;
3009 devr->c0 = cq;
3010
3011unlock:
3012 mutex_unlock(&devr->cq_lock);
3013 return ret;
3014}
3015
3016int mlx5_ib_dev_res_srq_init(struct mlx5_ib_dev *dev)
3017{
3018 struct mlx5_ib_resources *devr = &dev->devr;
3019 struct ib_srq_init_attr attr;
3020 struct ib_srq *s0, *s1;
3021 int ret = 0;
3022
3023 /*
3024 * devr->s1 is set once, never changed until device unload.
3025 * Avoid taking the mutex if initialization is already done.
3026 */
3027 if (devr->s1)
3028 return 0;
3029
3030 mutex_lock(&devr->srq_lock);
3031 if (devr->s1)
3032 goto unlock;
3033
3034 ret = mlx5_ib_dev_res_cq_init(dev);
f4375443 3035 if (ret)
5895e70f 3036 goto unlock;
3e1f000f 3037
e126ba97
EC		 3038 memset(&attr, 0, sizeof(attr));
3039 attr.attr.max_sge = 1;
3040 attr.attr.max_wr = 1;
3041 attr.srq_type = IB_SRQT_XRC;
1a56ff6d 3042 attr.ext.cq = devr->c0;
66247fbb 3043
5895e70f
JL		 3044 s0 = ib_create_srq(devr->p0, &attr);
3045 if (IS_ERR(s0)) {
3046 ret = PTR_ERR(s0);
3047 mlx5_ib_err(dev, "Couldn't create SRQ 0 for res init, err=%d\n", ret);
3048 goto unlock;
20da44df 3049 }
7c16f477 3050
4aa17b28
HA		 3051 memset(&attr, 0, sizeof(attr));
3052 attr.attr.max_sge = 1;
3053 attr.attr.max_wr = 1;
3054 attr.srq_type = IB_SRQT_BASIC;
58dcb60a 3055
5895e70f
JL		 3056 s1 = ib_create_srq(devr->p0, &attr);
3057 if (IS_ERR(s1)) {
3058 ret = PTR_ERR(s1);
3059 mlx5_ib_err(dev, "Couldn't create SRQ 1 for res init, err=%d\n", ret);
3060 ib_destroy_srq(s0);
3061 }
3062
3063 devr->s0 = s0;
3064 devr->s1 = s1;
3065
3066unlock:
3067 mutex_unlock(&devr->srq_lock);
3068 return ret;
3069}
3070
3071static int mlx5_ib_dev_res_init(struct mlx5_ib_dev *dev)
3072{
3073 struct mlx5_ib_resources *devr = &dev->devr;
5895e70f
JL		 3074 int ret;
3075
3076 if (!MLX5_CAP_GEN(dev->mdev, xrc))
3077 return -EOPNOTSUPP;
3078
3079 ret = mlx5_cmd_xrcd_alloc(dev->mdev, &devr->xrcdn0, 0);
3080 if (ret)
3081 return ret;
3082
3083 ret = mlx5_cmd_xrcd_alloc(dev->mdev, &devr->xrcdn1, 0);
3084 if (ret) {
3085 mlx5_cmd_xrcd_dealloc(dev->mdev, devr->xrcdn0, 0);
3086 return ret;
20da44df 3087 }
7c16f477 3088
5895e70f
JL		 3089 mutex_init(&devr->cq_lock);
3090 mutex_init(&devr->srq_lock);
7c16f477 3091
5895e70f 3092 return 0;
7c16f477
KH		 3093}
3094
1e2b5a90 3095static void mlx5_ib_dev_res_cleanup(struct mlx5_ib_dev *dev)
7c16f477 3096{
1e2b5a90 3097 struct mlx5_ib_resources *devr = &dev->devr;
b0791dbf 3098
5895e70f
JL		 3099 /* After s0/s1 init, they are not unset during the device lifetime. */
3100 if (devr->s1) {
3101 ib_destroy_srq(devr->s1);
3102 ib_destroy_srq(devr->s0);
3103 }
f4375443
LR		 3104 mlx5_cmd_xrcd_dealloc(dev->mdev, devr->xrcdn1, 0);
3105 mlx5_cmd_xrcd_dealloc(dev->mdev, devr->xrcdn0, 0);
5895e70f
JL		 3106 /* After p0/c0 init, they are not unset during the device lifetime. */
3107 if (devr->c0) {
3108 ib_destroy_cq(devr->c0);
3109 ib_dealloc_pd(devr->p0);
3110 }
3111 mutex_destroy(&devr->cq_lock);
3112 mutex_destroy(&devr->srq_lock);
0837e86a
MB		 3113}
3114
2e8e631d
YH		 3115static int
3116mlx5_ib_create_data_direct_resources(struct mlx5_ib_dev *dev)
3117{
3118 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
3119 struct mlx5_core_dev *mdev = dev->mdev;
3120 void *mkc;
3121 u32 mkey;
3122 u32 pdn;
3123 u32 *in;
3124 int err;
3125
3126 err = mlx5_core_alloc_pd(mdev, &pdn);
3127 if (err)
3128 return err;
3129
3130 in = kvzalloc(inlen, GFP_KERNEL);
3131 if (!in) {
3132 err = -ENOMEM;
3133 goto err;
3134 }
3135
3136 MLX5_SET(create_mkey_in, in, data_direct, 1);
3137 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
3138 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_PA);
3139 MLX5_SET(mkc, mkc, lw, 1);
3140 MLX5_SET(mkc, mkc, lr, 1);
3141 MLX5_SET(mkc, mkc, rw, 1);
3142 MLX5_SET(mkc, mkc, rr, 1);
3143 MLX5_SET(mkc, mkc, a, 1);
3144 MLX5_SET(mkc, mkc, pd, pdn);
3145 MLX5_SET(mkc, mkc, length64, 1);
3146 MLX5_SET(mkc, mkc, qpn, 0xffffff);
3147 err = mlx5_core_create_mkey(mdev, &mkey, in, inlen);
3148 kvfree(in);
3149 if (err)
3150 goto err;
3151
3152 dev->ddr.mkey = mkey;
3153 dev->ddr.pdn = pdn;
3154 return 0;
3155
3156err:
3157 mlx5_core_dealloc_pd(mdev, pdn);
3158 return err;
3159}
3160
3161static void
3162mlx5_ib_free_data_direct_resources(struct mlx5_ib_dev *dev)
3163{
3164 mlx5_core_destroy_mkey(dev->mdev, dev->ddr.mkey);
3165 mlx5_core_dealloc_pd(dev->mdev, dev->ddr.pdn);
3166}
3167
b02289b3
AK		 3168static u32 get_core_cap_flags(struct ib_device *ibdev,
3169 struct mlx5_hca_vport_context *rep)
0837e86a 3170{
e53505a8
AS		 3171 struct mlx5_ib_dev *dev = to_mdev(ibdev);
3172 enum rdma_link_layer ll = mlx5_ib_port_link_layer(ibdev, 1);
3173 u8 l3_type_cap = MLX5_CAP_ROCE(dev->mdev, l3_type);
3174 u8 roce_version_cap = MLX5_CAP_ROCE(dev->mdev, roce_version);
85c7c014 3175 bool raw_support = !mlx5_core_mp_enabled(dev->mdev);
e53505a8 3176 u32 ret = 0;
aac4492e 3177
b02289b3
AK		 3178 if (rep->grh_required)
3179 ret |= RDMA_CORE_CAP_IB_GRH_REQUIRED;
7c16f477 3180
2a5db20f
MZ		 3181 if (dev->num_plane)
3182 return ret | RDMA_CORE_CAP_PROT_IB | RDMA_CORE_CAP_IB_MAD |
3183 RDMA_CORE_CAP_IB_CM | RDMA_CORE_CAP_IB_SA |
3184 RDMA_CORE_CAP_AF_IB;
026a4259
MZ		 3185 else if (ibdev->type == RDMA_DEVICE_TYPE_SMI)
3186 return ret | RDMA_CORE_CAP_IB_MAD | RDMA_CORE_CAP_IB_SMI;
2a5db20f 3187
e53505a8 3188 if (ll == IB_LINK_LAYER_INFINIBAND)
b02289b3 3189 return ret | RDMA_CORE_PORT_IBA_IB;
bfd745f8 3190
85c7c014 3191 if (raw_support)
b02289b3 3192 ret |= RDMA_CORE_PORT_RAW_PACKET;
bfd745f8 3193
e53505a8 3194 if (!(l3_type_cap & MLX5_ROCE_L3_TYPE_IPV4_CAP))
72cd5717 3195 return ret;
0837e86a 3196
e53505a8 3197 if (!(l3_type_cap & MLX5_ROCE_L3_TYPE_IPV6_CAP))
72cd5717 3198 return ret;
0837e86a 3199
e53505a8
AS		 3200 if (roce_version_cap & MLX5_ROCE_VERSION_1_CAP)
3201 ret |= RDMA_CORE_PORT_IBA_ROCE;
3e1f000f 3202
e53505a8
AS		 3203 if (roce_version_cap & MLX5_ROCE_VERSION_2_CAP)
3204 ret |= RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
3e1f000f 3205
e53505a8 3206 return ret;
3e1f000f
PP		 3207}
3208
1fb7f897 3209static int mlx5_port_immutable(struct ib_device *ibdev, u32 port_num,
7738613e 3210 struct ib_port_immutable *immutable)
0ad17a8f 3211{
7738613e 3212 struct ib_port_attr attr;
7c16f477 3213 struct mlx5_ib_dev *dev = to_mdev(ibdev);
ca5b91d6 3214 enum rdma_link_layer ll = mlx5_ib_port_link_layer(ibdev, port_num);
b02289b3 3215 struct mlx5_hca_vport_context rep = {0};
7738613e 3216 int err;
0ad17a8f 3217
c4550c63 3218 err = ib_query_port(ibdev, port_num, &attr);
7738613e
IW		 3219 if (err)
3220 return err;
3221
b02289b3 3222 if (ll == IB_LINK_LAYER_INFINIBAND) {
026a4259
MZ		 3223 if (ibdev->type == RDMA_DEVICE_TYPE_SMI)
3224 port_num = smi_to_native_portnum(dev, port_num);
3225
b02289b3
AK		 3226 err = mlx5_query_hca_vport_context(dev->mdev, 0, port_num, 0,
3227 &rep);
3228 if (err)
3229 return err;
3230 }
0ad17a8f 3231
7738613e
IW		 3232 immutable->pkey_tbl_len = attr.pkey_tbl_len;
3233 immutable->gid_tbl_len = attr.gid_tbl_len;
b02289b3 3234 immutable->core_cap_flags = get_core_cap_flags(ibdev, &rep);
94de879c 3235 immutable->max_mad_size = IB_MGMT_MAD_SIZE;
3e1f000f 3236
7738613e 3237 return 0;
0ad17a8f
MB		 3238}
3239
1fb7f897 3240static int mlx5_port_rep_immutable(struct ib_device *ibdev, u32 port_num,
8e6efa3a 3241 struct ib_port_immutable *immutable)
0ad17a8f 3242{
8e6efa3a
MB		 3243 struct ib_port_attr attr;
3244 int err;
0ad17a8f 3245
8e6efa3a 3246 immutable->core_cap_flags = RDMA_CORE_PORT_RAW_PACKET;
0ad17a8f 3247
8e6efa3a
MB		 3248 err = ib_query_port(ibdev, port_num, &attr);
3249 if (err)
3250 return err;
3251
3252 immutable->pkey_tbl_len = attr.pkey_tbl_len;
3253 immutable->gid_tbl_len = attr.gid_tbl_len;
3254 immutable->core_cap_flags = RDMA_CORE_PORT_RAW_PACKET;
7c16f477 3255
66247fbb 3256 return 0;
e1f24a79
PP		 3257}
3258
9abb0d1b 3259static void get_dev_fw_str(struct ib_device *ibdev, char *str)
9f876f3d 3260{
c7342823
IW		 3261 struct mlx5_ib_dev *dev =
3262 container_of(ibdev, struct mlx5_ib_dev, ib_dev);
9abb0d1b
LR		 3263 snprintf(str, IB_FW_VERSION_NAME_MAX, "%d.%d.%04d",
3264 fw_rev_maj(dev->mdev), fw_rev_min(dev->mdev),
3265 fw_rev_sub(dev->mdev));
9f876f3d
TB		 3266}
3267
8d159eb2
CM		 3268static int lag_event(struct notifier_block *nb, unsigned long event, void *data)
3269{
3270 struct mlx5_ib_dev *dev = container_of(nb, struct mlx5_ib_dev,
3271 lag_events);
3272 struct mlx5_core_dev *mdev = dev->mdev;
0bd2c61d
CM		 3273 struct ib_device *ibdev = &dev->ib_dev;
3274 struct net_device *old_ndev = NULL;
8d159eb2
CM		 3275 struct mlx5_ib_port *port;
3276 struct net_device *ndev;
0bd2c61d
CM		 3277 u32 portnum = 0;
3278 int ret = 0;
3279 int i;
8d159eb2 3280
8d159eb2
CM		 3281 switch (event) {
3282 case MLX5_DRIVER_EVENT_ACTIVE_BACKUP_LAG_CHANGE_LOWERSTATE:
3283 ndev = data;
3284 if (ndev) {
3285 if (!mlx5_lag_is_roce(mdev)) {
3286 // sriov lag
3287 for (i = 0; i < dev->num_ports; i++) {
3288 port = &dev->port[i];
3289 if (port->rep && port->rep->vport ==
3290 MLX5_VPORT_UPLINK) {
3291 portnum = i;
3292 break;
3293 }
3294 }
3295 }
0bd2c61d
CM		 3296 old_ndev = ib_device_get_netdev(ibdev, portnum + 1);
3297 ret = ib_device_set_netdev(ibdev, ndev, portnum + 1);
3298 if (ret)
3299 goto out;
3300
3301 if (old_ndev)
3302 roce_del_all_netdev_gids(ibdev, portnum + 1,
3303 old_ndev);
3304 rdma_roce_rescan_port(ibdev, portnum + 1);
8d159eb2
CM		 3305 }
3306 break;
3307 default:
3308 return NOTIFY_DONE;
3309 }
0bd2c61d
CM		 3310
3311out:
3312 dev_put(old_ndev);
3313 return notifier_from_errno(ret);
8d159eb2
CM		 3314}
3315
3316static void mlx5e_lag_event_register(struct mlx5_ib_dev *dev)
3317{
3318 dev->lag_events.notifier_call = lag_event;
3319 blocking_notifier_chain_register(&dev->mdev->priv.lag_nh,
3320 &dev->lag_events);
3321}
3322
3323static void mlx5e_lag_event_unregister(struct mlx5_ib_dev *dev)
3324{
3325 blocking_notifier_chain_unregister(&dev->mdev->priv.lag_nh,
3326 &dev->lag_events);
3327}
3328
45f95acd 3329static int mlx5_eth_lag_init(struct mlx5_ib_dev *dev)
e1f24a79 3330{
9ef9c640
AH		 3331 struct mlx5_core_dev *mdev = dev->mdev;
3332 struct mlx5_flow_namespace *ns = mlx5_get_flow_namespace(mdev,
3333 MLX5_FLOW_NAMESPACE_LAG);
3334 struct mlx5_flow_table *ft;
3335 int err;
e1f24a79 3336
c446d9da 3337 if (!ns || !mlx5_lag_is_active(mdev))
9ef9c640 3338 return 0;
aac4492e 3339
9ef9c640
AH		 3340 err = mlx5_cmd_create_vport_lag(mdev);
3341 if (err)
3342 return err;
e1f24a79 3343
9ef9c640
AH		 3344 ft = mlx5_create_lag_demux_flow_table(ns, 0, 0);
3345 if (IS_ERR(ft)) {
3346 err = PTR_ERR(ft);
3347 goto err_destroy_vport_lag;
9f876f3d
TB		 3348 }
3349
8d159eb2 3350 mlx5e_lag_event_register(dev);
9a4ca38d 3351 dev->flow_db->lag_demux_ft = ft;
34a30d76 3352 dev->lag_ports = mlx5_lag_get_num_ports(mdev);
7c34ec19 3353 dev->lag_active = true;
9ef9c640 3354 return 0;
e1f24a79 3355
9ef9c640
AH		 3356err_destroy_vport_lag:
3357 mlx5_cmd_destroy_vport_lag(mdev);
3358 return err;
0ad17a8f
MB		 3359}
3360
45f95acd 3361static void mlx5_eth_lag_cleanup(struct mlx5_ib_dev *dev)
18d422ce 3362{
9ef9c640 3363 struct mlx5_core_dev *mdev = dev->mdev;
18d422ce 3364
7c34ec19
AH		 3365 if (dev->lag_active) {
3366 dev->lag_active = false;
18d422ce 3367
8d159eb2 3368 mlx5e_lag_event_unregister(dev);
9a4ca38d
MB		 3369 mlx5_destroy_flow_table(dev->flow_db->lag_demux_ft);
3370 dev->flow_db->lag_demux_ft = NULL;
18d422ce 3371
9ef9c640
AH		 3372 mlx5_cmd_destroy_vport_lag(mdev);
3373 }
18d422ce
MZ		 3374}
3375
dca55da0
JP		 3376static void mlx5_netdev_notifier_register(struct mlx5_roce *roce,
3377 struct net_device *netdev)
66247fbb 3378{
d012f5d6 3379 int err;
66247fbb 3380
dca55da0
JP		 3381 if (roce->tracking_netdev)
3382 return;
3383 roce->tracking_netdev = netdev;
3384 roce->nb.notifier_call = mlx5_netdev_event;
3385 err = register_netdevice_notifier_dev_net(netdev, &roce->nb, &roce->nn);
3386 WARN_ON(err);
3387}
66247fbb 3388
dca55da0
JP		 3389static void mlx5_netdev_notifier_unregister(struct mlx5_roce *roce)
3390{
3391 if (!roce->tracking_netdev)
3392 return;
3393 unregister_netdevice_notifier_dev_net(roce->tracking_netdev, &roce->nb,
3394 &roce->nn);
3395 roce->tracking_netdev = NULL;
66247fbb
LR		 3396}
3397
dca55da0
JP		 3398static int mlx5e_mdev_notifier_event(struct notifier_block *nb,
3399 unsigned long event, void *data)
45842fc6 3400{
dca55da0
JP		 3401 struct mlx5_roce *roce = container_of(nb, struct mlx5_roce, mdev_nb);
3402 struct net_device *netdev = data;
3403
3404 switch (event) {
3405 case MLX5_DRIVER_EVENT_UPLINK_NETDEV:
3406 if (netdev)
3407 mlx5_netdev_notifier_register(roce, netdev);
3408 else
3409 mlx5_netdev_notifier_unregister(roce);
3410 break;
3411 default:
3412 return NOTIFY_DONE;
5ec8c83e 3413 }
dca55da0
JP		 3414
3415 return NOTIFY_OK;
3416}
3417
3418static void mlx5_mdev_netdev_track(struct mlx5_ib_dev *dev, u32 port_num)
3419{
3420 struct mlx5_roce *roce = &dev->port[port_num].roce;
3421
3422 roce->mdev_nb.notifier_call = mlx5e_mdev_notifier_event;
3423 mlx5_blocking_notifier_register(dev->mdev, &roce->mdev_nb);
3424 mlx5_core_uplink_netdev_event_replay(dev->mdev);
3425}
3426
3427static void mlx5_mdev_netdev_untrack(struct mlx5_ib_dev *dev, u32 port_num)
3428{
3429 struct mlx5_roce *roce = &dev->port[port_num].roce;
3430
3431 mlx5_blocking_notifier_unregister(dev->mdev, &roce->mdev_nb);
3432 mlx5_netdev_notifier_unregister(roce);
5ec8c83e 3433}
45842fc6 3434
e3f1ed1f 3435static int mlx5_enable_eth(struct mlx5_ib_dev *dev)
fc24fc5e 3436{
e53505a8 3437 int err;
bfd745f8 3438
c446d9da
MB		 3439 if (!dev->is_rep && dev->profile != &raw_eth_profile) {
3440 err = mlx5_nic_vport_enable_roce(dev->mdev);
3441 if (err)
3442 return err;
3443 }
45842fc6 3444
45f95acd 3445 err = mlx5_eth_lag_init(dev);
45842fc6 3446 if (err)
9ef9c640 3447 goto err_disable_roce;
45842fc6
MZ		 3448
3449 return 0;
3450
9ef9c640 3451err_disable_roce:
c446d9da
MB		 3452 if (!dev->is_rep && dev->profile != &raw_eth_profile)
3453 mlx5_nic_vport_disable_roce(dev->mdev);
45842fc6
MZ		 3454
3455 return err;
3456}
3457
45f95acd 3458static void mlx5_disable_eth(struct mlx5_ib_dev *dev)
45842fc6 3459{
45f95acd 3460 mlx5_eth_lag_cleanup(dev);
c446d9da
MB		 3461 if (!dev->is_rep && dev->profile != &raw_eth_profile)
3462 mlx5_nic_vport_disable_roce(dev->mdev);
45842fc6
MZ		 3463}
3464
1fb7f897 3465static int mlx5_ib_rn_get_params(struct ib_device *device, u32 port_num,
f6a8a19b
DD		 3466 enum rdma_netdev_t type,
3467 struct rdma_netdev_alloc_params *params)
693dfd5a
ES		 3468{
3469 if (type != RDMA_NETDEV_IPOIB)
f6a8a19b 3470 return -EOPNOTSUPP;
693dfd5a 3471
f6a8a19b 3472 return mlx5_rdma_rn_get_params(to_mdev(device)->mdev, device, params);
693dfd5a
ES		 3473}
3474
fe248c3a
MG		 3475static ssize_t delay_drop_timeout_read(struct file *filp, char __user *buf,
3476 size_t count, loff_t *pos)
3477{
3478 struct mlx5_ib_delay_drop *delay_drop = filp->private_data;
3479 char lbuf[20];
3480 int len;
3481
3482 len = snprintf(lbuf, sizeof(lbuf), "%u\n", delay_drop->timeout);
3483 return simple_read_from_buffer(buf, count, pos, lbuf, len);
3484}
3485
3486static ssize_t delay_drop_timeout_write(struct file *filp, const char __user *buf,
3487 size_t count, loff_t *pos)
3488{
3489 struct mlx5_ib_delay_drop *delay_drop = filp->private_data;
3490 u32 timeout;
3491 u32 var;
3492
3493 if (kstrtouint_from_user(buf, count, 0, &var))
3494 return -EFAULT;
3495
3496 timeout = min_t(u32, roundup(var, 100), MLX5_MAX_DELAY_DROP_TIMEOUT_MS *
3497 1000);
3498 if (timeout != var)
3499 mlx5_ib_dbg(delay_drop->dev, "Round delay drop timeout to %u usec\n",
3500 timeout);
3501
3502 delay_drop->timeout = timeout;
3503
3504 return count;
3505}
3506
3507static const struct file_operations fops_delay_drop_timeout = {
3508 .owner = THIS_MODULE,
3509 .open = simple_open,
3510 .write = delay_drop_timeout_write,
3511 .read = delay_drop_timeout_read,
3512};
3513
32f69e4b
DJ		 3514static void mlx5_ib_unbind_slave_port(struct mlx5_ib_dev *ibdev,
3515 struct mlx5_ib_multiport_info *mpi)
3516{
1fb7f897 3517 u32 port_num = mlx5_core_native_port_num(mpi->mdev) - 1;
32f69e4b
DJ		 3518 struct mlx5_ib_port *port = &ibdev->port[port_num];
3519 int comps;
3520 int err;
3521 int i;
3522
9dc4cfff
LR		 3523 lockdep_assert_held(&mlx5_ib_multiport_mutex);
3524
a9a9e689
PH		 3525 mlx5_ib_disable_lb_mp(ibdev->mdev, mpi->mdev);
3526
0d293714
PH		 3527 mlx5_core_mp_event_replay(ibdev->mdev,
3528 MLX5_DRIVER_EVENT_AFFILIATION_REMOVED,
3529 NULL);
3530 mlx5_core_mp_event_replay(mpi->mdev,
3531 MLX5_DRIVER_EVENT_AFFILIATION_REMOVED,
3532 NULL);
3533
a9e546e7
PP		 3534 mlx5_ib_cleanup_cong_debugfs(ibdev, port_num);
3535
32f69e4b
DJ		 3536 spin_lock(&port->mp.mpi_lock);
3537 if (!mpi->ibdev) {
3538 spin_unlock(&port->mp.mpi_lock);
3539 return;
3540 }
df097a27 3541
32f69e4b
DJ		 3542 mpi->ibdev = NULL;
3543
3544 spin_unlock(&port->mp.mpi_lock);
23eaf3b5
LR		 3545 if (mpi->mdev_events.notifier_call)
3546 mlx5_notifier_unregister(mpi->mdev, &mpi->mdev_events);
3547 mpi->mdev_events.notifier_call = NULL;
dca55da0 3548 mlx5_mdev_netdev_untrack(ibdev, port_num);
32f69e4b
DJ		 3549 spin_lock(&port->mp.mpi_lock);
3550
3551 comps = mpi->mdev_refcnt;
3552 if (comps) {
3553 mpi->unaffiliate = true;
3554 init_completion(&mpi->unref_comp);
3555 spin_unlock(&port->mp.mpi_lock);
3556
3557 for (i = 0; i < comps; i++)
3558 wait_for_completion(&mpi->unref_comp);
3559
3560 spin_lock(&port->mp.mpi_lock);
3561 mpi->unaffiliate = false;
3562 }
3563
3564 port->mp.mpi = NULL;
3565
32f69e4b
DJ		 3566 spin_unlock(&port->mp.mpi_lock);
3567
3568 err = mlx5_nic_vport_unaffiliate_multiport(mpi->mdev);
3569
1fb7f897 3570 mlx5_ib_dbg(ibdev, "unaffiliated port %u\n", port_num + 1);
32f69e4b
DJ		 3571 /* Log an error, still needed to cleanup the pointers and add
3572 * it back to the list.
3573 */
3574 if (err)
3575 mlx5_ib_err(ibdev, "Failed to unaffiliate port %u\n",
3576 port_num + 1);
3577
95579e78 3578 ibdev->port[port_num].roce.last_port_state = IB_PORT_DOWN;
32f69e4b
DJ		 3579}
3580
32f69e4b
DJ		 3581static bool mlx5_ib_bind_slave_port(struct mlx5_ib_dev *ibdev,
3582 struct mlx5_ib_multiport_info *mpi)
3583{
1fb7f897 3584 u32 port_num = mlx5_core_native_port_num(mpi->mdev) - 1;
0d293714 3585 u64 key;
32f69e4b
DJ		 3586 int err;
3587
9dc4cfff
LR		 3588 lockdep_assert_held(&mlx5_ib_multiport_mutex);
3589
32f69e4b
DJ		 3590 spin_lock(&ibdev->port[port_num].mp.mpi_lock);
3591 if (ibdev->port[port_num].mp.mpi) {
1fb7f897 3592 mlx5_ib_dbg(ibdev, "port %u already affiliated.\n",
2577188e 3593 port_num + 1);
32f69e4b
DJ		 3594 spin_unlock(&ibdev->port[port_num].mp.mpi_lock);
3595 return false;
3596 }
3597
3598 ibdev->port[port_num].mp.mpi = mpi;
3599 mpi->ibdev = ibdev;
df097a27 3600 mpi->mdev_events.notifier_call = NULL;
32f69e4b
DJ		 3601 spin_unlock(&ibdev->port[port_num].mp.mpi_lock);
3602
3603 err = mlx5_nic_vport_affiliate_multiport(ibdev->mdev, mpi->mdev);
3604 if (err)
3605 goto unbind;
3606
dca55da0 3607 mlx5_mdev_netdev_track(ibdev, port_num);
32f69e4b 3608
df097a27
SM		 3609 mpi->mdev_events.notifier_call = mlx5_ib_event_slave_port;
3610 mlx5_notifier_register(mpi->mdev, &mpi->mdev_events);
3611
73eb8f03 3612 mlx5_ib_init_cong_debugfs(ibdev, port_num);
a9e546e7 3613
02e7d139 3614 key = mpi->mdev->priv.adev_idx;
0d293714
PH		 3615 mlx5_core_mp_event_replay(mpi->mdev,
3616 MLX5_DRIVER_EVENT_AFFILIATION_DONE,
3617 &key);
3618 mlx5_core_mp_event_replay(ibdev->mdev,
3619 MLX5_DRIVER_EVENT_AFFILIATION_DONE,
3620 &key);
3621
a9a9e689
PH		 3622 err = mlx5_ib_enable_lb_mp(ibdev->mdev, mpi->mdev);
3623 if (err)
3624 goto unbind;
3625
32f69e4b
DJ		 3626 return true;
3627
3628unbind:
3629 mlx5_ib_unbind_slave_port(ibdev, mpi);
3630 return false;
3631}
3632
2e8e631d
YH		 3633static int mlx5_ib_data_direct_init(struct mlx5_ib_dev *dev)
3634{
3635 char vuid[MLX5_ST_SZ_BYTES(array1024_auto) + 1] = {};
3636 int ret;
3637
c77aec65
YH		 3638 if (!MLX5_CAP_GEN(dev->mdev, data_direct) ||
3639 !MLX5_CAP_GEN_2(dev->mdev, query_vuid))
2e8e631d
YH		 3640 return 0;
3641
3642 ret = mlx5_cmd_query_vuid(dev->mdev, true, vuid);
3643 if (ret)
3644 return ret;
3645
3646 ret = mlx5_ib_create_data_direct_resources(dev);
3647 if (ret)
3648 return ret;
3649
de8f847a 3650 INIT_LIST_HEAD(&dev->data_direct_mr_list);
2e8e631d
YH		 3651 ret = mlx5_data_direct_ib_reg(dev, vuid);
3652 if (ret)
3653 mlx5_ib_free_data_direct_resources(dev);
3654
3655 return ret;
3656}
3657
3658static void mlx5_ib_data_direct_cleanup(struct mlx5_ib_dev *dev)
3659{
c77aec65
YH		 3660 if (!MLX5_CAP_GEN(dev->mdev, data_direct) ||
3661 !MLX5_CAP_GEN_2(dev->mdev, query_vuid))
2e8e631d
YH		 3662 return;
3663
3664 mlx5_data_direct_ib_unreg(dev);
3665 mlx5_ib_free_data_direct_resources(dev);
3666}
3667
32f69e4b
DJ		 3668static int mlx5_ib_init_multiport_master(struct mlx5_ib_dev *dev)
3669{
1fb7f897 3670 u32 port_num = mlx5_core_native_port_num(dev->mdev) - 1;
32f69e4b
DJ		 3671 enum rdma_link_layer ll = mlx5_ib_port_link_layer(&dev->ib_dev,
3672 port_num + 1);
3673 struct mlx5_ib_multiport_info *mpi;
3674 int err;
1fb7f897 3675 u32 i;
32f69e4b
DJ		 3676
3677 if (!mlx5_core_is_mp_master(dev->mdev) || ll != IB_LINK_LAYER_ETHERNET)
3678 return 0;
3679
3680 err = mlx5_query_nic_vport_system_image_guid(dev->mdev,
3681 &dev->sys_image_guid);
3682 if (err)
3683 return err;
3684
3685 err = mlx5_nic_vport_enable_roce(dev->mdev);
3686 if (err)
3687 return err;
3688
3689 mutex_lock(&mlx5_ib_multiport_mutex);
3690 for (i = 0; i < dev->num_ports; i++) {
3691 bool bound = false;
3692
3693 /* build a stub multiport info struct for the native port. */
3694 if (i == port_num) {
3695 mpi = kzalloc(sizeof(*mpi), GFP_KERNEL);
3696 if (!mpi) {
3697 mutex_unlock(&mlx5_ib_multiport_mutex);
3698 mlx5_nic_vport_disable_roce(dev->mdev);
3699 return -ENOMEM;
3700 }
3701
3702 mpi->is_master = true;
3703 mpi->mdev = dev->mdev;
3704 mpi->sys_image_guid = dev->sys_image_guid;
3705 dev->port[i].mp.mpi = mpi;
3706 mpi->ibdev = dev;
3707 mpi = NULL;
3708 continue;
3709 }
3710
3711 list_for_each_entry(mpi, &mlx5_ib_unaffiliated_port_list,
3712 list) {
3713 if (dev->sys_image_guid == mpi->sys_image_guid &&
e05feab2
PH		 3714 (mlx5_core_native_port_num(mpi->mdev) - 1) == i &&
3715 mlx5_core_same_coredev_type(dev->mdev, mpi->mdev)) {
32f69e4b
DJ		 3716 bound = mlx5_ib_bind_slave_port(dev, mpi);
3717 }
3718
3719 if (bound) {
c42260f1
VP		 3720 dev_dbg(mpi->mdev->device,
3721 "removing port from unaffiliated list.\n");
32f69e4b
DJ		 3722 mlx5_ib_dbg(dev, "port %d bound\n", i + 1);
3723 list_del(&mpi->list);
3724 break;
3725 }
3726 }
7416790e 3727 if (!bound)
32f69e4b
DJ		 3728 mlx5_ib_dbg(dev, "no free port found for port %d\n",
3729 i + 1);
32f69e4b
DJ		 3730 }
3731
3732 list_add_tail(&dev->ib_dev_list, &mlx5_ib_dev_list);
3733 mutex_unlock(&mlx5_ib_multiport_mutex);
3734 return err;
3735}
3736
3737static void mlx5_ib_cleanup_multiport_master(struct mlx5_ib_dev *dev)
3738{
1fb7f897 3739 u32 port_num = mlx5_core_native_port_num(dev->mdev) - 1;
32f69e4b
DJ		 3740 enum rdma_link_layer ll = mlx5_ib_port_link_layer(&dev->ib_dev,
3741 port_num + 1);
1fb7f897 3742 u32 i;
32f69e4b
DJ		 3743
3744 if (!mlx5_core_is_mp_master(dev->mdev) || ll != IB_LINK_LAYER_ETHERNET)
3745 return;
3746
3747 mutex_lock(&mlx5_ib_multiport_mutex);
3748 for (i = 0; i < dev->num_ports; i++) {
3749 if (dev->port[i].mp.mpi) {
3750 /* Destroy the native port stub */
3751 if (i == port_num) {
3752 kfree(dev->port[i].mp.mpi);
3753 dev->port[i].mp.mpi = NULL;
3754 } else {
1fb7f897
MB		 3755 mlx5_ib_dbg(dev, "unbinding port_num: %u\n",
3756 i + 1);
7ce6095e
LR		 3757 list_add_tail(&dev->port[i].mp.mpi->list,
3758 &mlx5_ib_unaffiliated_port_list);
4a754d76
LR		 3759 mlx5_ib_unbind_slave_port(dev,
3760 dev->port[i].mp.mpi);
32f69e4b
DJ		 3761 }
3762 }
3763 }
3764
3765 mlx5_ib_dbg(dev, "removing from devlist\n");
3766 list_del(&dev->ib_dev_list);
3767 mutex_unlock(&mlx5_ib_multiport_mutex);
3768
3769 mlx5_nic_vport_disable_roce(dev->mdev);
3770}
3771
342ee59d
YH		 3772static int mmap_obj_cleanup(struct ib_uobject *uobject,
3773 enum rdma_remove_reason why,
3774 struct uverbs_attr_bundle *attrs)
7be76bef
YH		 3775{
3776 struct mlx5_user_mmap_entry *obj = uobject->object;
3777
3778 rdma_user_mmap_entry_remove(&obj->rdma_entry);
3779 return 0;
3780}
3781
342ee59d
YH		 3782static int mlx5_rdma_user_mmap_entry_insert(struct mlx5_ib_ucontext *c,
3783 struct mlx5_user_mmap_entry *entry,
3784 size_t length)
3785{
3786 return rdma_user_mmap_entry_insert_range(
3787 &c->ibucontext, &entry->rdma_entry, length,
3788 (MLX5_IB_MMAP_OFFSET_START << 16),
3789 ((MLX5_IB_MMAP_OFFSET_END << 16) + (1UL << 16) - 1));
3790}
3791
7be76bef
YH		 3792static struct mlx5_user_mmap_entry *
3793alloc_var_entry(struct mlx5_ib_ucontext *c)
3794{
3795 struct mlx5_user_mmap_entry *entry;
3796 struct mlx5_var_table *var_table;
3797 u32 page_idx;
3798 int err;
3799
3800 var_table = &to_mdev(c->ibucontext.device)->var_table;
3801 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
3802 if (!entry)
3803 return ERR_PTR(-ENOMEM);
3804
3805 mutex_lock(&var_table->bitmap_lock);
3806 page_idx = find_first_zero_bit(var_table->bitmap,
3807 var_table->num_var_hw_entries);
3808 if (page_idx >= var_table->num_var_hw_entries) {
3809 err = -ENOSPC;
3810 mutex_unlock(&var_table->bitmap_lock);
3811 goto end;
3812 }
3813
3814 set_bit(page_idx, var_table->bitmap);
3815 mutex_unlock(&var_table->bitmap_lock);
3816
3817 entry->address = var_table->hw_start_addr +
3818 (page_idx * var_table->stride_size);
3819 entry->page_idx = page_idx;
3820 entry->mmap_flag = MLX5_IB_MMAP_TYPE_VAR;
3821
342ee59d
YH		 3822 err = mlx5_rdma_user_mmap_entry_insert(c, entry,
3823 var_table->stride_size);
7be76bef
YH		 3824 if (err)
3825 goto err_insert;
3826
3827 return entry;
3828
3829err_insert:
3830 mutex_lock(&var_table->bitmap_lock);
3831 clear_bit(page_idx, var_table->bitmap);
3832 mutex_unlock(&var_table->bitmap_lock);
3833end:
3834 kfree(entry);
3835 return ERR_PTR(err);
3836}
3837
3838static int UVERBS_HANDLER(MLX5_IB_METHOD_VAR_OBJ_ALLOC)(
3839 struct uverbs_attr_bundle *attrs)
3840{
3841 struct ib_uobject *uobj = uverbs_attr_get_uobject(
3842 attrs, MLX5_IB_ATTR_VAR_OBJ_ALLOC_HANDLE);
3843 struct mlx5_ib_ucontext *c;
3844 struct mlx5_user_mmap_entry *entry;
3845 u64 mmap_offset;
3846 u32 length;
3847 int err;
3848
3849 c = to_mucontext(ib_uverbs_get_ucontext(attrs));
3850 if (IS_ERR(c))
3851 return PTR_ERR(c);
3852
3853 entry = alloc_var_entry(c);
3854 if (IS_ERR(entry))
3855 return PTR_ERR(entry);
3856
3857 mmap_offset = mlx5_entry_to_mmap_offset(entry);
3858 length = entry->rdma_entry.npages * PAGE_SIZE;
3859 uobj->object = entry;
0ac8903c 3860 uverbs_finalize_uobj_create(attrs, MLX5_IB_ATTR_VAR_OBJ_ALLOC_HANDLE);
7be76bef
YH		 3861
3862 err = uverbs_copy_to(attrs, MLX5_IB_ATTR_VAR_OBJ_ALLOC_MMAP_OFFSET,
3863 &mmap_offset, sizeof(mmap_offset));
3864 if (err)
0ac8903c 3865 return err;
7be76bef
YH		 3866
3867 err = uverbs_copy_to(attrs, MLX5_IB_ATTR_VAR_OBJ_ALLOC_PAGE_ID,
3868 &entry->page_idx, sizeof(entry->page_idx));
3869 if (err)
0ac8903c 3870 return err;
7be76bef
YH		 3871
3872 err = uverbs_copy_to(attrs, MLX5_IB_ATTR_VAR_OBJ_ALLOC_MMAP_LENGTH,
3873 &length, sizeof(length));
7be76bef
YH		 3874 return err;
3875}
3876
3877DECLARE_UVERBS_NAMED_METHOD(
3878 MLX5_IB_METHOD_VAR_OBJ_ALLOC,
3879 UVERBS_ATTR_IDR(MLX5_IB_ATTR_VAR_OBJ_ALLOC_HANDLE,
3880 MLX5_IB_OBJECT_VAR,
3881 UVERBS_ACCESS_NEW,
3882 UA_MANDATORY),
3883 UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_VAR_OBJ_ALLOC_PAGE_ID,
3884 UVERBS_ATTR_TYPE(u32),
3885 UA_MANDATORY),
3886 UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_VAR_OBJ_ALLOC_MMAP_LENGTH,
3887 UVERBS_ATTR_TYPE(u32),
3888 UA_MANDATORY),
3889 UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_VAR_OBJ_ALLOC_MMAP_OFFSET,
3890 UVERBS_ATTR_TYPE(u64),
3891 UA_MANDATORY));
3892
3893DECLARE_UVERBS_NAMED_METHOD_DESTROY(
3894 MLX5_IB_METHOD_VAR_OBJ_DESTROY,
3895 UVERBS_ATTR_IDR(MLX5_IB_ATTR_VAR_OBJ_DESTROY_HANDLE,
3896 MLX5_IB_OBJECT_VAR,
3897 UVERBS_ACCESS_DESTROY,
3898 UA_MANDATORY));
3899
3900DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_VAR,
342ee59d 3901 UVERBS_TYPE_ALLOC_IDR(mmap_obj_cleanup),
7be76bef
YH		 3902 &UVERBS_METHOD(MLX5_IB_METHOD_VAR_OBJ_ALLOC),
3903 &UVERBS_METHOD(MLX5_IB_METHOD_VAR_OBJ_DESTROY));
3904
3905static bool var_is_supported(struct ib_device *device)
3906{
3907 struct mlx5_ib_dev *dev = to_mdev(device);
3908
3909 return (MLX5_CAP_GEN_64(dev->mdev, general_obj_types) &
3910 MLX5_GENERAL_OBJ_TYPES_CAP_VIRTIO_NET_Q);
3911}
3912
342ee59d
YH		 3913static struct mlx5_user_mmap_entry *
3914alloc_uar_entry(struct mlx5_ib_ucontext *c,
3915 enum mlx5_ib_uapi_uar_alloc_type alloc_type)
3916{
3917 struct mlx5_user_mmap_entry *entry;
3918 struct mlx5_ib_dev *dev;
3919 u32 uar_index;
3920 int err;
3921
3922 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
3923 if (!entry)
3924 return ERR_PTR(-ENOMEM);
3925
3926 dev = to_mdev(c->ibucontext.device);
d2c8a155 3927 err = mlx5_cmd_uar_alloc(dev->mdev, &uar_index, c->devx_uid);
342ee59d
YH		 3928 if (err)
3929 goto end;
3930
3931 entry->page_idx = uar_index;
3932 entry->address = uar_index2paddress(dev, uar_index);
3933 if (alloc_type == MLX5_IB_UAPI_UAR_ALLOC_TYPE_BF)
3934 entry->mmap_flag = MLX5_IB_MMAP_TYPE_UAR_WC;
3935 else
3936 entry->mmap_flag = MLX5_IB_MMAP_TYPE_UAR_NC;
3937
3938 err = mlx5_rdma_user_mmap_entry_insert(c, entry, PAGE_SIZE);
3939 if (err)
3940 goto err_insert;
3941
3942 return entry;
3943
3944err_insert:
d2c8a155 3945 mlx5_cmd_uar_dealloc(dev->mdev, uar_index, c->devx_uid);
342ee59d
YH		 3946end:
3947 kfree(entry);
3948 return ERR_PTR(err);
3949}
3950
3951static int UVERBS_HANDLER(MLX5_IB_METHOD_UAR_OBJ_ALLOC)(
3952 struct uverbs_attr_bundle *attrs)
3953{
3954 struct ib_uobject *uobj = uverbs_attr_get_uobject(
3955 attrs, MLX5_IB_ATTR_UAR_OBJ_ALLOC_HANDLE);
3956 enum mlx5_ib_uapi_uar_alloc_type alloc_type;
3957 struct mlx5_ib_ucontext *c;
3958 struct mlx5_user_mmap_entry *entry;
3959 u64 mmap_offset;
3960 u32 length;
3961 int err;
3962
3963 c = to_mucontext(ib_uverbs_get_ucontext(attrs));
3964 if (IS_ERR(c))
3965 return PTR_ERR(c);
3966
3967 err = uverbs_get_const(&alloc_type, attrs,
3968 MLX5_IB_ATTR_UAR_OBJ_ALLOC_TYPE);
3969 if (err)
3970 return err;
3971
3972 if (alloc_type != MLX5_IB_UAPI_UAR_ALLOC_TYPE_BF &&
3973 alloc_type != MLX5_IB_UAPI_UAR_ALLOC_TYPE_NC)
3974 return -EOPNOTSUPP;
3975
d98995b4 3976 if (!mlx5_wc_support_get(to_mdev(c->ibucontext.device)->mdev) &&
342ee59d
YH		 3977 alloc_type == MLX5_IB_UAPI_UAR_ALLOC_TYPE_BF)
3978 return -EOPNOTSUPP;
3979
3980 entry = alloc_uar_entry(c, alloc_type);
3981 if (IS_ERR(entry))
3982 return PTR_ERR(entry);
3983
3984 mmap_offset = mlx5_entry_to_mmap_offset(entry);
3985 length = entry->rdma_entry.npages * PAGE_SIZE;
3986 uobj->object = entry;
0ac8903c 3987 uverbs_finalize_uobj_create(attrs, MLX5_IB_ATTR_UAR_OBJ_ALLOC_HANDLE);
342ee59d
YH		 3988
3989 err = uverbs_copy_to(attrs, MLX5_IB_ATTR_UAR_OBJ_ALLOC_MMAP_OFFSET,
3990 &mmap_offset, sizeof(mmap_offset));
3991 if (err)
0ac8903c 3992 return err;
342ee59d
YH		 3993
3994 err = uverbs_copy_to(attrs, MLX5_IB_ATTR_UAR_OBJ_ALLOC_PAGE_ID,
3995 &entry->page_idx, sizeof(entry->page_idx));
3996 if (err)
0ac8903c 3997 return err;
342ee59d
YH		 3998
3999 err = uverbs_copy_to(attrs, MLX5_IB_ATTR_UAR_OBJ_ALLOC_MMAP_LENGTH,
4000 &length, sizeof(length));
342ee59d
YH		 4001 return err;
4002}
4003
4004DECLARE_UVERBS_NAMED_METHOD(
4005 MLX5_IB_METHOD_UAR_OBJ_ALLOC,
4006 UVERBS_ATTR_IDR(MLX5_IB_ATTR_UAR_OBJ_ALLOC_HANDLE,
4007 MLX5_IB_OBJECT_UAR,
4008 UVERBS_ACCESS_NEW,
4009 UA_MANDATORY),
4010 UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_UAR_OBJ_ALLOC_TYPE,
4011 enum mlx5_ib_uapi_uar_alloc_type,
4012 UA_MANDATORY),
4013 UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_UAR_OBJ_ALLOC_PAGE_ID,
4014 UVERBS_ATTR_TYPE(u32),
4015 UA_MANDATORY),
4016 UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_UAR_OBJ_ALLOC_MMAP_LENGTH,
4017 UVERBS_ATTR_TYPE(u32),
4018 UA_MANDATORY),
4019 UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_UAR_OBJ_ALLOC_MMAP_OFFSET,
4020 UVERBS_ATTR_TYPE(u64),
4021 UA_MANDATORY));
4022
4023DECLARE_UVERBS_NAMED_METHOD_DESTROY(
4024 MLX5_IB_METHOD_UAR_OBJ_DESTROY,
4025 UVERBS_ATTR_IDR(MLX5_IB_ATTR_UAR_OBJ_DESTROY_HANDLE,
4026 MLX5_IB_OBJECT_UAR,
4027 UVERBS_ACCESS_DESTROY,
4028 UA_MANDATORY));
4029
4030DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_UAR,
4031 UVERBS_TYPE_ALLOC_IDR(mmap_obj_cleanup),
4032 &UVERBS_METHOD(MLX5_IB_METHOD_UAR_OBJ_ALLOC),
4033 &UVERBS_METHOD(MLX5_IB_METHOD_UAR_OBJ_DESTROY));
4034
0fb556b2
YH		 4035ADD_UVERBS_ATTRIBUTES_SIMPLE(
4036 mlx5_ib_query_context,
4037 UVERBS_OBJECT_DEVICE,
4038 UVERBS_METHOD_QUERY_CONTEXT,
4039 UVERBS_ATTR_PTR_OUT(
4040 MLX5_IB_ATTR_QUERY_CONTEXT_RESP_UCTX,
4041 UVERBS_ATTR_STRUCT(struct mlx5_ib_alloc_ucontext_resp,
4042 dump_fill_mkey),
4043 UA_MANDATORY));
4044
de8f847a
YH		 4045ADD_UVERBS_ATTRIBUTES_SIMPLE(
4046 mlx5_ib_reg_dmabuf_mr,
4047 UVERBS_OBJECT_MR,
4048 UVERBS_METHOD_REG_DMABUF_MR,
4049 UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_REG_DMABUF_MR_ACCESS_FLAGS,
4050 enum mlx5_ib_uapi_reg_dmabuf_flags,
4051 UA_OPTIONAL));
4052
0cbf432d 4053static const struct uapi_definition mlx5_ib_defs[] = {
36e235c8 4054 UAPI_DEF_CHAIN(mlx5_ib_devx_defs),
0cbf432d 4055 UAPI_DEF_CHAIN(mlx5_ib_flow_defs),
30f2fe40 4056 UAPI_DEF_CHAIN(mlx5_ib_qos_defs),
05f71ef9 4057 UAPI_DEF_CHAIN(mlx5_ib_std_types_defs),
831df883 4058 UAPI_DEF_CHAIN(mlx5_ib_dm_defs),
589b844f 4059 UAPI_DEF_CHAIN(mlx5_ib_create_cq_defs),
8c84660b 4060
0fb556b2 4061 UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_DEVICE, &mlx5_ib_query_context),
de8f847a 4062 UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_MR, &mlx5_ib_reg_dmabuf_mr),
7be76bef
YH		 4063 UAPI_DEF_CHAIN_OBJ_TREE_NAMED(MLX5_IB_OBJECT_VAR,
4064 UAPI_DEF_IS_OBJ_SUPPORTED(var_is_supported)),
342ee59d 4065 UAPI_DEF_CHAIN_OBJ_TREE_NAMED(MLX5_IB_OBJECT_UAR),
0cbf432d
JG		 4066 {}
4067};
8c84660b 4068
fb652d32 4069static void mlx5_ib_stage_init_cleanup(struct mlx5_ib_dev *dev)
e126ba97 4070{
2e8e631d 4071 mlx5_ib_data_direct_cleanup(dev);
32f69e4b 4072 mlx5_ib_cleanup_multiport_master(dev);
806b101b 4073 WARN_ON(!xa_empty(&dev->odp_mkeys));
ab40530a 4074 mutex_destroy(&dev->cap_mask_mutex);
50211ec9 4075 WARN_ON(!xa_empty(&dev->sig_mrs));
4056b12e 4076 WARN_ON(!bitmap_empty(dev->dm.memic_alloc_pages, MLX5_MAX_MEMIC_PAGES));
758ce14a 4077 mlx5r_macsec_dealloc_gids(dev);
16c1975f
MB		 4078}
4079
fb652d32 4080static int mlx5_ib_stage_init_init(struct mlx5_ib_dev *dev)
16c1975f
MB		 4081{
4082 struct mlx5_core_dev *mdev = dev->mdev;
758ce14a 4083 int err, i;
e126ba97 4084
13179652
PP		 4085 dev->ib_dev.node_type = RDMA_NODE_IB_CA;
4086 dev->ib_dev.local_dma_lkey = 0 /* not supported for now */;
13179652
PP		 4087 dev->ib_dev.dev.parent = mdev->device;
4088 dev->ib_dev.lag_flags = RDMA_LAG_FLAGS_HASH_ALL_SLAVES;
4089
32f69e4b
DJ		 4090 for (i = 0; i < dev->num_ports; i++) {
4091 spin_lock_init(&dev->port[i].mp.mpi_lock);
d3b5cc1c
MB		 4092 dev->port[i].roce.dev = dev;
4093 dev->port[i].roce.native_port_num = i + 1;
4094 dev->port[i].roce.last_port_state = IB_PORT_DOWN;
32f69e4b
DJ		 4095 }
4096
594cac11
OHT		 4097 err = mlx5r_cmd_query_special_mkeys(dev);
4098 if (err)
4099 return err;
4100
58dbd642 4101 err = mlx5r_macsec_init_gids_and_devlist(dev);
e126ba97 4102 if (err)
da796ccb 4103 return err;
e126ba97 4104
758ce14a
PH		 4105 err = mlx5_ib_init_multiport_master(dev);
4106 if (err)
4107 goto err;
4108
a989ea01
MB		 4109 err = set_has_smi_cap(dev);
4110 if (err)
2cb091f6 4111 goto err_mp;
e126ba97 4112
2019d70e
PP		 4113 err = mlx5_query_max_pkeys(&dev->ib_dev, &dev->pkey_table_len);
4114 if (err)
4115 goto err_mp;
4116
1b5daf11
MD		 4117 if (mlx5_use_mad_ifc(dev))
4118 get_ext_port_caps(dev);
e126ba97 4119
674dd4e2 4120 dev->ib_dev.num_comp_vectors = mlx5_comp_vectors_max(mdev);
e126ba97 4121
3cc297db 4122 mutex_init(&dev->cap_mask_mutex);
6910e366 4123 mutex_init(&dev->data_direct_lock);
3cc297db
MB		 4124 INIT_LIST_HEAD(&dev->qp_list);
4125 spin_lock_init(&dev->reset_flow_resource_lock);
806b101b 4126 xa_init(&dev->odp_mkeys);
50211ec9 4127 xa_init(&dev->sig_mrs);
f743ff3b 4128 atomic_set(&dev->mkey_var, 0);
3cc297db 4129
3b113a1e
AL		 4130 spin_lock_init(&dev->dm.lock);
4131 dev->dm.dev = mdev;
2e8e631d
YH		 4132 err = mlx5_ib_data_direct_init(dev);
4133 if (err)
4134 goto err_mp;
4135
16c1975f 4136 return 0;
32f69e4b
DJ		 4137err_mp:
4138 mlx5_ib_cleanup_multiport_master(dev);
81497c14
YH		 4139err:
4140 mlx5r_macsec_dealloc_gids(dev);
d286ac1d 4141 return err;
16c1975f
MB		 4142}
4143
026a4259
MZ		 4144static struct ib_device *mlx5_ib_add_sub_dev(struct ib_device *parent,
4145 enum rdma_nl_dev_type type,
4146 const char *name);
4147static void mlx5_ib_del_sub_dev(struct ib_device *sub_dev);
4148
96458233 4149static const struct ib_device_ops mlx5_ib_dev_ops = {
7a154142 4150 .owner = THIS_MODULE,
b9560a41 4151 .driver_id = RDMA_DRIVER_MLX5,
72c6ec18 4152 .uverbs_abi_ver = MLX5_IB_UVERBS_ABI_VERSION,
b9560a41 4153
96458233 4154 .add_gid = mlx5_ib_add_gid,
026a4259 4155 .add_sub_dev = mlx5_ib_add_sub_dev,
96458233 4156 .alloc_mr = mlx5_ib_alloc_mr,
6c984472 4157 .alloc_mr_integrity = mlx5_ib_alloc_mr_integrity,
96458233
KH		 4158 .alloc_pd = mlx5_ib_alloc_pd,
4159 .alloc_ucontext = mlx5_ib_alloc_ucontext,
4160 .attach_mcast = mlx5_ib_mcg_attach,
4161 .check_mr_status = mlx5_ib_check_mr_status,
4162 .create_ah = mlx5_ib_create_ah,
96458233 4163 .create_cq = mlx5_ib_create_cq,
96458233
KH		 4164 .create_qp = mlx5_ib_create_qp,
4165 .create_srq = mlx5_ib_create_srq,
676a80ad 4166 .create_user_ah = mlx5_ib_create_ah,
96458233
KH		 4167 .dealloc_pd = mlx5_ib_dealloc_pd,
4168 .dealloc_ucontext = mlx5_ib_dealloc_ucontext,
4169 .del_gid = mlx5_ib_del_gid,
026a4259 4170 .del_sub_dev = mlx5_ib_del_sub_dev,
96458233
KH		 4171 .dereg_mr = mlx5_ib_dereg_mr,
4172 .destroy_ah = mlx5_ib_destroy_ah,
96458233 4173 .destroy_cq = mlx5_ib_destroy_cq,
96458233
KH		 4174 .destroy_qp = mlx5_ib_destroy_qp,
4175 .destroy_srq = mlx5_ib_destroy_srq,
4176 .detach_mcast = mlx5_ib_mcg_detach,
4177 .disassociate_ucontext = mlx5_ib_disassociate_ucontext,
4178 .drain_rq = mlx5_ib_drain_rq,
4179 .drain_sq = mlx5_ib_drain_sq,
915e4af5 4180 .device_group = &mlx5_attr_group,
96458233
KH		 4181 .get_dev_fw_str = get_dev_fw_str,
4182 .get_dma_mr = mlx5_ib_get_dma_mr,
4183 .get_link_layer = mlx5_ib_port_link_layer,
4184 .map_mr_sg = mlx5_ib_map_mr_sg,
6c984472 4185 .map_mr_sg_pi = mlx5_ib_map_mr_sg_pi,
96458233 4186 .mmap = mlx5_ib_mmap,
dc2316eb 4187 .mmap_free = mlx5_ib_mmap_free,
96458233
KH		 4188 .modify_cq = mlx5_ib_modify_cq,
4189 .modify_device = mlx5_ib_modify_device,
4190 .modify_port = mlx5_ib_modify_port,
4191 .modify_qp = mlx5_ib_modify_qp,
4192 .modify_srq = mlx5_ib_modify_srq,
4193 .poll_cq = mlx5_ib_poll_cq,
029e88fd
LR		 4194 .post_recv = mlx5_ib_post_recv_nodrain,
4195 .post_send = mlx5_ib_post_send_nodrain,
96458233
KH		 4196 .post_srq_recv = mlx5_ib_post_srq_recv,
4197 .process_mad = mlx5_ib_process_mad,
4198 .query_ah = mlx5_ib_query_ah,
4199 .query_device = mlx5_ib_query_device,
4200 .query_gid = mlx5_ib_query_gid,
4201 .query_pkey = mlx5_ib_query_pkey,
4202 .query_qp = mlx5_ib_query_qp,
4203 .query_srq = mlx5_ib_query_srq,
0fb556b2 4204 .query_ucontext = mlx5_ib_query_ucontext,
96458233 4205 .reg_user_mr = mlx5_ib_reg_user_mr,
90da7dc8 4206 .reg_user_mr_dmabuf = mlx5_ib_reg_user_mr_dmabuf,
96458233
KH		 4207 .req_notify_cq = mlx5_ib_arm_cq,
4208 .rereg_user_mr = mlx5_ib_rereg_user_mr,
4209 .resize_cq = mlx5_ib_resize_cq,
7c891a4d 4210 .ufile_hw_cleanup = mlx5_ib_ufile_hw_cleanup,
d3456914
LR		 4211
4212 INIT_RDMA_OBJ_SIZE(ib_ah, mlx5_ib_ah, ibah),
3b023e1b 4213 INIT_RDMA_OBJ_SIZE(ib_counters, mlx5_ib_mcounters, ibcntrs),
e39afe3d 4214 INIT_RDMA_OBJ_SIZE(ib_cq, mlx5_ib_cq, ibcq),
21a428a0 4215 INIT_RDMA_OBJ_SIZE(ib_pd, mlx5_ib_pd, ibpd),
514aee66 4216 INIT_RDMA_OBJ_SIZE(ib_qp, mlx5_ib_qp, ibqp),
68e326de 4217 INIT_RDMA_OBJ_SIZE(ib_srq, mlx5_ib_srq, ibsrq),
a2a074ef 4218 INIT_RDMA_OBJ_SIZE(ib_ucontext, mlx5_ib_ucontext, ibucontext),
96458233
KH		 4219};
4220
96458233
KH		 4221static const struct ib_device_ops mlx5_ib_dev_ipoib_enhanced_ops = {
4222 .rdma_netdev_get_params = mlx5_ib_rn_get_params,
4223};
4224
4225static const struct ib_device_ops mlx5_ib_dev_sriov_ops = {
4226 .get_vf_config = mlx5_ib_get_vf_config,
9c0015ef 4227 .get_vf_guid = mlx5_ib_get_vf_guid,
96458233
KH		 4228 .get_vf_stats = mlx5_ib_get_vf_stats,
4229 .set_vf_guid = mlx5_ib_set_vf_guid,
4230 .set_vf_link_state = mlx5_ib_set_vf_link_state,
4231};
4232
4233static const struct ib_device_ops mlx5_ib_dev_mw_ops = {
4234 .alloc_mw = mlx5_ib_alloc_mw,
4235 .dealloc_mw = mlx5_ib_dealloc_mw,
d18bb3e1
LR		 4236
4237 INIT_RDMA_OBJ_SIZE(ib_mw, mlx5_ib_mw, ibmw),
96458233
KH		 4238};
4239
4240static const struct ib_device_ops mlx5_ib_dev_xrc_ops = {
4241 .alloc_xrcd = mlx5_ib_alloc_xrcd,
4242 .dealloc_xrcd = mlx5_ib_dealloc_xrcd,
28ad5f65
LR		 4243
4244 INIT_RDMA_OBJ_SIZE(ib_xrcd, mlx5_ib_xrcd, ibxrcd),
96458233
KH		 4245};
4246
f164be8c
YH		 4247static int mlx5_ib_init_var_table(struct mlx5_ib_dev *dev)
4248{
4249 struct mlx5_core_dev *mdev = dev->mdev;
4250 struct mlx5_var_table *var_table = &dev->var_table;
4251 u8 log_doorbell_bar_size;
4252 u8 log_doorbell_stride;
4253 u64 bar_size;
4254
4255 log_doorbell_bar_size = MLX5_CAP_DEV_VDPA_EMULATION(mdev,
4256 log_doorbell_bar_size);
4257 log_doorbell_stride = MLX5_CAP_DEV_VDPA_EMULATION(mdev,
4258 log_doorbell_stride);
4259 var_table->hw_start_addr = dev->mdev->bar_addr +
4260 MLX5_CAP64_DEV_VDPA_EMULATION(mdev,
4261 doorbell_bar_offset);
4262 bar_size = (1ULL << log_doorbell_bar_size) * 4096;
4263 var_table->stride_size = 1ULL << log_doorbell_stride;
91b74bf5
AL		 4264 var_table->num_var_hw_entries = div_u64(bar_size,
4265 var_table->stride_size);
f164be8c
YH		 4266 mutex_init(&var_table->bitmap_lock);
4267 var_table->bitmap = bitmap_zalloc(var_table->num_var_hw_entries,
4268 GFP_KERNEL);
4269 return (var_table->bitmap) ? 0 : -ENOMEM;
4270}
4271
cf7174e8
CM		 4272static void mlx5_ib_cleanup_ucaps(struct mlx5_ib_dev *dev)
4273{
4274 if (MLX5_CAP_GEN(dev->mdev, uctx_cap) & MLX5_UCTX_CAP_RDMA_CTRL)
4275 ib_remove_ucap(RDMA_UCAP_MLX5_CTRL_LOCAL);
4276
4277 if (MLX5_CAP_GEN(dev->mdev, uctx_cap) &
4278 MLX5_UCTX_CAP_RDMA_CTRL_OTHER_VHCA)
4279 ib_remove_ucap(RDMA_UCAP_MLX5_CTRL_OTHER_VHCA);
4280}
4281
4282static int mlx5_ib_init_ucaps(struct mlx5_ib_dev *dev)
4283{
4284 int ret;
4285
4286 if (MLX5_CAP_GEN(dev->mdev, uctx_cap) & MLX5_UCTX_CAP_RDMA_CTRL) {
4287 ret = ib_create_ucap(RDMA_UCAP_MLX5_CTRL_LOCAL);
4288 if (ret)
4289 return ret;
4290 }
4291
4292 if (MLX5_CAP_GEN(dev->mdev, uctx_cap) &
4293 MLX5_UCTX_CAP_RDMA_CTRL_OTHER_VHCA) {
4294 ret = ib_create_ucap(RDMA_UCAP_MLX5_CTRL_OTHER_VHCA);
4295 if (ret)
4296 goto remove_local;
4297 }
4298
4299 return 0;
4300
4301remove_local:
4302 if (MLX5_CAP_GEN(dev->mdev, uctx_cap) & MLX5_UCTX_CAP_RDMA_CTRL)
4303 ib_remove_ucap(RDMA_UCAP_MLX5_CTRL_LOCAL);
4304 return ret;
4305}
4306
f164be8c
YH		 4307static void mlx5_ib_stage_caps_cleanup(struct mlx5_ib_dev *dev)
4308{
cf7174e8
CM		 4309 if (MLX5_CAP_GEN_2_64(dev->mdev, general_obj_types_127_64) &
4310 MLX5_HCA_CAP_2_GENERAL_OBJECT_TYPES_RDMA_CTRL)
4311 mlx5_ib_cleanup_ucaps(dev);
4312
f164be8c
YH		 4313 bitmap_free(dev->var_table.bitmap);
4314}
4315
fb652d32 4316static int mlx5_ib_stage_caps_init(struct mlx5_ib_dev *dev)
16c1975f
MB		 4317{
4318 struct mlx5_core_dev *mdev = dev->mdev;
16c1975f
MB		 4319 int err;
4320
f6a8a19b
DD		 4321 if (MLX5_CAP_GEN(mdev, ipoib_enhanced_offloads) &&
4322 IS_ENABLED(CONFIG_MLX5_CORE_IPOIB))
96458233
KH		 4323 ib_set_device_ops(&dev->ib_dev,
4324 &mlx5_ib_dev_ipoib_enhanced_ops);
8e959601 4325
96458233
KH		 4326 if (mlx5_core_is_pf(mdev))
4327 ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_sriov_ops);
7c2344c3 4328
6e8484c5
MG		 4329 dev->umr_fence = mlx5_get_umr_fence(MLX5_CAP_GEN(mdev, umr_fence));
4330
44ce37bc 4331 if (MLX5_CAP_GEN(mdev, imaicl))
96458233 4332 ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_mw_ops);
d2370e0a 4333
44ce37bc 4334 if (MLX5_CAP_GEN(mdev, xrc))
96458233 4335 ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_xrc_ops);
e126ba97 4336
25c13324
AL		 4337 if (MLX5_CAP_DEV_MEM(mdev, memic) ||
4338 MLX5_CAP_GEN_64(dev->mdev, general_obj_types) &
4339 MLX5_GENERAL_OBJ_TYPES_CAP_SW_ICM)
96458233 4340 ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_dm_ops);
24da0016 4341
96458233 4342 ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_ops);
81e30880 4343
36e235c8
JG		 4344 if (IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS))
4345 dev->ib_dev.driver_def = mlx5_ib_defs;
81e30880 4346
e126ba97
EC		 4347 err = init_node_data(dev);
4348 if (err)
16c1975f 4349 return err;
e126ba97 4350
c8b89924 4351 if ((MLX5_CAP_GEN(dev->mdev, port_type) == MLX5_CAP_PORT_TYPE_ETH) &&
e7996a9a
JG		 4352 (MLX5_CAP_GEN(dev->mdev, disable_local_lb_uc) ||
4353 MLX5_CAP_GEN(dev->mdev, disable_local_lb_mc)))
a560f1d9 4354 mutex_init(&dev->lb.mutex);
c8b89924 4355
f164be8c
YH		 4356 if (MLX5_CAP_GEN_64(dev->mdev, general_obj_types) &
4357 MLX5_GENERAL_OBJ_TYPES_CAP_VIRTIO_NET_Q) {
4358 err = mlx5_ib_init_var_table(dev);
4359 if (err)
4360 return err;
4361 }
4362
cf7174e8
CM		 4363 if (MLX5_CAP_GEN_2_64(dev->mdev, general_obj_types_127_64) &
4364 MLX5_HCA_CAP_2_GENERAL_OBJECT_TYPES_RDMA_CTRL) {
4365 err = mlx5_ib_init_ucaps(dev);
4366 if (err)
4367 return err;
4368 }
4369
96e2fd73
LR		 4370 dev->ib_dev.use_cq_dim = true;
4371
16c1975f
MB		 4372 return 0;
4373}
4374
96458233
KH		 4375static const struct ib_device_ops mlx5_ib_dev_port_ops = {
4376 .get_port_immutable = mlx5_port_immutable,
4377 .query_port = mlx5_ib_query_port,
4378};
4379
8e6efa3a
MB		 4380static int mlx5_ib_stage_non_default_cb(struct mlx5_ib_dev *dev)
4381{
96458233 4382 ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_port_ops);
8e6efa3a
MB		 4383 return 0;
4384}
4385
96458233
KH		 4386static const struct ib_device_ops mlx5_ib_dev_port_rep_ops = {
4387 .get_port_immutable = mlx5_port_rep_immutable,
4388 .query_port = mlx5_ib_rep_query_port,
d6fd59e1 4389 .query_pkey = mlx5_ib_rep_query_pkey,
96458233
KH		 4390};
4391
b5a498ba 4392static int mlx5_ib_stage_raw_eth_non_default_cb(struct mlx5_ib_dev *dev)
8e6efa3a 4393{
96458233 4394 ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_port_rep_ops);
8e6efa3a
MB		 4395 return 0;
4396}
4397
96458233
KH		 4398static const struct ib_device_ops mlx5_ib_dev_common_roce_ops = {
4399 .create_rwq_ind_table = mlx5_ib_create_rwq_ind_table,
4400 .create_wq = mlx5_ib_create_wq,
4401 .destroy_rwq_ind_table = mlx5_ib_destroy_rwq_ind_table,
4402 .destroy_wq = mlx5_ib_destroy_wq,
96458233 4403 .modify_wq = mlx5_ib_modify_wq,
c0a6b5ec
LR		 4404
4405 INIT_RDMA_OBJ_SIZE(ib_rwq_ind_table, mlx5_ib_rwq_ind_table,
4406 ib_rwq_ind_tbl),
96458233
KH		 4407};
4408
1e2b5a90 4409static int mlx5_ib_roce_init(struct mlx5_ib_dev *dev)
16c1975f
MB		 4410{
4411 struct mlx5_core_dev *mdev = dev->mdev;
4412 enum rdma_link_layer ll;
4413 int port_type_cap;
1fb7f897 4414 u32 port_num = 0;
16c1975f
MB		 4415 int err;
4416
4417 port_type_cap = MLX5_CAP_GEN(mdev, port_type);
4418 ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap);
4419
fc24fc5e 4420 if (ll == IB_LINK_LAYER_ETHERNET) {
1e2b5a90
LR		 4421 ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_common_roce_ops);
4422
4423 port_num = mlx5_core_native_port_num(dev->mdev) - 1;
4424
4425 /* Register only for native ports */
dca55da0 4426 mlx5_mdev_netdev_track(dev, port_num);
7fd8aefb 4427
e3f1ed1f 4428 err = mlx5_enable_eth(dev);
fc24fc5e 4429 if (err)
8e6efa3a 4430 goto cleanup;
fc24fc5e
AS		 4431 }
4432
16c1975f 4433 return 0;
8e6efa3a 4434cleanup:
dca55da0 4435 mlx5_mdev_netdev_untrack(dev, port_num);
8e6efa3a 4436 return err;
16c1975f 4437}
e126ba97 4438
1e2b5a90 4439static void mlx5_ib_roce_cleanup(struct mlx5_ib_dev *dev)
16c1975f
MB		 4440{
4441 struct mlx5_core_dev *mdev = dev->mdev;
4442 enum rdma_link_layer ll;
4443 int port_type_cap;
1fb7f897 4444 u32 port_num;
e126ba97 4445
16c1975f
MB		 4446 port_type_cap = MLX5_CAP_GEN(mdev, port_type);
4447 ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap);
4448
4449 if (ll == IB_LINK_LAYER_ETHERNET) {
c446d9da 4450 mlx5_disable_eth(dev);
5e1e7612 4451
1e2b5a90 4452 port_num = mlx5_core_native_port_num(dev->mdev) - 1;
dca55da0 4453 mlx5_mdev_netdev_untrack(dev, port_num);
5e1e7612 4454 }
16c1975f
MB		 4455}
4456
4457static int mlx5_ib_stage_cong_debugfs_init(struct mlx5_ib_dev *dev)
4458{
73eb8f03
GKH		 4459 mlx5_ib_init_cong_debugfs(dev,
4460 mlx5_core_native_port_num(dev->mdev) - 1);
4461 return 0;
16c1975f
MB		 4462}
4463
4464static void mlx5_ib_stage_cong_debugfs_cleanup(struct mlx5_ib_dev *dev)
4465{
a9e546e7
PP		 4466 mlx5_ib_cleanup_cong_debugfs(dev,
4467 mlx5_core_native_port_num(dev->mdev) - 1);
16c1975f
MB		 4468}
4469
fb652d32 4470static int mlx5_ib_stage_bfrag_init(struct mlx5_ib_dev *dev)
16c1975f
MB		 4471{
4472 int err;
5fe9dec0
EC		 4473
4474 err = mlx5_alloc_bfreg(dev->mdev, &dev->bfreg, false, false);
4475 if (err)
16c1975f 4476 return err;
5fe9dec0
EC		 4477
4478 err = mlx5_alloc_bfreg(dev->mdev, &dev->fp_bfreg, false, true);
4479 if (err)
1c3aa6bd 4480 mlx5_free_bfreg(dev->mdev, &dev->bfreg);
5fe9dec0 4481
16c1975f
MB		 4482 return err;
4483}
0837e86a 4484
fb652d32 4485static void mlx5_ib_stage_bfrag_cleanup(struct mlx5_ib_dev *dev)
16c1975f
MB		 4486{
4487 mlx5_free_bfreg(dev->mdev, &dev->fp_bfreg);
4488 mlx5_free_bfreg(dev->mdev, &dev->bfreg);
4489}
e126ba97 4490
fb652d32 4491static int mlx5_ib_stage_ib_reg_init(struct mlx5_ib_dev *dev)
16c1975f 4492{
e349f858
JG		 4493 const char *name;
4494
af48f954 4495 if (dev->sub_dev_name) {
026a4259 4496 name = dev->sub_dev_name;
af48f954
MZ		 4497 ib_mark_name_assigned_by_user(&dev->ib_dev);
4498 } else if (!mlx5_lag_is_active(dev->mdev))
e349f858
JG		 4499 name = "mlx5_%d";
4500 else
4501 name = "mlx5_bond_%d";
e0477b34 4502 return ib_register_device(&dev->ib_dev, name, &dev->mdev->pdev->dev);
16c1975f
MB		 4503}
4504
fb652d32 4505static void mlx5_ib_stage_pre_ib_reg_umr_cleanup(struct mlx5_ib_dev *dev)
16c1975f 4506{
85f9e38a 4507 mlx5_mkey_cache_cleanup(dev);
04876c12 4508 mlx5r_umr_resource_cleanup(dev);
63842011 4509 mlx5r_umr_cleanup(dev);
16c1975f
MB		 4510}
4511
fb652d32 4512static void mlx5_ib_stage_ib_reg_cleanup(struct mlx5_ib_dev *dev)
16c1975f 4513{
42cea83f 4514 ib_unregister_device(&dev->ib_dev);
16c1975f
MB		 4515}
4516
fb652d32 4517static int mlx5_ib_stage_post_ib_reg_umr_init(struct mlx5_ib_dev *dev)
16c1975f 4518{
1e2b5a90
LR		 4519 int ret;
4520
63842011 4521 ret = mlx5r_umr_init(dev);
04876c12
AL		 4522 if (ret)
4523 return ret;
1e2b5a90 4524
01137808 4525 ret = mlx5_mkey_cache_init(dev);
2ef422f0 4526 if (ret)
1e2b5a90 4527 mlx5_ib_warn(dev, "mr cache init failed %d\n", ret);
1e2b5a90 4528 return ret;
16c1975f
MB		 4529}
4530
4531static int mlx5_ib_stage_delay_drop_init(struct mlx5_ib_dev *dev)
4532{
1e2b5a90
LR		 4533 struct dentry *root;
4534
4535 if (!(dev->ib_dev.attrs.raw_packet_caps & IB_RAW_PACKET_CAP_DELAY_DROP))
4536 return 0;
4537
4538 mutex_init(&dev->delay_drop.lock);
4539 dev->delay_drop.dev = dev;
4540 dev->delay_drop.activate = false;
4541 dev->delay_drop.timeout = MLX5_MAX_DELAY_DROP_TIMEOUT_MS * 1000;
4542 INIT_WORK(&dev->delay_drop.delay_drop_work, delay_drop_handler);
4543 atomic_set(&dev->delay_drop.rqs_cnt, 0);
4544 atomic_set(&dev->delay_drop.events_cnt, 0);
4545
4546 if (!mlx5_debugfs_root)
4547 return 0;
03404e8a 4548
66771a1c 4549 root = debugfs_create_dir("delay_drop", mlx5_debugfs_get_dev_root(dev->mdev));
1e2b5a90 4550 dev->delay_drop.dir_debugfs = root;
03404e8a 4551
1e2b5a90
LR		 4552 debugfs_create_atomic_t("num_timeout_events", 0400, root,
4553 &dev->delay_drop.events_cnt);
4554 debugfs_create_atomic_t("num_rqs", 0400, root,
4555 &dev->delay_drop.rqs_cnt);
4556 debugfs_create_file("timeout", 0600, root, &dev->delay_drop,
4557 &fops_delay_drop_timeout);
16c1975f
MB		 4558 return 0;
4559}
4560
4561static void mlx5_ib_stage_delay_drop_cleanup(struct mlx5_ib_dev *dev)
4562{
1e2b5a90
LR		 4563 if (!(dev->ib_dev.attrs.raw_packet_caps & IB_RAW_PACKET_CAP_DELAY_DROP))
4564 return;
4565
4566 cancel_work_sync(&dev->delay_drop.delay_drop_work);
4567 if (!dev->delay_drop.dir_debugfs)
4568 return;
4569
4570 debugfs_remove_recursive(dev->delay_drop.dir_debugfs);
4571 dev->delay_drop.dir_debugfs = NULL;
16c1975f
MB		 4572}
4573
df097a27
SM		 4574static int mlx5_ib_stage_dev_notifier_init(struct mlx5_ib_dev *dev)
4575{
ede132a5
PH		 4576 struct mlx5_ib_resources *devr = &dev->devr;
4577 int port;
4578
4579 for (port = 0; port < ARRAY_SIZE(devr->ports); ++port)
4580 INIT_WORK(&devr->ports[port].pkey_change_work,
4581 pkey_change_handler);
4582
df097a27
SM		 4583 dev->mdev_events.notifier_call = mlx5_ib_event;
4584 mlx5_notifier_register(dev->mdev, &dev->mdev_events);
58dbd642
PH		 4585
4586 mlx5r_macsec_event_register(dev);
4587
df097a27
SM		 4588 return 0;
4589}
4590
4591static void mlx5_ib_stage_dev_notifier_cleanup(struct mlx5_ib_dev *dev)
4592{
ede132a5
PH		 4593 struct mlx5_ib_resources *devr = &dev->devr;
4594 int port;
4595
58dbd642 4596 mlx5r_macsec_event_unregister(dev);
df097a27 4597 mlx5_notifier_unregister(dev->mdev, &dev->mdev_events);
ede132a5
PH		 4598
4599 for (port = 0; port < ARRAY_SIZE(devr->ports); ++port)
4600 cancel_work_sync(&devr->ports[port].pkey_change_work);
df097a27
SM		 4601}
4602
6910e366
YH		 4603void mlx5_ib_data_direct_bind(struct mlx5_ib_dev *ibdev,
4604 struct mlx5_data_direct_dev *dev)
4605{
4606 mutex_lock(&ibdev->data_direct_lock);
4607 ibdev->data_direct_dev = dev;
4608 mutex_unlock(&ibdev->data_direct_lock);
4609}
4610
4611void mlx5_ib_data_direct_unbind(struct mlx5_ib_dev *ibdev)
4612{
4613 mutex_lock(&ibdev->data_direct_lock);
de8f847a 4614 mlx5_ib_revoke_data_direct_mrs(ibdev);
6910e366
YH		 4615 ibdev->data_direct_dev = NULL;
4616 mutex_unlock(&ibdev->data_direct_lock);
4617}
4618
b5ca15ad
MB		 4619void __mlx5_ib_remove(struct mlx5_ib_dev *dev,
4620 const struct mlx5_ib_profile *profile,
4621 int stage)
16c1975f 4622{
4cca96a8
PP		 4623 dev->ib_active = false;
4624
16c1975f
MB		 4625 /* Number of stages to cleanup */
4626 while (stage) {
4627 stage--;
4628 if (profile->stage[stage].cleanup)
4629 profile->stage[stage].cleanup(dev);
4630 }
4a6dc855 4631
da796ccb 4632 kfree(dev->port);
4a6dc855 4633 ib_dealloc_device(&dev->ib_dev);
16c1975f 4634}
e126ba97 4635
93f82444
LR		 4636int __mlx5_ib_add(struct mlx5_ib_dev *dev,
4637 const struct mlx5_ib_profile *profile)
16c1975f 4638{
16c1975f
MB		 4639 int err;
4640 int i;
5fe9dec0 4641
8d93efb8
MB		 4642 dev->profile = profile;
4643
16c1975f
MB		 4644 for (i = 0; i < MLX5_IB_STAGE_MAX; i++) {
4645 if (profile->stage[i].init) {
4646 err = profile->stage[i].init(dev);
4647 if (err)
4648 goto err_out;
4649 }
4650 }
0837e86a 4651
16c1975f 4652 dev->ib_active = true;
93f82444 4653 return 0;
e126ba97 4654
16c1975f 4655err_out:
93f82444
LR		 4656 /* Clean up stages which were initialized */
4657 while (i) {
4658 i--;
4659 if (profile->stage[i].cleanup)
4660 profile->stage[i].cleanup(dev);
4661 }
4662 return -ENOMEM;
16c1975f 4663}
0837e86a 4664
16c1975f
MB		 4665static const struct mlx5_ib_profile pf_profile = {
4666 STAGE_CREATE(MLX5_IB_STAGE_INIT,
4667 mlx5_ib_stage_init_init,
4668 mlx5_ib_stage_init_cleanup),
f7c4ffda
LR		 4669 STAGE_CREATE(MLX5_IB_STAGE_FS,
4670 mlx5_ib_fs_init,
4671 mlx5_ib_fs_cleanup),
16c1975f
MB		 4672 STAGE_CREATE(MLX5_IB_STAGE_CAPS,
4673 mlx5_ib_stage_caps_init,
f164be8c 4674 mlx5_ib_stage_caps_cleanup),
8e6efa3a
MB		 4675 STAGE_CREATE(MLX5_IB_STAGE_NON_DEFAULT_CB,
4676 mlx5_ib_stage_non_default_cb,
4677 NULL),
16c1975f 4678 STAGE_CREATE(MLX5_IB_STAGE_ROCE,
1e2b5a90
LR		 4679 mlx5_ib_roce_init,
4680 mlx5_ib_roce_cleanup),
333fbaa0
LR		 4681 STAGE_CREATE(MLX5_IB_STAGE_QP,
4682 mlx5_init_qp_table,
4683 mlx5_cleanup_qp_table),
f3da6577
LR		 4684 STAGE_CREATE(MLX5_IB_STAGE_SRQ,
4685 mlx5_init_srq_table,
4686 mlx5_cleanup_srq_table),
16c1975f 4687 STAGE_CREATE(MLX5_IB_STAGE_DEVICE_RESOURCES,
1e2b5a90
LR		 4688 mlx5_ib_dev_res_init,
4689 mlx5_ib_dev_res_cleanup),
16c1975f 4690 STAGE_CREATE(MLX5_IB_STAGE_ODP,
1e2b5a90
LR		 4691 mlx5_ib_odp_init_one,
4692 mlx5_ib_odp_cleanup_one),
16c1975f 4693 STAGE_CREATE(MLX5_IB_STAGE_COUNTERS,
64825827
LR		 4694 mlx5_ib_counters_init,
4695 mlx5_ib_counters_cleanup),
16c1975f
MB		 4696 STAGE_CREATE(MLX5_IB_STAGE_CONG_DEBUGFS,
4697 mlx5_ib_stage_cong_debugfs_init,
4698 mlx5_ib_stage_cong_debugfs_cleanup),
16c1975f
MB		 4699 STAGE_CREATE(MLX5_IB_STAGE_BFREG,
4700 mlx5_ib_stage_bfrag_init,
4701 mlx5_ib_stage_bfrag_cleanup),
42cea83f
MB		 4702 STAGE_CREATE(MLX5_IB_STAGE_PRE_IB_REG_UMR,
4703 NULL,
4704 mlx5_ib_stage_pre_ib_reg_umr_cleanup),
81773ce5 4705 STAGE_CREATE(MLX5_IB_STAGE_WHITELIST_UID,
d8b7515e
LR		 4706 mlx5_ib_devx_init,
4707 mlx5_ib_devx_cleanup),
16c1975f
MB		 4708 STAGE_CREATE(MLX5_IB_STAGE_IB_REG,
4709 mlx5_ib_stage_ib_reg_init,
4710 mlx5_ib_stage_ib_reg_cleanup),
ede132a5
PH		 4711 STAGE_CREATE(MLX5_IB_STAGE_DEVICE_NOTIFIER,
4712 mlx5_ib_stage_dev_notifier_init,
4713 mlx5_ib_stage_dev_notifier_cleanup),
42cea83f
MB		 4714 STAGE_CREATE(MLX5_IB_STAGE_POST_IB_REG_UMR,
4715 mlx5_ib_stage_post_ib_reg_umr_init,
4716 NULL),
16c1975f
MB		 4717 STAGE_CREATE(MLX5_IB_STAGE_DELAY_DROP,
4718 mlx5_ib_stage_delay_drop_init,
4719 mlx5_ib_stage_delay_drop_cleanup),
b572ebe6
LR		 4720 STAGE_CREATE(MLX5_IB_STAGE_RESTRACK,
4721 mlx5_ib_restrack_init,
4722 NULL),
16c1975f 4723};
e126ba97 4724
b5a498ba 4725const struct mlx5_ib_profile raw_eth_profile = {
b5ca15ad
MB		 4726 STAGE_CREATE(MLX5_IB_STAGE_INIT,
4727 mlx5_ib_stage_init_init,
4728 mlx5_ib_stage_init_cleanup),
f7c4ffda
LR		 4729 STAGE_CREATE(MLX5_IB_STAGE_FS,
4730 mlx5_ib_fs_init,
4731 mlx5_ib_fs_cleanup),
b5ca15ad
MB		 4732 STAGE_CREATE(MLX5_IB_STAGE_CAPS,
4733 mlx5_ib_stage_caps_init,
f164be8c 4734 mlx5_ib_stage_caps_cleanup),
b5ca15ad 4735 STAGE_CREATE(MLX5_IB_STAGE_NON_DEFAULT_CB,
b5a498ba 4736 mlx5_ib_stage_raw_eth_non_default_cb,
b5ca15ad
MB		 4737 NULL),
4738 STAGE_CREATE(MLX5_IB_STAGE_ROCE,
1e2b5a90
LR		 4739 mlx5_ib_roce_init,
4740 mlx5_ib_roce_cleanup),
333fbaa0
LR		 4741 STAGE_CREATE(MLX5_IB_STAGE_QP,
4742 mlx5_init_qp_table,
4743 mlx5_cleanup_qp_table),
f3da6577
LR		 4744 STAGE_CREATE(MLX5_IB_STAGE_SRQ,
4745 mlx5_init_srq_table,
4746 mlx5_cleanup_srq_table),
b5ca15ad 4747 STAGE_CREATE(MLX5_IB_STAGE_DEVICE_RESOURCES,
1e2b5a90
LR		 4748 mlx5_ib_dev_res_init,
4749 mlx5_ib_dev_res_cleanup),
b5ca15ad 4750 STAGE_CREATE(MLX5_IB_STAGE_COUNTERS,
64825827
LR		 4751 mlx5_ib_counters_init,
4752 mlx5_ib_counters_cleanup),
79db784e
PP		 4753 STAGE_CREATE(MLX5_IB_STAGE_CONG_DEBUGFS,
4754 mlx5_ib_stage_cong_debugfs_init,
4755 mlx5_ib_stage_cong_debugfs_cleanup),
b5ca15ad
MB		 4756 STAGE_CREATE(MLX5_IB_STAGE_BFREG,
4757 mlx5_ib_stage_bfrag_init,
4758 mlx5_ib_stage_bfrag_cleanup),
03fe2deb
DM		 4759 STAGE_CREATE(MLX5_IB_STAGE_PRE_IB_REG_UMR,
4760 NULL,
4761 mlx5_ib_stage_pre_ib_reg_umr_cleanup),
7f575103 4762 STAGE_CREATE(MLX5_IB_STAGE_WHITELIST_UID,
d8b7515e
LR		 4763 mlx5_ib_devx_init,
4764 mlx5_ib_devx_cleanup),
b5ca15ad
MB		 4765 STAGE_CREATE(MLX5_IB_STAGE_IB_REG,
4766 mlx5_ib_stage_ib_reg_init,
4767 mlx5_ib_stage_ib_reg_cleanup),
ede132a5
PH		 4768 STAGE_CREATE(MLX5_IB_STAGE_DEVICE_NOTIFIER,
4769 mlx5_ib_stage_dev_notifier_init,
4770 mlx5_ib_stage_dev_notifier_cleanup),
03fe2deb
DM		 4771 STAGE_CREATE(MLX5_IB_STAGE_POST_IB_REG_UMR,
4772 mlx5_ib_stage_post_ib_reg_umr_init,
4773 NULL),
ee4d269e
MS		 4774 STAGE_CREATE(MLX5_IB_STAGE_DELAY_DROP,
4775 mlx5_ib_stage_delay_drop_init,
4776 mlx5_ib_stage_delay_drop_cleanup),
b572ebe6
LR		 4777 STAGE_CREATE(MLX5_IB_STAGE_RESTRACK,
4778 mlx5_ib_restrack_init,
4779 NULL),
b5ca15ad
MB		 4780};
4781
026a4259
MZ		 4782static const struct mlx5_ib_profile plane_profile = {
4783 STAGE_CREATE(MLX5_IB_STAGE_INIT,
4784 mlx5_ib_stage_init_init,
4785 mlx5_ib_stage_init_cleanup),
4786 STAGE_CREATE(MLX5_IB_STAGE_CAPS,
4787 mlx5_ib_stage_caps_init,
4788 mlx5_ib_stage_caps_cleanup),
4789 STAGE_CREATE(MLX5_IB_STAGE_NON_DEFAULT_CB,
4790 mlx5_ib_stage_non_default_cb,
4791 NULL),
4792 STAGE_CREATE(MLX5_IB_STAGE_QP,
4793 mlx5_init_qp_table,
4794 mlx5_cleanup_qp_table),
4795 STAGE_CREATE(MLX5_IB_STAGE_SRQ,
4796 mlx5_init_srq_table,
4797 mlx5_cleanup_srq_table),
4798 STAGE_CREATE(MLX5_IB_STAGE_DEVICE_RESOURCES,
4799 mlx5_ib_dev_res_init,
4800 mlx5_ib_dev_res_cleanup),
4801 STAGE_CREATE(MLX5_IB_STAGE_BFREG,
4802 mlx5_ib_stage_bfrag_init,
4803 mlx5_ib_stage_bfrag_cleanup),
4804 STAGE_CREATE(MLX5_IB_STAGE_IB_REG,
4805 mlx5_ib_stage_ib_reg_init,
4806 mlx5_ib_stage_ib_reg_cleanup),
4807};
4808
4809static struct ib_device *mlx5_ib_add_sub_dev(struct ib_device *parent,
4810 enum rdma_nl_dev_type type,
4811 const char *name)
4812{
4813 struct mlx5_ib_dev *mparent = to_mdev(parent), *mplane;
4814 enum rdma_link_layer ll;
4815 int ret;
4816
4817 if (mparent->smi_dev)
4818 return ERR_PTR(-EEXIST);
4819
4820 ll = mlx5_port_type_cap_to_rdma_ll(MLX5_CAP_GEN(mparent->mdev,
4821 port_type));
4822 if (type != RDMA_DEVICE_TYPE_SMI || !mparent->num_plane ||
4823 ll != IB_LINK_LAYER_INFINIBAND ||
4824 !MLX5_CAP_GEN_2(mparent->mdev, multiplane_qp_ud))
4825 return ERR_PTR(-EOPNOTSUPP);
4826
4827 mplane = ib_alloc_device(mlx5_ib_dev, ib_dev);
4828 if (!mplane)
4829 return ERR_PTR(-ENOMEM);
4830
4831 mplane->port = kcalloc(mparent->num_plane * mparent->num_ports,
4832 sizeof(*mplane->port), GFP_KERNEL);
4833 if (!mplane->port) {
4834 ret = -ENOMEM;
4835 goto fail_kcalloc;
4836 }
4837
4838 mplane->ib_dev.type = type;
4839 mplane->mdev = mparent->mdev;
4840 mplane->num_ports = mparent->num_plane;
4841 mplane->sub_dev_name = name;
91b4b2c6 4842 mplane->ib_dev.phys_port_cnt = mplane->num_ports;
026a4259
MZ		 4843
4844 ret = __mlx5_ib_add(mplane, &plane_profile);
4845 if (ret)
4846 goto fail_ib_add;
4847
4848 mparent->smi_dev = mplane;
4849 return &mplane->ib_dev;
4850
4851fail_ib_add:
4852 kfree(mplane->port);
4853fail_kcalloc:
4854 ib_dealloc_device(&mplane->ib_dev);
4855 return ERR_PTR(ret);
4856}
4857
4858static void mlx5_ib_del_sub_dev(struct ib_device *sub_dev)
4859{
4860 struct mlx5_ib_dev *mdev = to_mdev(sub_dev);
4861
4862 to_mdev(sub_dev->parent)->smi_dev = NULL;
4863 __mlx5_ib_remove(mdev, mdev->profile, MLX5_IB_STAGE_MAX);
4864}
4865
93f82444
LR		 4866static int mlx5r_mp_probe(struct auxiliary_device *adev,
4867 const struct auxiliary_device_id *id)
32f69e4b 4868{
93f82444
LR		 4869 struct mlx5_adev *idev = container_of(adev, struct mlx5_adev, adev);
4870 struct mlx5_core_dev *mdev = idev->mdev;
32f69e4b
DJ		 4871 struct mlx5_ib_multiport_info *mpi;
4872 struct mlx5_ib_dev *dev;
4873 bool bound = false;
4874 int err;
4875
4876 mpi = kzalloc(sizeof(*mpi), GFP_KERNEL);
4877 if (!mpi)
93f82444 4878 return -ENOMEM;
32f69e4b
DJ		 4879
4880 mpi->mdev = mdev;
32f69e4b
DJ		 4881 err = mlx5_query_nic_vport_system_image_guid(mdev,
4882 &mpi->sys_image_guid);
4883 if (err) {
4884 kfree(mpi);
93f82444 4885 return err;
32f69e4b
DJ		 4886 }
4887
4888 mutex_lock(&mlx5_ib_multiport_mutex);
4889 list_for_each_entry(dev, &mlx5_ib_dev_list, ib_dev_list) {
e05feab2
PH		 4890 if (dev->sys_image_guid == mpi->sys_image_guid &&
4891 mlx5_core_same_coredev_type(dev->mdev, mpi->mdev))
32f69e4b
DJ		 4892 bound = mlx5_ib_bind_slave_port(dev, mpi);
4893
4894 if (bound) {
4895 rdma_roce_rescan_device(&dev->ib_dev);
97f30d32 4896 mpi->ibdev->ib_active = true;
32f69e4b
DJ		 4897 break;
4898 }
4899 }
4900
4901 if (!bound) {
4902 list_add_tail(&mpi->list, &mlx5_ib_unaffiliated_port_list);
c42260f1
VP		 4903 dev_dbg(mdev->device,
4904 "no suitable IB device found to bind to, added to unaffiliated list.\n");
32f69e4b
DJ		 4905 }
4906 mutex_unlock(&mlx5_ib_multiport_mutex);
4907
27963d3d 4908 auxiliary_set_drvdata(adev, mpi);
93f82444
LR		 4909 return 0;
4910}
4911
4912static void mlx5r_mp_remove(struct auxiliary_device *adev)
4913{
4914 struct mlx5_ib_multiport_info *mpi;
4915
27963d3d 4916 mpi = auxiliary_get_drvdata(adev);
93f82444
LR		 4917 mutex_lock(&mlx5_ib_multiport_mutex);
4918 if (mpi->ibdev)
4919 mlx5_ib_unbind_slave_port(mpi->ibdev, mpi);
da78fe5f
MG		 4920 else
4921 list_del(&mpi->list);
93f82444
LR		 4922 mutex_unlock(&mlx5_ib_multiport_mutex);
4923 kfree(mpi);
32f69e4b
DJ		 4924}
4925
93f82444
LR		 4926static int mlx5r_probe(struct auxiliary_device *adev,
4927 const struct auxiliary_device_id *id)
16c1975f 4928{
93f82444
LR		 4929 struct mlx5_adev *idev = container_of(adev, struct mlx5_adev, adev);
4930 struct mlx5_core_dev *mdev = idev->mdev;
94de879c 4931 const struct mlx5_ib_profile *profile;
93f82444 4932 int port_type_cap, num_ports, ret;
32f69e4b 4933 enum rdma_link_layer ll;
b5ca15ad 4934 struct mlx5_ib_dev *dev;
f0666f1f 4935
32f69e4b
DJ		 4936 port_type_cap = MLX5_CAP_GEN(mdev, port_type);
4937 ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap);
4938
da796ccb
MB		 4939 num_ports = max(MLX5_CAP_GEN(mdev, num_ports),
4940 MLX5_CAP_GEN(mdev, num_vhca_ports));
459cc69f 4941 dev = ib_alloc_device(mlx5_ib_dev, ib_dev);
b5ca15ad 4942 if (!dev)
93f82444 4943 return -ENOMEM;
2a5db20f
MZ		 4944
4945 if (ll == IB_LINK_LAYER_INFINIBAND) {
4946 ret = mlx5_ib_get_plane_num(mdev, &dev->num_plane);
4947 if (ret)
4948 goto fail;
4949 }
4950
da796ccb
MB		 4951 dev->port = kcalloc(num_ports, sizeof(*dev->port),
4952 GFP_KERNEL);
4953 if (!dev->port) {
2a5db20f
MZ		 4954 ret = -ENOMEM;
4955 goto fail;
da796ccb 4956 }
b5ca15ad
MB		 4957
4958 dev->mdev = mdev;
da796ccb 4959 dev->num_ports = num_ports;
91b4b2c6 4960 dev->ib_dev.phys_port_cnt = num_ports;
b5ca15ad 4961
9ca05b0f 4962 if (ll == IB_LINK_LAYER_ETHERNET && !mlx5_get_roce_state(mdev))
94de879c
MG		 4963 profile = &raw_eth_profile;
4964 else
4965 profile = &pf_profile;
4966
93f82444 4967 ret = __mlx5_ib_add(dev, profile);
2a5db20f
MZ		 4968 if (ret)
4969 goto fail_ib_add;
93f82444 4970
27963d3d 4971 auxiliary_set_drvdata(adev, dev);
93f82444 4972 return 0;
2a5db20f
MZ		 4973
4974fail_ib_add:
4975 kfree(dev->port);
4976fail:
4977 ib_dealloc_device(&dev->ib_dev);
4978 return ret;
e126ba97
EC		 4979}
4980
93f82444 4981static void mlx5r_remove(struct auxiliary_device *adev)
e126ba97 4982{
32f69e4b
DJ		 4983 struct mlx5_ib_dev *dev;
4984
27963d3d 4985 dev = auxiliary_get_drvdata(adev);
f0666f1f 4986 __mlx5_ib_remove(dev, dev->profile, MLX5_IB_STAGE_MAX);
e126ba97
EC		 4987}
4988
93f82444
LR		 4989static const struct auxiliary_device_id mlx5r_mp_id_table[] = {
4990 { .name = MLX5_ADEV_NAME ".multiport", },
4991 {},
4992};
4993
4994static const struct auxiliary_device_id mlx5r_id_table[] = {
4995 { .name = MLX5_ADEV_NAME ".rdma", },
4996 {},
4997};
4998
4999MODULE_DEVICE_TABLE(auxiliary, mlx5r_mp_id_table);
5000MODULE_DEVICE_TABLE(auxiliary, mlx5r_id_table);
5001
5002static struct auxiliary_driver mlx5r_mp_driver = {
5003 .name = "multiport",
5004 .probe = mlx5r_mp_probe,
5005 .remove = mlx5r_mp_remove,
5006 .id_table = mlx5r_mp_id_table,
5007};
5008
5009static struct auxiliary_driver mlx5r_driver = {
5010 .name = "rdma",
5011 .probe = mlx5r_probe,
5012 .remove = mlx5r_remove,
5013 .id_table = mlx5r_id_table,
e126ba97
EC		 5014};
5015
5016static int __init mlx5_ib_init(void)
5017{
93f82444 5018 int ret;
6aec21f6 5019
8010d74b 5020 xlt_emergency_page = (void *)__get_free_page(GFP_KERNEL);
c44ef998
IL		 5021 if (!xlt_emergency_page)
5022 return -ENOMEM;
5023
d69a24e0 5024 mlx5_ib_event_wq = alloc_ordered_workqueue("mlx5_ib_event_wq", 0);
c44ef998 5025 if (!mlx5_ib_event_wq) {
8010d74b 5026 free_page((unsigned long)xlt_emergency_page);
d69a24e0 5027 return -ENOMEM;
c44ef998 5028 }
d69a24e0 5029
312b8f79
MZ		 5030 ret = mlx5_ib_qp_event_init();
5031 if (ret)
5032 goto qp_event_err;
5033
81713d37 5034 mlx5_ib_odp_init();
93f82444
LR		 5035 ret = mlx5r_rep_init();
5036 if (ret)
5037 goto rep_err;
6910e366
YH		 5038 ret = mlx5_data_direct_driver_register();
5039 if (ret)
5040 goto dd_err;
93f82444
LR		 5041 ret = auxiliary_driver_register(&mlx5r_mp_driver);
5042 if (ret)
5043 goto mp_err;
5044 ret = auxiliary_driver_register(&mlx5r_driver);
5045 if (ret)
5046 goto drv_err;
6910e366 5047
93f82444 5048 return 0;
9603b61d 5049
93f82444
LR		 5050drv_err:
5051 auxiliary_driver_unregister(&mlx5r_mp_driver);
5052mp_err:
6910e366
YH		 5053 mlx5_data_direct_driver_unregister();
5054dd_err:
93f82444
LR		 5055 mlx5r_rep_cleanup();
5056rep_err:
312b8f79
MZ		 5057 mlx5_ib_qp_event_cleanup();
5058qp_event_err:
93f82444
LR		 5059 destroy_workqueue(mlx5_ib_event_wq);
5060 free_page((unsigned long)xlt_emergency_page);
5061 return ret;
e126ba97
EC		 5062}
5063
5064static void __exit mlx5_ib_cleanup(void)
5065{
6910e366 5066 mlx5_data_direct_driver_unregister();
93f82444
LR		 5067 auxiliary_driver_unregister(&mlx5r_driver);
5068 auxiliary_driver_unregister(&mlx5r_mp_driver);
5069 mlx5r_rep_cleanup();
5070
312b8f79 5071 mlx5_ib_qp_event_cleanup();
d69a24e0 5072 destroy_workqueue(mlx5_ib_event_wq);
8010d74b 5073 free_page((unsigned long)xlt_emergency_page);
e126ba97
EC		 5074}
5075
5076module_init(mlx5_ib_init);
5077module_exit(mlx5_ib_cleanup);
Linux 6.x block layer and io_uring tree(s)