drivers/infiniband/hw/mlx5/main.c

   1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
   2 /*
   3  * Copyright (c) 2013-2020, Mellanox Technologies inc. All rights reserved.
   4  * Copyright (c) 2020, Intel Corporation. All rights reserved.
   5  */
   6
   7 #include <linux/debugfs.h>
   8 #include <linux/highmem.h>
   9 #include <linux/module.h>
  10 #include <linux/init.h>
  11 #include <linux/errno.h>
  12 #include <linux/pci.h>
  13 #include <linux/dma-mapping.h>
  14 #include <linux/slab.h>
  15 #include <linux/bitmap.h>
  16 #include <linux/sched.h>
  17 #include <linux/sched/mm.h>
  18 #include <linux/sched/task.h>
  19 #include <linux/delay.h>
  20 #include <rdma/ib_user_verbs.h>
  21 #include <rdma/ib_addr.h>
  22 #include <rdma/ib_cache.h>
  23 #include <linux/mlx5/port.h>
  24 #include <linux/mlx5/vport.h>
  25 #include <linux/mlx5/fs.h>
  26 #include <linux/mlx5/eswitch.h>
  27 #include <linux/mlx5/driver.h>
  28 #include <linux/list.h>
  29 #include <rdma/ib_smi.h>
  30 #include <rdma/ib_umem_odp.h>
  31 #include <rdma/lag.h>
  32 #include <linux/in.h>
  33 #include <linux/etherdevice.h>
  34 #include "mlx5_ib.h"
  35 #include "ib_rep.h"
  36 #include "cmd.h"
  37 #include "devx.h"
  38 #include "dm.h"
  39 #include "fs.h"
  40 #include "srq.h"
  41 #include "qp.h"
  42 #include "wr.h"
  43 #include "restrack.h"
  44 #include "counters.h"
  45 #include "umr.h"
  46 #include <rdma/uverbs_std_types.h>
  47 #include <rdma/uverbs_ioctl.h>
  48 #include <rdma/mlx5_user_ioctl_verbs.h>
  49 #include <rdma/mlx5_user_ioctl_cmds.h>
  50 #include <rdma/ib_ucaps.h>
  51 #include "macsec.h"
  52 #include "data_direct.h"
  53
  54 #define UVERBS_MODULE_NAME mlx5_ib
  55 #include <rdma/uverbs_named_ioctl.h>
  56
  57 MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>");
  58 MODULE_DESCRIPTION("Mellanox 5th generation network adapters (ConnectX series) IB driver");
  59 MODULE_LICENSE("Dual BSD/GPL");
  60
  61 struct mlx5_ib_event_work {
  62         struct work_struct      work;
  63         union {
  64                 struct mlx5_ib_dev            *dev;
  65                 struct mlx5_ib_multiport_info *mpi;
  66         };
  67         bool                    is_slave;
  68         unsigned int            event;
  69         void                    *param;
  70 };
  71
  72 enum {
  73         MLX5_ATOMIC_SIZE_QP_8BYTES = 1 << 3,
  74 };
  75
  76 static struct workqueue_struct *mlx5_ib_event_wq;
  77 static LIST_HEAD(mlx5_ib_unaffiliated_port_list);
  78 static LIST_HEAD(mlx5_ib_dev_list);
  79 /*
  80  * This mutex should be held when accessing either of the above lists
  81  */
  82 static DEFINE_MUTEX(mlx5_ib_multiport_mutex);
  83
  84 struct mlx5_ib_dev *mlx5_ib_get_ibdev_from_mpi(struct mlx5_ib_multiport_info *mpi)
  85 {
  86         struct mlx5_ib_dev *dev;
  87
  88         mutex_lock(&mlx5_ib_multiport_mutex);
  89         dev = mpi->ibdev;
  90         mutex_unlock(&mlx5_ib_multiport_mutex);
  91         return dev;
  92 }
  93
  94 static enum rdma_link_layer
  95 mlx5_port_type_cap_to_rdma_ll(int port_type_cap)
  96 {
  97         switch (port_type_cap) {
  98         case MLX5_CAP_PORT_TYPE_IB:
  99                 return IB_LINK_LAYER_INFINIBAND;
 100         case MLX5_CAP_PORT_TYPE_ETH:
 101                 return IB_LINK_LAYER_ETHERNET;
 102         default:
 103                 return IB_LINK_LAYER_UNSPECIFIED;
 104         }
 105 }
 106
 107 static enum rdma_link_layer
 108 mlx5_ib_port_link_layer(struct ib_device *device, u32 port_num)
 109 {
 110         struct mlx5_ib_dev *dev = to_mdev(device);
 111         int port_type_cap = MLX5_CAP_GEN(dev->mdev, port_type);
 112
 113         return mlx5_port_type_cap_to_rdma_ll(port_type_cap);
 114 }
 115
 116 static int get_port_state(struct ib_device *ibdev,
 117                           u32 port_num,
 118                           enum ib_port_state *state)
 119 {
 120         struct ib_port_attr attr;
 121         int ret;
 122
 123         memset(&attr, 0, sizeof(attr));
 124         ret = ibdev->ops.query_port(ibdev, port_num, &attr);
 125         if (!ret)
 126                 *state = attr.state;
 127         return ret;
 128 }
 129
 130 static struct mlx5_roce *mlx5_get_rep_roce(struct mlx5_ib_dev *dev,
 131                                            struct net_device *ndev,
 132                                            struct net_device *upper,
 133                                            u32 *port_num)
 134 {
 135         struct net_device *rep_ndev;
 136         struct mlx5_ib_port *port;
 137         int i;
 138
 139         for (i = 0; i < dev->num_ports; i++) {
 140                 port  = &dev->port[i];
 141                 if (!port->rep)
 142                         continue;
 143
 144                 if (upper == ndev && port->rep->vport == MLX5_VPORT_UPLINK) {
 145                         *port_num = i + 1;
 146                         return &port->roce;
 147                 }
 148
 149                 if (upper && port->rep->vport == MLX5_VPORT_UPLINK)
 150                         continue;
 151                 rep_ndev = ib_device_get_netdev(&dev->ib_dev, i + 1);
 152                 if (rep_ndev && rep_ndev == ndev) {
 153                         dev_put(rep_ndev);
 154                         *port_num = i + 1;
 155                         return &port->roce;
 156                 }
 157
 158                 dev_put(rep_ndev);
 159         }
 160
 161         return NULL;
 162 }
 163
 164 static bool mlx5_netdev_send_event(struct mlx5_ib_dev *dev,
 165                                    struct net_device *ndev,
 166                                    struct net_device *upper,
 167                                    struct net_device *ib_ndev)
 168 {
 169         if (!dev->ib_active)
 170                 return false;
 171
 172         /* Event is about our upper device */
 173         if (upper == ndev)
 174                 return true;
 175
 176         /* RDMA device is not in lag and not in switchdev */
 177         if (!dev->is_rep && !upper && ndev == ib_ndev)
 178                 return true;
 179
 180         /* RDMA devie is in switchdev */
 181         if (dev->is_rep && ndev == ib_ndev)
 182                 return true;
 183
 184         return false;
 185 }
 186
 187 static struct net_device *mlx5_ib_get_rep_uplink_netdev(struct mlx5_ib_dev *ibdev)
 188 {
 189         struct mlx5_ib_port *port;
 190         int i;
 191
 192         for (i = 0; i < ibdev->num_ports; i++) {
 193                 port = &ibdev->port[i];
 194                 if (port->rep && port->rep->vport == MLX5_VPORT_UPLINK) {
 195                         return ib_device_get_netdev(&ibdev->ib_dev, i + 1);
 196                 }
 197         }
 198
 199         return NULL;
 200 }
 201
 202 static int mlx5_netdev_event(struct notifier_block *this,
 203                              unsigned long event, void *ptr)
 204 {
 205         struct mlx5_roce *roce = container_of(this, struct mlx5_roce, nb);
 206         struct net_device *ndev = netdev_notifier_info_to_dev(ptr);
 207         u32 port_num = roce->native_port_num;
 208         struct net_device *ib_ndev = NULL;
 209         struct mlx5_core_dev *mdev;
 210         struct mlx5_ib_dev *ibdev;
 211
 212         ibdev = roce->dev;
 213         mdev = mlx5_ib_get_native_port_mdev(ibdev, port_num, NULL);
 214         if (!mdev)
 215                 return NOTIFY_DONE;
 216
 217         switch (event) {
 218         case NETDEV_REGISTER:
 219                 /* Should already be registered during the load */
 220                 if (ibdev->is_rep)
 221                         break;
 222
 223                 ib_ndev = ib_device_get_netdev(&ibdev->ib_dev, port_num);
 224                 /* Exit if already registered */
 225                 if (ib_ndev)
 226                         goto put_ndev;
 227
 228                 if (ndev->dev.parent == mdev->device)
 229                         ib_device_set_netdev(&ibdev->ib_dev, ndev, port_num);
 230                 break;
 231
 232         case NETDEV_UNREGISTER:
 233                 /* In case of reps, ib device goes away before the netdevs */
 234                 if (ibdev->is_rep)
 235                         break;
 236                 ib_ndev = ib_device_get_netdev(&ibdev->ib_dev, port_num);
 237                 if (ib_ndev == ndev)
 238                         ib_device_set_netdev(&ibdev->ib_dev, NULL, port_num);
 239                 goto put_ndev;
 240
 241         case NETDEV_CHANGE:
 242         case NETDEV_UP:
 243         case NETDEV_DOWN: {
 244                 struct net_device *upper = NULL;
 245
 246                 if (!netif_is_lag_master(ndev) && !netif_is_lag_port(ndev) &&
 247                     !mlx5_core_mp_enabled(mdev))
 248                         return NOTIFY_DONE;
 249
 250                 if (mlx5_lag_is_roce(mdev) || mlx5_lag_is_sriov(mdev)) {
 251                         struct net_device *lag_ndev;
 252
 253                         if(mlx5_lag_is_roce(mdev))
 254                                 lag_ndev = ib_device_get_netdev(&ibdev->ib_dev, 1);
 255                         else /* sriov lag */
 256                                 lag_ndev = mlx5_ib_get_rep_uplink_netdev(ibdev);
 257
 258                         if (lag_ndev) {
 259                                 upper = netdev_master_upper_dev_get(lag_ndev);
 260                                 dev_put(lag_ndev);
 261                         } else {
 262                                 goto done;
 263                         }
 264                 }
 265
 266                 if (ibdev->is_rep)
 267                         roce = mlx5_get_rep_roce(ibdev, ndev, upper, &port_num);
 268                 if (!roce)
 269                         return NOTIFY_DONE;
 270
 271                 ib_ndev = ib_device_get_netdev(&ibdev->ib_dev, port_num);
 272
 273                 if (mlx5_netdev_send_event(ibdev, ndev, upper, ib_ndev)) {
 274                         struct ib_event ibev = { };
 275                         enum ib_port_state port_state;
 276
 277                         if (get_port_state(&ibdev->ib_dev, port_num,
 278                                            &port_state))
 279                                 goto put_ndev;
 280
 281                         if (roce->last_port_state == port_state)
 282                                 goto put_ndev;
 283
 284                         roce->last_port_state = port_state;
 285                         ibev.device = &ibdev->ib_dev;
 286                         if (port_state == IB_PORT_DOWN)
 287                                 ibev.event = IB_EVENT_PORT_ERR;
 288                         else if (port_state == IB_PORT_ACTIVE)
 289                                 ibev.event = IB_EVENT_PORT_ACTIVE;
 290                         else
 291                                 goto put_ndev;
 292
 293                         ibev.element.port_num = port_num;
 294                         ib_dispatch_event(&ibev);
 295                 }
 296                 break;
 297         }
 298
 299         default:
 300                 break;
 301         }
 302 put_ndev:
 303         dev_put(ib_ndev);
 304 done:
 305         mlx5_ib_put_native_port_mdev(ibdev, port_num);
 306         return NOTIFY_DONE;
 307 }
 308
 309 struct mlx5_core_dev *mlx5_ib_get_native_port_mdev(struct mlx5_ib_dev *ibdev,
 310                                                    u32 ib_port_num,
 311                                                    u32 *native_port_num)
 312 {
 313         enum rdma_link_layer ll = mlx5_ib_port_link_layer(&ibdev->ib_dev,
 314                                                           ib_port_num);
 315         struct mlx5_core_dev *mdev = NULL;
 316         struct mlx5_ib_multiport_info *mpi;
 317         struct mlx5_ib_port *port;
 318
 319         if (ibdev->ib_dev.type == RDMA_DEVICE_TYPE_SMI) {
 320                 if (native_port_num)
 321                         *native_port_num = smi_to_native_portnum(ibdev,
 322                                                                  ib_port_num);
 323                 return ibdev->mdev;
 324
 325         }
 326
 327         if (!mlx5_core_mp_enabled(ibdev->mdev) ||
 328             ll != IB_LINK_LAYER_ETHERNET) {
 329                 if (native_port_num)
 330                         *native_port_num = ib_port_num;
 331                 return ibdev->mdev;
 332         }
 333
 334         if (native_port_num)
 335                 *native_port_num = 1;
 336
 337         port = &ibdev->port[ib_port_num - 1];
 338         spin_lock(&port->mp.mpi_lock);
 339         mpi = ibdev->port[ib_port_num - 1].mp.mpi;
 340         if (mpi && !mpi->unaffiliate) {
 341                 mdev = mpi->mdev;
 342                 /* If it's the master no need to refcount, it'll exist
 343                  * as long as the ib_dev exists.
 344                  */
 345                 if (!mpi->is_master)
 346                         mpi->mdev_refcnt++;
 347         }
 348         spin_unlock(&port->mp.mpi_lock);
 349
 350         return mdev;
 351 }
 352
 353 void mlx5_ib_put_native_port_mdev(struct mlx5_ib_dev *ibdev, u32 port_num)
 354 {
 355         enum rdma_link_layer ll = mlx5_ib_port_link_layer(&ibdev->ib_dev,
 356                                                           port_num);
 357         struct mlx5_ib_multiport_info *mpi;
 358         struct mlx5_ib_port *port;
 359
 360         if (!mlx5_core_mp_enabled(ibdev->mdev) || ll != IB_LINK_LAYER_ETHERNET)
 361                 return;
 362
 363         port = &ibdev->port[port_num - 1];
 364
 365         spin_lock(&port->mp.mpi_lock);
 366         mpi = ibdev->port[port_num - 1].mp.mpi;
 367         if (mpi->is_master)
 368                 goto out;
 369
 370         mpi->mdev_refcnt--;
 371         if (mpi->unaffiliate)
 372                 complete(&mpi->unref_comp);
 373 out:
 374         spin_unlock(&port->mp.mpi_lock);
 375 }
 376
 377 static int translate_eth_legacy_proto_oper(u32 eth_proto_oper,
 378                                            u16 *active_speed, u8 *active_width)
 379 {
 380         switch (eth_proto_oper) {
 381         case MLX5E_PROT_MASK(MLX5E_1000BASE_CX_SGMII):
 382         case MLX5E_PROT_MASK(MLX5E_1000BASE_KX):
 383         case MLX5E_PROT_MASK(MLX5E_100BASE_TX):
 384         case MLX5E_PROT_MASK(MLX5E_1000BASE_T):
 385                 *active_width = IB_WIDTH_1X;
 386                 *active_speed = IB_SPEED_SDR;
 387                 break;
 388         case MLX5E_PROT_MASK(MLX5E_10GBASE_T):
 389         case MLX5E_PROT_MASK(MLX5E_10GBASE_CX4):
 390         case MLX5E_PROT_MASK(MLX5E_10GBASE_KX4):
 391         case MLX5E_PROT_MASK(MLX5E_10GBASE_KR):
 392         case MLX5E_PROT_MASK(MLX5E_10GBASE_CR):
 393         case MLX5E_PROT_MASK(MLX5E_10GBASE_SR):
 394         case MLX5E_PROT_MASK(MLX5E_10GBASE_ER):
 395                 *active_width = IB_WIDTH_1X;
 396                 *active_speed = IB_SPEED_QDR;
 397                 break;
 398         case MLX5E_PROT_MASK(MLX5E_25GBASE_CR):
 399         case MLX5E_PROT_MASK(MLX5E_25GBASE_KR):
 400         case MLX5E_PROT_MASK(MLX5E_25GBASE_SR):
 401                 *active_width = IB_WIDTH_1X;
 402                 *active_speed = IB_SPEED_EDR;
 403                 break;
 404         case MLX5E_PROT_MASK(MLX5E_40GBASE_CR4):
 405         case MLX5E_PROT_MASK(MLX5E_40GBASE_KR4):
 406         case MLX5E_PROT_MASK(MLX5E_40GBASE_SR4):
 407         case MLX5E_PROT_MASK(MLX5E_40GBASE_LR4):
 408                 *active_width = IB_WIDTH_4X;
 409                 *active_speed = IB_SPEED_QDR;
 410                 break;
 411         case MLX5E_PROT_MASK(MLX5E_50GBASE_CR2):
 412         case MLX5E_PROT_MASK(MLX5E_50GBASE_KR2):
 413         case MLX5E_PROT_MASK(MLX5E_50GBASE_SR2):
 414                 *active_width = IB_WIDTH_1X;
 415                 *active_speed = IB_SPEED_HDR;
 416                 break;
 417         case MLX5E_PROT_MASK(MLX5E_56GBASE_R4):
 418                 *active_width = IB_WIDTH_4X;
 419                 *active_speed = IB_SPEED_FDR;
 420                 break;
 421         case MLX5E_PROT_MASK(MLX5E_100GBASE_CR4):
 422         case MLX5E_PROT_MASK(MLX5E_100GBASE_SR4):
 423         case MLX5E_PROT_MASK(MLX5E_100GBASE_KR4):
 424         case MLX5E_PROT_MASK(MLX5E_100GBASE_LR4):
 425                 *active_width = IB_WIDTH_4X;
 426                 *active_speed = IB_SPEED_EDR;
 427                 break;
 428         default:
 429                 return -EINVAL;
 430         }
 431
 432         return 0;
 433 }
 434
 435 static int translate_eth_ext_proto_oper(u32 eth_proto_oper, u16 *active_speed,
 436                                         u8 *active_width)
 437 {
 438         switch (eth_proto_oper) {
 439         case MLX5E_PROT_MASK(MLX5E_SGMII_100M):
 440         case MLX5E_PROT_MASK(MLX5E_1000BASE_X_SGMII):
 441                 *active_width = IB_WIDTH_1X;
 442                 *active_speed = IB_SPEED_SDR;
 443                 break;
 444         case MLX5E_PROT_MASK(MLX5E_5GBASE_R):
 445                 *active_width = IB_WIDTH_1X;
 446                 *active_speed = IB_SPEED_DDR;
 447                 break;
 448         case MLX5E_PROT_MASK(MLX5E_10GBASE_XFI_XAUI_1):
 449                 *active_width = IB_WIDTH_1X;
 450                 *active_speed = IB_SPEED_QDR;
 451                 break;
 452         case MLX5E_PROT_MASK(MLX5E_40GBASE_XLAUI_4_XLPPI_4):
 453                 *active_width = IB_WIDTH_4X;
 454                 *active_speed = IB_SPEED_QDR;
 455                 break;
 456         case MLX5E_PROT_MASK(MLX5E_25GAUI_1_25GBASE_CR_KR):
 457                 *active_width = IB_WIDTH_1X;
 458                 *active_speed = IB_SPEED_EDR;
 459                 break;
 460         case MLX5E_PROT_MASK(MLX5E_50GAUI_2_LAUI_2_50GBASE_CR2_KR2):
 461                 *active_width = IB_WIDTH_2X;
 462                 *active_speed = IB_SPEED_EDR;
 463                 break;
 464         case MLX5E_PROT_MASK(MLX5E_50GAUI_1_LAUI_1_50GBASE_CR_KR):
 465                 *active_width = IB_WIDTH_1X;
 466                 *active_speed = IB_SPEED_HDR;
 467                 break;
 468         case MLX5E_PROT_MASK(MLX5E_CAUI_4_100GBASE_CR4_KR4):
 469                 *active_width = IB_WIDTH_4X;
 470                 *active_speed = IB_SPEED_EDR;
 471                 break;
 472         case MLX5E_PROT_MASK(MLX5E_100GAUI_2_100GBASE_CR2_KR2):
 473                 *active_width = IB_WIDTH_2X;
 474                 *active_speed = IB_SPEED_HDR;
 475                 break;
 476         case MLX5E_PROT_MASK(MLX5E_100GAUI_1_100GBASE_CR_KR):
 477                 *active_width = IB_WIDTH_1X;
 478                 *active_speed = IB_SPEED_NDR;
 479                 break;
 480         case MLX5E_PROT_MASK(MLX5E_200GAUI_4_200GBASE_CR4_KR4):
 481                 *active_width = IB_WIDTH_4X;
 482                 *active_speed = IB_SPEED_HDR;
 483                 break;
 484         case MLX5E_PROT_MASK(MLX5E_200GAUI_2_200GBASE_CR2_KR2):
 485                 *active_width = IB_WIDTH_2X;
 486                 *active_speed = IB_SPEED_NDR;
 487                 break;
 488         case MLX5E_PROT_MASK(MLX5E_200GAUI_1_200GBASE_CR1_KR1):
 489                 *active_width = IB_WIDTH_1X;
 490                 *active_speed = IB_SPEED_XDR;
 491                 break;
 492         case MLX5E_PROT_MASK(MLX5E_400GAUI_8_400GBASE_CR8):
 493                 *active_width = IB_WIDTH_8X;
 494                 *active_speed = IB_SPEED_HDR;
 495                 break;
 496         case MLX5E_PROT_MASK(MLX5E_400GAUI_4_400GBASE_CR4_KR4):
 497                 *active_width = IB_WIDTH_4X;
 498                 *active_speed = IB_SPEED_NDR;
 499                 break;
 500         case MLX5E_PROT_MASK(MLX5E_400GAUI_2_400GBASE_CR2_KR2):
 501                 *active_width = IB_WIDTH_2X;
 502                 *active_speed = IB_SPEED_XDR;
 503                 break;
 504         case MLX5E_PROT_MASK(MLX5E_800GAUI_8_800GBASE_CR8_KR8):
 505                 *active_width = IB_WIDTH_8X;
 506                 *active_speed = IB_SPEED_NDR;
 507                 break;
 508         case MLX5E_PROT_MASK(MLX5E_800GAUI_4_800GBASE_CR4_KR4):
 509                 *active_width = IB_WIDTH_4X;
 510                 *active_speed = IB_SPEED_XDR;
 511                 break;
 512         default:
 513                 return -EINVAL;
 514         }
 515
 516         return 0;
 517 }
 518
 519 static int translate_eth_proto_oper(u32 eth_proto_oper, u16 *active_speed,
 520                                     u8 *active_width, bool ext)
 521 {
 522         return ext ?
 523                 translate_eth_ext_proto_oper(eth_proto_oper, active_speed,
 524                                              active_width) :
 525                 translate_eth_legacy_proto_oper(eth_proto_oper, active_speed,
 526                                                 active_width);
 527 }
 528
 529 static int mlx5_query_port_roce(struct ib_device *device, u32 port_num,
 530                                 struct ib_port_attr *props)
 531 {
 532         struct mlx5_ib_dev *dev = to_mdev(device);
 533         u32 out[MLX5_ST_SZ_DW(ptys_reg)] = {0};
 534         struct mlx5_core_dev *mdev;
 535         struct net_device *ndev, *upper;
 536         enum ib_mtu ndev_ib_mtu;
 537         bool put_mdev = true;
 538         u32 eth_prot_oper;
 539         u32 mdev_port_num;
 540         bool ext;
 541         int err;
 542
 543         mdev = mlx5_ib_get_native_port_mdev(dev, port_num, &mdev_port_num);
 544         if (!mdev) {
 545                 /* This means the port isn't affiliated yet. Get the
 546                  * info for the master port instead.
 547                  */
 548                 put_mdev = false;
 549                 mdev = dev->mdev;
 550                 mdev_port_num = 1;
 551                 port_num = 1;
 552         }
 553
 554         /* Possible bad flows are checked before filling out props so in case
 555          * of an error it will still be zeroed out.
 556          * Use native port in case of reps
 557          */
 558         if (dev->is_rep)
 559                 err = mlx5_query_port_ptys(mdev, out, sizeof(out), MLX5_PTYS_EN,
 560                                            1, 0);
 561         else
 562                 err = mlx5_query_port_ptys(mdev, out, sizeof(out), MLX5_PTYS_EN,
 563                                            mdev_port_num, 0);
 564         if (err)
 565                 goto out;
 566         ext = !!MLX5_GET_ETH_PROTO(ptys_reg, out, true, eth_proto_capability);
 567         eth_prot_oper = MLX5_GET_ETH_PROTO(ptys_reg, out, ext, eth_proto_oper);
 568
 569         props->active_width     = IB_WIDTH_4X;
 570         props->active_speed     = IB_SPEED_QDR;
 571
 572         translate_eth_proto_oper(eth_prot_oper, &props->active_speed,
 573                                  &props->active_width, ext);
 574
 575         if (!dev->is_rep && dev->mdev->roce.roce_en) {
 576                 u16 qkey_viol_cntr;
 577
 578                 props->port_cap_flags |= IB_PORT_CM_SUP;
 579                 props->ip_gids = true;
 580                 props->gid_tbl_len = MLX5_CAP_ROCE(dev->mdev,
 581                                                    roce_address_table_size);
 582                 mlx5_query_nic_vport_qkey_viol_cntr(mdev, &qkey_viol_cntr);
 583                 props->qkey_viol_cntr = qkey_viol_cntr;
 584         }
 585         props->max_mtu          = IB_MTU_4096;
 586         props->max_msg_sz       = 1 << MLX5_CAP_GEN(dev->mdev, log_max_msg);
 587         props->pkey_tbl_len     = 1;
 588         props->state            = IB_PORT_DOWN;
 589         props->phys_state       = IB_PORT_PHYS_STATE_DISABLED;
 590
 591         /* If this is a stub query for an unaffiliated port stop here */
 592         if (!put_mdev)
 593                 goto out;
 594
 595         ndev = ib_device_get_netdev(device, port_num);
 596         if (!ndev)
 597                 goto out;
 598
 599         if (mlx5_lag_is_roce(mdev) || mlx5_lag_is_sriov(mdev)) {
 600                 rcu_read_lock();
 601                 upper = netdev_master_upper_dev_get_rcu(ndev);
 602                 if (upper) {
 603                         dev_put(ndev);
 604                         ndev = upper;
 605                         dev_hold(ndev);
 606                 }
 607                 rcu_read_unlock();
 608         }
 609
 610         if (netif_running(ndev) && netif_carrier_ok(ndev)) {
 611                 props->state      = IB_PORT_ACTIVE;
 612                 props->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
 613         }
 614
 615         ndev_ib_mtu = iboe_get_mtu(ndev->mtu);
 616
 617         dev_put(ndev);
 618
 619         props->active_mtu       = min(props->max_mtu, ndev_ib_mtu);
 620 out:
 621         if (put_mdev)
 622                 mlx5_ib_put_native_port_mdev(dev, port_num);
 623         return err;
 624 }
 625
 626 int set_roce_addr(struct mlx5_ib_dev *dev, u32 port_num,
 627                   unsigned int index, const union ib_gid *gid,
 628                   const struct ib_gid_attr *attr)
 629 {
 630         enum ib_gid_type gid_type;
 631         u16 vlan_id = 0xffff;
 632         u8 roce_version = 0;
 633         u8 roce_l3_type = 0;
 634         u8 mac[ETH_ALEN];
 635         int ret;
 636
 637         gid_type = attr->gid_type;
 638         if (gid) {
 639                 ret = rdma_read_gid_l2_fields(attr, &vlan_id, &mac[0]);
 640                 if (ret)
 641                         return ret;
 642         }
 643
 644         switch (gid_type) {
 645         case IB_GID_TYPE_ROCE:
 646                 roce_version = MLX5_ROCE_VERSION_1;
 647                 break;
 648         case IB_GID_TYPE_ROCE_UDP_ENCAP:
 649                 roce_version = MLX5_ROCE_VERSION_2;
 650                 if (gid && ipv6_addr_v4mapped((void *)gid))
 651                         roce_l3_type = MLX5_ROCE_L3_TYPE_IPV4;
 652                 else
 653                         roce_l3_type = MLX5_ROCE_L3_TYPE_IPV6;
 654                 break;
 655
 656         default:
 657                 mlx5_ib_warn(dev, "Unexpected GID type %u\n", gid_type);
 658         }
 659
 660         return mlx5_core_roce_gid_set(dev->mdev, index, roce_version,
 661                                       roce_l3_type, gid->raw, mac,
 662                                       vlan_id < VLAN_CFI_MASK, vlan_id,
 663                                       port_num);
 664 }
 665
 666 static int mlx5_ib_add_gid(const struct ib_gid_attr *attr,
 667                            __always_unused void **context)
 668 {
 669         int ret;
 670
 671         ret = mlx5r_add_gid_macsec_operations(attr);
 672         if (ret)
 673                 return ret;
 674
 675         return set_roce_addr(to_mdev(attr->device), attr->port_num,
 676                              attr->index, &attr->gid, attr);
 677 }
 678
 679 static int mlx5_ib_del_gid(const struct ib_gid_attr *attr,
 680                            __always_unused void **context)
 681 {
 682         int ret;
 683
 684         ret = set_roce_addr(to_mdev(attr->device), attr->port_num,
 685                             attr->index, NULL, attr);
 686         if (ret)
 687                 return ret;
 688
 689         mlx5r_del_gid_macsec_operations(attr);
 690         return 0;
 691 }
 692
 693 __be16 mlx5_get_roce_udp_sport_min(const struct mlx5_ib_dev *dev,
 694                                    const struct ib_gid_attr *attr)
 695 {
 696         if (attr->gid_type != IB_GID_TYPE_ROCE_UDP_ENCAP)
 697                 return 0;
 698
 699         return cpu_to_be16(MLX5_CAP_ROCE(dev->mdev, r_roce_min_src_udp_port));
 700 }
 701
 702 static int mlx5_use_mad_ifc(struct mlx5_ib_dev *dev)
 703 {
 704         if (MLX5_CAP_GEN(dev->mdev, port_type) == MLX5_CAP_PORT_TYPE_IB)
 705                 return !MLX5_CAP_GEN(dev->mdev, ib_virt);
 706         return 0;
 707 }
 708
 709 enum {
 710         MLX5_VPORT_ACCESS_METHOD_MAD,
 711         MLX5_VPORT_ACCESS_METHOD_HCA,
 712         MLX5_VPORT_ACCESS_METHOD_NIC,
 713 };
 714
 715 static int mlx5_get_vport_access_method(struct ib_device *ibdev)
 716 {
 717         if (mlx5_use_mad_ifc(to_mdev(ibdev)))
 718                 return MLX5_VPORT_ACCESS_METHOD_MAD;
 719
 720         if (mlx5_ib_port_link_layer(ibdev, 1) ==
 721             IB_LINK_LAYER_ETHERNET)
 722                 return MLX5_VPORT_ACCESS_METHOD_NIC;
 723
 724         return MLX5_VPORT_ACCESS_METHOD_HCA;
 725 }
 726
 727 static void get_atomic_caps(struct mlx5_ib_dev *dev,
 728                             u8 atomic_size_qp,
 729                             struct ib_device_attr *props)
 730 {
 731         u8 tmp;
 732         u8 atomic_operations = MLX5_CAP_ATOMIC(dev->mdev, atomic_operations);
 733         u8 atomic_req_8B_endianness_mode =
 734                 MLX5_CAP_ATOMIC(dev->mdev, atomic_req_8B_endianness_mode);
 735
 736         /* Check if HW supports 8 bytes standard atomic operations and capable
 737          * of host endianness respond
 738          */
 739         tmp = MLX5_ATOMIC_OPS_CMP_SWAP | MLX5_ATOMIC_OPS_FETCH_ADD;
 740         if (((atomic_operations & tmp) == tmp) &&
 741             (atomic_size_qp & MLX5_ATOMIC_SIZE_QP_8BYTES) &&
 742             (atomic_req_8B_endianness_mode)) {
 743                 props->atomic_cap = IB_ATOMIC_HCA;
 744         } else {
 745                 props->atomic_cap = IB_ATOMIC_NONE;
 746         }
 747 }
 748
 749 static void get_atomic_caps_qp(struct mlx5_ib_dev *dev,
 750                                struct ib_device_attr *props)
 751 {
 752         u8 atomic_size_qp = MLX5_CAP_ATOMIC(dev->mdev, atomic_size_qp);
 753
 754         get_atomic_caps(dev, atomic_size_qp, props);
 755 }
 756
 757 static int mlx5_query_system_image_guid(struct ib_device *ibdev,
 758                                         __be64 *sys_image_guid)
 759 {
 760         struct mlx5_ib_dev *dev = to_mdev(ibdev);
 761         struct mlx5_core_dev *mdev = dev->mdev;
 762         u64 tmp;
 763         int err;
 764
 765         switch (mlx5_get_vport_access_method(ibdev)) {
 766         case MLX5_VPORT_ACCESS_METHOD_MAD:
 767                 return mlx5_query_mad_ifc_system_image_guid(ibdev,
 768                                                             sys_image_guid);
 769
 770         case MLX5_VPORT_ACCESS_METHOD_HCA:
 771                 err = mlx5_query_hca_vport_system_image_guid(mdev, &tmp);
 772                 break;
 773
 774         case MLX5_VPORT_ACCESS_METHOD_NIC:
 775                 err = mlx5_query_nic_vport_system_image_guid(mdev, &tmp);
 776                 break;
 777
 778         default:
 779                 return -EINVAL;
 780         }
 781
 782         if (!err)
 783                 *sys_image_guid = cpu_to_be64(tmp);
 784
 785         return err;
 786
 787 }
 788
 789 static int mlx5_query_max_pkeys(struct ib_device *ibdev,
 790                                 u16 *max_pkeys)
 791 {
 792         struct mlx5_ib_dev *dev = to_mdev(ibdev);
 793         struct mlx5_core_dev *mdev = dev->mdev;
 794
 795         switch (mlx5_get_vport_access_method(ibdev)) {
 796         case MLX5_VPORT_ACCESS_METHOD_MAD:
 797                 return mlx5_query_mad_ifc_max_pkeys(ibdev, max_pkeys);
 798
 799         case MLX5_VPORT_ACCESS_METHOD_HCA:
 800         case MLX5_VPORT_ACCESS_METHOD_NIC:
 801                 *max_pkeys = mlx5_to_sw_pkey_sz(MLX5_CAP_GEN(mdev,
 802                                                 pkey_table_size));
 803                 return 0;
 804
 805         default:
 806                 return -EINVAL;
 807         }
 808 }
 809
 810 static int mlx5_query_vendor_id(struct ib_device *ibdev,
 811                                 u32 *vendor_id)
 812 {
 813         struct mlx5_ib_dev *dev = to_mdev(ibdev);
 814
 815         switch (mlx5_get_vport_access_method(ibdev)) {
 816         case MLX5_VPORT_ACCESS_METHOD_MAD:
 817                 return mlx5_query_mad_ifc_vendor_id(ibdev, vendor_id);
 818
 819         case MLX5_VPORT_ACCESS_METHOD_HCA:
 820         case MLX5_VPORT_ACCESS_METHOD_NIC:
 821                 return mlx5_core_query_vendor_id(dev->mdev, vendor_id);
 822
 823         default:
 824                 return -EINVAL;
 825         }
 826 }
 827
 828 static int mlx5_query_node_guid(struct mlx5_ib_dev *dev,
 829                                 __be64 *node_guid)
 830 {
 831         u64 tmp;
 832         int err;
 833
 834         switch (mlx5_get_vport_access_method(&dev->ib_dev)) {
 835         case MLX5_VPORT_ACCESS_METHOD_MAD:
 836                 return mlx5_query_mad_ifc_node_guid(dev, node_guid);
 837
 838         case MLX5_VPORT_ACCESS_METHOD_HCA:
 839                 err = mlx5_query_hca_vport_node_guid(dev->mdev, &tmp);
 840                 break;
 841
 842         case MLX5_VPORT_ACCESS_METHOD_NIC:
 843                 err = mlx5_query_nic_vport_node_guid(dev->mdev, &tmp);
 844                 break;
 845
 846         default:
 847                 return -EINVAL;
 848         }
 849
 850         if (!err)
 851                 *node_guid = cpu_to_be64(tmp);
 852
 853         return err;
 854 }
 855
 856 struct mlx5_reg_node_desc {
 857         u8      desc[IB_DEVICE_NODE_DESC_MAX];
 858 };
 859
 860 static int mlx5_query_node_desc(struct mlx5_ib_dev *dev, char *node_desc)
 861 {
 862         struct mlx5_reg_node_desc in;
 863
 864         if (mlx5_use_mad_ifc(dev))
 865                 return mlx5_query_mad_ifc_node_desc(dev, node_desc);
 866
 867         memset(&in, 0, sizeof(in));
 868
 869         return mlx5_core_access_reg(dev->mdev, &in, sizeof(in), node_desc,
 870                                     sizeof(struct mlx5_reg_node_desc),
 871                                     MLX5_REG_NODE_DESC, 0, 0);
 872 }
 873
 874 static void fill_esw_mgr_reg_c0(struct mlx5_core_dev *mdev,
 875                                 struct mlx5_ib_query_device_resp *resp)
 876 {
 877         struct mlx5_eswitch *esw = mdev->priv.eswitch;
 878         u16 vport = mlx5_eswitch_manager_vport(mdev);
 879
 880         resp->reg_c0.value = mlx5_eswitch_get_vport_metadata_for_match(esw,
 881                                                                       vport);
 882         resp->reg_c0.mask = mlx5_eswitch_get_vport_metadata_mask();
 883 }
 884
 885 static int mlx5_ib_query_device(struct ib_device *ibdev,
 886                                 struct ib_device_attr *props,
 887                                 struct ib_udata *uhw)
 888 {
 889         size_t uhw_outlen = (uhw) ? uhw->outlen : 0;
 890         struct mlx5_ib_dev *dev = to_mdev(ibdev);
 891         struct mlx5_core_dev *mdev = dev->mdev;
 892         int err = -ENOMEM;
 893         int max_sq_desc;
 894         int max_rq_sg;
 895         int max_sq_sg;
 896         u64 min_page_size = 1ull << MLX5_CAP_GEN(mdev, log_pg_sz);
 897         bool raw_support = !mlx5_core_mp_enabled(mdev);
 898         struct mlx5_ib_query_device_resp resp = {};
 899         size_t resp_len;
 900         u64 max_tso;
 901
 902         resp_len = sizeof(resp.comp_mask) + sizeof(resp.response_length);
 903         if (uhw_outlen && uhw_outlen < resp_len)
 904                 return -EINVAL;
 905
 906         resp.response_length = resp_len;
 907
 908         if (uhw && uhw->inlen && !ib_is_udata_cleared(uhw, 0, uhw->inlen))
 909                 return -EINVAL;
 910
 911         memset(props, 0, sizeof(*props));
 912         err = mlx5_query_system_image_guid(ibdev,
 913                                            &props->sys_image_guid);
 914         if (err)
 915                 return err;
 916
 917         props->max_pkeys = dev->pkey_table_len;
 918
 919         err = mlx5_query_vendor_id(ibdev, &props->vendor_id);
 920         if (err)
 921                 return err;
 922
 923         props->fw_ver = ((u64)fw_rev_maj(dev->mdev) << 32) |
 924                 (fw_rev_min(dev->mdev) << 16) |
 925                 fw_rev_sub(dev->mdev);
 926         props->device_cap_flags    = IB_DEVICE_CHANGE_PHY_PORT |
 927                 IB_DEVICE_PORT_ACTIVE_EVENT             |
 928                 IB_DEVICE_SYS_IMAGE_GUID                |
 929                 IB_DEVICE_RC_RNR_NAK_GEN;
 930
 931         if (MLX5_CAP_GEN(mdev, pkv))
 932                 props->device_cap_flags |= IB_DEVICE_BAD_PKEY_CNTR;
 933         if (MLX5_CAP_GEN(mdev, qkv))
 934                 props->device_cap_flags |= IB_DEVICE_BAD_QKEY_CNTR;
 935         if (MLX5_CAP_GEN(mdev, apm))
 936                 props->device_cap_flags |= IB_DEVICE_AUTO_PATH_MIG;
 937         if (MLX5_CAP_GEN(mdev, xrc))
 938                 props->device_cap_flags |= IB_DEVICE_XRC;
 939         if (MLX5_CAP_GEN(mdev, imaicl)) {
 940                 props->device_cap_flags |= IB_DEVICE_MEM_WINDOW |
 941                                            IB_DEVICE_MEM_WINDOW_TYPE_2B;
 942                 props->max_mw = 1 << MLX5_CAP_GEN(mdev, log_max_mkey);
 943                 /* We support 'Gappy' memory registration too */
 944                 props->kernel_cap_flags |= IBK_SG_GAPS_REG;
 945         }
 946         /* IB_WR_REG_MR always requires changing the entity size with UMR */
 947         if (!MLX5_CAP_GEN(dev->mdev, umr_modify_entity_size_disabled))
 948                 props->device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS;
 949         if (MLX5_CAP_GEN(mdev, sho)) {
 950                 props->kernel_cap_flags |= IBK_INTEGRITY_HANDOVER;
 951                 /* At this stage no support for signature handover */
 952                 props->sig_prot_cap = IB_PROT_T10DIF_TYPE_1 |
 953                                       IB_PROT_T10DIF_TYPE_2 |
 954                                       IB_PROT_T10DIF_TYPE_3;
 955                 props->sig_guard_cap = IB_GUARD_T10DIF_CRC |
 956                                        IB_GUARD_T10DIF_CSUM;
 957         }
 958         if (MLX5_CAP_GEN(mdev, block_lb_mc))
 959                 props->kernel_cap_flags |= IBK_BLOCK_MULTICAST_LOOPBACK;
 960
 961         if (MLX5_CAP_GEN(dev->mdev, eth_net_offloads) && raw_support) {
 962                 if (MLX5_CAP_ETH(mdev, csum_cap)) {
 963                         /* Legacy bit to support old userspace libraries */
 964                         props->device_cap_flags |= IB_DEVICE_RAW_IP_CSUM;
 965                         props->raw_packet_caps |= IB_RAW_PACKET_CAP_IP_CSUM;
 966                 }
 967
 968                 if (MLX5_CAP_ETH(dev->mdev, vlan_cap))
 969                         props->raw_packet_caps |=
 970                                 IB_RAW_PACKET_CAP_CVLAN_STRIPPING;
 971
 972                 if (offsetofend(typeof(resp), tso_caps) <= uhw_outlen) {
 973                         max_tso = MLX5_CAP_ETH(mdev, max_lso_cap);
 974                         if (max_tso) {
 975                                 resp.tso_caps.max_tso = 1 << max_tso;
 976                                 resp.tso_caps.supported_qpts |=
 977                                         1 << IB_QPT_RAW_PACKET;
 978                                 resp.response_length += sizeof(resp.tso_caps);
 979                         }
 980                 }
 981
 982                 if (offsetofend(typeof(resp), rss_caps) <= uhw_outlen) {
 983                         resp.rss_caps.rx_hash_function =
 984                                                 MLX5_RX_HASH_FUNC_TOEPLITZ;
 985                         resp.rss_caps.rx_hash_fields_mask =
 986                                                 MLX5_RX_HASH_SRC_IPV4 |
 987                                                 MLX5_RX_HASH_DST_IPV4 |
 988                                                 MLX5_RX_HASH_SRC_IPV6 |
 989                                                 MLX5_RX_HASH_DST_IPV6 |
 990                                                 MLX5_RX_HASH_SRC_PORT_TCP |
 991                                                 MLX5_RX_HASH_DST_PORT_TCP |
 992                                                 MLX5_RX_HASH_SRC_PORT_UDP |
 993                                                 MLX5_RX_HASH_DST_PORT_UDP |
 994                                                 MLX5_RX_HASH_INNER;
 995                         resp.response_length += sizeof(resp.rss_caps);
 996                 }
 997         } else {
 998                 if (offsetofend(typeof(resp), tso_caps) <= uhw_outlen)
 999                         resp.response_length += sizeof(resp.tso_caps);
1000                 if (offsetofend(typeof(resp), rss_caps) <= uhw_outlen)
1001                         resp.response_length += sizeof(resp.rss_caps);
1002         }
1003
1004         if (MLX5_CAP_GEN(mdev, ipoib_basic_offloads)) {
1005                 props->device_cap_flags |= IB_DEVICE_UD_IP_CSUM;
1006                 props->kernel_cap_flags |= IBK_UD_TSO;
1007         }
1008
1009         if (MLX5_CAP_GEN(dev->mdev, rq_delay_drop) &&
1010             MLX5_CAP_GEN(dev->mdev, general_notification_event) &&
1011             raw_support)
1012                 props->raw_packet_caps |= IB_RAW_PACKET_CAP_DELAY_DROP;
1013
1014         if (MLX5_CAP_GEN(mdev, ipoib_enhanced_offloads) &&
1015             MLX5_CAP_IPOIB_ENHANCED(mdev, csum_cap))
1016                 props->device_cap_flags |= IB_DEVICE_UD_IP_CSUM;
1017
1018         if (MLX5_CAP_GEN(dev->mdev, eth_net_offloads) &&
1019             MLX5_CAP_ETH(dev->mdev, scatter_fcs) &&
1020             raw_support) {
1021                 /* Legacy bit to support old userspace libraries */
1022                 props->device_cap_flags |= IB_DEVICE_RAW_SCATTER_FCS;
1023                 props->raw_packet_caps |= IB_RAW_PACKET_CAP_SCATTER_FCS;
1024         }
1025
1026         if (MLX5_CAP_DEV_MEM(mdev, memic)) {
1027                 props->max_dm_size =
1028                         MLX5_CAP_DEV_MEM(mdev, max_memic_size);
1029         }
1030
1031         if (mlx5_get_flow_namespace(dev->mdev, MLX5_FLOW_NAMESPACE_BYPASS))
1032                 props->device_cap_flags |= IB_DEVICE_MANAGED_FLOW_STEERING;
1033
1034         if (MLX5_CAP_GEN(mdev, end_pad))
1035                 props->device_cap_flags |= IB_DEVICE_PCI_WRITE_END_PADDING;
1036
1037         props->vendor_part_id      = mdev->pdev->device;
1038         props->hw_ver              = mdev->pdev->revision;
1039
1040         props->max_mr_size         = ~0ull;
1041         props->page_size_cap       = ~(min_page_size - 1);
1042         props->max_qp              = 1 << MLX5_CAP_GEN(mdev, log_max_qp);
1043         props->max_qp_wr           = 1 << MLX5_CAP_GEN(mdev, log_max_qp_sz);
1044         max_rq_sg =  MLX5_CAP_GEN(mdev, max_wqe_sz_rq) /
1045                      sizeof(struct mlx5_wqe_data_seg);
1046         max_sq_desc = min_t(int, MLX5_CAP_GEN(mdev, max_wqe_sz_sq), 512);
1047         max_sq_sg = (max_sq_desc - sizeof(struct mlx5_wqe_ctrl_seg) -
1048                      sizeof(struct mlx5_wqe_raddr_seg)) /
1049                 sizeof(struct mlx5_wqe_data_seg);
1050         props->max_send_sge = max_sq_sg;
1051         props->max_recv_sge = max_rq_sg;
1052         props->max_sge_rd          = MLX5_MAX_SGE_RD;
1053         props->max_cq              = 1 << MLX5_CAP_GEN(mdev, log_max_cq);
1054         props->max_cqe = (1 << MLX5_CAP_GEN(mdev, log_max_cq_sz)) - 1;
1055         props->max_mr              = 1 << MLX5_CAP_GEN(mdev, log_max_mkey);
1056         props->max_pd              = 1 << MLX5_CAP_GEN(mdev, log_max_pd);
1057         props->max_qp_rd_atom      = 1 << MLX5_CAP_GEN(mdev, log_max_ra_req_qp);
1058         props->max_qp_init_rd_atom = 1 << MLX5_CAP_GEN(mdev, log_max_ra_res_qp);
1059         props->max_srq             = 1 << MLX5_CAP_GEN(mdev, log_max_srq);
1060         props->max_srq_wr = (1 << MLX5_CAP_GEN(mdev, log_max_srq_sz)) - 1;
1061         props->local_ca_ack_delay  = MLX5_CAP_GEN(mdev, local_ca_ack_delay);
1062         props->max_res_rd_atom     = props->max_qp_rd_atom * props->max_qp;
1063         props->max_srq_sge         = max_rq_sg - 1;
1064         props->max_fast_reg_page_list_len =
1065                 1 << MLX5_CAP_GEN(mdev, log_max_klm_list_size);
1066         props->max_pi_fast_reg_page_list_len =
1067                 props->max_fast_reg_page_list_len / 2;
1068         props->max_sgl_rd =
1069                 MLX5_CAP_GEN(mdev, max_sgl_for_optimized_performance);
1070         get_atomic_caps_qp(dev, props);
1071         props->masked_atomic_cap   = IB_ATOMIC_NONE;
1072         props->max_mcast_grp       = 1 << MLX5_CAP_GEN(mdev, log_max_mcg);
1073         props->max_mcast_qp_attach = MLX5_CAP_GEN(mdev, max_qp_mcg);
1074         props->max_total_mcast_qp_attach = props->max_mcast_qp_attach *
1075                                            props->max_mcast_grp;
1076         props->max_ah = INT_MAX;
1077         props->hca_core_clock = MLX5_CAP_GEN(mdev, device_frequency_khz);
1078         props->timestamp_mask = 0x7FFFFFFFFFFFFFFFULL;
1079
1080         if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING)) {
1081                 if (dev->odp_caps.general_caps & IB_ODP_SUPPORT)
1082                         props->kernel_cap_flags |= IBK_ON_DEMAND_PAGING;
1083                 props->odp_caps = dev->odp_caps;
1084                 if (!uhw) {
1085                         /* ODP for kernel QPs is not implemented for receive
1086                          * WQEs and SRQ WQEs
1087                          */
1088                         props->odp_caps.per_transport_caps.rc_odp_caps &=
1089                                 ~(IB_ODP_SUPPORT_READ |
1090                                   IB_ODP_SUPPORT_SRQ_RECV);
1091                         props->odp_caps.per_transport_caps.uc_odp_caps &=
1092                                 ~(IB_ODP_SUPPORT_READ |
1093                                   IB_ODP_SUPPORT_SRQ_RECV);
1094                         props->odp_caps.per_transport_caps.ud_odp_caps &=
1095                                 ~(IB_ODP_SUPPORT_READ |
1096                                   IB_ODP_SUPPORT_SRQ_RECV);
1097                         props->odp_caps.per_transport_caps.xrc_odp_caps &=
1098                                 ~(IB_ODP_SUPPORT_READ |
1099                                   IB_ODP_SUPPORT_SRQ_RECV);
1100                 }
1101         }
1102
1103         if (mlx5_core_is_vf(mdev))
1104                 props->kernel_cap_flags |= IBK_VIRTUAL_FUNCTION;
1105
1106         if (mlx5_ib_port_link_layer(ibdev, 1) ==
1107             IB_LINK_LAYER_ETHERNET && raw_support) {
1108                 props->rss_caps.max_rwq_indirection_tables =
1109                         1 << MLX5_CAP_GEN(dev->mdev, log_max_rqt);
1110                 props->rss_caps.max_rwq_indirection_table_size =
1111                         1 << MLX5_CAP_GEN(dev->mdev, log_max_rqt_size);
1112                 props->rss_caps.supported_qpts = 1 << IB_QPT_RAW_PACKET;
1113                 props->max_wq_type_rq =
1114                         1 << MLX5_CAP_GEN(dev->mdev, log_max_rq);
1115         }
1116
1117         if (MLX5_CAP_GEN(mdev, tag_matching)) {
1118                 props->tm_caps.max_num_tags =
1119                         (1 << MLX5_CAP_GEN(mdev, log_tag_matching_list_sz)) - 1;
1120                 props->tm_caps.max_ops =
1121                         1 << MLX5_CAP_GEN(mdev, log_max_qp_sz);
1122                 props->tm_caps.max_sge = MLX5_TM_MAX_SGE;
1123         }
1124
1125         if (MLX5_CAP_GEN(mdev, tag_matching) &&
1126             MLX5_CAP_GEN(mdev, rndv_offload_rc)) {
1127                 props->tm_caps.flags = IB_TM_CAP_RNDV_RC;
1128                 props->tm_caps.max_rndv_hdr_size = MLX5_TM_MAX_RNDV_MSG_SIZE;
1129         }
1130
1131         if (MLX5_CAP_GEN(dev->mdev, cq_moderation)) {
1132                 props->cq_caps.max_cq_moderation_count =
1133                                                 MLX5_MAX_CQ_COUNT;
1134                 props->cq_caps.max_cq_moderation_period =
1135                                                 MLX5_MAX_CQ_PERIOD;
1136         }
1137
1138         if (offsetofend(typeof(resp), cqe_comp_caps) <= uhw_outlen) {
1139                 resp.response_length += sizeof(resp.cqe_comp_caps);
1140
1141                 if (MLX5_CAP_GEN(dev->mdev, cqe_compression)) {
1142                         resp.cqe_comp_caps.max_num =
1143                                 MLX5_CAP_GEN(dev->mdev,
1144                                              cqe_compression_max_num);
1145
1146                         resp.cqe_comp_caps.supported_format =
1147                                 MLX5_IB_CQE_RES_FORMAT_HASH |
1148                                 MLX5_IB_CQE_RES_FORMAT_CSUM;
1149
1150                         if (MLX5_CAP_GEN(dev->mdev, mini_cqe_resp_stride_index))
1151                                 resp.cqe_comp_caps.supported_format |=
1152                                         MLX5_IB_CQE_RES_FORMAT_CSUM_STRIDX;
1153                 }
1154         }
1155
1156         if (offsetofend(typeof(resp), packet_pacing_caps) <= uhw_outlen &&
1157             raw_support) {
1158                 if (MLX5_CAP_QOS(mdev, packet_pacing) &&
1159                     MLX5_CAP_GEN(mdev, qos)) {
1160                         resp.packet_pacing_caps.qp_rate_limit_max =
1161                                 MLX5_CAP_QOS(mdev, packet_pacing_max_rate);
1162                         resp.packet_pacing_caps.qp_rate_limit_min =
1163                                 MLX5_CAP_QOS(mdev, packet_pacing_min_rate);
1164                         resp.packet_pacing_caps.supported_qpts |=
1165                                 1 << IB_QPT_RAW_PACKET;
1166                         if (MLX5_CAP_QOS(mdev, packet_pacing_burst_bound) &&
1167                             MLX5_CAP_QOS(mdev, packet_pacing_typical_size))
1168                                 resp.packet_pacing_caps.cap_flags |=
1169                                         MLX5_IB_PP_SUPPORT_BURST;
1170                 }
1171                 resp.response_length += sizeof(resp.packet_pacing_caps);
1172         }
1173
1174         if (offsetofend(typeof(resp), mlx5_ib_support_multi_pkt_send_wqes) <=
1175             uhw_outlen) {
1176                 if (MLX5_CAP_ETH(mdev, multi_pkt_send_wqe))
1177                         resp.mlx5_ib_support_multi_pkt_send_wqes =
1178                                 MLX5_IB_ALLOW_MPW;
1179
1180                 if (MLX5_CAP_ETH(mdev, enhanced_multi_pkt_send_wqe))
1181                         resp.mlx5_ib_support_multi_pkt_send_wqes |=
1182                                 MLX5_IB_SUPPORT_EMPW;
1183
1184                 resp.response_length +=
1185                         sizeof(resp.mlx5_ib_support_multi_pkt_send_wqes);
1186         }
1187
1188         if (offsetofend(typeof(resp), flags) <= uhw_outlen) {
1189                 resp.response_length += sizeof(resp.flags);
1190
1191                 if (MLX5_CAP_GEN(mdev, cqe_compression_128))
1192                         resp.flags |=
1193                                 MLX5_IB_QUERY_DEV_RESP_FLAGS_CQE_128B_COMP;
1194
1195                 if (MLX5_CAP_GEN(mdev, cqe_128_always))
1196                         resp.flags |= MLX5_IB_QUERY_DEV_RESP_FLAGS_CQE_128B_PAD;
1197                 if (MLX5_CAP_GEN(mdev, qp_packet_based))
1198                         resp.flags |=
1199                                 MLX5_IB_QUERY_DEV_RESP_PACKET_BASED_CREDIT_MODE;
1200
1201                 resp.flags |= MLX5_IB_QUERY_DEV_RESP_FLAGS_SCAT2CQE_DCT;
1202
1203                 if (MLX5_CAP_GEN_2(mdev, dp_ordering_force) &&
1204                     (MLX5_CAP_GEN(mdev, dp_ordering_ooo_all_xrc) ||
1205                     MLX5_CAP_GEN(mdev, dp_ordering_ooo_all_dc) ||
1206                     MLX5_CAP_GEN(mdev, dp_ordering_ooo_all_rc) ||
1207                     MLX5_CAP_GEN(mdev, dp_ordering_ooo_all_ud) ||
1208                     MLX5_CAP_GEN(mdev, dp_ordering_ooo_all_uc)))
1209                         resp.flags |= MLX5_IB_QUERY_DEV_RESP_FLAGS_OOO_DP;
1210         }
1211
1212         if (offsetofend(typeof(resp), sw_parsing_caps) <= uhw_outlen) {
1213                 resp.response_length += sizeof(resp.sw_parsing_caps);
1214                 if (MLX5_CAP_ETH(mdev, swp)) {
1215                         resp.sw_parsing_caps.sw_parsing_offloads |=
1216                                 MLX5_IB_SW_PARSING;
1217
1218                         if (MLX5_CAP_ETH(mdev, swp_csum))
1219                                 resp.sw_parsing_caps.sw_parsing_offloads |=
1220                                         MLX5_IB_SW_PARSING_CSUM;
1221
1222                         if (MLX5_CAP_ETH(mdev, swp_lso))
1223                                 resp.sw_parsing_caps.sw_parsing_offloads |=
1224                                         MLX5_IB_SW_PARSING_LSO;
1225
1226                         if (resp.sw_parsing_caps.sw_parsing_offloads)
1227                                 resp.sw_parsing_caps.supported_qpts =
1228                                         BIT(IB_QPT_RAW_PACKET);
1229                 }
1230         }
1231
1232         if (offsetofend(typeof(resp), striding_rq_caps) <= uhw_outlen &&
1233             raw_support) {
1234                 resp.response_length += sizeof(resp.striding_rq_caps);
1235                 if (MLX5_CAP_GEN(mdev, striding_rq)) {
1236                         resp.striding_rq_caps.min_single_stride_log_num_of_bytes =
1237                                 MLX5_MIN_SINGLE_STRIDE_LOG_NUM_BYTES;
1238                         resp.striding_rq_caps.max_single_stride_log_num_of_bytes =
1239                                 MLX5_MAX_SINGLE_STRIDE_LOG_NUM_BYTES;
1240                         if (MLX5_CAP_GEN(dev->mdev, ext_stride_num_range))
1241                                 resp.striding_rq_caps
1242                                         .min_single_wqe_log_num_of_strides =
1243                                         MLX5_EXT_MIN_SINGLE_WQE_LOG_NUM_STRIDES;
1244                         else
1245                                 resp.striding_rq_caps
1246                                         .min_single_wqe_log_num_of_strides =
1247                                         MLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES;
1248                         resp.striding_rq_caps.max_single_wqe_log_num_of_strides =
1249                                 MLX5_MAX_SINGLE_WQE_LOG_NUM_STRIDES;
1250                         resp.striding_rq_caps.supported_qpts =
1251                                 BIT(IB_QPT_RAW_PACKET);
1252                 }
1253         }
1254
1255         if (offsetofend(typeof(resp), tunnel_offloads_caps) <= uhw_outlen) {
1256                 resp.response_length += sizeof(resp.tunnel_offloads_caps);
1257                 if (MLX5_CAP_ETH(mdev, tunnel_stateless_vxlan))
1258                         resp.tunnel_offloads_caps |=
1259                                 MLX5_IB_TUNNELED_OFFLOADS_VXLAN;
1260                 if (MLX5_CAP_ETH(mdev, tunnel_stateless_geneve_rx))
1261                         resp.tunnel_offloads_caps |=
1262                                 MLX5_IB_TUNNELED_OFFLOADS_GENEVE;
1263                 if (MLX5_CAP_ETH(mdev, tunnel_stateless_gre))
1264                         resp.tunnel_offloads_caps |=
1265                                 MLX5_IB_TUNNELED_OFFLOADS_GRE;
1266                 if (MLX5_CAP_ETH(mdev, tunnel_stateless_mpls_over_gre))
1267                         resp.tunnel_offloads_caps |=
1268                                 MLX5_IB_TUNNELED_OFFLOADS_MPLS_GRE;
1269                 if (MLX5_CAP_ETH(mdev, tunnel_stateless_mpls_over_udp))
1270                         resp.tunnel_offloads_caps |=
1271                                 MLX5_IB_TUNNELED_OFFLOADS_MPLS_UDP;
1272         }
1273
1274         if (offsetofend(typeof(resp), dci_streams_caps) <= uhw_outlen) {
1275                 resp.response_length += sizeof(resp.dci_streams_caps);
1276
1277                 resp.dci_streams_caps.max_log_num_concurent =
1278                         MLX5_CAP_GEN(mdev, log_max_dci_stream_channels);
1279
1280                 resp.dci_streams_caps.max_log_num_errored =
1281                         MLX5_CAP_GEN(mdev, log_max_dci_errored_streams);
1282         }
1283
1284         if (offsetofend(typeof(resp), reserved) <= uhw_outlen)
1285                 resp.response_length += sizeof(resp.reserved);
1286
1287         if (offsetofend(typeof(resp), reg_c0) <= uhw_outlen) {
1288                 struct mlx5_eswitch *esw = mdev->priv.eswitch;
1289
1290                 resp.response_length += sizeof(resp.reg_c0);
1291
1292                 if (mlx5_eswitch_mode(mdev) == MLX5_ESWITCH_OFFLOADS &&
1293                     mlx5_eswitch_vport_match_metadata_enabled(esw))
1294                         fill_esw_mgr_reg_c0(mdev, &resp);
1295         }
1296
1297         if (uhw_outlen) {
1298                 err = ib_copy_to_udata(uhw, &resp, resp.response_length);
1299
1300                 if (err)
1301                         return err;
1302         }
1303
1304         return 0;
1305 }
1306
1307 static void translate_active_width(struct ib_device *ibdev, u16 active_width,
1308                                    u8 *ib_width)
1309 {
1310         struct mlx5_ib_dev *dev = to_mdev(ibdev);
1311
1312         if (active_width & MLX5_PTYS_WIDTH_1X)
1313                 *ib_width = IB_WIDTH_1X;
1314         else if (active_width & MLX5_PTYS_WIDTH_2X)
1315                 *ib_width = IB_WIDTH_2X;
1316         else if (active_width & MLX5_PTYS_WIDTH_4X)
1317                 *ib_width = IB_WIDTH_4X;
1318         else if (active_width & MLX5_PTYS_WIDTH_8X)
1319                 *ib_width = IB_WIDTH_8X;
1320         else if (active_width & MLX5_PTYS_WIDTH_12X)
1321                 *ib_width = IB_WIDTH_12X;
1322         else {
1323                 mlx5_ib_dbg(dev, "Invalid active_width %d, setting width to default value: 4x\n",
1324                             active_width);
1325                 *ib_width = IB_WIDTH_4X;
1326         }
1327
1328         return;
1329 }
1330
1331 static int mlx5_mtu_to_ib_mtu(int mtu)
1332 {
1333         switch (mtu) {
1334         case 256: return 1;
1335         case 512: return 2;
1336         case 1024: return 3;
1337         case 2048: return 4;
1338         case 4096: return 5;
1339         default:
1340                 pr_warn("invalid mtu\n");
1341                 return -1;
1342         }
1343 }
1344
1345 enum ib_max_vl_num {
1346         __IB_MAX_VL_0           = 1,
1347         __IB_MAX_VL_0_1         = 2,
1348         __IB_MAX_VL_0_3         = 3,
1349         __IB_MAX_VL_0_7         = 4,
1350         __IB_MAX_VL_0_14        = 5,
1351 };
1352
1353 enum mlx5_vl_hw_cap {
1354         MLX5_VL_HW_0    = 1,
1355         MLX5_VL_HW_0_1  = 2,
1356         MLX5_VL_HW_0_2  = 3,
1357         MLX5_VL_HW_0_3  = 4,
1358         MLX5_VL_HW_0_4  = 5,
1359         MLX5_VL_HW_0_5  = 6,
1360         MLX5_VL_HW_0_6  = 7,
1361         MLX5_VL_HW_0_7  = 8,
1362         MLX5_VL_HW_0_14 = 15
1363 };
1364
1365 static int translate_max_vl_num(struct ib_device *ibdev, u8 vl_hw_cap,
1366                                 u8 *max_vl_num)
1367 {
1368         switch (vl_hw_cap) {
1369         case MLX5_VL_HW_0:
1370                 *max_vl_num = __IB_MAX_VL_0;
1371                 break;
1372         case MLX5_VL_HW_0_1:
1373                 *max_vl_num = __IB_MAX_VL_0_1;
1374                 break;
1375         case MLX5_VL_HW_0_3:
1376                 *max_vl_num = __IB_MAX_VL_0_3;
1377                 break;
1378         case MLX5_VL_HW_0_7:
1379                 *max_vl_num = __IB_MAX_VL_0_7;
1380                 break;
1381         case MLX5_VL_HW_0_14:
1382                 *max_vl_num = __IB_MAX_VL_0_14;
1383                 break;
1384
1385         default:
1386                 return -EINVAL;
1387         }
1388
1389         return 0;
1390 }
1391
1392 static int mlx5_query_hca_port(struct ib_device *ibdev, u32 port,
1393                                struct ib_port_attr *props)
1394 {
1395         struct mlx5_ib_dev *dev = to_mdev(ibdev);
1396         struct mlx5_core_dev *mdev = dev->mdev;
1397         struct mlx5_hca_vport_context *rep;
1398         u8 vl_hw_cap, plane_index = 0;
1399         u16 max_mtu;
1400         u16 oper_mtu;
1401         int err;
1402         u16 ib_link_width_oper;
1403
1404         rep = kzalloc(sizeof(*rep), GFP_KERNEL);
1405         if (!rep) {
1406                 err = -ENOMEM;
1407                 goto out;
1408         }
1409
1410         /* props being zeroed by the caller, avoid zeroing it here */
1411
1412         if (ibdev->type == RDMA_DEVICE_TYPE_SMI) {
1413                 plane_index = port;
1414                 port = smi_to_native_portnum(dev, port);
1415         }
1416
1417         err = mlx5_query_hca_vport_context(mdev, 0, port, 0, rep);
1418         if (err)
1419                 goto out;
1420
1421         props->lid              = rep->lid;
1422         props->lmc              = rep->lmc;
1423         props->sm_lid           = rep->sm_lid;
1424         props->sm_sl            = rep->sm_sl;
1425         props->state            = rep->vport_state;
1426         props->phys_state       = rep->port_physical_state;
1427
1428         props->port_cap_flags = rep->cap_mask1;
1429         if (dev->num_plane) {
1430                 props->port_cap_flags |= IB_PORT_SM_DISABLED;
1431                 props->port_cap_flags &= ~IB_PORT_SM;
1432         } else if (ibdev->type == RDMA_DEVICE_TYPE_SMI)
1433                 props->port_cap_flags &= ~IB_PORT_CM_SUP;
1434
1435         props->gid_tbl_len      = mlx5_get_gid_table_len(MLX5_CAP_GEN(mdev, gid_table_size));
1436         props->max_msg_sz       = 1 << MLX5_CAP_GEN(mdev, log_max_msg);
1437         props->pkey_tbl_len     = mlx5_to_sw_pkey_sz(MLX5_CAP_GEN(mdev, pkey_table_size));
1438         props->bad_pkey_cntr    = rep->pkey_violation_counter;
1439         props->qkey_viol_cntr   = rep->qkey_violation_counter;
1440         props->subnet_timeout   = rep->subnet_timeout;
1441         props->init_type_reply  = rep->init_type_reply;
1442
1443         if (props->port_cap_flags & IB_PORT_CAP_MASK2_SUP)
1444                 props->port_cap_flags2 = rep->cap_mask2;
1445
1446         err = mlx5_query_ib_port_oper(mdev, &ib_link_width_oper,
1447                                       &props->active_speed, port, plane_index);
1448         if (err)
1449                 goto out;
1450
1451         translate_active_width(ibdev, ib_link_width_oper, &props->active_width);
1452
1453         mlx5_query_port_max_mtu(mdev, &max_mtu, port);
1454
1455         props->max_mtu = mlx5_mtu_to_ib_mtu(max_mtu);
1456
1457         mlx5_query_port_oper_mtu(mdev, &oper_mtu, port);
1458
1459         props->active_mtu = mlx5_mtu_to_ib_mtu(oper_mtu);
1460
1461         err = mlx5_query_port_vl_hw_cap(mdev, &vl_hw_cap, port);
1462         if (err)
1463                 goto out;
1464
1465         err = translate_max_vl_num(ibdev, vl_hw_cap,
1466                                    &props->max_vl_num);
1467 out:
1468         kfree(rep);
1469         return err;
1470 }
1471
1472 int mlx5_ib_query_port(struct ib_device *ibdev, u32 port,
1473                        struct ib_port_attr *props)
1474 {
1475         unsigned int count;
1476         int ret;
1477
1478         switch (mlx5_get_vport_access_method(ibdev)) {
1479         case MLX5_VPORT_ACCESS_METHOD_MAD:
1480                 ret = mlx5_query_mad_ifc_port(ibdev, port, props);
1481                 break;
1482
1483         case MLX5_VPORT_ACCESS_METHOD_HCA:
1484                 ret = mlx5_query_hca_port(ibdev, port, props);
1485                 break;
1486
1487         case MLX5_VPORT_ACCESS_METHOD_NIC:
1488                 ret = mlx5_query_port_roce(ibdev, port, props);
1489                 break;
1490
1491         default:
1492                 ret = -EINVAL;
1493         }
1494
1495         if (!ret && props) {
1496                 struct mlx5_ib_dev *dev = to_mdev(ibdev);
1497                 struct mlx5_core_dev *mdev;
1498                 bool put_mdev = true;
1499
1500                 mdev = mlx5_ib_get_native_port_mdev(dev, port, NULL);
1501                 if (!mdev) {
1502                         /* If the port isn't affiliated yet query the master.
1503                          * The master and slave will have the same values.
1504                          */
1505                         mdev = dev->mdev;
1506                         port = 1;
1507                         put_mdev = false;
1508                 }
1509                 count = mlx5_core_reserved_gids_count(mdev);
1510                 if (put_mdev)
1511                         mlx5_ib_put_native_port_mdev(dev, port);
1512                 props->gid_tbl_len -= count;
1513         }
1514         return ret;
1515 }
1516
1517 static int mlx5_ib_rep_query_port(struct ib_device *ibdev, u32 port,
1518                                   struct ib_port_attr *props)
1519 {
1520         return mlx5_query_port_roce(ibdev, port, props);
1521 }
1522
1523 static int mlx5_ib_rep_query_pkey(struct ib_device *ibdev, u32 port, u16 index,
1524                                   u16 *pkey)
1525 {
1526         /* Default special Pkey for representor device port as per the
1527          * IB specification 1.3 section 10.9.1.2.
1528          */
1529         *pkey = 0xffff;
1530         return 0;
1531 }
1532
1533 static int mlx5_ib_query_gid(struct ib_device *ibdev, u32 port, int index,
1534                              union ib_gid *gid)
1535 {
1536         struct mlx5_ib_dev *dev = to_mdev(ibdev);
1537         struct mlx5_core_dev *mdev = dev->mdev;
1538
1539         switch (mlx5_get_vport_access_method(ibdev)) {
1540         case MLX5_VPORT_ACCESS_METHOD_MAD:
1541                 return mlx5_query_mad_ifc_gids(ibdev, port, index, gid);
1542
1543         case MLX5_VPORT_ACCESS_METHOD_HCA:
1544                 return mlx5_query_hca_vport_gid(mdev, 0, port, 0, index, gid);
1545
1546         default:
1547                 return -EINVAL;
1548         }
1549
1550 }
1551
1552 static int mlx5_query_hca_nic_pkey(struct ib_device *ibdev, u32 port,
1553                                    u16 index, u16 *pkey)
1554 {
1555         struct mlx5_ib_dev *dev = to_mdev(ibdev);
1556         struct mlx5_core_dev *mdev;
1557         bool put_mdev = true;
1558         u32 mdev_port_num;
1559         int err;
1560
1561         mdev = mlx5_ib_get_native_port_mdev(dev, port, &mdev_port_num);
1562         if (!mdev) {
1563                 /* The port isn't affiliated yet, get the PKey from the master
1564                  * port. For RoCE the PKey tables will be the same.
1565                  */
1566                 put_mdev = false;
1567                 mdev = dev->mdev;
1568                 mdev_port_num = 1;
1569         }
1570
1571         err = mlx5_query_hca_vport_pkey(mdev, 0, mdev_port_num, 0,
1572                                         index, pkey);
1573         if (put_mdev)
1574                 mlx5_ib_put_native_port_mdev(dev, port);
1575
1576         return err;
1577 }
1578
1579 static int mlx5_ib_query_pkey(struct ib_device *ibdev, u32 port, u16 index,
1580                               u16 *pkey)
1581 {
1582         switch (mlx5_get_vport_access_method(ibdev)) {
1583         case MLX5_VPORT_ACCESS_METHOD_MAD:
1584                 return mlx5_query_mad_ifc_pkey(ibdev, port, index, pkey);
1585
1586         case MLX5_VPORT_ACCESS_METHOD_HCA:
1587         case MLX5_VPORT_ACCESS_METHOD_NIC:
1588                 return mlx5_query_hca_nic_pkey(ibdev, port, index, pkey);
1589         default:
1590                 return -EINVAL;
1591         }
1592 }
1593
1594 static int mlx5_ib_modify_device(struct ib_device *ibdev, int mask,
1595                                  struct ib_device_modify *props)
1596 {
1597         struct mlx5_ib_dev *dev = to_mdev(ibdev);
1598         struct mlx5_reg_node_desc in;
1599         struct mlx5_reg_node_desc out;
1600         int err;
1601
1602         if (mask & ~IB_DEVICE_MODIFY_NODE_DESC)
1603                 return -EOPNOTSUPP;
1604
1605         if (!(mask & IB_DEVICE_MODIFY_NODE_DESC))
1606                 return 0;
1607
1608         /*
1609          * If possible, pass node desc to FW, so it can generate
1610          * a 144 trap.  If cmd fails, just ignore.
1611          */
1612         memcpy(&in, props->node_desc, IB_DEVICE_NODE_DESC_MAX);
1613         err = mlx5_core_access_reg(dev->mdev, &in, sizeof(in), &out,
1614                                    sizeof(out), MLX5_REG_NODE_DESC, 0, 1);
1615         if (err)
1616                 return err;
1617
1618         memcpy(ibdev->node_desc, props->node_desc, IB_DEVICE_NODE_DESC_MAX);
1619
1620         return err;
1621 }
1622
1623 static int set_port_caps_atomic(struct mlx5_ib_dev *dev, u32 port_num, u32 mask,
1624                                 u32 value)
1625 {
1626         struct mlx5_hca_vport_context ctx = {};
1627         struct mlx5_core_dev *mdev;
1628         u32 mdev_port_num;
1629         int err;
1630
1631         mdev = mlx5_ib_get_native_port_mdev(dev, port_num, &mdev_port_num);
1632         if (!mdev)
1633                 return -ENODEV;
1634
1635         err = mlx5_query_hca_vport_context(mdev, 0, mdev_port_num, 0, &ctx);
1636         if (err)
1637                 goto out;
1638
1639         if (~ctx.cap_mask1_perm & mask) {
1640                 mlx5_ib_warn(dev, "trying to change bitmask 0x%X but change supported 0x%X\n",
1641                              mask, ctx.cap_mask1_perm);
1642                 err = -EINVAL;
1643                 goto out;
1644         }
1645
1646         ctx.cap_mask1 = value;
1647         ctx.cap_mask1_perm = mask;
1648         err = mlx5_core_modify_hca_vport_context(mdev, 0, mdev_port_num,
1649                                                  0, &ctx);
1650
1651 out:
1652         mlx5_ib_put_native_port_mdev(dev, port_num);
1653
1654         return err;
1655 }
1656
1657 static int mlx5_ib_modify_port(struct ib_device *ibdev, u32 port, int mask,
1658                                struct ib_port_modify *props)
1659 {
1660         struct mlx5_ib_dev *dev = to_mdev(ibdev);
1661         struct ib_port_attr attr;
1662         u32 tmp;
1663         int err;
1664         u32 change_mask;
1665         u32 value;
1666         bool is_ib = (mlx5_ib_port_link_layer(ibdev, port) ==
1667                       IB_LINK_LAYER_INFINIBAND);
1668
1669         /* CM layer calls ib_modify_port() regardless of the link layer. For
1670          * Ethernet ports, qkey violation and Port capabilities are meaningless.
1671          */
1672         if (!is_ib)
1673                 return 0;
1674
1675         if (MLX5_CAP_GEN(dev->mdev, ib_virt) && is_ib) {
1676                 change_mask = props->clr_port_cap_mask | props->set_port_cap_mask;
1677                 value = ~props->clr_port_cap_mask | props->set_port_cap_mask;
1678                 return set_port_caps_atomic(dev, port, change_mask, value);
1679         }
1680
1681         mutex_lock(&dev->cap_mask_mutex);
1682
1683         err = ib_query_port(ibdev, port, &attr);
1684         if (err)
1685                 goto out;
1686
1687         tmp = (attr.port_cap_flags | props->set_port_cap_mask) &
1688                 ~props->clr_port_cap_mask;
1689
1690         err = mlx5_set_port_caps(dev->mdev, port, tmp);
1691
1692 out:
1693         mutex_unlock(&dev->cap_mask_mutex);
1694         return err;
1695 }
1696
1697 static void print_lib_caps(struct mlx5_ib_dev *dev, u64 caps)
1698 {
1699         mlx5_ib_dbg(dev, "MLX5_LIB_CAP_4K_UAR = %s\n",
1700                     caps & MLX5_LIB_CAP_4K_UAR ? "y" : "n");
1701 }
1702
1703 static u16 calc_dynamic_bfregs(int uars_per_sys_page)
1704 {
1705         /* Large page with non 4k uar support might limit the dynamic size */
1706         if (uars_per_sys_page == 1  && PAGE_SIZE > 4096)
1707                 return MLX5_MIN_DYN_BFREGS;
1708
1709         return MLX5_MAX_DYN_BFREGS;
1710 }
1711
1712 static int calc_total_bfregs(struct mlx5_ib_dev *dev, bool lib_uar_4k,
1713                              struct mlx5_ib_alloc_ucontext_req_v2 *req,
1714                              struct mlx5_bfreg_info *bfregi)
1715 {
1716         int uars_per_sys_page;
1717         int bfregs_per_sys_page;
1718         int ref_bfregs = req->total_num_bfregs;
1719
1720         if (req->total_num_bfregs == 0)
1721                 return -EINVAL;
1722
1723         BUILD_BUG_ON(MLX5_MAX_BFREGS % MLX5_NON_FP_BFREGS_IN_PAGE);
1724         BUILD_BUG_ON(MLX5_MAX_BFREGS < MLX5_NON_FP_BFREGS_IN_PAGE);
1725
1726         if (req->total_num_bfregs > MLX5_MAX_BFREGS)
1727                 return -ENOMEM;
1728
1729         uars_per_sys_page = get_uars_per_sys_page(dev, lib_uar_4k);
1730         bfregs_per_sys_page = uars_per_sys_page * MLX5_NON_FP_BFREGS_PER_UAR;
1731         /* This holds the required static allocation asked by the user */
1732         req->total_num_bfregs = ALIGN(req->total_num_bfregs, bfregs_per_sys_page);
1733         if (req->num_low_latency_bfregs > req->total_num_bfregs - 1)
1734                 return -EINVAL;
1735
1736         bfregi->num_static_sys_pages = req->total_num_bfregs / bfregs_per_sys_page;
1737         bfregi->num_dyn_bfregs = ALIGN(calc_dynamic_bfregs(uars_per_sys_page), bfregs_per_sys_page);
1738         bfregi->total_num_bfregs = req->total_num_bfregs + bfregi->num_dyn_bfregs;
1739         bfregi->num_sys_pages = bfregi->total_num_bfregs / bfregs_per_sys_page;
1740
1741         mlx5_ib_dbg(dev, "uar_4k: fw support %s, lib support %s, user requested %d bfregs, allocated %d, total bfregs %d, using %d sys pages\n",
1742                     MLX5_CAP_GEN(dev->mdev, uar_4k) ? "yes" : "no",
1743                     lib_uar_4k ? "yes" : "no", ref_bfregs,
1744                     req->total_num_bfregs, bfregi->total_num_bfregs,
1745                     bfregi->num_sys_pages);
1746
1747         return 0;
1748 }
1749
1750 static int allocate_uars(struct mlx5_ib_dev *dev, struct mlx5_ib_ucontext *context)
1751 {
1752         struct mlx5_bfreg_info *bfregi;
1753         int err;
1754         int i;
1755
1756         bfregi = &context->bfregi;
1757         for (i = 0; i < bfregi->num_static_sys_pages; i++) {
1758                 err = mlx5_cmd_uar_alloc(dev->mdev, &bfregi->sys_pages[i],
1759                                          context->devx_uid);
1760                 if (err)
1761                         goto error;
1762
1763                 mlx5_ib_dbg(dev, "allocated uar %d\n", bfregi->sys_pages[i]);
1764         }
1765
1766         for (i = bfregi->num_static_sys_pages; i < bfregi->num_sys_pages; i++)
1767                 bfregi->sys_pages[i] = MLX5_IB_INVALID_UAR_INDEX;
1768
1769         return 0;
1770
1771 error:
1772         for (--i; i >= 0; i--)
1773                 if (mlx5_cmd_uar_dealloc(dev->mdev, bfregi->sys_pages[i],
1774                                          context->devx_uid))
1775                         mlx5_ib_warn(dev, "failed to free uar %d\n", i);
1776
1777         return err;
1778 }
1779
1780 static void deallocate_uars(struct mlx5_ib_dev *dev,
1781                             struct mlx5_ib_ucontext *context)
1782 {
1783         struct mlx5_bfreg_info *bfregi;
1784         int i;
1785
1786         bfregi = &context->bfregi;
1787         for (i = 0; i < bfregi->num_sys_pages; i++)
1788                 if (i < bfregi->num_static_sys_pages ||
1789                     bfregi->sys_pages[i] != MLX5_IB_INVALID_UAR_INDEX)
1790                         mlx5_cmd_uar_dealloc(dev->mdev, bfregi->sys_pages[i],
1791                                              context->devx_uid);
1792 }
1793
1794 static int mlx5_ib_enable_lb_mp(struct mlx5_core_dev *master,
1795                                 struct mlx5_core_dev *slave)
1796 {
1797         int err;
1798
1799         err = mlx5_nic_vport_update_local_lb(master, true);
1800         if (err)
1801                 return err;
1802
1803         err = mlx5_nic_vport_update_local_lb(slave, true);
1804         if (err)
1805                 goto out;
1806
1807         return 0;
1808
1809 out:
1810         mlx5_nic_vport_update_local_lb(master, false);
1811         return err;
1812 }
1813
1814 static void mlx5_ib_disable_lb_mp(struct mlx5_core_dev *master,
1815                                   struct mlx5_core_dev *slave)
1816 {
1817         mlx5_nic_vport_update_local_lb(slave, false);
1818         mlx5_nic_vport_update_local_lb(master, false);
1819 }
1820
1821 int mlx5_ib_enable_lb(struct mlx5_ib_dev *dev, bool td, bool qp)
1822 {
1823         int err = 0;
1824
1825         mutex_lock(&dev->lb.mutex);
1826         if (td)
1827                 dev->lb.user_td++;
1828         if (qp)
1829                 dev->lb.qps++;
1830
1831         if (dev->lb.user_td == 2 ||
1832             dev->lb.qps == 1) {
1833                 if (!dev->lb.enabled) {
1834                         err = mlx5_nic_vport_update_local_lb(dev->mdev, true);
1835                         dev->lb.enabled = true;
1836                 }
1837         }
1838
1839         mutex_unlock(&dev->lb.mutex);
1840
1841         return err;
1842 }
1843
1844 void mlx5_ib_disable_lb(struct mlx5_ib_dev *dev, bool td, bool qp)
1845 {
1846         mutex_lock(&dev->lb.mutex);
1847         if (td)
1848                 dev->lb.user_td--;
1849         if (qp)
1850                 dev->lb.qps--;
1851
1852         if (dev->lb.user_td == 1 &&
1853             dev->lb.qps == 0) {
1854                 if (dev->lb.enabled) {
1855                         mlx5_nic_vport_update_local_lb(dev->mdev, false);
1856                         dev->lb.enabled = false;
1857                 }
1858         }
1859
1860         mutex_unlock(&dev->lb.mutex);
1861 }
1862
1863 static int mlx5_ib_alloc_transport_domain(struct mlx5_ib_dev *dev, u32 *tdn,
1864                                           u16 uid)
1865 {
1866         int err;
1867
1868         if (!MLX5_CAP_GEN(dev->mdev, log_max_transport_domain))
1869                 return 0;
1870
1871         err = mlx5_cmd_alloc_transport_domain(dev->mdev, tdn, uid);
1872         if (err)
1873                 return err;
1874
1875         if ((MLX5_CAP_GEN(dev->mdev, port_type) != MLX5_CAP_PORT_TYPE_ETH) ||
1876             (!MLX5_CAP_GEN(dev->mdev, disable_local_lb_uc) &&
1877              !MLX5_CAP_GEN(dev->mdev, disable_local_lb_mc)))
1878                 return err;
1879
1880         return mlx5_ib_enable_lb(dev, true, false);
1881 }
1882
1883 static void mlx5_ib_dealloc_transport_domain(struct mlx5_ib_dev *dev, u32 tdn,
1884                                              u16 uid)
1885 {
1886         if (!MLX5_CAP_GEN(dev->mdev, log_max_transport_domain))
1887                 return;
1888
1889         mlx5_cmd_dealloc_transport_domain(dev->mdev, tdn, uid);
1890
1891         if ((MLX5_CAP_GEN(dev->mdev, port_type) != MLX5_CAP_PORT_TYPE_ETH) ||
1892             (!MLX5_CAP_GEN(dev->mdev, disable_local_lb_uc) &&
1893              !MLX5_CAP_GEN(dev->mdev, disable_local_lb_mc)))
1894                 return;
1895
1896         mlx5_ib_disable_lb(dev, true, false);
1897 }
1898
1899 static int set_ucontext_resp(struct ib_ucontext *uctx,
1900                              struct mlx5_ib_alloc_ucontext_resp *resp)
1901 {
1902         struct ib_device *ibdev = uctx->device;
1903         struct mlx5_ib_dev *dev = to_mdev(ibdev);
1904         struct mlx5_ib_ucontext *context = to_mucontext(uctx);
1905         struct mlx5_bfreg_info *bfregi = &context->bfregi;
1906
1907         if (MLX5_CAP_GEN(dev->mdev, dump_fill_mkey)) {
1908                 resp->dump_fill_mkey = dev->mkeys.dump_fill_mkey;
1909                 resp->comp_mask |=
1910                         MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_DUMP_FILL_MKEY;
1911         }
1912
1913         resp->qp_tab_size = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp);
1914         if (mlx5_wc_support_get(dev->mdev))
1915                 resp->bf_reg_size = 1 << MLX5_CAP_GEN(dev->mdev,
1916                                                       log_bf_reg_size);
1917         resp->cache_line_size = cache_line_size();
1918         resp->max_sq_desc_sz = MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq);
1919         resp->max_rq_desc_sz = MLX5_CAP_GEN(dev->mdev, max_wqe_sz_rq);
1920         resp->max_send_wqebb = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz);
1921         resp->max_recv_wr = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz);
1922         resp->max_srq_recv_wr = 1 << MLX5_CAP_GEN(dev->mdev, log_max_srq_sz);
1923         resp->cqe_version = context->cqe_version;
1924         resp->log_uar_size = MLX5_CAP_GEN(dev->mdev, uar_4k) ?
1925                                 MLX5_ADAPTER_PAGE_SHIFT : PAGE_SHIFT;
1926         resp->num_uars_per_page = MLX5_CAP_GEN(dev->mdev, uar_4k) ?
1927                                         MLX5_CAP_GEN(dev->mdev,
1928                                                      num_of_uars_per_page) : 1;
1929         resp->tot_bfregs = bfregi->lib_uar_dyn ? 0 :
1930                         bfregi->total_num_bfregs - bfregi->num_dyn_bfregs;
1931         resp->num_ports = dev->num_ports;
1932         resp->cmds_supp_uhw |= MLX5_USER_CMDS_SUPP_UHW_QUERY_DEVICE |
1933                                       MLX5_USER_CMDS_SUPP_UHW_CREATE_AH;
1934
1935         if (mlx5_ib_port_link_layer(ibdev, 1) == IB_LINK_LAYER_ETHERNET) {
1936                 mlx5_query_min_inline(dev->mdev, &resp->eth_min_inline);
1937                 resp->eth_min_inline++;
1938         }
1939
1940         if (dev->mdev->clock_info)
1941                 resp->clock_info_versions = BIT(MLX5_IB_CLOCK_INFO_V1);
1942
1943         /*
1944          * We don't want to expose information from the PCI bar that is located
1945          * after 4096 bytes, so if the arch only supports larger pages, let's
1946          * pretend we don't support reading the HCA's core clock. This is also
1947          * forced by mmap function.
1948          */
1949         if (PAGE_SIZE <= 4096) {
1950                 resp->comp_mask |=
1951                         MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_CORE_CLOCK_OFFSET;
1952                 resp->hca_core_clock_offset =
1953                         offsetof(struct mlx5_init_seg,
1954                                  internal_timer_h) % PAGE_SIZE;
1955         }
1956
1957         if (MLX5_CAP_GEN(dev->mdev, ece_support))
1958                 resp->comp_mask |= MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_ECE;
1959
1960         if (rt_supported(MLX5_CAP_GEN(dev->mdev, sq_ts_format)) &&
1961             rt_supported(MLX5_CAP_GEN(dev->mdev, rq_ts_format)) &&
1962             rt_supported(MLX5_CAP_ROCE(dev->mdev, qp_ts_format)))
1963                 resp->comp_mask |=
1964                         MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_REAL_TIME_TS;
1965
1966         resp->num_dyn_bfregs = bfregi->num_dyn_bfregs;
1967
1968         if (MLX5_CAP_GEN(dev->mdev, drain_sigerr))
1969                 resp->comp_mask |= MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_SQD2RTS;
1970
1971         resp->comp_mask |=
1972                 MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_MKEY_UPDATE_TAG;
1973
1974         return 0;
1975 }
1976
1977 static bool uctx_rdma_ctrl_is_enabled(u64 enabled_caps)
1978 {
1979         return UCAP_ENABLED(enabled_caps, RDMA_UCAP_MLX5_CTRL_LOCAL) ||
1980                UCAP_ENABLED(enabled_caps, RDMA_UCAP_MLX5_CTRL_OTHER_VHCA);
1981 }
1982
1983 static int mlx5_ib_alloc_ucontext(struct ib_ucontext *uctx,
1984                                   struct ib_udata *udata)
1985 {
1986         struct ib_device *ibdev = uctx->device;
1987         struct mlx5_ib_dev *dev = to_mdev(ibdev);
1988         struct mlx5_ib_alloc_ucontext_req_v2 req = {};
1989         struct mlx5_ib_alloc_ucontext_resp resp = {};
1990         struct mlx5_ib_ucontext *context = to_mucontext(uctx);
1991         struct mlx5_bfreg_info *bfregi;
1992         int ver;
1993         int err;
1994         size_t min_req_v2 = offsetof(struct mlx5_ib_alloc_ucontext_req_v2,
1995                                      max_cqe_version);
1996         bool lib_uar_4k;
1997         bool lib_uar_dyn;
1998
1999         if (!dev->ib_active)
2000                 return -EAGAIN;
2001
2002         if (udata->inlen == sizeof(struct mlx5_ib_alloc_ucontext_req))
2003                 ver = 0;
2004         else if (udata->inlen >= min_req_v2)
2005                 ver = 2;
2006         else
2007                 return -EINVAL;
2008
2009         err = ib_copy_from_udata(&req, udata, min(udata->inlen, sizeof(req)));
2010         if (err)
2011                 return err;
2012
2013         if (req.flags & ~MLX5_IB_ALLOC_UCTX_DEVX)
2014                 return -EOPNOTSUPP;
2015
2016         if (req.comp_mask || req.reserved0 || req.reserved1 || req.reserved2)
2017                 return -EOPNOTSUPP;
2018
2019         req.total_num_bfregs = ALIGN(req.total_num_bfregs,
2020                                     MLX5_NON_FP_BFREGS_PER_UAR);
2021         if (req.num_low_latency_bfregs > req.total_num_bfregs - 1)
2022                 return -EINVAL;
2023
2024         if (req.flags & MLX5_IB_ALLOC_UCTX_DEVX) {
2025                 err = mlx5_ib_devx_create(dev, true, uctx->enabled_caps);
2026                 if (err < 0)
2027                         goto out_ctx;
2028                 context->devx_uid = err;
2029
2030                 if (uctx_rdma_ctrl_is_enabled(uctx->enabled_caps)) {
2031                         err = mlx5_cmd_add_privileged_uid(dev->mdev,
2032                                                           context->devx_uid);
2033                         if (err)
2034                                 goto out_devx;
2035                 }
2036         }
2037
2038         lib_uar_4k = req.lib_caps & MLX5_LIB_CAP_4K_UAR;
2039         lib_uar_dyn = req.lib_caps & MLX5_LIB_CAP_DYN_UAR;
2040         bfregi = &context->bfregi;
2041
2042         if (lib_uar_dyn) {
2043                 bfregi->lib_uar_dyn = lib_uar_dyn;
2044                 goto uar_done;
2045         }
2046
2047         /* updates req->total_num_bfregs */
2048         err = calc_total_bfregs(dev, lib_uar_4k, &req, bfregi);
2049         if (err)
2050                 goto out_ucap;
2051
2052         mutex_init(&bfregi->lock);
2053         bfregi->lib_uar_4k = lib_uar_4k;
2054         bfregi->count = kcalloc(bfregi->total_num_bfregs, sizeof(*bfregi->count),
2055                                 GFP_KERNEL);
2056         if (!bfregi->count) {
2057                 err = -ENOMEM;
2058                 goto out_ucap;
2059         }
2060
2061         bfregi->sys_pages = kcalloc(bfregi->num_sys_pages,
2062                                     sizeof(*bfregi->sys_pages),
2063                                     GFP_KERNEL);
2064         if (!bfregi->sys_pages) {
2065                 err = -ENOMEM;
2066                 goto out_count;
2067         }
2068
2069         err = allocate_uars(dev, context);
2070         if (err)
2071                 goto out_sys_pages;
2072
2073 uar_done:
2074         err = mlx5_ib_alloc_transport_domain(dev, &context->tdn,
2075                                              context->devx_uid);
2076         if (err)
2077                 goto out_uars;
2078
2079         INIT_LIST_HEAD(&context->db_page_list);
2080         mutex_init(&context->db_page_mutex);
2081
2082         context->cqe_version = min_t(__u8,
2083                                  (__u8)MLX5_CAP_GEN(dev->mdev, cqe_version),
2084                                  req.max_cqe_version);
2085
2086         err = set_ucontext_resp(uctx, &resp);
2087         if (err)
2088                 goto out_mdev;
2089
2090         resp.response_length = min(udata->outlen, sizeof(resp));
2091         err = ib_copy_to_udata(udata, &resp, resp.response_length);
2092         if (err)
2093                 goto out_mdev;
2094
2095         bfregi->ver = ver;
2096         bfregi->num_low_latency_bfregs = req.num_low_latency_bfregs;
2097         context->lib_caps = req.lib_caps;
2098         print_lib_caps(dev, context->lib_caps);
2099
2100         if (mlx5_ib_lag_should_assign_affinity(dev)) {
2101                 u32 port = mlx5_core_native_port_num(dev->mdev) - 1;
2102
2103                 atomic_set(&context->tx_port_affinity,
2104                            atomic_add_return(
2105                                    1, &dev->port[port].roce.tx_port_affinity));
2106         }
2107
2108         return 0;
2109
2110 out_mdev:
2111         mlx5_ib_dealloc_transport_domain(dev, context->tdn, context->devx_uid);
2112
2113 out_uars:
2114         deallocate_uars(dev, context);
2115
2116 out_sys_pages:
2117         kfree(bfregi->sys_pages);
2118
2119 out_count:
2120         kfree(bfregi->count);
2121
2122 out_ucap:
2123         if (req.flags & MLX5_IB_ALLOC_UCTX_DEVX &&
2124             uctx_rdma_ctrl_is_enabled(uctx->enabled_caps))
2125                 mlx5_cmd_remove_privileged_uid(dev->mdev, context->devx_uid);
2126
2127 out_devx:
2128         if (req.flags & MLX5_IB_ALLOC_UCTX_DEVX)
2129                 mlx5_ib_devx_destroy(dev, context->devx_uid);
2130
2131 out_ctx:
2132         return err;
2133 }
2134
2135 static int mlx5_ib_query_ucontext(struct ib_ucontext *ibcontext,
2136                                   struct uverbs_attr_bundle *attrs)
2137 {
2138         struct mlx5_ib_alloc_ucontext_resp uctx_resp = {};
2139         int ret;
2140
2141         ret = set_ucontext_resp(ibcontext, &uctx_resp);
2142         if (ret)
2143                 return ret;
2144
2145         uctx_resp.response_length =
2146                 min_t(size_t,
2147                       uverbs_attr_get_len(attrs,
2148                                 MLX5_IB_ATTR_QUERY_CONTEXT_RESP_UCTX),
2149                       sizeof(uctx_resp));
2150
2151         ret = uverbs_copy_to_struct_or_zero(attrs,
2152                                         MLX5_IB_ATTR_QUERY_CONTEXT_RESP_UCTX,
2153                                         &uctx_resp,
2154                                         sizeof(uctx_resp));
2155         return ret;
2156 }
2157
2158 static void mlx5_ib_dealloc_ucontext(struct ib_ucontext *ibcontext)
2159 {
2160         struct mlx5_ib_ucontext *context = to_mucontext(ibcontext);
2161         struct mlx5_ib_dev *dev = to_mdev(ibcontext->device);
2162         struct mlx5_bfreg_info *bfregi;
2163
2164         bfregi = &context->bfregi;
2165         mlx5_ib_dealloc_transport_domain(dev, context->tdn, context->devx_uid);
2166
2167         deallocate_uars(dev, context);
2168         kfree(bfregi->sys_pages);
2169         kfree(bfregi->count);
2170
2171         if (context->devx_uid) {
2172                 if (uctx_rdma_ctrl_is_enabled(ibcontext->enabled_caps))
2173                         mlx5_cmd_remove_privileged_uid(dev->mdev,
2174                                                        context->devx_uid);
2175                 mlx5_ib_devx_destroy(dev, context->devx_uid);
2176         }
2177 }
2178
2179 static phys_addr_t uar_index2pfn(struct mlx5_ib_dev *dev,
2180                                  int uar_idx)
2181 {
2182         int fw_uars_per_page;
2183
2184         fw_uars_per_page = MLX5_CAP_GEN(dev->mdev, uar_4k) ? MLX5_UARS_IN_PAGE : 1;
2185
2186         return (dev->mdev->bar_addr >> PAGE_SHIFT) + uar_idx / fw_uars_per_page;
2187 }
2188
2189 static u64 uar_index2paddress(struct mlx5_ib_dev *dev,
2190                                  int uar_idx)
2191 {
2192         unsigned int fw_uars_per_page;
2193
2194         fw_uars_per_page = MLX5_CAP_GEN(dev->mdev, uar_4k) ?
2195                                 MLX5_UARS_IN_PAGE : 1;
2196
2197         return (dev->mdev->bar_addr + (uar_idx / fw_uars_per_page) * PAGE_SIZE);
2198 }
2199
2200 static int get_command(unsigned long offset)
2201 {
2202         return (offset >> MLX5_IB_MMAP_CMD_SHIFT) & MLX5_IB_MMAP_CMD_MASK;
2203 }
2204
2205 static int get_arg(unsigned long offset)
2206 {
2207         return offset & ((1 << MLX5_IB_MMAP_CMD_SHIFT) - 1);
2208 }
2209
2210 static int get_index(unsigned long offset)
2211 {
2212         return get_arg(offset);
2213 }
2214
2215 /* Index resides in an extra byte to enable larger values than 255 */
2216 static int get_extended_index(unsigned long offset)
2217 {
2218         return get_arg(offset) | ((offset >> 16) & 0xff) << 8;
2219 }
2220
2221
2222 static void mlx5_ib_disassociate_ucontext(struct ib_ucontext *ibcontext)
2223 {
2224 }
2225
2226 static inline char *mmap_cmd2str(enum mlx5_ib_mmap_cmd cmd)
2227 {
2228         switch (cmd) {
2229         case MLX5_IB_MMAP_WC_PAGE:
2230                 return "WC";
2231         case MLX5_IB_MMAP_REGULAR_PAGE:
2232                 return "best effort WC";
2233         case MLX5_IB_MMAP_NC_PAGE:
2234                 return "NC";
2235         case MLX5_IB_MMAP_DEVICE_MEM:
2236                 return "Device Memory";
2237         default:
2238                 return "Unknown";
2239         }
2240 }
2241
2242 static int mlx5_ib_mmap_clock_info_page(struct mlx5_ib_dev *dev,
2243                                         struct vm_area_struct *vma,
2244                                         struct mlx5_ib_ucontext *context)
2245 {
2246         if ((vma->vm_end - vma->vm_start != PAGE_SIZE) ||
2247             !(vma->vm_flags & VM_SHARED))
2248                 return -EINVAL;
2249
2250         if (get_index(vma->vm_pgoff) != MLX5_IB_CLOCK_INFO_V1)
2251                 return -EOPNOTSUPP;
2252
2253         if (vma->vm_flags & (VM_WRITE | VM_EXEC))
2254                 return -EPERM;
2255         vm_flags_clear(vma, VM_MAYWRITE);
2256
2257         if (!dev->mdev->clock_info)
2258                 return -EOPNOTSUPP;
2259
2260         return vm_insert_page(vma, vma->vm_start,
2261                               virt_to_page(dev->mdev->clock_info));
2262 }
2263
2264 static void mlx5_ib_mmap_free(struct rdma_user_mmap_entry *entry)
2265 {
2266         struct mlx5_user_mmap_entry *mentry = to_mmmap(entry);
2267         struct mlx5_ib_dev *dev = to_mdev(entry->ucontext->device);
2268         struct mlx5_var_table *var_table = &dev->var_table;
2269         struct mlx5_ib_ucontext *context = to_mucontext(entry->ucontext);
2270
2271         switch (mentry->mmap_flag) {
2272         case MLX5_IB_MMAP_TYPE_MEMIC:
2273         case MLX5_IB_MMAP_TYPE_MEMIC_OP:
2274                 mlx5_ib_dm_mmap_free(dev, mentry);
2275                 break;
2276         case MLX5_IB_MMAP_TYPE_VAR:
2277                 mutex_lock(&var_table->bitmap_lock);
2278                 clear_bit(mentry->page_idx, var_table->bitmap);
2279                 mutex_unlock(&var_table->bitmap_lock);
2280                 kfree(mentry);
2281                 break;
2282         case MLX5_IB_MMAP_TYPE_UAR_WC:
2283         case MLX5_IB_MMAP_TYPE_UAR_NC:
2284                 mlx5_cmd_uar_dealloc(dev->mdev, mentry->page_idx,
2285                                      context->devx_uid);
2286                 kfree(mentry);
2287                 break;
2288         default:
2289                 WARN_ON(true);
2290         }
2291 }
2292
2293 static int uar_mmap(struct mlx5_ib_dev *dev, enum mlx5_ib_mmap_cmd cmd,
2294                     struct vm_area_struct *vma,
2295                     struct mlx5_ib_ucontext *context)
2296 {
2297         struct mlx5_bfreg_info *bfregi = &context->bfregi;
2298         int err;
2299         unsigned long idx;
2300         phys_addr_t pfn;
2301         pgprot_t prot;
2302         u32 bfreg_dyn_idx = 0;
2303         u32 uar_index;
2304         int dyn_uar = (cmd == MLX5_IB_MMAP_ALLOC_WC);
2305         int max_valid_idx = dyn_uar ? bfregi->num_sys_pages :
2306                                 bfregi->num_static_sys_pages;
2307
2308         if (bfregi->lib_uar_dyn)
2309                 return -EINVAL;
2310
2311         if (vma->vm_end - vma->vm_start != PAGE_SIZE)
2312                 return -EINVAL;
2313
2314         if (dyn_uar)
2315                 idx = get_extended_index(vma->vm_pgoff) + bfregi->num_static_sys_pages;
2316         else
2317                 idx = get_index(vma->vm_pgoff);
2318
2319         if (idx >= max_valid_idx) {
2320                 mlx5_ib_warn(dev, "invalid uar index %lu, max=%d\n",
2321                              idx, max_valid_idx);
2322                 return -EINVAL;
2323         }
2324
2325         switch (cmd) {
2326         case MLX5_IB_MMAP_WC_PAGE:
2327         case MLX5_IB_MMAP_ALLOC_WC:
2328         case MLX5_IB_MMAP_REGULAR_PAGE:
2329                 /* For MLX5_IB_MMAP_REGULAR_PAGE do the best effort to get WC */
2330                 prot = pgprot_writecombine(vma->vm_page_prot);
2331                 break;
2332         case MLX5_IB_MMAP_NC_PAGE:
2333                 prot = pgprot_noncached(vma->vm_page_prot);
2334                 break;
2335         default:
2336                 return -EINVAL;
2337         }
2338
2339         if (dyn_uar) {
2340                 int uars_per_page;
2341
2342                 uars_per_page = get_uars_per_sys_page(dev, bfregi->lib_uar_4k);
2343                 bfreg_dyn_idx = idx * (uars_per_page * MLX5_NON_FP_BFREGS_PER_UAR);
2344                 if (bfreg_dyn_idx >= bfregi->total_num_bfregs) {
2345                         mlx5_ib_warn(dev, "invalid bfreg_dyn_idx %u, max=%u\n",
2346                                      bfreg_dyn_idx, bfregi->total_num_bfregs);
2347                         return -EINVAL;
2348                 }
2349
2350                 mutex_lock(&bfregi->lock);
2351                 /* Fail if uar already allocated, first bfreg index of each
2352                  * page holds its count.
2353                  */
2354                 if (bfregi->count[bfreg_dyn_idx]) {
2355                         mlx5_ib_warn(dev, "wrong offset, idx %lu is busy, bfregn=%u\n", idx, bfreg_dyn_idx);
2356                         mutex_unlock(&bfregi->lock);
2357                         return -EINVAL;
2358                 }
2359
2360                 bfregi->count[bfreg_dyn_idx]++;
2361                 mutex_unlock(&bfregi->lock);
2362
2363                 err = mlx5_cmd_uar_alloc(dev->mdev, &uar_index,
2364                                          context->devx_uid);
2365                 if (err) {
2366                         mlx5_ib_warn(dev, "UAR alloc failed\n");
2367                         goto free_bfreg;
2368                 }
2369         } else {
2370                 uar_index = bfregi->sys_pages[idx];
2371         }
2372
2373         pfn = uar_index2pfn(dev, uar_index);
2374         mlx5_ib_dbg(dev, "uar idx 0x%lx, pfn %pa\n", idx, &pfn);
2375
2376         err = rdma_user_mmap_io(&context->ibucontext, vma, pfn, PAGE_SIZE,
2377                                 prot, NULL);
2378         if (err) {
2379                 mlx5_ib_err(dev,
2380                             "rdma_user_mmap_io failed with error=%d, mmap_cmd=%s\n",
2381                             err, mmap_cmd2str(cmd));
2382                 goto err;
2383         }
2384
2385         if (dyn_uar)
2386                 bfregi->sys_pages[idx] = uar_index;
2387         return 0;
2388
2389 err:
2390         if (!dyn_uar)
2391                 return err;
2392
2393         mlx5_cmd_uar_dealloc(dev->mdev, idx, context->devx_uid);
2394
2395 free_bfreg:
2396         mlx5_ib_free_bfreg(dev, bfregi, bfreg_dyn_idx);
2397
2398         return err;
2399 }
2400
2401 static unsigned long mlx5_vma_to_pgoff(struct vm_area_struct *vma)
2402 {
2403         unsigned long idx;
2404         u8 command;
2405
2406         command = get_command(vma->vm_pgoff);
2407         idx = get_extended_index(vma->vm_pgoff);
2408
2409         return (command << 16 | idx);
2410 }
2411
2412 static int mlx5_ib_mmap_offset(struct mlx5_ib_dev *dev,
2413                                struct vm_area_struct *vma,
2414                                struct ib_ucontext *ucontext)
2415 {
2416         struct mlx5_user_mmap_entry *mentry;
2417         struct rdma_user_mmap_entry *entry;
2418         unsigned long pgoff;
2419         pgprot_t prot;
2420         phys_addr_t pfn;
2421         int ret;
2422
2423         pgoff = mlx5_vma_to_pgoff(vma);
2424         entry = rdma_user_mmap_entry_get_pgoff(ucontext, pgoff);
2425         if (!entry)
2426                 return -EINVAL;
2427
2428         mentry = to_mmmap(entry);
2429         pfn = (mentry->address >> PAGE_SHIFT);
2430         if (mentry->mmap_flag == MLX5_IB_MMAP_TYPE_VAR ||
2431             mentry->mmap_flag == MLX5_IB_MMAP_TYPE_UAR_NC)
2432                 prot = pgprot_noncached(vma->vm_page_prot);
2433         else
2434                 prot = pgprot_writecombine(vma->vm_page_prot);
2435         ret = rdma_user_mmap_io(ucontext, vma, pfn,
2436                                 entry->npages * PAGE_SIZE,
2437                                 prot,
2438                                 entry);
2439         rdma_user_mmap_entry_put(&mentry->rdma_entry);
2440         return ret;
2441 }
2442
2443 static u64 mlx5_entry_to_mmap_offset(struct mlx5_user_mmap_entry *entry)
2444 {
2445         u64 cmd = (entry->rdma_entry.start_pgoff >> 16) & 0xFFFF;
2446         u64 index = entry->rdma_entry.start_pgoff & 0xFFFF;
2447
2448         return (((index >> 8) << 16) | (cmd << MLX5_IB_MMAP_CMD_SHIFT) |
2449                 (index & 0xFF)) << PAGE_SHIFT;
2450 }
2451
2452 static int mlx5_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma)
2453 {
2454         struct mlx5_ib_ucontext *context = to_mucontext(ibcontext);
2455         struct mlx5_ib_dev *dev = to_mdev(ibcontext->device);
2456         unsigned long command;
2457         phys_addr_t pfn;
2458
2459         command = get_command(vma->vm_pgoff);
2460         switch (command) {
2461         case MLX5_IB_MMAP_WC_PAGE:
2462         case MLX5_IB_MMAP_ALLOC_WC:
2463                 if (!mlx5_wc_support_get(dev->mdev))
2464                         return -EPERM;
2465                 fallthrough;
2466         case MLX5_IB_MMAP_NC_PAGE:
2467         case MLX5_IB_MMAP_REGULAR_PAGE:
2468                 return uar_mmap(dev, command, vma, context);
2469
2470         case MLX5_IB_MMAP_GET_CONTIGUOUS_PAGES:
2471                 return -ENOSYS;
2472
2473         case MLX5_IB_MMAP_CORE_CLOCK:
2474                 if (vma->vm_end - vma->vm_start != PAGE_SIZE)
2475                         return -EINVAL;
2476
2477                 if (vma->vm_flags & VM_WRITE)
2478                         return -EPERM;
2479                 vm_flags_clear(vma, VM_MAYWRITE);
2480
2481                 /* Don't expose to user-space information it shouldn't have */
2482                 if (PAGE_SIZE > 4096)
2483                         return -EOPNOTSUPP;
2484
2485                 pfn = (dev->mdev->iseg_base +
2486                        offsetof(struct mlx5_init_seg, internal_timer_h)) >>
2487                         PAGE_SHIFT;
2488                 return rdma_user_mmap_io(&context->ibucontext, vma, pfn,
2489                                          PAGE_SIZE,
2490                                          pgprot_noncached(vma->vm_page_prot),
2491                                          NULL);
2492         case MLX5_IB_MMAP_CLOCK_INFO:
2493                 return mlx5_ib_mmap_clock_info_page(dev, vma, context);
2494
2495         default:
2496                 return mlx5_ib_mmap_offset(dev, vma, ibcontext);
2497         }
2498
2499         return 0;
2500 }
2501
2502 static int mlx5_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
2503 {
2504         struct mlx5_ib_pd *pd = to_mpd(ibpd);
2505         struct ib_device *ibdev = ibpd->device;
2506         struct mlx5_ib_alloc_pd_resp resp;
2507         int err;
2508         u32 out[MLX5_ST_SZ_DW(alloc_pd_out)] = {};
2509         u32 in[MLX5_ST_SZ_DW(alloc_pd_in)] = {};
2510         u16 uid = 0;
2511         struct mlx5_ib_ucontext *context = rdma_udata_to_drv_context(
2512                 udata, struct mlx5_ib_ucontext, ibucontext);
2513
2514         uid = context ? context->devx_uid : 0;
2515         MLX5_SET(alloc_pd_in, in, opcode, MLX5_CMD_OP_ALLOC_PD);
2516         MLX5_SET(alloc_pd_in, in, uid, uid);
2517         err = mlx5_cmd_exec_inout(to_mdev(ibdev)->mdev, alloc_pd, in, out);
2518         if (err)
2519                 return err;
2520
2521         pd->pdn = MLX5_GET(alloc_pd_out, out, pd);
2522         pd->uid = uid;
2523         if (udata) {
2524                 resp.pdn = pd->pdn;
2525                 if (ib_copy_to_udata(udata, &resp, sizeof(resp))) {
2526                         mlx5_cmd_dealloc_pd(to_mdev(ibdev)->mdev, pd->pdn, uid);
2527                         return -EFAULT;
2528                 }
2529         }
2530
2531         return 0;
2532 }
2533
2534 static int mlx5_ib_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata)
2535 {
2536         struct mlx5_ib_dev *mdev = to_mdev(pd->device);
2537         struct mlx5_ib_pd *mpd = to_mpd(pd);
2538
2539         return mlx5_cmd_dealloc_pd(mdev->mdev, mpd->pdn, mpd->uid);
2540 }
2541
2542 static int mlx5_ib_mcg_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
2543 {
2544         struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
2545         struct mlx5_ib_qp *mqp = to_mqp(ibqp);
2546         int err;
2547         u16 uid;
2548
2549         uid = ibqp->pd ?
2550                 to_mpd(ibqp->pd)->uid : 0;
2551
2552         if (mqp->flags & IB_QP_CREATE_SOURCE_QPN) {
2553                 mlx5_ib_dbg(dev, "Attaching a multi cast group to underlay QP is not supported\n");
2554                 return -EOPNOTSUPP;
2555         }
2556
2557         err = mlx5_cmd_attach_mcg(dev->mdev, gid, ibqp->qp_num, uid);
2558         if (err)
2559                 mlx5_ib_warn(dev, "failed attaching QPN 0x%x, MGID %pI6\n",
2560                              ibqp->qp_num, gid->raw);
2561
2562         return err;
2563 }
2564
2565 static int mlx5_ib_mcg_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
2566 {
2567         struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
2568         int err;
2569         u16 uid;
2570
2571         uid = ibqp->pd ?
2572                 to_mpd(ibqp->pd)->uid : 0;
2573         err = mlx5_cmd_detach_mcg(dev->mdev, gid, ibqp->qp_num, uid);
2574         if (err)
2575                 mlx5_ib_warn(dev, "failed detaching QPN 0x%x, MGID %pI6\n",
2576                              ibqp->qp_num, gid->raw);
2577
2578         return err;
2579 }
2580
2581 static int init_node_data(struct mlx5_ib_dev *dev)
2582 {
2583         int err;
2584
2585         err = mlx5_query_node_desc(dev, dev->ib_dev.node_desc);
2586         if (err)
2587                 return err;
2588
2589         dev->mdev->rev_id = dev->mdev->pdev->revision;
2590
2591         return mlx5_query_node_guid(dev, &dev->ib_dev.node_guid);
2592 }
2593
2594 static ssize_t fw_pages_show(struct device *device,
2595                              struct device_attribute *attr, char *buf)
2596 {
2597         struct mlx5_ib_dev *dev =
2598                 rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev);
2599
2600         return sysfs_emit(buf, "%d\n", dev->mdev->priv.fw_pages);
2601 }
2602 static DEVICE_ATTR_RO(fw_pages);
2603
2604 static ssize_t reg_pages_show(struct device *device,
2605                               struct device_attribute *attr, char *buf)
2606 {
2607         struct mlx5_ib_dev *dev =
2608                 rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev);
2609
2610         return sysfs_emit(buf, "%d\n", atomic_read(&dev->mdev->priv.reg_pages));
2611 }
2612 static DEVICE_ATTR_RO(reg_pages);
2613
2614 static ssize_t hca_type_show(struct device *device,
2615                              struct device_attribute *attr, char *buf)
2616 {
2617         struct mlx5_ib_dev *dev =
2618                 rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev);
2619
2620         return sysfs_emit(buf, "MT%d\n", dev->mdev->pdev->device);
2621 }
2622 static DEVICE_ATTR_RO(hca_type);
2623
2624 static ssize_t hw_rev_show(struct device *device,
2625                            struct device_attribute *attr, char *buf)
2626 {
2627         struct mlx5_ib_dev *dev =
2628                 rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev);
2629
2630         return sysfs_emit(buf, "%x\n", dev->mdev->rev_id);
2631 }
2632 static DEVICE_ATTR_RO(hw_rev);
2633
2634 static ssize_t board_id_show(struct device *device,
2635                              struct device_attribute *attr, char *buf)
2636 {
2637         struct mlx5_ib_dev *dev =
2638                 rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev);
2639
2640         return sysfs_emit(buf, "%.*s\n", MLX5_BOARD_ID_LEN,
2641                           dev->mdev->board_id);
2642 }
2643 static DEVICE_ATTR_RO(board_id);
2644
2645 static struct attribute *mlx5_class_attributes[] = {
2646         &dev_attr_hw_rev.attr,
2647         &dev_attr_hca_type.attr,
2648         &dev_attr_board_id.attr,
2649         &dev_attr_fw_pages.attr,
2650         &dev_attr_reg_pages.attr,
2651         NULL,
2652 };
2653
2654 static const struct attribute_group mlx5_attr_group = {
2655         .attrs = mlx5_class_attributes,
2656 };
2657
2658 static void pkey_change_handler(struct work_struct *work)
2659 {
2660         struct mlx5_ib_port_resources *ports =
2661                 container_of(work, struct mlx5_ib_port_resources,
2662                              pkey_change_work);
2663
2664         if (!ports->gsi)
2665                 /*
2666                  * We got this event before device was fully configured
2667                  * and MAD registration code wasn't called/finished yet.
2668                  */
2669                 return;
2670
2671         mlx5_ib_gsi_pkey_change(ports->gsi);
2672 }
2673
2674 static void mlx5_ib_handle_internal_error(struct mlx5_ib_dev *ibdev)
2675 {
2676         struct mlx5_ib_qp *mqp;
2677         struct mlx5_ib_cq *send_mcq, *recv_mcq;
2678         struct mlx5_core_cq *mcq;
2679         struct list_head cq_armed_list;
2680         unsigned long flags_qp;
2681         unsigned long flags_cq;
2682         unsigned long flags;
2683
2684         INIT_LIST_HEAD(&cq_armed_list);
2685
2686         /* Go over qp list reside on that ibdev, sync with create/destroy qp.*/
2687         spin_lock_irqsave(&ibdev->reset_flow_resource_lock, flags);
2688         list_for_each_entry(mqp, &ibdev->qp_list, qps_list) {
2689                 spin_lock_irqsave(&mqp->sq.lock, flags_qp);
2690                 if (mqp->sq.tail != mqp->sq.head) {
2691                         send_mcq = to_mcq(mqp->ibqp.send_cq);
2692                         spin_lock_irqsave(&send_mcq->lock, flags_cq);
2693                         if (send_mcq->mcq.comp &&
2694                             mqp->ibqp.send_cq->comp_handler) {
2695                                 if (!send_mcq->mcq.reset_notify_added) {
2696                                         send_mcq->mcq.reset_notify_added = 1;
2697                                         list_add_tail(&send_mcq->mcq.reset_notify,
2698                                                       &cq_armed_list);
2699                                 }
2700                         }
2701                         spin_unlock_irqrestore(&send_mcq->lock, flags_cq);
2702                 }
2703                 spin_unlock_irqrestore(&mqp->sq.lock, flags_qp);
2704                 spin_lock_irqsave(&mqp->rq.lock, flags_qp);
2705                 /* no handling is needed for SRQ */
2706                 if (!mqp->ibqp.srq) {
2707                         if (mqp->rq.tail != mqp->rq.head) {
2708                                 recv_mcq = to_mcq(mqp->ibqp.recv_cq);
2709                                 spin_lock_irqsave(&recv_mcq->lock, flags_cq);
2710                                 if (recv_mcq->mcq.comp &&
2711                                     mqp->ibqp.recv_cq->comp_handler) {
2712                                         if (!recv_mcq->mcq.reset_notify_added) {
2713                                                 recv_mcq->mcq.reset_notify_added = 1;
2714                                                 list_add_tail(&recv_mcq->mcq.reset_notify,
2715                                                               &cq_armed_list);
2716                                         }
2717                                 }
2718                                 spin_unlock_irqrestore(&recv_mcq->lock,
2719                                                        flags_cq);
2720                         }
2721                 }
2722                 spin_unlock_irqrestore(&mqp->rq.lock, flags_qp);
2723         }
2724         /*At that point all inflight post send were put to be executed as of we
2725          * lock/unlock above locks Now need to arm all involved CQs.
2726          */
2727         list_for_each_entry(mcq, &cq_armed_list, reset_notify) {
2728                 mcq->comp(mcq, NULL);
2729         }
2730         spin_unlock_irqrestore(&ibdev->reset_flow_resource_lock, flags);
2731 }
2732
2733 static void delay_drop_handler(struct work_struct *work)
2734 {
2735         int err;
2736         struct mlx5_ib_delay_drop *delay_drop =
2737                 container_of(work, struct mlx5_ib_delay_drop,
2738                              delay_drop_work);
2739
2740         atomic_inc(&delay_drop->events_cnt);
2741
2742         mutex_lock(&delay_drop->lock);
2743         err = mlx5_core_set_delay_drop(delay_drop->dev, delay_drop->timeout);
2744         if (err) {
2745                 mlx5_ib_warn(delay_drop->dev, "Failed to set delay drop, timeout=%u\n",
2746                              delay_drop->timeout);
2747                 delay_drop->activate = false;
2748         }
2749         mutex_unlock(&delay_drop->lock);
2750 }
2751
2752 static void handle_general_event(struct mlx5_ib_dev *ibdev, struct mlx5_eqe *eqe,
2753                                  struct ib_event *ibev)
2754 {
2755         u32 port = (eqe->data.port.port >> 4) & 0xf;
2756
2757         switch (eqe->sub_type) {
2758         case MLX5_GENERAL_SUBTYPE_DELAY_DROP_TIMEOUT:
2759                 if (mlx5_ib_port_link_layer(&ibdev->ib_dev, port) ==
2760                                             IB_LINK_LAYER_ETHERNET)
2761                         schedule_work(&ibdev->delay_drop.delay_drop_work);
2762                 break;
2763         default: /* do nothing */
2764                 return;
2765         }
2766 }
2767
2768 static int handle_port_change(struct mlx5_ib_dev *ibdev, struct mlx5_eqe *eqe,
2769                               struct ib_event *ibev)
2770 {
2771         u32 port = (eqe->data.port.port >> 4) & 0xf;
2772
2773         ibev->element.port_num = port;
2774
2775         switch (eqe->sub_type) {
2776         case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE:
2777         case MLX5_PORT_CHANGE_SUBTYPE_DOWN:
2778         case MLX5_PORT_CHANGE_SUBTYPE_INITIALIZED:
2779                 /* In RoCE, port up/down events are handled in
2780                  * mlx5_netdev_event().
2781                  */
2782                 if (mlx5_ib_port_link_layer(&ibdev->ib_dev, port) ==
2783                                             IB_LINK_LAYER_ETHERNET)
2784                         return -EINVAL;
2785
2786                 ibev->event = (eqe->sub_type == MLX5_PORT_CHANGE_SUBTYPE_ACTIVE) ?
2787                                 IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR;
2788                 break;
2789
2790         case MLX5_PORT_CHANGE_SUBTYPE_LID:
2791                 ibev->event = IB_EVENT_LID_CHANGE;
2792                 break;
2793
2794         case MLX5_PORT_CHANGE_SUBTYPE_PKEY:
2795                 ibev->event = IB_EVENT_PKEY_CHANGE;
2796                 schedule_work(&ibdev->devr.ports[port - 1].pkey_change_work);
2797                 break;
2798
2799         case MLX5_PORT_CHANGE_SUBTYPE_GUID:
2800                 ibev->event = IB_EVENT_GID_CHANGE;
2801                 break;
2802
2803         case MLX5_PORT_CHANGE_SUBTYPE_CLIENT_REREG:
2804                 ibev->event = IB_EVENT_CLIENT_REREGISTER;
2805                 break;
2806         default:
2807                 return -EINVAL;
2808         }
2809
2810         return 0;
2811 }
2812
2813 static void mlx5_ib_handle_event(struct work_struct *_work)
2814 {
2815         struct mlx5_ib_event_work *work =
2816                 container_of(_work, struct mlx5_ib_event_work, work);
2817         struct mlx5_ib_dev *ibdev;
2818         struct ib_event ibev;
2819         bool fatal = false;
2820
2821         if (work->is_slave) {
2822                 ibdev = mlx5_ib_get_ibdev_from_mpi(work->mpi);
2823                 if (!ibdev)
2824                         goto out;
2825         } else {
2826                 ibdev = work->dev;
2827         }
2828
2829         switch (work->event) {
2830         case MLX5_DEV_EVENT_SYS_ERROR:
2831                 ibev.event = IB_EVENT_DEVICE_FATAL;
2832                 mlx5_ib_handle_internal_error(ibdev);
2833                 ibev.element.port_num  = (u8)(unsigned long)work->param;
2834                 fatal = true;
2835                 break;
2836         case MLX5_EVENT_TYPE_PORT_CHANGE:
2837                 if (handle_port_change(ibdev, work->param, &ibev))
2838                         goto out;
2839                 break;
2840         case MLX5_EVENT_TYPE_GENERAL_EVENT:
2841                 handle_general_event(ibdev, work->param, &ibev);
2842                 fallthrough;
2843         default:
2844                 goto out;
2845         }
2846
2847         ibev.device = &ibdev->ib_dev;
2848
2849         if (!rdma_is_port_valid(&ibdev->ib_dev, ibev.element.port_num)) {
2850                 mlx5_ib_warn(ibdev, "warning: event on port %d\n",  ibev.element.port_num);
2851                 goto out;
2852         }
2853
2854         if (ibdev->ib_active)
2855                 ib_dispatch_event(&ibev);
2856
2857         if (fatal)
2858                 ibdev->ib_active = false;
2859 out:
2860         kfree(work);
2861 }
2862
2863 static int mlx5_ib_event(struct notifier_block *nb,
2864                          unsigned long event, void *param)
2865 {
2866         struct mlx5_ib_event_work *work;
2867
2868         work = kmalloc(sizeof(*work), GFP_ATOMIC);
2869         if (!work)
2870                 return NOTIFY_DONE;
2871
2872         INIT_WORK(&work->work, mlx5_ib_handle_event);
2873         work->dev = container_of(nb, struct mlx5_ib_dev, mdev_events);
2874         work->is_slave = false;
2875         work->param = param;
2876         work->event = event;
2877
2878         queue_work(mlx5_ib_event_wq, &work->work);
2879
2880         return NOTIFY_OK;
2881 }
2882
2883 static int mlx5_ib_event_slave_port(struct notifier_block *nb,
2884                                     unsigned long event, void *param)
2885 {
2886         struct mlx5_ib_event_work *work;
2887
2888         work = kmalloc(sizeof(*work), GFP_ATOMIC);
2889         if (!work)
2890                 return NOTIFY_DONE;
2891
2892         INIT_WORK(&work->work, mlx5_ib_handle_event);
2893         work->mpi = container_of(nb, struct mlx5_ib_multiport_info, mdev_events);
2894         work->is_slave = true;
2895         work->param = param;
2896         work->event = event;
2897         queue_work(mlx5_ib_event_wq, &work->work);
2898
2899         return NOTIFY_OK;
2900 }
2901
2902 static int mlx5_ib_get_plane_num(struct mlx5_core_dev *mdev, u8 *num_plane)
2903 {
2904         struct mlx5_hca_vport_context vport_ctx;
2905         int err;
2906
2907         *num_plane = 0;
2908         if (!MLX5_CAP_GEN(mdev, ib_virt) || !MLX5_CAP_GEN_2(mdev, multiplane))
2909                 return 0;
2910
2911         err = mlx5_query_hca_vport_context(mdev, 0, 1, 0, &vport_ctx);
2912         if (err)
2913                 return err;
2914
2915         *num_plane = vport_ctx.num_plane;
2916         return 0;
2917 }
2918
2919 static int set_has_smi_cap(struct mlx5_ib_dev *dev)
2920 {
2921         struct mlx5_hca_vport_context vport_ctx;
2922         int err;
2923         int port;
2924
2925         if (MLX5_CAP_GEN(dev->mdev, port_type) != MLX5_CAP_PORT_TYPE_IB)
2926                 return 0;
2927
2928         for (port = 1; port <= dev->num_ports; port++) {
2929                 if (dev->num_plane) {
2930                         dev->port_caps[port - 1].has_smi = false;
2931                         continue;
2932                 } else if (!MLX5_CAP_GEN(dev->mdev, ib_virt) ||
2933                         dev->ib_dev.type == RDMA_DEVICE_TYPE_SMI) {
2934                         dev->port_caps[port - 1].has_smi = true;
2935                         continue;
2936                 }
2937
2938                 err = mlx5_query_hca_vport_context(dev->mdev, 0, port, 0,
2939                                                    &vport_ctx);
2940                 if (err) {
2941                         mlx5_ib_err(dev, "query_hca_vport_context for port=%d failed %d\n",
2942                                     port, err);
2943                         return err;
2944                 }
2945                 dev->port_caps[port - 1].has_smi = vport_ctx.has_smi;
2946         }
2947
2948         return 0;
2949 }
2950
2951 static void get_ext_port_caps(struct mlx5_ib_dev *dev)
2952 {
2953         unsigned int port;
2954
2955         rdma_for_each_port (&dev->ib_dev, port)
2956                 mlx5_query_ext_port_caps(dev, port);
2957 }
2958
2959 static u8 mlx5_get_umr_fence(u8 umr_fence_cap)
2960 {
2961         switch (umr_fence_cap) {
2962         case MLX5_CAP_UMR_FENCE_NONE:
2963                 return MLX5_FENCE_MODE_NONE;
2964         case MLX5_CAP_UMR_FENCE_SMALL:
2965                 return MLX5_FENCE_MODE_INITIATOR_SMALL;
2966         default:
2967                 return MLX5_FENCE_MODE_STRONG_ORDERING;
2968         }
2969 }
2970
2971 int mlx5_ib_dev_res_cq_init(struct mlx5_ib_dev *dev)
2972 {
2973         struct mlx5_ib_resources *devr = &dev->devr;
2974         struct ib_cq_init_attr cq_attr = {.cqe = 1};
2975         struct ib_device *ibdev;
2976         struct ib_pd *pd;
2977         struct ib_cq *cq;
2978         int ret = 0;
2979
2980
2981         /*
2982          * devr->c0 is set once, never changed until device unload.
2983          * Avoid taking the mutex if initialization is already done.
2984          */
2985         if (devr->c0)
2986                 return 0;
2987
2988         mutex_lock(&devr->cq_lock);
2989         if (devr->c0)
2990                 goto unlock;
2991
2992         ibdev = &dev->ib_dev;
2993         pd = ib_alloc_pd(ibdev, 0);
2994         if (IS_ERR(pd)) {
2995                 ret = PTR_ERR(pd);
2996                 mlx5_ib_err(dev, "Couldn't allocate PD for res init, err=%d\n", ret);
2997                 goto unlock;
2998         }
2999
3000         cq = ib_create_cq(ibdev, NULL, NULL, NULL, &cq_attr);
3001         if (IS_ERR(cq)) {
3002                 ret = PTR_ERR(cq);
3003                 mlx5_ib_err(dev, "Couldn't create CQ for res init, err=%d\n", ret);
3004                 ib_dealloc_pd(pd);
3005                 goto unlock;
3006         }
3007
3008         devr->p0 = pd;
3009         devr->c0 = cq;
3010
3011 unlock:
3012         mutex_unlock(&devr->cq_lock);
3013         return ret;
3014 }
3015
3016 int mlx5_ib_dev_res_srq_init(struct mlx5_ib_dev *dev)
3017 {
3018         struct mlx5_ib_resources *devr = &dev->devr;
3019         struct ib_srq_init_attr attr;
3020         struct ib_srq *s0, *s1;
3021         int ret = 0;
3022
3023         /*
3024          * devr->s1 is set once, never changed until device unload.
3025          * Avoid taking the mutex if initialization is already done.
3026          */
3027         if (devr->s1)
3028                 return 0;
3029
3030         mutex_lock(&devr->srq_lock);
3031         if (devr->s1)
3032                 goto unlock;
3033
3034         ret = mlx5_ib_dev_res_cq_init(dev);
3035         if (ret)
3036                 goto unlock;
3037
3038         memset(&attr, 0, sizeof(attr));
3039         attr.attr.max_sge = 1;
3040         attr.attr.max_wr = 1;
3041         attr.srq_type = IB_SRQT_XRC;
3042         attr.ext.cq = devr->c0;
3043
3044         s0 = ib_create_srq(devr->p0, &attr);
3045         if (IS_ERR(s0)) {
3046                 ret = PTR_ERR(s0);
3047                 mlx5_ib_err(dev, "Couldn't create SRQ 0 for res init, err=%d\n", ret);
3048                 goto unlock;
3049         }
3050
3051         memset(&attr, 0, sizeof(attr));
3052         attr.attr.max_sge = 1;
3053         attr.attr.max_wr = 1;
3054         attr.srq_type = IB_SRQT_BASIC;
3055
3056         s1 = ib_create_srq(devr->p0, &attr);
3057         if (IS_ERR(s1)) {
3058                 ret = PTR_ERR(s1);
3059                 mlx5_ib_err(dev, "Couldn't create SRQ 1 for res init, err=%d\n", ret);
3060                 ib_destroy_srq(s0);
3061         }
3062
3063         devr->s0 = s0;
3064         devr->s1 = s1;
3065
3066 unlock:
3067         mutex_unlock(&devr->srq_lock);
3068         return ret;
3069 }
3070
3071 static int mlx5_ib_dev_res_init(struct mlx5_ib_dev *dev)
3072 {
3073         struct mlx5_ib_resources *devr = &dev->devr;
3074         int ret;
3075
3076         if (!MLX5_CAP_GEN(dev->mdev, xrc))
3077                 return -EOPNOTSUPP;
3078
3079         ret = mlx5_cmd_xrcd_alloc(dev->mdev, &devr->xrcdn0, 0);
3080         if (ret)
3081                 return ret;
3082
3083         ret = mlx5_cmd_xrcd_alloc(dev->mdev, &devr->xrcdn1, 0);
3084         if (ret) {
3085                 mlx5_cmd_xrcd_dealloc(dev->mdev, devr->xrcdn0, 0);
3086                 return ret;
3087         }
3088
3089         mutex_init(&devr->cq_lock);
3090         mutex_init(&devr->srq_lock);
3091
3092         return 0;
3093 }
3094
3095 static void mlx5_ib_dev_res_cleanup(struct mlx5_ib_dev *dev)
3096 {
3097         struct mlx5_ib_resources *devr = &dev->devr;
3098
3099         /* After s0/s1 init, they are not unset during the device lifetime. */
3100         if (devr->s1) {
3101                 ib_destroy_srq(devr->s1);
3102                 ib_destroy_srq(devr->s0);
3103         }
3104         mlx5_cmd_xrcd_dealloc(dev->mdev, devr->xrcdn1, 0);
3105         mlx5_cmd_xrcd_dealloc(dev->mdev, devr->xrcdn0, 0);
3106         /* After p0/c0 init, they are not unset during the device lifetime. */
3107         if (devr->c0) {
3108                 ib_destroy_cq(devr->c0);
3109                 ib_dealloc_pd(devr->p0);
3110         }
3111         mutex_destroy(&devr->cq_lock);
3112         mutex_destroy(&devr->srq_lock);
3113 }
3114
3115 static int
3116 mlx5_ib_create_data_direct_resources(struct mlx5_ib_dev *dev)
3117 {
3118         int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
3119         struct mlx5_core_dev *mdev = dev->mdev;
3120         void *mkc;
3121         u32 mkey;
3122         u32 pdn;
3123         u32 *in;
3124         int err;
3125
3126         err = mlx5_core_alloc_pd(mdev, &pdn);
3127         if (err)
3128                 return err;
3129
3130         in = kvzalloc(inlen, GFP_KERNEL);
3131         if (!in) {
3132                 err = -ENOMEM;
3133                 goto err;
3134         }
3135
3136         MLX5_SET(create_mkey_in, in, data_direct, 1);
3137         mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
3138         MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_PA);
3139         MLX5_SET(mkc, mkc, lw, 1);
3140         MLX5_SET(mkc, mkc, lr, 1);
3141         MLX5_SET(mkc, mkc, rw, 1);
3142         MLX5_SET(mkc, mkc, rr, 1);
3143         MLX5_SET(mkc, mkc, a, 1);
3144         MLX5_SET(mkc, mkc, pd, pdn);
3145         MLX5_SET(mkc, mkc, length64, 1);
3146         MLX5_SET(mkc, mkc, qpn, 0xffffff);
3147         err = mlx5_core_create_mkey(mdev, &mkey, in, inlen);
3148         kvfree(in);
3149         if (err)
3150                 goto err;
3151
3152         dev->ddr.mkey = mkey;
3153         dev->ddr.pdn = pdn;
3154         return 0;
3155
3156 err:
3157         mlx5_core_dealloc_pd(mdev, pdn);
3158         return err;
3159 }
3160
3161 static void
3162 mlx5_ib_free_data_direct_resources(struct mlx5_ib_dev *dev)
3163 {
3164         mlx5_core_destroy_mkey(dev->mdev, dev->ddr.mkey);
3165         mlx5_core_dealloc_pd(dev->mdev, dev->ddr.pdn);
3166 }
3167
3168 static u32 get_core_cap_flags(struct ib_device *ibdev,
3169                               struct mlx5_hca_vport_context *rep)
3170 {
3171         struct mlx5_ib_dev *dev = to_mdev(ibdev);
3172         enum rdma_link_layer ll = mlx5_ib_port_link_layer(ibdev, 1);
3173         u8 l3_type_cap = MLX5_CAP_ROCE(dev->mdev, l3_type);
3174         u8 roce_version_cap = MLX5_CAP_ROCE(dev->mdev, roce_version);
3175         bool raw_support = !mlx5_core_mp_enabled(dev->mdev);
3176         u32 ret = 0;
3177
3178         if (rep->grh_required)
3179                 ret |= RDMA_CORE_CAP_IB_GRH_REQUIRED;
3180
3181         if (dev->num_plane)
3182                 return ret | RDMA_CORE_CAP_PROT_IB | RDMA_CORE_CAP_IB_MAD |
3183                         RDMA_CORE_CAP_IB_CM | RDMA_CORE_CAP_IB_SA |
3184                         RDMA_CORE_CAP_AF_IB;
3185         else if (ibdev->type == RDMA_DEVICE_TYPE_SMI)
3186                 return ret | RDMA_CORE_CAP_IB_MAD | RDMA_CORE_CAP_IB_SMI;
3187
3188         if (ll == IB_LINK_LAYER_INFINIBAND)
3189                 return ret | RDMA_CORE_PORT_IBA_IB;
3190
3191         if (raw_support)
3192                 ret |= RDMA_CORE_PORT_RAW_PACKET;
3193
3194         if (!(l3_type_cap & MLX5_ROCE_L3_TYPE_IPV4_CAP))
3195                 return ret;
3196
3197         if (!(l3_type_cap & MLX5_ROCE_L3_TYPE_IPV6_CAP))
3198                 return ret;
3199
3200         if (roce_version_cap & MLX5_ROCE_VERSION_1_CAP)
3201                 ret |= RDMA_CORE_PORT_IBA_ROCE;
3202
3203         if (roce_version_cap & MLX5_ROCE_VERSION_2_CAP)
3204                 ret |= RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
3205
3206         return ret;
3207 }
3208
3209 static int mlx5_port_immutable(struct ib_device *ibdev, u32 port_num,
3210                                struct ib_port_immutable *immutable)
3211 {
3212         struct ib_port_attr attr;
3213         struct mlx5_ib_dev *dev = to_mdev(ibdev);
3214         enum rdma_link_layer ll = mlx5_ib_port_link_layer(ibdev, port_num);
3215         struct mlx5_hca_vport_context rep = {0};
3216         int err;
3217
3218         err = ib_query_port(ibdev, port_num, &attr);
3219         if (err)
3220                 return err;
3221
3222         if (ll == IB_LINK_LAYER_INFINIBAND) {
3223                 if (ibdev->type == RDMA_DEVICE_TYPE_SMI)
3224                         port_num = smi_to_native_portnum(dev, port_num);
3225
3226                 err = mlx5_query_hca_vport_context(dev->mdev, 0, port_num, 0,
3227                                                    &rep);
3228                 if (err)
3229                         return err;
3230         }
3231
3232         immutable->pkey_tbl_len = attr.pkey_tbl_len;
3233         immutable->gid_tbl_len = attr.gid_tbl_len;
3234         immutable->core_cap_flags = get_core_cap_flags(ibdev, &rep);
3235         immutable->max_mad_size = IB_MGMT_MAD_SIZE;
3236
3237         return 0;
3238 }
3239
3240 static int mlx5_port_rep_immutable(struct ib_device *ibdev, u32 port_num,
3241                                    struct ib_port_immutable *immutable)
3242 {
3243         struct ib_port_attr attr;
3244         int err;
3245
3246         immutable->core_cap_flags = RDMA_CORE_PORT_RAW_PACKET;
3247
3248         err = ib_query_port(ibdev, port_num, &attr);
3249         if (err)
3250                 return err;
3251
3252         immutable->pkey_tbl_len = attr.pkey_tbl_len;
3253         immutable->gid_tbl_len = attr.gid_tbl_len;
3254         immutable->core_cap_flags = RDMA_CORE_PORT_RAW_PACKET;
3255
3256         return 0;
3257 }
3258
3259 static void get_dev_fw_str(struct ib_device *ibdev, char *str)
3260 {
3261         struct mlx5_ib_dev *dev =
3262                 container_of(ibdev, struct mlx5_ib_dev, ib_dev);
3263         snprintf(str, IB_FW_VERSION_NAME_MAX, "%d.%d.%04d",
3264                  fw_rev_maj(dev->mdev), fw_rev_min(dev->mdev),
3265                  fw_rev_sub(dev->mdev));
3266 }
3267
3268 static int lag_event(struct notifier_block *nb, unsigned long event, void *data)
3269 {
3270         struct mlx5_ib_dev *dev = container_of(nb, struct mlx5_ib_dev,
3271                                                lag_events);
3272         struct mlx5_core_dev *mdev = dev->mdev;
3273         struct ib_device *ibdev = &dev->ib_dev;
3274         struct net_device *old_ndev = NULL;
3275         struct mlx5_ib_port *port;
3276         struct net_device *ndev;
3277         u32 portnum = 0;
3278         int ret = 0;
3279         int i;
3280
3281         switch (event) {
3282         case MLX5_DRIVER_EVENT_ACTIVE_BACKUP_LAG_CHANGE_LOWERSTATE:
3283                 ndev = data;
3284                 if (ndev) {
3285                         if (!mlx5_lag_is_roce(mdev)) {
3286                                 // sriov lag
3287                                 for (i = 0; i < dev->num_ports; i++) {
3288                                         port = &dev->port[i];
3289                                         if (port->rep && port->rep->vport ==
3290                                             MLX5_VPORT_UPLINK) {
3291                                                 portnum = i;
3292                                                 break;
3293                                         }
3294                                 }
3295                         }
3296                         old_ndev = ib_device_get_netdev(ibdev, portnum + 1);
3297                         ret = ib_device_set_netdev(ibdev, ndev, portnum + 1);
3298                         if (ret)
3299                                 goto out;
3300
3301                         if (old_ndev)
3302                                 roce_del_all_netdev_gids(ibdev, portnum + 1,
3303                                                          old_ndev);
3304                         rdma_roce_rescan_port(ibdev, portnum + 1);
3305                 }
3306                 break;
3307         default:
3308                 return NOTIFY_DONE;
3309         }
3310
3311 out:
3312         dev_put(old_ndev);
3313         return notifier_from_errno(ret);
3314 }
3315
3316 static void mlx5e_lag_event_register(struct mlx5_ib_dev *dev)
3317 {
3318         dev->lag_events.notifier_call = lag_event;
3319         blocking_notifier_chain_register(&dev->mdev->priv.lag_nh,
3320                                          &dev->lag_events);
3321 }
3322
3323 static void mlx5e_lag_event_unregister(struct mlx5_ib_dev *dev)
3324 {
3325         blocking_notifier_chain_unregister(&dev->mdev->priv.lag_nh,
3326                                            &dev->lag_events);
3327 }
3328
3329 static int mlx5_eth_lag_init(struct mlx5_ib_dev *dev)
3330 {
3331         struct mlx5_core_dev *mdev = dev->mdev;
3332         struct mlx5_flow_namespace *ns = mlx5_get_flow_namespace(mdev,
3333                                                                  MLX5_FLOW_NAMESPACE_LAG);
3334         struct mlx5_flow_table *ft;
3335         int err;
3336
3337         if (!ns || !mlx5_lag_is_active(mdev))
3338                 return 0;
3339
3340         err = mlx5_cmd_create_vport_lag(mdev);
3341         if (err)
3342                 return err;
3343
3344         ft = mlx5_create_lag_demux_flow_table(ns, 0, 0);
3345         if (IS_ERR(ft)) {
3346                 err = PTR_ERR(ft);
3347                 goto err_destroy_vport_lag;
3348         }
3349
3350         mlx5e_lag_event_register(dev);
3351         dev->flow_db->lag_demux_ft = ft;
3352         dev->lag_ports = mlx5_lag_get_num_ports(mdev);
3353         dev->lag_active = true;
3354         return 0;
3355
3356 err_destroy_vport_lag:
3357         mlx5_cmd_destroy_vport_lag(mdev);
3358         return err;
3359 }
3360
3361 static void mlx5_eth_lag_cleanup(struct mlx5_ib_dev *dev)
3362 {
3363         struct mlx5_core_dev *mdev = dev->mdev;
3364
3365         if (dev->lag_active) {
3366                 dev->lag_active = false;
3367
3368                 mlx5e_lag_event_unregister(dev);
3369                 mlx5_destroy_flow_table(dev->flow_db->lag_demux_ft);
3370                 dev->flow_db->lag_demux_ft = NULL;
3371
3372                 mlx5_cmd_destroy_vport_lag(mdev);
3373         }
3374 }
3375
3376 static void mlx5_netdev_notifier_register(struct mlx5_roce *roce,
3377                                           struct net_device *netdev)
3378 {
3379         int err;
3380
3381         if (roce->tracking_netdev)
3382                 return;
3383         roce->tracking_netdev = netdev;
3384         roce->nb.notifier_call = mlx5_netdev_event;
3385         err = register_netdevice_notifier_dev_net(netdev, &roce->nb, &roce->nn);
3386         WARN_ON(err);
3387 }
3388
3389 static void mlx5_netdev_notifier_unregister(struct mlx5_roce *roce)
3390 {
3391         if (!roce->tracking_netdev)
3392                 return;
3393         unregister_netdevice_notifier_dev_net(roce->tracking_netdev, &roce->nb,
3394                                               &roce->nn);
3395         roce->tracking_netdev = NULL;
3396 }
3397
3398 static int mlx5e_mdev_notifier_event(struct notifier_block *nb,
3399                                      unsigned long event, void *data)
3400 {
3401         struct mlx5_roce *roce = container_of(nb, struct mlx5_roce, mdev_nb);
3402         struct net_device *netdev = data;
3403
3404         switch (event) {
3405         case MLX5_DRIVER_EVENT_UPLINK_NETDEV:
3406                 if (netdev)
3407                         mlx5_netdev_notifier_register(roce, netdev);
3408                 else
3409                         mlx5_netdev_notifier_unregister(roce);
3410                 break;
3411         default:
3412                 return NOTIFY_DONE;
3413         }
3414
3415         return NOTIFY_OK;
3416 }
3417
3418 static void mlx5_mdev_netdev_track(struct mlx5_ib_dev *dev, u32 port_num)
3419 {
3420         struct mlx5_roce *roce = &dev->port[port_num].roce;
3421
3422         roce->mdev_nb.notifier_call = mlx5e_mdev_notifier_event;
3423         mlx5_blocking_notifier_register(dev->mdev, &roce->mdev_nb);
3424         mlx5_core_uplink_netdev_event_replay(dev->mdev);
3425 }
3426
3427 static void mlx5_mdev_netdev_untrack(struct mlx5_ib_dev *dev, u32 port_num)
3428 {
3429         struct mlx5_roce *roce = &dev->port[port_num].roce;
3430
3431         mlx5_blocking_notifier_unregister(dev->mdev, &roce->mdev_nb);
3432         mlx5_netdev_notifier_unregister(roce);
3433 }
3434
3435 static int mlx5_enable_eth(struct mlx5_ib_dev *dev)
3436 {
3437         int err;
3438
3439         if (!dev->is_rep && dev->profile != &raw_eth_profile) {
3440                 err = mlx5_nic_vport_enable_roce(dev->mdev);
3441                 if (err)
3442                         return err;
3443         }
3444
3445         err = mlx5_eth_lag_init(dev);
3446         if (err)
3447                 goto err_disable_roce;
3448
3449         return 0;
3450
3451 err_disable_roce:
3452         if (!dev->is_rep && dev->profile != &raw_eth_profile)
3453                 mlx5_nic_vport_disable_roce(dev->mdev);
3454
3455         return err;
3456 }
3457
3458 static void mlx5_disable_eth(struct mlx5_ib_dev *dev)
3459 {
3460         mlx5_eth_lag_cleanup(dev);
3461         if (!dev->is_rep && dev->profile != &raw_eth_profile)
3462                 mlx5_nic_vport_disable_roce(dev->mdev);
3463 }
3464
3465 static int mlx5_ib_rn_get_params(struct ib_device *device, u32 port_num,
3466                                  enum rdma_netdev_t type,
3467                                  struct rdma_netdev_alloc_params *params)
3468 {
3469         if (type != RDMA_NETDEV_IPOIB)
3470                 return -EOPNOTSUPP;
3471
3472         return mlx5_rdma_rn_get_params(to_mdev(device)->mdev, device, params);
3473 }
3474
3475 static ssize_t delay_drop_timeout_read(struct file *filp, char __user *buf,
3476                                        size_t count, loff_t *pos)
3477 {
3478         struct mlx5_ib_delay_drop *delay_drop = filp->private_data;
3479         char lbuf[20];
3480         int len;
3481
3482         len = snprintf(lbuf, sizeof(lbuf), "%u\n", delay_drop->timeout);
3483         return simple_read_from_buffer(buf, count, pos, lbuf, len);
3484 }
3485
3486 static ssize_t delay_drop_timeout_write(struct file *filp, const char __user *buf,
3487                                         size_t count, loff_t *pos)
3488 {
3489         struct mlx5_ib_delay_drop *delay_drop = filp->private_data;
3490         u32 timeout;
3491         u32 var;
3492
3493         if (kstrtouint_from_user(buf, count, 0, &var))
3494                 return -EFAULT;
3495
3496         timeout = min_t(u32, roundup(var, 100), MLX5_MAX_DELAY_DROP_TIMEOUT_MS *
3497                         1000);
3498         if (timeout != var)
3499                 mlx5_ib_dbg(delay_drop->dev, "Round delay drop timeout to %u usec\n",
3500                             timeout);
3501
3502         delay_drop->timeout = timeout;
3503
3504         return count;
3505 }
3506
3507 static const struct file_operations fops_delay_drop_timeout = {
3508         .owner  = THIS_MODULE,
3509         .open   = simple_open,
3510         .write  = delay_drop_timeout_write,
3511         .read   = delay_drop_timeout_read,
3512 };
3513
3514 static void mlx5_ib_unbind_slave_port(struct mlx5_ib_dev *ibdev,
3515                                       struct mlx5_ib_multiport_info *mpi)
3516 {
3517         u32 port_num = mlx5_core_native_port_num(mpi->mdev) - 1;
3518         struct mlx5_ib_port *port = &ibdev->port[port_num];
3519         int comps;
3520         int err;
3521         int i;
3522
3523         lockdep_assert_held(&mlx5_ib_multiport_mutex);
3524
3525         mlx5_ib_disable_lb_mp(ibdev->mdev, mpi->mdev);
3526
3527         mlx5_core_mp_event_replay(ibdev->mdev,
3528                                   MLX5_DRIVER_EVENT_AFFILIATION_REMOVED,
3529                                   NULL);
3530         mlx5_core_mp_event_replay(mpi->mdev,
3531                                   MLX5_DRIVER_EVENT_AFFILIATION_REMOVED,
3532                                   NULL);
3533
3534         mlx5_ib_cleanup_cong_debugfs(ibdev, port_num);
3535
3536         spin_lock(&port->mp.mpi_lock);
3537         if (!mpi->ibdev) {
3538                 spin_unlock(&port->mp.mpi_lock);
3539                 return;
3540         }
3541
3542         mpi->ibdev = NULL;
3543
3544         spin_unlock(&port->mp.mpi_lock);
3545         if (mpi->mdev_events.notifier_call)
3546                 mlx5_notifier_unregister(mpi->mdev, &mpi->mdev_events);
3547         mpi->mdev_events.notifier_call = NULL;
3548         mlx5_mdev_netdev_untrack(ibdev, port_num);
3549         spin_lock(&port->mp.mpi_lock);
3550
3551         comps = mpi->mdev_refcnt;
3552         if (comps) {
3553                 mpi->unaffiliate = true;
3554                 init_completion(&mpi->unref_comp);
3555                 spin_unlock(&port->mp.mpi_lock);
3556
3557                 for (i = 0; i < comps; i++)
3558                         wait_for_completion(&mpi->unref_comp);
3559
3560                 spin_lock(&port->mp.mpi_lock);
3561                 mpi->unaffiliate = false;
3562         }
3563
3564         port->mp.mpi = NULL;
3565
3566         spin_unlock(&port->mp.mpi_lock);
3567
3568         err = mlx5_nic_vport_unaffiliate_multiport(mpi->mdev);
3569
3570         mlx5_ib_dbg(ibdev, "unaffiliated port %u\n", port_num + 1);
3571         /* Log an error, still needed to cleanup the pointers and add
3572          * it back to the list.
3573          */
3574         if (err)
3575                 mlx5_ib_err(ibdev, "Failed to unaffiliate port %u\n",
3576                             port_num + 1);
3577
3578         ibdev->port[port_num].roce.last_port_state = IB_PORT_DOWN;
3579 }
3580
3581 static bool mlx5_ib_bind_slave_port(struct mlx5_ib_dev *ibdev,
3582                                     struct mlx5_ib_multiport_info *mpi)
3583 {
3584         u32 port_num = mlx5_core_native_port_num(mpi->mdev) - 1;
3585         u64 key;
3586         int err;
3587
3588         lockdep_assert_held(&mlx5_ib_multiport_mutex);
3589
3590         spin_lock(&ibdev->port[port_num].mp.mpi_lock);
3591         if (ibdev->port[port_num].mp.mpi) {
3592                 mlx5_ib_dbg(ibdev, "port %u already affiliated.\n",
3593                             port_num + 1);
3594                 spin_unlock(&ibdev->port[port_num].mp.mpi_lock);
3595                 return false;
3596         }
3597
3598         ibdev->port[port_num].mp.mpi = mpi;
3599         mpi->ibdev = ibdev;
3600         mpi->mdev_events.notifier_call = NULL;
3601         spin_unlock(&ibdev->port[port_num].mp.mpi_lock);
3602
3603         err = mlx5_nic_vport_affiliate_multiport(ibdev->mdev, mpi->mdev);
3604         if (err)
3605                 goto unbind;
3606
3607         mlx5_mdev_netdev_track(ibdev, port_num);
3608
3609         mpi->mdev_events.notifier_call = mlx5_ib_event_slave_port;
3610         mlx5_notifier_register(mpi->mdev, &mpi->mdev_events);
3611
3612         mlx5_ib_init_cong_debugfs(ibdev, port_num);
3613
3614         key = mpi->mdev->priv.adev_idx;
3615         mlx5_core_mp_event_replay(mpi->mdev,
3616                                   MLX5_DRIVER_EVENT_AFFILIATION_DONE,
3617                                   &key);
3618         mlx5_core_mp_event_replay(ibdev->mdev,
3619                                   MLX5_DRIVER_EVENT_AFFILIATION_DONE,
3620                                   &key);
3621
3622         err = mlx5_ib_enable_lb_mp(ibdev->mdev, mpi->mdev);
3623         if (err)
3624                 goto unbind;
3625
3626         return true;
3627
3628 unbind:
3629         mlx5_ib_unbind_slave_port(ibdev, mpi);
3630         return false;
3631 }
3632
3633 static int mlx5_ib_data_direct_init(struct mlx5_ib_dev *dev)
3634 {
3635         char vuid[MLX5_ST_SZ_BYTES(array1024_auto) + 1] = {};
3636         int ret;
3637
3638         if (!MLX5_CAP_GEN(dev->mdev, data_direct) ||
3639             !MLX5_CAP_GEN_2(dev->mdev, query_vuid))
3640                 return 0;
3641
3642         ret = mlx5_cmd_query_vuid(dev->mdev, true, vuid);
3643         if (ret)
3644                 return ret;
3645
3646         ret = mlx5_ib_create_data_direct_resources(dev);
3647         if (ret)
3648                 return ret;
3649
3650         INIT_LIST_HEAD(&dev->data_direct_mr_list);
3651         ret = mlx5_data_direct_ib_reg(dev, vuid);
3652         if (ret)
3653                 mlx5_ib_free_data_direct_resources(dev);
3654
3655         return ret;
3656 }
3657
3658 static void mlx5_ib_data_direct_cleanup(struct mlx5_ib_dev *dev)
3659 {
3660         if (!MLX5_CAP_GEN(dev->mdev, data_direct) ||
3661             !MLX5_CAP_GEN_2(dev->mdev, query_vuid))
3662                 return;
3663
3664         mlx5_data_direct_ib_unreg(dev);
3665         mlx5_ib_free_data_direct_resources(dev);
3666 }
3667
3668 static int mlx5_ib_init_multiport_master(struct mlx5_ib_dev *dev)
3669 {
3670         u32 port_num = mlx5_core_native_port_num(dev->mdev) - 1;
3671         enum rdma_link_layer ll = mlx5_ib_port_link_layer(&dev->ib_dev,
3672                                                           port_num + 1);
3673         struct mlx5_ib_multiport_info *mpi;
3674         int err;
3675         u32 i;
3676
3677         if (!mlx5_core_is_mp_master(dev->mdev) || ll != IB_LINK_LAYER_ETHERNET)
3678                 return 0;
3679
3680         err = mlx5_query_nic_vport_system_image_guid(dev->mdev,
3681                                                      &dev->sys_image_guid);
3682         if (err)
3683                 return err;
3684
3685         err = mlx5_nic_vport_enable_roce(dev->mdev);
3686         if (err)
3687                 return err;
3688
3689         mutex_lock(&mlx5_ib_multiport_mutex);
3690         for (i = 0; i < dev->num_ports; i++) {
3691                 bool bound = false;
3692
3693                 /* build a stub multiport info struct for the native port. */
3694                 if (i == port_num) {
3695                         mpi = kzalloc(sizeof(*mpi), GFP_KERNEL);
3696                         if (!mpi) {
3697                                 mutex_unlock(&mlx5_ib_multiport_mutex);
3698                                 mlx5_nic_vport_disable_roce(dev->mdev);
3699                                 return -ENOMEM;
3700                         }
3701
3702                         mpi->is_master = true;
3703                         mpi->mdev = dev->mdev;
3704                         mpi->sys_image_guid = dev->sys_image_guid;
3705                         dev->port[i].mp.mpi = mpi;
3706                         mpi->ibdev = dev;
3707                         mpi = NULL;
3708                         continue;
3709                 }
3710
3711                 list_for_each_entry(mpi, &mlx5_ib_unaffiliated_port_list,
3712                                     list) {
3713                         if (dev->sys_image_guid == mpi->sys_image_guid &&
3714                             (mlx5_core_native_port_num(mpi->mdev) - 1) == i &&
3715                             mlx5_core_same_coredev_type(dev->mdev, mpi->mdev)) {
3716                                 bound = mlx5_ib_bind_slave_port(dev, mpi);
3717                         }
3718
3719                         if (bound) {
3720                                 dev_dbg(mpi->mdev->device,
3721                                         "removing port from unaffiliated list.\n");
3722                                 mlx5_ib_dbg(dev, "port %d bound\n", i + 1);
3723                                 list_del(&mpi->list);
3724                                 break;
3725                         }
3726                 }
3727                 if (!bound)
3728                         mlx5_ib_dbg(dev, "no free port found for port %d\n",
3729                                     i + 1);
3730         }
3731
3732         list_add_tail(&dev->ib_dev_list, &mlx5_ib_dev_list);
3733         mutex_unlock(&mlx5_ib_multiport_mutex);
3734         return err;
3735 }
3736
3737 static void mlx5_ib_cleanup_multiport_master(struct mlx5_ib_dev *dev)
3738 {
3739         u32 port_num = mlx5_core_native_port_num(dev->mdev) - 1;
3740         enum rdma_link_layer ll = mlx5_ib_port_link_layer(&dev->ib_dev,
3741                                                           port_num + 1);
3742         u32 i;
3743
3744         if (!mlx5_core_is_mp_master(dev->mdev) || ll != IB_LINK_LAYER_ETHERNET)
3745                 return;
3746
3747         mutex_lock(&mlx5_ib_multiport_mutex);
3748         for (i = 0; i < dev->num_ports; i++) {
3749                 if (dev->port[i].mp.mpi) {
3750                         /* Destroy the native port stub */
3751                         if (i == port_num) {
3752                                 kfree(dev->port[i].mp.mpi);
3753                                 dev->port[i].mp.mpi = NULL;
3754                         } else {
3755                                 mlx5_ib_dbg(dev, "unbinding port_num: %u\n",
3756                                             i + 1);
3757                                 list_add_tail(&dev->port[i].mp.mpi->list,
3758                                               &mlx5_ib_unaffiliated_port_list);
3759                                 mlx5_ib_unbind_slave_port(dev,
3760                                                           dev->port[i].mp.mpi);
3761                         }
3762                 }
3763         }
3764
3765         mlx5_ib_dbg(dev, "removing from devlist\n");
3766         list_del(&dev->ib_dev_list);
3767         mutex_unlock(&mlx5_ib_multiport_mutex);
3768
3769         mlx5_nic_vport_disable_roce(dev->mdev);
3770 }
3771
3772 static int mmap_obj_cleanup(struct ib_uobject *uobject,
3773                             enum rdma_remove_reason why,
3774                             struct uverbs_attr_bundle *attrs)
3775 {
3776         struct mlx5_user_mmap_entry *obj = uobject->object;
3777
3778         rdma_user_mmap_entry_remove(&obj->rdma_entry);
3779         return 0;
3780 }
3781
3782 static int mlx5_rdma_user_mmap_entry_insert(struct mlx5_ib_ucontext *c,
3783                                             struct mlx5_user_mmap_entry *entry,
3784                                             size_t length)
3785 {
3786         return rdma_user_mmap_entry_insert_range(
3787                 &c->ibucontext, &entry->rdma_entry, length,
3788                 (MLX5_IB_MMAP_OFFSET_START << 16),
3789                 ((MLX5_IB_MMAP_OFFSET_END << 16) + (1UL << 16) - 1));
3790 }
3791
3792 static struct mlx5_user_mmap_entry *
3793 alloc_var_entry(struct mlx5_ib_ucontext *c)
3794 {
3795         struct mlx5_user_mmap_entry *entry;
3796         struct mlx5_var_table *var_table;
3797         u32 page_idx;
3798         int err;
3799
3800         var_table = &to_mdev(c->ibucontext.device)->var_table;
3801         entry = kzalloc(sizeof(*entry), GFP_KERNEL);
3802         if (!entry)
3803                 return ERR_PTR(-ENOMEM);
3804
3805         mutex_lock(&var_table->bitmap_lock);
3806         page_idx = find_first_zero_bit(var_table->bitmap,
3807                                        var_table->num_var_hw_entries);
3808         if (page_idx >= var_table->num_var_hw_entries) {
3809                 err = -ENOSPC;
3810                 mutex_unlock(&var_table->bitmap_lock);
3811                 goto end;
3812         }
3813
3814         set_bit(page_idx, var_table->bitmap);
3815         mutex_unlock(&var_table->bitmap_lock);
3816
3817         entry->address = var_table->hw_start_addr +
3818                                 (page_idx * var_table->stride_size);
3819         entry->page_idx = page_idx;
3820         entry->mmap_flag = MLX5_IB_MMAP_TYPE_VAR;
3821
3822         err = mlx5_rdma_user_mmap_entry_insert(c, entry,
3823                                                var_table->stride_size);
3824         if (err)
3825                 goto err_insert;
3826
3827         return entry;
3828
3829 err_insert:
3830         mutex_lock(&var_table->bitmap_lock);
3831         clear_bit(page_idx, var_table->bitmap);
3832         mutex_unlock(&var_table->bitmap_lock);
3833 end:
3834         kfree(entry);
3835         return ERR_PTR(err);
3836 }
3837
3838 static int UVERBS_HANDLER(MLX5_IB_METHOD_VAR_OBJ_ALLOC)(
3839         struct uverbs_attr_bundle *attrs)
3840 {
3841         struct ib_uobject *uobj = uverbs_attr_get_uobject(
3842                 attrs, MLX5_IB_ATTR_VAR_OBJ_ALLOC_HANDLE);
3843         struct mlx5_ib_ucontext *c;
3844         struct mlx5_user_mmap_entry *entry;
3845         u64 mmap_offset;
3846         u32 length;
3847         int err;
3848
3849         c = to_mucontext(ib_uverbs_get_ucontext(attrs));
3850         if (IS_ERR(c))
3851                 return PTR_ERR(c);
3852
3853         entry = alloc_var_entry(c);
3854         if (IS_ERR(entry))
3855                 return PTR_ERR(entry);
3856
3857         mmap_offset = mlx5_entry_to_mmap_offset(entry);
3858         length = entry->rdma_entry.npages * PAGE_SIZE;
3859         uobj->object = entry;
3860         uverbs_finalize_uobj_create(attrs, MLX5_IB_ATTR_VAR_OBJ_ALLOC_HANDLE);
3861
3862         err = uverbs_copy_to(attrs, MLX5_IB_ATTR_VAR_OBJ_ALLOC_MMAP_OFFSET,
3863                              &mmap_offset, sizeof(mmap_offset));
3864         if (err)
3865                 return err;
3866
3867         err = uverbs_copy_to(attrs, MLX5_IB_ATTR_VAR_OBJ_ALLOC_PAGE_ID,
3868                              &entry->page_idx, sizeof(entry->page_idx));
3869         if (err)
3870                 return err;
3871
3872         err = uverbs_copy_to(attrs, MLX5_IB_ATTR_VAR_OBJ_ALLOC_MMAP_LENGTH,
3873                              &length, sizeof(length));
3874         return err;
3875 }
3876
3877 DECLARE_UVERBS_NAMED_METHOD(
3878         MLX5_IB_METHOD_VAR_OBJ_ALLOC,
3879         UVERBS_ATTR_IDR(MLX5_IB_ATTR_VAR_OBJ_ALLOC_HANDLE,
3880                         MLX5_IB_OBJECT_VAR,
3881                         UVERBS_ACCESS_NEW,
3882                         UA_MANDATORY),
3883         UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_VAR_OBJ_ALLOC_PAGE_ID,
3884                            UVERBS_ATTR_TYPE(u32),
3885                            UA_MANDATORY),
3886         UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_VAR_OBJ_ALLOC_MMAP_LENGTH,
3887                            UVERBS_ATTR_TYPE(u32),
3888                            UA_MANDATORY),
3889         UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_VAR_OBJ_ALLOC_MMAP_OFFSET,
3890                             UVERBS_ATTR_TYPE(u64),
3891                             UA_MANDATORY));
3892
3893 DECLARE_UVERBS_NAMED_METHOD_DESTROY(
3894         MLX5_IB_METHOD_VAR_OBJ_DESTROY,
3895         UVERBS_ATTR_IDR(MLX5_IB_ATTR_VAR_OBJ_DESTROY_HANDLE,
3896                         MLX5_IB_OBJECT_VAR,
3897                         UVERBS_ACCESS_DESTROY,
3898                         UA_MANDATORY));
3899
3900 DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_VAR,
3901                             UVERBS_TYPE_ALLOC_IDR(mmap_obj_cleanup),
3902                             &UVERBS_METHOD(MLX5_IB_METHOD_VAR_OBJ_ALLOC),
3903                             &UVERBS_METHOD(MLX5_IB_METHOD_VAR_OBJ_DESTROY));
3904
3905 static bool var_is_supported(struct ib_device *device)
3906 {
3907         struct mlx5_ib_dev *dev = to_mdev(device);
3908
3909         return (MLX5_CAP_GEN_64(dev->mdev, general_obj_types) &
3910                         MLX5_GENERAL_OBJ_TYPES_CAP_VIRTIO_NET_Q);
3911 }
3912
3913 static struct mlx5_user_mmap_entry *
3914 alloc_uar_entry(struct mlx5_ib_ucontext *c,
3915                 enum mlx5_ib_uapi_uar_alloc_type alloc_type)
3916 {
3917         struct mlx5_user_mmap_entry *entry;
3918         struct mlx5_ib_dev *dev;
3919         u32 uar_index;
3920         int err;
3921
3922         entry = kzalloc(sizeof(*entry), GFP_KERNEL);
3923         if (!entry)
3924                 return ERR_PTR(-ENOMEM);
3925
3926         dev = to_mdev(c->ibucontext.device);
3927         err = mlx5_cmd_uar_alloc(dev->mdev, &uar_index, c->devx_uid);
3928         if (err)
3929                 goto end;
3930
3931         entry->page_idx = uar_index;
3932         entry->address = uar_index2paddress(dev, uar_index);
3933         if (alloc_type == MLX5_IB_UAPI_UAR_ALLOC_TYPE_BF)
3934                 entry->mmap_flag = MLX5_IB_MMAP_TYPE_UAR_WC;
3935         else
3936                 entry->mmap_flag = MLX5_IB_MMAP_TYPE_UAR_NC;
3937
3938         err = mlx5_rdma_user_mmap_entry_insert(c, entry, PAGE_SIZE);
3939         if (err)
3940                 goto err_insert;
3941
3942         return entry;
3943
3944 err_insert:
3945         mlx5_cmd_uar_dealloc(dev->mdev, uar_index, c->devx_uid);
3946 end:
3947         kfree(entry);
3948         return ERR_PTR(err);
3949 }
3950
3951 static int UVERBS_HANDLER(MLX5_IB_METHOD_UAR_OBJ_ALLOC)(
3952         struct uverbs_attr_bundle *attrs)
3953 {
3954         struct ib_uobject *uobj = uverbs_attr_get_uobject(
3955                 attrs, MLX5_IB_ATTR_UAR_OBJ_ALLOC_HANDLE);
3956         enum mlx5_ib_uapi_uar_alloc_type alloc_type;
3957         struct mlx5_ib_ucontext *c;
3958         struct mlx5_user_mmap_entry *entry;
3959         u64 mmap_offset;
3960         u32 length;
3961         int err;
3962
3963         c = to_mucontext(ib_uverbs_get_ucontext(attrs));
3964         if (IS_ERR(c))
3965                 return PTR_ERR(c);
3966
3967         err = uverbs_get_const(&alloc_type, attrs,
3968                                MLX5_IB_ATTR_UAR_OBJ_ALLOC_TYPE);
3969         if (err)
3970                 return err;
3971
3972         if (alloc_type != MLX5_IB_UAPI_UAR_ALLOC_TYPE_BF &&
3973             alloc_type != MLX5_IB_UAPI_UAR_ALLOC_TYPE_NC)
3974                 return -EOPNOTSUPP;
3975
3976         if (!mlx5_wc_support_get(to_mdev(c->ibucontext.device)->mdev) &&
3977             alloc_type == MLX5_IB_UAPI_UAR_ALLOC_TYPE_BF)
3978                 return -EOPNOTSUPP;
3979
3980         entry = alloc_uar_entry(c, alloc_type);
3981         if (IS_ERR(entry))
3982                 return PTR_ERR(entry);
3983
3984         mmap_offset = mlx5_entry_to_mmap_offset(entry);
3985         length = entry->rdma_entry.npages * PAGE_SIZE;
3986         uobj->object = entry;
3987         uverbs_finalize_uobj_create(attrs, MLX5_IB_ATTR_UAR_OBJ_ALLOC_HANDLE);
3988
3989         err = uverbs_copy_to(attrs, MLX5_IB_ATTR_UAR_OBJ_ALLOC_MMAP_OFFSET,
3990                              &mmap_offset, sizeof(mmap_offset));
3991         if (err)
3992                 return err;
3993
3994         err = uverbs_copy_to(attrs, MLX5_IB_ATTR_UAR_OBJ_ALLOC_PAGE_ID,
3995                              &entry->page_idx, sizeof(entry->page_idx));
3996         if (err)
3997                 return err;
3998
3999         err = uverbs_copy_to(attrs, MLX5_IB_ATTR_UAR_OBJ_ALLOC_MMAP_LENGTH,
4000                              &length, sizeof(length));
4001         return err;
4002 }
4003
4004 DECLARE_UVERBS_NAMED_METHOD(
4005         MLX5_IB_METHOD_UAR_OBJ_ALLOC,
4006         UVERBS_ATTR_IDR(MLX5_IB_ATTR_UAR_OBJ_ALLOC_HANDLE,
4007                         MLX5_IB_OBJECT_UAR,
4008                         UVERBS_ACCESS_NEW,
4009                         UA_MANDATORY),
4010         UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_UAR_OBJ_ALLOC_TYPE,
4011                              enum mlx5_ib_uapi_uar_alloc_type,
4012                              UA_MANDATORY),
4013         UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_UAR_OBJ_ALLOC_PAGE_ID,
4014                            UVERBS_ATTR_TYPE(u32),
4015                            UA_MANDATORY),
4016         UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_UAR_OBJ_ALLOC_MMAP_LENGTH,
4017                            UVERBS_ATTR_TYPE(u32),
4018                            UA_MANDATORY),
4019         UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_UAR_OBJ_ALLOC_MMAP_OFFSET,
4020                             UVERBS_ATTR_TYPE(u64),
4021                             UA_MANDATORY));
4022
4023 DECLARE_UVERBS_NAMED_METHOD_DESTROY(
4024         MLX5_IB_METHOD_UAR_OBJ_DESTROY,
4025         UVERBS_ATTR_IDR(MLX5_IB_ATTR_UAR_OBJ_DESTROY_HANDLE,
4026                         MLX5_IB_OBJECT_UAR,
4027                         UVERBS_ACCESS_DESTROY,
4028                         UA_MANDATORY));
4029
4030 DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_UAR,
4031                             UVERBS_TYPE_ALLOC_IDR(mmap_obj_cleanup),
4032                             &UVERBS_METHOD(MLX5_IB_METHOD_UAR_OBJ_ALLOC),
4033                             &UVERBS_METHOD(MLX5_IB_METHOD_UAR_OBJ_DESTROY));
4034
4035 ADD_UVERBS_ATTRIBUTES_SIMPLE(
4036         mlx5_ib_query_context,
4037         UVERBS_OBJECT_DEVICE,
4038         UVERBS_METHOD_QUERY_CONTEXT,
4039         UVERBS_ATTR_PTR_OUT(
4040                 MLX5_IB_ATTR_QUERY_CONTEXT_RESP_UCTX,
4041                 UVERBS_ATTR_STRUCT(struct mlx5_ib_alloc_ucontext_resp,
4042                                    dump_fill_mkey),
4043                 UA_MANDATORY));
4044
4045 ADD_UVERBS_ATTRIBUTES_SIMPLE(
4046         mlx5_ib_reg_dmabuf_mr,
4047         UVERBS_OBJECT_MR,
4048         UVERBS_METHOD_REG_DMABUF_MR,
4049         UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_REG_DMABUF_MR_ACCESS_FLAGS,
4050                              enum mlx5_ib_uapi_reg_dmabuf_flags,
4051                              UA_OPTIONAL));
4052
4053 static const struct uapi_definition mlx5_ib_defs[] = {
4054         UAPI_DEF_CHAIN(mlx5_ib_devx_defs),
4055         UAPI_DEF_CHAIN(mlx5_ib_flow_defs),
4056         UAPI_DEF_CHAIN(mlx5_ib_qos_defs),
4057         UAPI_DEF_CHAIN(mlx5_ib_std_types_defs),
4058         UAPI_DEF_CHAIN(mlx5_ib_dm_defs),
4059         UAPI_DEF_CHAIN(mlx5_ib_create_cq_defs),
4060
4061         UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_DEVICE, &mlx5_ib_query_context),
4062         UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_MR, &mlx5_ib_reg_dmabuf_mr),
4063         UAPI_DEF_CHAIN_OBJ_TREE_NAMED(MLX5_IB_OBJECT_VAR,
4064                                 UAPI_DEF_IS_OBJ_SUPPORTED(var_is_supported)),
4065         UAPI_DEF_CHAIN_OBJ_TREE_NAMED(MLX5_IB_OBJECT_UAR),
4066         {}
4067 };
4068
4069 static void mlx5_ib_stage_init_cleanup(struct mlx5_ib_dev *dev)
4070 {
4071         mlx5_ib_data_direct_cleanup(dev);
4072         mlx5_ib_cleanup_multiport_master(dev);
4073         WARN_ON(!xa_empty(&dev->odp_mkeys));
4074         mutex_destroy(&dev->cap_mask_mutex);
4075         WARN_ON(!xa_empty(&dev->sig_mrs));
4076         WARN_ON(!bitmap_empty(dev->dm.memic_alloc_pages, MLX5_MAX_MEMIC_PAGES));
4077         mlx5r_macsec_dealloc_gids(dev);
4078 }
4079
4080 static int mlx5_ib_stage_init_init(struct mlx5_ib_dev *dev)
4081 {
4082         struct mlx5_core_dev *mdev = dev->mdev;
4083         int err, i;
4084
4085         dev->ib_dev.node_type = RDMA_NODE_IB_CA;
4086         dev->ib_dev.local_dma_lkey = 0 /* not supported for now */;
4087         dev->ib_dev.dev.parent = mdev->device;
4088         dev->ib_dev.lag_flags = RDMA_LAG_FLAGS_HASH_ALL_SLAVES;
4089
4090         for (i = 0; i < dev->num_ports; i++) {
4091                 spin_lock_init(&dev->port[i].mp.mpi_lock);
4092                 dev->port[i].roce.dev = dev;
4093                 dev->port[i].roce.native_port_num = i + 1;
4094                 dev->port[i].roce.last_port_state = IB_PORT_DOWN;
4095         }
4096
4097         err = mlx5r_cmd_query_special_mkeys(dev);
4098         if (err)
4099                 return err;
4100
4101         err = mlx5r_macsec_init_gids_and_devlist(dev);
4102         if (err)
4103                 return err;
4104
4105         err = mlx5_ib_init_multiport_master(dev);
4106         if (err)
4107                 goto err;
4108
4109         err = set_has_smi_cap(dev);
4110         if (err)
4111                 goto err_mp;
4112
4113         err = mlx5_query_max_pkeys(&dev->ib_dev, &dev->pkey_table_len);
4114         if (err)
4115                 goto err_mp;
4116
4117         if (mlx5_use_mad_ifc(dev))
4118                 get_ext_port_caps(dev);
4119
4120         dev->ib_dev.num_comp_vectors    = mlx5_comp_vectors_max(mdev);
4121
4122         mutex_init(&dev->cap_mask_mutex);
4123         mutex_init(&dev->data_direct_lock);
4124         INIT_LIST_HEAD(&dev->qp_list);
4125         spin_lock_init(&dev->reset_flow_resource_lock);
4126         xa_init(&dev->odp_mkeys);
4127         xa_init(&dev->sig_mrs);
4128         atomic_set(&dev->mkey_var, 0);
4129
4130         spin_lock_init(&dev->dm.lock);
4131         dev->dm.dev = mdev;
4132         err = mlx5_ib_data_direct_init(dev);
4133         if (err)
4134                 goto err_mp;
4135
4136         return 0;
4137 err_mp:
4138         mlx5_ib_cleanup_multiport_master(dev);
4139 err:
4140         mlx5r_macsec_dealloc_gids(dev);
4141         return err;
4142 }
4143
4144 static struct ib_device *mlx5_ib_add_sub_dev(struct ib_device *parent,
4145                                              enum rdma_nl_dev_type type,
4146                                              const char *name);
4147 static void mlx5_ib_del_sub_dev(struct ib_device *sub_dev);
4148
4149 static const struct ib_device_ops mlx5_ib_dev_ops = {
4150         .owner = THIS_MODULE,
4151         .driver_id = RDMA_DRIVER_MLX5,
4152         .uverbs_abi_ver = MLX5_IB_UVERBS_ABI_VERSION,
4153
4154         .add_gid = mlx5_ib_add_gid,
4155         .add_sub_dev = mlx5_ib_add_sub_dev,
4156         .alloc_mr = mlx5_ib_alloc_mr,
4157         .alloc_mr_integrity = mlx5_ib_alloc_mr_integrity,
4158         .alloc_pd = mlx5_ib_alloc_pd,
4159         .alloc_ucontext = mlx5_ib_alloc_ucontext,
4160         .attach_mcast = mlx5_ib_mcg_attach,
4161         .check_mr_status = mlx5_ib_check_mr_status,
4162         .create_ah = mlx5_ib_create_ah,
4163         .create_cq = mlx5_ib_create_cq,
4164         .create_qp = mlx5_ib_create_qp,
4165         .create_srq = mlx5_ib_create_srq,
4166         .create_user_ah = mlx5_ib_create_ah,
4167         .dealloc_pd = mlx5_ib_dealloc_pd,
4168         .dealloc_ucontext = mlx5_ib_dealloc_ucontext,
4169         .del_gid = mlx5_ib_del_gid,
4170         .del_sub_dev = mlx5_ib_del_sub_dev,
4171         .dereg_mr = mlx5_ib_dereg_mr,
4172         .destroy_ah = mlx5_ib_destroy_ah,
4173         .destroy_cq = mlx5_ib_destroy_cq,
4174         .destroy_qp = mlx5_ib_destroy_qp,
4175         .destroy_srq = mlx5_ib_destroy_srq,
4176         .detach_mcast = mlx5_ib_mcg_detach,
4177         .disassociate_ucontext = mlx5_ib_disassociate_ucontext,
4178         .drain_rq = mlx5_ib_drain_rq,
4179         .drain_sq = mlx5_ib_drain_sq,
4180         .device_group = &mlx5_attr_group,
4181         .get_dev_fw_str = get_dev_fw_str,
4182         .get_dma_mr = mlx5_ib_get_dma_mr,
4183         .get_link_layer = mlx5_ib_port_link_layer,
4184         .map_mr_sg = mlx5_ib_map_mr_sg,
4185         .map_mr_sg_pi = mlx5_ib_map_mr_sg_pi,
4186         .mmap = mlx5_ib_mmap,
4187         .mmap_free = mlx5_ib_mmap_free,
4188         .modify_cq = mlx5_ib_modify_cq,
4189         .modify_device = mlx5_ib_modify_device,
4190         .modify_port = mlx5_ib_modify_port,
4191         .modify_qp = mlx5_ib_modify_qp,
4192         .modify_srq = mlx5_ib_modify_srq,
4193         .poll_cq = mlx5_ib_poll_cq,
4194         .post_recv = mlx5_ib_post_recv_nodrain,
4195         .post_send = mlx5_ib_post_send_nodrain,
4196         .post_srq_recv = mlx5_ib_post_srq_recv,
4197         .process_mad = mlx5_ib_process_mad,
4198         .query_ah = mlx5_ib_query_ah,
4199         .query_device = mlx5_ib_query_device,
4200         .query_gid = mlx5_ib_query_gid,
4201         .query_pkey = mlx5_ib_query_pkey,
4202         .query_qp = mlx5_ib_query_qp,
4203         .query_srq = mlx5_ib_query_srq,
4204         .query_ucontext = mlx5_ib_query_ucontext,
4205         .reg_user_mr = mlx5_ib_reg_user_mr,
4206         .reg_user_mr_dmabuf = mlx5_ib_reg_user_mr_dmabuf,
4207         .req_notify_cq = mlx5_ib_arm_cq,
4208         .rereg_user_mr = mlx5_ib_rereg_user_mr,
4209         .resize_cq = mlx5_ib_resize_cq,
4210         .ufile_hw_cleanup = mlx5_ib_ufile_hw_cleanup,
4211
4212         INIT_RDMA_OBJ_SIZE(ib_ah, mlx5_ib_ah, ibah),
4213         INIT_RDMA_OBJ_SIZE(ib_counters, mlx5_ib_mcounters, ibcntrs),
4214         INIT_RDMA_OBJ_SIZE(ib_cq, mlx5_ib_cq, ibcq),
4215         INIT_RDMA_OBJ_SIZE(ib_pd, mlx5_ib_pd, ibpd),
4216         INIT_RDMA_OBJ_SIZE(ib_qp, mlx5_ib_qp, ibqp),
4217         INIT_RDMA_OBJ_SIZE(ib_srq, mlx5_ib_srq, ibsrq),
4218         INIT_RDMA_OBJ_SIZE(ib_ucontext, mlx5_ib_ucontext, ibucontext),
4219 };
4220
4221 static const struct ib_device_ops mlx5_ib_dev_ipoib_enhanced_ops = {
4222         .rdma_netdev_get_params = mlx5_ib_rn_get_params,
4223 };
4224
4225 static const struct ib_device_ops mlx5_ib_dev_sriov_ops = {
4226         .get_vf_config = mlx5_ib_get_vf_config,
4227         .get_vf_guid = mlx5_ib_get_vf_guid,
4228         .get_vf_stats = mlx5_ib_get_vf_stats,
4229         .set_vf_guid = mlx5_ib_set_vf_guid,
4230         .set_vf_link_state = mlx5_ib_set_vf_link_state,
4231 };
4232
4233 static const struct ib_device_ops mlx5_ib_dev_mw_ops = {
4234         .alloc_mw = mlx5_ib_alloc_mw,
4235         .dealloc_mw = mlx5_ib_dealloc_mw,
4236
4237         INIT_RDMA_OBJ_SIZE(ib_mw, mlx5_ib_mw, ibmw),
4238 };
4239
4240 static const struct ib_device_ops mlx5_ib_dev_xrc_ops = {
4241         .alloc_xrcd = mlx5_ib_alloc_xrcd,
4242         .dealloc_xrcd = mlx5_ib_dealloc_xrcd,
4243
4244         INIT_RDMA_OBJ_SIZE(ib_xrcd, mlx5_ib_xrcd, ibxrcd),
4245 };
4246
4247 static int mlx5_ib_init_var_table(struct mlx5_ib_dev *dev)
4248 {
4249         struct mlx5_core_dev *mdev = dev->mdev;
4250         struct mlx5_var_table *var_table = &dev->var_table;
4251         u8 log_doorbell_bar_size;
4252         u8 log_doorbell_stride;
4253         u64 bar_size;
4254
4255         log_doorbell_bar_size = MLX5_CAP_DEV_VDPA_EMULATION(mdev,
4256                                         log_doorbell_bar_size);
4257         log_doorbell_stride = MLX5_CAP_DEV_VDPA_EMULATION(mdev,
4258                                         log_doorbell_stride);
4259         var_table->hw_start_addr = dev->mdev->bar_addr +
4260                                 MLX5_CAP64_DEV_VDPA_EMULATION(mdev,
4261                                         doorbell_bar_offset);
4262         bar_size = (1ULL << log_doorbell_bar_size) * 4096;
4263         var_table->stride_size = 1ULL << log_doorbell_stride;
4264         var_table->num_var_hw_entries = div_u64(bar_size,
4265                                                 var_table->stride_size);
4266         mutex_init(&var_table->bitmap_lock);
4267         var_table->bitmap = bitmap_zalloc(var_table->num_var_hw_entries,
4268                                           GFP_KERNEL);
4269         return (var_table->bitmap) ? 0 : -ENOMEM;
4270 }
4271
4272 static void mlx5_ib_cleanup_ucaps(struct mlx5_ib_dev *dev)
4273 {
4274         if (MLX5_CAP_GEN(dev->mdev, uctx_cap) & MLX5_UCTX_CAP_RDMA_CTRL)
4275                 ib_remove_ucap(RDMA_UCAP_MLX5_CTRL_LOCAL);
4276
4277         if (MLX5_CAP_GEN(dev->mdev, uctx_cap) &
4278             MLX5_UCTX_CAP_RDMA_CTRL_OTHER_VHCA)
4279                 ib_remove_ucap(RDMA_UCAP_MLX5_CTRL_OTHER_VHCA);
4280 }
4281
4282 static int mlx5_ib_init_ucaps(struct mlx5_ib_dev *dev)
4283 {
4284         int ret;
4285
4286         if (MLX5_CAP_GEN(dev->mdev, uctx_cap) & MLX5_UCTX_CAP_RDMA_CTRL) {
4287                 ret = ib_create_ucap(RDMA_UCAP_MLX5_CTRL_LOCAL);
4288                 if (ret)
4289                         return ret;
4290         }
4291
4292         if (MLX5_CAP_GEN(dev->mdev, uctx_cap) &
4293             MLX5_UCTX_CAP_RDMA_CTRL_OTHER_VHCA) {
4294                 ret = ib_create_ucap(RDMA_UCAP_MLX5_CTRL_OTHER_VHCA);
4295                 if (ret)
4296                         goto remove_local;
4297         }
4298
4299         return 0;
4300
4301 remove_local:
4302         if (MLX5_CAP_GEN(dev->mdev, uctx_cap) & MLX5_UCTX_CAP_RDMA_CTRL)
4303                 ib_remove_ucap(RDMA_UCAP_MLX5_CTRL_LOCAL);
4304         return ret;
4305 }
4306
4307 static void mlx5_ib_stage_caps_cleanup(struct mlx5_ib_dev *dev)
4308 {
4309         if (MLX5_CAP_GEN_2_64(dev->mdev, general_obj_types_127_64) &
4310             MLX5_HCA_CAP_2_GENERAL_OBJECT_TYPES_RDMA_CTRL)
4311                 mlx5_ib_cleanup_ucaps(dev);
4312
4313         bitmap_free(dev->var_table.bitmap);
4314 }
4315
4316 static int mlx5_ib_stage_caps_init(struct mlx5_ib_dev *dev)
4317 {
4318         struct mlx5_core_dev *mdev = dev->mdev;
4319         int err;
4320
4321         if (MLX5_CAP_GEN(mdev, ipoib_enhanced_offloads) &&
4322             IS_ENABLED(CONFIG_MLX5_CORE_IPOIB))
4323                 ib_set_device_ops(&dev->ib_dev,
4324                                   &mlx5_ib_dev_ipoib_enhanced_ops);
4325
4326         if (mlx5_core_is_pf(mdev))
4327                 ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_sriov_ops);
4328
4329         dev->umr_fence = mlx5_get_umr_fence(MLX5_CAP_GEN(mdev, umr_fence));
4330
4331         if (MLX5_CAP_GEN(mdev, imaicl))
4332                 ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_mw_ops);
4333
4334         if (MLX5_CAP_GEN(mdev, xrc))
4335                 ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_xrc_ops);
4336
4337         if (MLX5_CAP_DEV_MEM(mdev, memic) ||
4338             MLX5_CAP_GEN_64(dev->mdev, general_obj_types) &
4339             MLX5_GENERAL_OBJ_TYPES_CAP_SW_ICM)
4340                 ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_dm_ops);
4341
4342         ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_ops);
4343
4344         if (IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS))
4345                 dev->ib_dev.driver_def = mlx5_ib_defs;
4346
4347         err = init_node_data(dev);
4348         if (err)
4349                 return err;
4350
4351         if ((MLX5_CAP_GEN(dev->mdev, port_type) == MLX5_CAP_PORT_TYPE_ETH) &&
4352             (MLX5_CAP_GEN(dev->mdev, disable_local_lb_uc) ||
4353              MLX5_CAP_GEN(dev->mdev, disable_local_lb_mc)))
4354                 mutex_init(&dev->lb.mutex);
4355
4356         if (MLX5_CAP_GEN_64(dev->mdev, general_obj_types) &
4357                         MLX5_GENERAL_OBJ_TYPES_CAP_VIRTIO_NET_Q) {
4358                 err = mlx5_ib_init_var_table(dev);
4359                 if (err)
4360                         return err;
4361         }
4362
4363         if (MLX5_CAP_GEN_2_64(dev->mdev, general_obj_types_127_64) &
4364             MLX5_HCA_CAP_2_GENERAL_OBJECT_TYPES_RDMA_CTRL) {
4365                 err = mlx5_ib_init_ucaps(dev);
4366                 if (err)
4367                         return err;
4368         }
4369
4370         dev->ib_dev.use_cq_dim = true;
4371
4372         return 0;
4373 }
4374
4375 static const struct ib_device_ops mlx5_ib_dev_port_ops = {
4376         .get_port_immutable = mlx5_port_immutable,
4377         .query_port = mlx5_ib_query_port,
4378 };
4379
4380 static int mlx5_ib_stage_non_default_cb(struct mlx5_ib_dev *dev)
4381 {
4382         ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_port_ops);
4383         return 0;
4384 }
4385
4386 static const struct ib_device_ops mlx5_ib_dev_port_rep_ops = {
4387         .get_port_immutable = mlx5_port_rep_immutable,
4388         .query_port = mlx5_ib_rep_query_port,
4389         .query_pkey = mlx5_ib_rep_query_pkey,
4390 };
4391
4392 static int mlx5_ib_stage_raw_eth_non_default_cb(struct mlx5_ib_dev *dev)
4393 {
4394         ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_port_rep_ops);
4395         return 0;
4396 }
4397
4398 static const struct ib_device_ops mlx5_ib_dev_common_roce_ops = {
4399         .create_rwq_ind_table = mlx5_ib_create_rwq_ind_table,
4400         .create_wq = mlx5_ib_create_wq,
4401         .destroy_rwq_ind_table = mlx5_ib_destroy_rwq_ind_table,
4402         .destroy_wq = mlx5_ib_destroy_wq,
4403         .modify_wq = mlx5_ib_modify_wq,
4404
4405         INIT_RDMA_OBJ_SIZE(ib_rwq_ind_table, mlx5_ib_rwq_ind_table,
4406                            ib_rwq_ind_tbl),
4407 };
4408
4409 static int mlx5_ib_roce_init(struct mlx5_ib_dev *dev)
4410 {
4411         struct mlx5_core_dev *mdev = dev->mdev;
4412         enum rdma_link_layer ll;
4413         int port_type_cap;
4414         u32 port_num = 0;
4415         int err;
4416
4417         port_type_cap = MLX5_CAP_GEN(mdev, port_type);
4418         ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap);
4419
4420         if (ll == IB_LINK_LAYER_ETHERNET) {
4421                 ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_common_roce_ops);
4422
4423                 port_num = mlx5_core_native_port_num(dev->mdev) - 1;
4424
4425                 /* Register only for native ports */
4426                 mlx5_mdev_netdev_track(dev, port_num);
4427
4428                 err = mlx5_enable_eth(dev);
4429                 if (err)
4430                         goto cleanup;
4431         }
4432
4433         return 0;
4434 cleanup:
4435         mlx5_mdev_netdev_untrack(dev, port_num);
4436         return err;
4437 }
4438
4439 static void mlx5_ib_roce_cleanup(struct mlx5_ib_dev *dev)
4440 {
4441         struct mlx5_core_dev *mdev = dev->mdev;
4442         enum rdma_link_layer ll;
4443         int port_type_cap;
4444         u32 port_num;
4445
4446         port_type_cap = MLX5_CAP_GEN(mdev, port_type);
4447         ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap);
4448
4449         if (ll == IB_LINK_LAYER_ETHERNET) {
4450                 mlx5_disable_eth(dev);
4451
4452                 port_num = mlx5_core_native_port_num(dev->mdev) - 1;
4453                 mlx5_mdev_netdev_untrack(dev, port_num);
4454         }
4455 }
4456
4457 static int mlx5_ib_stage_cong_debugfs_init(struct mlx5_ib_dev *dev)
4458 {
4459         mlx5_ib_init_cong_debugfs(dev,
4460                                   mlx5_core_native_port_num(dev->mdev) - 1);
4461         return 0;
4462 }
4463
4464 static void mlx5_ib_stage_cong_debugfs_cleanup(struct mlx5_ib_dev *dev)
4465 {
4466         mlx5_ib_cleanup_cong_debugfs(dev,
4467                                      mlx5_core_native_port_num(dev->mdev) - 1);
4468 }
4469
4470 static int mlx5_ib_stage_bfrag_init(struct mlx5_ib_dev *dev)
4471 {
4472         int err;
4473
4474         err = mlx5_alloc_bfreg(dev->mdev, &dev->bfreg, false, false);
4475         if (err)
4476                 return err;
4477
4478         err = mlx5_alloc_bfreg(dev->mdev, &dev->fp_bfreg, false, true);
4479         if (err)
4480                 mlx5_free_bfreg(dev->mdev, &dev->bfreg);
4481
4482         return err;
4483 }
4484
4485 static void mlx5_ib_stage_bfrag_cleanup(struct mlx5_ib_dev *dev)
4486 {
4487         mlx5_free_bfreg(dev->mdev, &dev->fp_bfreg);
4488         mlx5_free_bfreg(dev->mdev, &dev->bfreg);
4489 }
4490
4491 static int mlx5_ib_stage_ib_reg_init(struct mlx5_ib_dev *dev)
4492 {
4493         const char *name;
4494
4495         if (dev->sub_dev_name) {
4496                 name = dev->sub_dev_name;
4497                 ib_mark_name_assigned_by_user(&dev->ib_dev);
4498         } else if (!mlx5_lag_is_active(dev->mdev))
4499                 name = "mlx5_%d";
4500         else
4501                 name = "mlx5_bond_%d";
4502         return ib_register_device(&dev->ib_dev, name, &dev->mdev->pdev->dev);
4503 }
4504
4505 static void mlx5_ib_stage_pre_ib_reg_umr_cleanup(struct mlx5_ib_dev *dev)
4506 {
4507         mlx5_mkey_cache_cleanup(dev);
4508         mlx5r_umr_resource_cleanup(dev);
4509         mlx5r_umr_cleanup(dev);
4510 }
4511
4512 static void mlx5_ib_stage_ib_reg_cleanup(struct mlx5_ib_dev *dev)
4513 {
4514         ib_unregister_device(&dev->ib_dev);
4515 }
4516
4517 static int mlx5_ib_stage_post_ib_reg_umr_init(struct mlx5_ib_dev *dev)
4518 {
4519         int ret;
4520
4521         ret = mlx5r_umr_init(dev);
4522         if (ret)
4523                 return ret;
4524
4525         ret = mlx5_mkey_cache_init(dev);
4526         if (ret)
4527                 mlx5_ib_warn(dev, "mr cache init failed %d\n", ret);
4528         return ret;
4529 }
4530
4531 static int mlx5_ib_stage_delay_drop_init(struct mlx5_ib_dev *dev)
4532 {
4533         struct dentry *root;
4534
4535         if (!(dev->ib_dev.attrs.raw_packet_caps & IB_RAW_PACKET_CAP_DELAY_DROP))
4536                 return 0;
4537
4538         mutex_init(&dev->delay_drop.lock);
4539         dev->delay_drop.dev = dev;
4540         dev->delay_drop.activate = false;
4541         dev->delay_drop.timeout = MLX5_MAX_DELAY_DROP_TIMEOUT_MS * 1000;
4542         INIT_WORK(&dev->delay_drop.delay_drop_work, delay_drop_handler);
4543         atomic_set(&dev->delay_drop.rqs_cnt, 0);
4544         atomic_set(&dev->delay_drop.events_cnt, 0);
4545
4546         if (!mlx5_debugfs_root)
4547                 return 0;
4548
4549         root = debugfs_create_dir("delay_drop", mlx5_debugfs_get_dev_root(dev->mdev));
4550         dev->delay_drop.dir_debugfs = root;
4551
4552         debugfs_create_atomic_t("num_timeout_events", 0400, root,
4553                                 &dev->delay_drop.events_cnt);
4554         debugfs_create_atomic_t("num_rqs", 0400, root,
4555                                 &dev->delay_drop.rqs_cnt);
4556         debugfs_create_file("timeout", 0600, root, &dev->delay_drop,
4557                             &fops_delay_drop_timeout);
4558         return 0;
4559 }
4560
4561 static void mlx5_ib_stage_delay_drop_cleanup(struct mlx5_ib_dev *dev)
4562 {
4563         if (!(dev->ib_dev.attrs.raw_packet_caps & IB_RAW_PACKET_CAP_DELAY_DROP))
4564                 return;
4565
4566         cancel_work_sync(&dev->delay_drop.delay_drop_work);
4567         if (!dev->delay_drop.dir_debugfs)
4568                 return;
4569
4570         debugfs_remove_recursive(dev->delay_drop.dir_debugfs);
4571         dev->delay_drop.dir_debugfs = NULL;
4572 }
4573
4574 static int mlx5_ib_stage_dev_notifier_init(struct mlx5_ib_dev *dev)
4575 {
4576         struct mlx5_ib_resources *devr = &dev->devr;
4577         int port;
4578
4579         for (port = 0; port < ARRAY_SIZE(devr->ports); ++port)
4580                 INIT_WORK(&devr->ports[port].pkey_change_work,
4581                           pkey_change_handler);
4582
4583         dev->mdev_events.notifier_call = mlx5_ib_event;
4584         mlx5_notifier_register(dev->mdev, &dev->mdev_events);
4585
4586         mlx5r_macsec_event_register(dev);
4587
4588         return 0;
4589 }
4590
4591 static void mlx5_ib_stage_dev_notifier_cleanup(struct mlx5_ib_dev *dev)
4592 {
4593         struct mlx5_ib_resources *devr = &dev->devr;
4594         int port;
4595
4596         mlx5r_macsec_event_unregister(dev);
4597         mlx5_notifier_unregister(dev->mdev, &dev->mdev_events);
4598
4599         for (port = 0; port < ARRAY_SIZE(devr->ports); ++port)
4600                 cancel_work_sync(&devr->ports[port].pkey_change_work);
4601 }
4602
4603 void mlx5_ib_data_direct_bind(struct mlx5_ib_dev *ibdev,
4604                               struct mlx5_data_direct_dev *dev)
4605 {
4606         mutex_lock(&ibdev->data_direct_lock);
4607         ibdev->data_direct_dev = dev;
4608         mutex_unlock(&ibdev->data_direct_lock);
4609 }
4610
4611 void mlx5_ib_data_direct_unbind(struct mlx5_ib_dev *ibdev)
4612 {
4613         mutex_lock(&ibdev->data_direct_lock);
4614         mlx5_ib_revoke_data_direct_mrs(ibdev);
4615         ibdev->data_direct_dev = NULL;
4616         mutex_unlock(&ibdev->data_direct_lock);
4617 }
4618
4619 void __mlx5_ib_remove(struct mlx5_ib_dev *dev,
4620                       const struct mlx5_ib_profile *profile,
4621                       int stage)
4622 {
4623         dev->ib_active = false;
4624
4625         /* Number of stages to cleanup */
4626         while (stage) {
4627                 stage--;
4628                 if (profile->stage[stage].cleanup)
4629                         profile->stage[stage].cleanup(dev);
4630         }
4631
4632         kfree(dev->port);
4633         ib_dealloc_device(&dev->ib_dev);
4634 }
4635
4636 int __mlx5_ib_add(struct mlx5_ib_dev *dev,
4637                   const struct mlx5_ib_profile *profile)
4638 {
4639         int err;
4640         int i;
4641
4642         dev->profile = profile;
4643
4644         for (i = 0; i < MLX5_IB_STAGE_MAX; i++) {
4645                 if (profile->stage[i].init) {
4646                         err = profile->stage[i].init(dev);
4647                         if (err)
4648                                 goto err_out;
4649                 }
4650         }
4651
4652         dev->ib_active = true;
4653         return 0;
4654
4655 err_out:
4656         /* Clean up stages which were initialized */
4657         while (i) {
4658                 i--;
4659                 if (profile->stage[i].cleanup)
4660                         profile->stage[i].cleanup(dev);
4661         }
4662         return -ENOMEM;
4663 }
4664
4665 static const struct mlx5_ib_profile pf_profile = {
4666         STAGE_CREATE(MLX5_IB_STAGE_INIT,
4667                      mlx5_ib_stage_init_init,
4668                      mlx5_ib_stage_init_cleanup),
4669         STAGE_CREATE(MLX5_IB_STAGE_FS,
4670                      mlx5_ib_fs_init,
4671                      mlx5_ib_fs_cleanup),
4672         STAGE_CREATE(MLX5_IB_STAGE_CAPS,
4673                      mlx5_ib_stage_caps_init,
4674                      mlx5_ib_stage_caps_cleanup),
4675         STAGE_CREATE(MLX5_IB_STAGE_NON_DEFAULT_CB,
4676                      mlx5_ib_stage_non_default_cb,
4677                      NULL),
4678         STAGE_CREATE(MLX5_IB_STAGE_ROCE,
4679                      mlx5_ib_roce_init,
4680                      mlx5_ib_roce_cleanup),
4681         STAGE_CREATE(MLX5_IB_STAGE_QP,
4682                      mlx5_init_qp_table,
4683                      mlx5_cleanup_qp_table),
4684         STAGE_CREATE(MLX5_IB_STAGE_SRQ,
4685                      mlx5_init_srq_table,
4686                      mlx5_cleanup_srq_table),
4687         STAGE_CREATE(MLX5_IB_STAGE_DEVICE_RESOURCES,
4688                      mlx5_ib_dev_res_init,
4689                      mlx5_ib_dev_res_cleanup),
4690         STAGE_CREATE(MLX5_IB_STAGE_ODP,
4691                      mlx5_ib_odp_init_one,
4692                      mlx5_ib_odp_cleanup_one),
4693         STAGE_CREATE(MLX5_IB_STAGE_COUNTERS,
4694                      mlx5_ib_counters_init,
4695                      mlx5_ib_counters_cleanup),
4696         STAGE_CREATE(MLX5_IB_STAGE_CONG_DEBUGFS,
4697                      mlx5_ib_stage_cong_debugfs_init,
4698                      mlx5_ib_stage_cong_debugfs_cleanup),
4699         STAGE_CREATE(MLX5_IB_STAGE_BFREG,
4700                      mlx5_ib_stage_bfrag_init,
4701                      mlx5_ib_stage_bfrag_cleanup),
4702         STAGE_CREATE(MLX5_IB_STAGE_PRE_IB_REG_UMR,
4703                      NULL,
4704                      mlx5_ib_stage_pre_ib_reg_umr_cleanup),
4705         STAGE_CREATE(MLX5_IB_STAGE_WHITELIST_UID,
4706                      mlx5_ib_devx_init,
4707                      mlx5_ib_devx_cleanup),
4708         STAGE_CREATE(MLX5_IB_STAGE_IB_REG,
4709                      mlx5_ib_stage_ib_reg_init,
4710                      mlx5_ib_stage_ib_reg_cleanup),
4711         STAGE_CREATE(MLX5_IB_STAGE_DEVICE_NOTIFIER,
4712                      mlx5_ib_stage_dev_notifier_init,
4713                      mlx5_ib_stage_dev_notifier_cleanup),
4714         STAGE_CREATE(MLX5_IB_STAGE_POST_IB_REG_UMR,
4715                      mlx5_ib_stage_post_ib_reg_umr_init,
4716                      NULL),
4717         STAGE_CREATE(MLX5_IB_STAGE_DELAY_DROP,
4718                      mlx5_ib_stage_delay_drop_init,
4719                      mlx5_ib_stage_delay_drop_cleanup),
4720         STAGE_CREATE(MLX5_IB_STAGE_RESTRACK,
4721                      mlx5_ib_restrack_init,
4722                      NULL),
4723 };
4724
4725 const struct mlx5_ib_profile raw_eth_profile = {
4726         STAGE_CREATE(MLX5_IB_STAGE_INIT,
4727                      mlx5_ib_stage_init_init,
4728                      mlx5_ib_stage_init_cleanup),
4729         STAGE_CREATE(MLX5_IB_STAGE_FS,
4730                      mlx5_ib_fs_init,
4731                      mlx5_ib_fs_cleanup),
4732         STAGE_CREATE(MLX5_IB_STAGE_CAPS,
4733                      mlx5_ib_stage_caps_init,
4734                      mlx5_ib_stage_caps_cleanup),
4735         STAGE_CREATE(MLX5_IB_STAGE_NON_DEFAULT_CB,
4736                      mlx5_ib_stage_raw_eth_non_default_cb,
4737                      NULL),
4738         STAGE_CREATE(MLX5_IB_STAGE_ROCE,
4739                      mlx5_ib_roce_init,
4740                      mlx5_ib_roce_cleanup),
4741         STAGE_CREATE(MLX5_IB_STAGE_QP,
4742                      mlx5_init_qp_table,
4743                      mlx5_cleanup_qp_table),
4744         STAGE_CREATE(MLX5_IB_STAGE_SRQ,
4745                      mlx5_init_srq_table,
4746                      mlx5_cleanup_srq_table),
4747         STAGE_CREATE(MLX5_IB_STAGE_DEVICE_RESOURCES,
4748                      mlx5_ib_dev_res_init,
4749                      mlx5_ib_dev_res_cleanup),
4750         STAGE_CREATE(MLX5_IB_STAGE_COUNTERS,
4751                      mlx5_ib_counters_init,
4752                      mlx5_ib_counters_cleanup),
4753         STAGE_CREATE(MLX5_IB_STAGE_CONG_DEBUGFS,
4754                      mlx5_ib_stage_cong_debugfs_init,
4755                      mlx5_ib_stage_cong_debugfs_cleanup),
4756         STAGE_CREATE(MLX5_IB_STAGE_BFREG,
4757                      mlx5_ib_stage_bfrag_init,
4758                      mlx5_ib_stage_bfrag_cleanup),
4759         STAGE_CREATE(MLX5_IB_STAGE_PRE_IB_REG_UMR,
4760                      NULL,
4761                      mlx5_ib_stage_pre_ib_reg_umr_cleanup),
4762         STAGE_CREATE(MLX5_IB_STAGE_WHITELIST_UID,
4763                      mlx5_ib_devx_init,
4764                      mlx5_ib_devx_cleanup),
4765         STAGE_CREATE(MLX5_IB_STAGE_IB_REG,
4766                      mlx5_ib_stage_ib_reg_init,
4767                      mlx5_ib_stage_ib_reg_cleanup),
4768         STAGE_CREATE(MLX5_IB_STAGE_DEVICE_NOTIFIER,
4769                      mlx5_ib_stage_dev_notifier_init,
4770                      mlx5_ib_stage_dev_notifier_cleanup),
4771         STAGE_CREATE(MLX5_IB_STAGE_POST_IB_REG_UMR,
4772                      mlx5_ib_stage_post_ib_reg_umr_init,
4773                      NULL),
4774         STAGE_CREATE(MLX5_IB_STAGE_DELAY_DROP,
4775                      mlx5_ib_stage_delay_drop_init,
4776                      mlx5_ib_stage_delay_drop_cleanup),
4777         STAGE_CREATE(MLX5_IB_STAGE_RESTRACK,
4778                      mlx5_ib_restrack_init,
4779                      NULL),
4780 };
4781
4782 static const struct mlx5_ib_profile plane_profile = {
4783         STAGE_CREATE(MLX5_IB_STAGE_INIT,
4784                      mlx5_ib_stage_init_init,
4785                      mlx5_ib_stage_init_cleanup),
4786         STAGE_CREATE(MLX5_IB_STAGE_CAPS,
4787                      mlx5_ib_stage_caps_init,
4788                      mlx5_ib_stage_caps_cleanup),
4789         STAGE_CREATE(MLX5_IB_STAGE_NON_DEFAULT_CB,
4790                      mlx5_ib_stage_non_default_cb,
4791                      NULL),
4792         STAGE_CREATE(MLX5_IB_STAGE_QP,
4793                      mlx5_init_qp_table,
4794                      mlx5_cleanup_qp_table),
4795         STAGE_CREATE(MLX5_IB_STAGE_SRQ,
4796                      mlx5_init_srq_table,
4797                      mlx5_cleanup_srq_table),
4798         STAGE_CREATE(MLX5_IB_STAGE_DEVICE_RESOURCES,
4799                      mlx5_ib_dev_res_init,
4800                      mlx5_ib_dev_res_cleanup),
4801         STAGE_CREATE(MLX5_IB_STAGE_BFREG,
4802                      mlx5_ib_stage_bfrag_init,
4803                      mlx5_ib_stage_bfrag_cleanup),
4804         STAGE_CREATE(MLX5_IB_STAGE_IB_REG,
4805                      mlx5_ib_stage_ib_reg_init,
4806                      mlx5_ib_stage_ib_reg_cleanup),
4807 };
4808
4809 static struct ib_device *mlx5_ib_add_sub_dev(struct ib_device *parent,
4810                                              enum rdma_nl_dev_type type,
4811                                              const char *name)
4812 {
4813         struct mlx5_ib_dev *mparent = to_mdev(parent), *mplane;
4814         enum rdma_link_layer ll;
4815         int ret;
4816
4817         if (mparent->smi_dev)
4818                 return ERR_PTR(-EEXIST);
4819
4820         ll = mlx5_port_type_cap_to_rdma_ll(MLX5_CAP_GEN(mparent->mdev,
4821                                                         port_type));
4822         if (type != RDMA_DEVICE_TYPE_SMI || !mparent->num_plane ||
4823             ll != IB_LINK_LAYER_INFINIBAND ||
4824             !MLX5_CAP_GEN_2(mparent->mdev, multiplane_qp_ud))
4825                 return ERR_PTR(-EOPNOTSUPP);
4826
4827         mplane = ib_alloc_device(mlx5_ib_dev, ib_dev);
4828         if (!mplane)
4829                 return ERR_PTR(-ENOMEM);
4830
4831         mplane->port = kcalloc(mparent->num_plane * mparent->num_ports,
4832                                sizeof(*mplane->port), GFP_KERNEL);
4833         if (!mplane->port) {
4834                 ret = -ENOMEM;
4835                 goto fail_kcalloc;
4836         }
4837
4838         mplane->ib_dev.type = type;
4839         mplane->mdev = mparent->mdev;
4840         mplane->num_ports = mparent->num_plane;
4841         mplane->sub_dev_name = name;
4842         mplane->ib_dev.phys_port_cnt = mplane->num_ports;
4843
4844         ret = __mlx5_ib_add(mplane, &plane_profile);
4845         if (ret)
4846                 goto fail_ib_add;
4847
4848         mparent->smi_dev = mplane;
4849         return &mplane->ib_dev;
4850
4851 fail_ib_add:
4852         kfree(mplane->port);
4853 fail_kcalloc:
4854         ib_dealloc_device(&mplane->ib_dev);
4855         return ERR_PTR(ret);
4856 }
4857
4858 static void mlx5_ib_del_sub_dev(struct ib_device *sub_dev)
4859 {
4860         struct mlx5_ib_dev *mdev = to_mdev(sub_dev);
4861
4862         to_mdev(sub_dev->parent)->smi_dev = NULL;
4863         __mlx5_ib_remove(mdev, mdev->profile, MLX5_IB_STAGE_MAX);
4864 }
4865
4866 static int mlx5r_mp_probe(struct auxiliary_device *adev,
4867                           const struct auxiliary_device_id *id)
4868 {
4869         struct mlx5_adev *idev = container_of(adev, struct mlx5_adev, adev);
4870         struct mlx5_core_dev *mdev = idev->mdev;
4871         struct mlx5_ib_multiport_info *mpi;
4872         struct mlx5_ib_dev *dev;
4873         bool bound = false;
4874         int err;
4875
4876         mpi = kzalloc(sizeof(*mpi), GFP_KERNEL);
4877         if (!mpi)
4878                 return -ENOMEM;
4879
4880         mpi->mdev = mdev;
4881         err = mlx5_query_nic_vport_system_image_guid(mdev,
4882                                                      &mpi->sys_image_guid);
4883         if (err) {
4884                 kfree(mpi);
4885                 return err;
4886         }
4887
4888         mutex_lock(&mlx5_ib_multiport_mutex);
4889         list_for_each_entry(dev, &mlx5_ib_dev_list, ib_dev_list) {
4890                 if (dev->sys_image_guid == mpi->sys_image_guid &&
4891                     mlx5_core_same_coredev_type(dev->mdev, mpi->mdev))
4892                         bound = mlx5_ib_bind_slave_port(dev, mpi);
4893
4894                 if (bound) {
4895                         rdma_roce_rescan_device(&dev->ib_dev);
4896                         mpi->ibdev->ib_active = true;
4897                         break;
4898                 }
4899         }
4900
4901         if (!bound) {
4902                 list_add_tail(&mpi->list, &mlx5_ib_unaffiliated_port_list);
4903                 dev_dbg(mdev->device,
4904                         "no suitable IB device found to bind to, added to unaffiliated list.\n");
4905         }
4906         mutex_unlock(&mlx5_ib_multiport_mutex);
4907
4908         auxiliary_set_drvdata(adev, mpi);
4909         return 0;
4910 }
4911
4912 static void mlx5r_mp_remove(struct auxiliary_device *adev)
4913 {
4914         struct mlx5_ib_multiport_info *mpi;
4915
4916         mpi = auxiliary_get_drvdata(adev);
4917         mutex_lock(&mlx5_ib_multiport_mutex);
4918         if (mpi->ibdev)
4919                 mlx5_ib_unbind_slave_port(mpi->ibdev, mpi);
4920         else
4921                 list_del(&mpi->list);
4922         mutex_unlock(&mlx5_ib_multiport_mutex);
4923         kfree(mpi);
4924 }
4925
4926 static int mlx5r_probe(struct auxiliary_device *adev,
4927                        const struct auxiliary_device_id *id)
4928 {
4929         struct mlx5_adev *idev = container_of(adev, struct mlx5_adev, adev);
4930         struct mlx5_core_dev *mdev = idev->mdev;
4931         const struct mlx5_ib_profile *profile;
4932         int port_type_cap, num_ports, ret;
4933         enum rdma_link_layer ll;
4934         struct mlx5_ib_dev *dev;
4935
4936         port_type_cap = MLX5_CAP_GEN(mdev, port_type);
4937         ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap);
4938
4939         num_ports = max(MLX5_CAP_GEN(mdev, num_ports),
4940                         MLX5_CAP_GEN(mdev, num_vhca_ports));
4941         dev = ib_alloc_device(mlx5_ib_dev, ib_dev);
4942         if (!dev)
4943                 return -ENOMEM;
4944
4945         if (ll == IB_LINK_LAYER_INFINIBAND) {
4946                 ret = mlx5_ib_get_plane_num(mdev, &dev->num_plane);
4947                 if (ret)
4948                         goto fail;
4949         }
4950
4951         dev->port = kcalloc(num_ports, sizeof(*dev->port),
4952                              GFP_KERNEL);
4953         if (!dev->port) {
4954                 ret = -ENOMEM;
4955                 goto fail;
4956         }
4957
4958         dev->mdev = mdev;
4959         dev->num_ports = num_ports;
4960         dev->ib_dev.phys_port_cnt = num_ports;
4961
4962         if (ll == IB_LINK_LAYER_ETHERNET && !mlx5_get_roce_state(mdev))
4963                 profile = &raw_eth_profile;
4964         else
4965                 profile = &pf_profile;
4966
4967         ret = __mlx5_ib_add(dev, profile);
4968         if (ret)
4969                 goto fail_ib_add;
4970
4971         auxiliary_set_drvdata(adev, dev);
4972         return 0;
4973
4974 fail_ib_add:
4975         kfree(dev->port);
4976 fail:
4977         ib_dealloc_device(&dev->ib_dev);
4978         return ret;
4979 }
4980
4981 static void mlx5r_remove(struct auxiliary_device *adev)
4982 {
4983         struct mlx5_ib_dev *dev;
4984
4985         dev = auxiliary_get_drvdata(adev);
4986         __mlx5_ib_remove(dev, dev->profile, MLX5_IB_STAGE_MAX);
4987 }
4988
4989 static const struct auxiliary_device_id mlx5r_mp_id_table[] = {
4990         { .name = MLX5_ADEV_NAME ".multiport", },
4991         {},
4992 };
4993
4994 static const struct auxiliary_device_id mlx5r_id_table[] = {
4995         { .name = MLX5_ADEV_NAME ".rdma", },
4996         {},
4997 };
4998
4999 MODULE_DEVICE_TABLE(auxiliary, mlx5r_mp_id_table);
5000 MODULE_DEVICE_TABLE(auxiliary, mlx5r_id_table);
5001
5002 static struct auxiliary_driver mlx5r_mp_driver = {
5003         .name = "multiport",
5004         .probe = mlx5r_mp_probe,
5005         .remove = mlx5r_mp_remove,
5006         .id_table = mlx5r_mp_id_table,
5007 };
5008
5009 static struct auxiliary_driver mlx5r_driver = {
5010         .name = "rdma",
5011         .probe = mlx5r_probe,
5012         .remove = mlx5r_remove,
5013         .id_table = mlx5r_id_table,
5014 };
5015
5016 static int __init mlx5_ib_init(void)
5017 {
5018         int ret;
5019
5020         xlt_emergency_page = (void *)__get_free_page(GFP_KERNEL);
5021         if (!xlt_emergency_page)
5022                 return -ENOMEM;
5023
5024         mlx5_ib_event_wq = alloc_ordered_workqueue("mlx5_ib_event_wq", 0);
5025         if (!mlx5_ib_event_wq) {
5026                 free_page((unsigned long)xlt_emergency_page);
5027                 return -ENOMEM;
5028         }
5029
5030         ret = mlx5_ib_qp_event_init();
5031         if (ret)
5032                 goto qp_event_err;
5033
5034         mlx5_ib_odp_init();
5035         ret = mlx5r_rep_init();
5036         if (ret)
5037                 goto rep_err;
5038         ret = mlx5_data_direct_driver_register();
5039         if (ret)
5040                 goto dd_err;
5041         ret = auxiliary_driver_register(&mlx5r_mp_driver);
5042         if (ret)
5043                 goto mp_err;
5044         ret = auxiliary_driver_register(&mlx5r_driver);
5045         if (ret)
5046                 goto drv_err;
5047
5048         return 0;
5049
5050 drv_err:
5051         auxiliary_driver_unregister(&mlx5r_mp_driver);
5052 mp_err:
5053         mlx5_data_direct_driver_unregister();
5054 dd_err:
5055         mlx5r_rep_cleanup();
5056 rep_err:
5057         mlx5_ib_qp_event_cleanup();
5058 qp_event_err:
5059         destroy_workqueue(mlx5_ib_event_wq);
5060         free_page((unsigned long)xlt_emergency_page);
5061         return ret;
5062 }
5063
5064 static void __exit mlx5_ib_cleanup(void)
5065 {
5066         mlx5_data_direct_driver_unregister();
5067         auxiliary_driver_unregister(&mlx5r_driver);
5068         auxiliary_driver_unregister(&mlx5r_mp_driver);
5069         mlx5r_rep_cleanup();
5070
5071         mlx5_ib_qp_event_cleanup();
5072         destroy_workqueue(mlx5_ib_event_wq);
5073         free_page((unsigned long)xlt_emergency_page);
5074 }
5075
5076 module_init(mlx5_ib_init);
5077 module_exit(mlx5_ib_cleanup);