projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
IB/mlx5: Add hca_core_clock_offset to udata in init_ucontext
[linux-2.6-block.git] / drivers / infiniband / hw / mlx5 / main.c
CommitLineData
e126ba97 1/*
6cf0a15f 2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
e126ba97
EC		 3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
adec640e 33#include <linux/highmem.h>
e126ba97
EC		 34#include <linux/module.h>
35#include <linux/init.h>
36#include <linux/errno.h>
37#include <linux/pci.h>
38#include <linux/dma-mapping.h>
39#include <linux/slab.h>
40#include <linux/io-mapping.h>
41#include <linux/sched.h>
42#include <rdma/ib_user_verbs.h>
3f89a643 43#include <rdma/ib_addr.h>
2811ba51 44#include <rdma/ib_cache.h>
1b5daf11 45#include <linux/mlx5/vport.h>
e126ba97
EC		 46#include <rdma/ib_smi.h>
47#include <rdma/ib_umem.h>
48#include "user.h"
49#include "mlx5_ib.h"
50
51#define DRIVER_NAME "mlx5_ib"
169a1d85
AV		 52#define DRIVER_VERSION "2.2-1"
53#define DRIVER_RELDATE "Feb 2014"
e126ba97
EC		 54
55MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>");
56MODULE_DESCRIPTION("Mellanox Connect-IB HCA IB driver");
57MODULE_LICENSE("Dual BSD/GPL");
58MODULE_VERSION(DRIVER_VERSION);
59
9603b61d
JM		 60static int deprecated_prof_sel = 2;
61module_param_named(prof_sel, deprecated_prof_sel, int, 0444);
62MODULE_PARM_DESC(prof_sel, "profile selector. Deprecated here. Moved to module mlx5_core");
e126ba97
EC		 63
64static char mlx5_version[] =
65 DRIVER_NAME ": Mellanox Connect-IB Infiniband driver v"
66 DRIVER_VERSION " (" DRIVER_RELDATE ")\n";
67
1b5daf11 68static enum rdma_link_layer
ebd61f68 69mlx5_port_type_cap_to_rdma_ll(int port_type_cap)
1b5daf11 70{
ebd61f68 71 switch (port_type_cap) {
1b5daf11
MD		 72 case MLX5_CAP_PORT_TYPE_IB:
73 return IB_LINK_LAYER_INFINIBAND;
74 case MLX5_CAP_PORT_TYPE_ETH:
75 return IB_LINK_LAYER_ETHERNET;
76 default:
77 return IB_LINK_LAYER_UNSPECIFIED;
78 }
79}
80
ebd61f68
AS		 81static enum rdma_link_layer
82mlx5_ib_port_link_layer(struct ib_device *device, u8 port_num)
83{
84 struct mlx5_ib_dev *dev = to_mdev(device);
85 int port_type_cap = MLX5_CAP_GEN(dev->mdev, port_type);
86
87 return mlx5_port_type_cap_to_rdma_ll(port_type_cap);
88}
89
fc24fc5e
AS		 90static int mlx5_netdev_event(struct notifier_block *this,
91 unsigned long event, void *ptr)
92{
93 struct net_device *ndev = netdev_notifier_info_to_dev(ptr);
94 struct mlx5_ib_dev *ibdev = container_of(this, struct mlx5_ib_dev,
95 roce.nb);
96
97 if ((event != NETDEV_UNREGISTER) && (event != NETDEV_REGISTER))
98 return NOTIFY_DONE;
99
100 write_lock(&ibdev->roce.netdev_lock);
101 if (ndev->dev.parent == &ibdev->mdev->pdev->dev)
102 ibdev->roce.netdev = (event == NETDEV_UNREGISTER) ? NULL : ndev;
103 write_unlock(&ibdev->roce.netdev_lock);
104
105 return NOTIFY_DONE;
106}
107
108static struct net_device *mlx5_ib_get_netdev(struct ib_device *device,
109 u8 port_num)
110{
111 struct mlx5_ib_dev *ibdev = to_mdev(device);
112 struct net_device *ndev;
113
114 /* Ensure ndev does not disappear before we invoke dev_hold()
115 */
116 read_lock(&ibdev->roce.netdev_lock);
117 ndev = ibdev->roce.netdev;
118 if (ndev)
119 dev_hold(ndev);
120 read_unlock(&ibdev->roce.netdev_lock);
121
122 return ndev;
123}
124
3f89a643
AS		 125static int mlx5_query_port_roce(struct ib_device *device, u8 port_num,
126 struct ib_port_attr *props)
127{
128 struct mlx5_ib_dev *dev = to_mdev(device);
129 struct net_device *ndev;
130 enum ib_mtu ndev_ib_mtu;
131
132 memset(props, 0, sizeof(*props));
133
134 props->port_cap_flags |= IB_PORT_CM_SUP;
135 props->port_cap_flags |= IB_PORT_IP_BASED_GIDS;
136
137 props->gid_tbl_len = MLX5_CAP_ROCE(dev->mdev,
138 roce_address_table_size);
139 props->max_mtu = IB_MTU_4096;
140 props->max_msg_sz = 1 << MLX5_CAP_GEN(dev->mdev, log_max_msg);
141 props->pkey_tbl_len = 1;
142 props->state = IB_PORT_DOWN;
143 props->phys_state = 3;
144
145 mlx5_query_nic_vport_qkey_viol_cntr(dev->mdev,
146 (u16 *)&props->qkey_viol_cntr);
147
148 ndev = mlx5_ib_get_netdev(device, port_num);
149 if (!ndev)
150 return 0;
151
152 if (netif_running(ndev) && netif_carrier_ok(ndev)) {
153 props->state = IB_PORT_ACTIVE;
154 props->phys_state = 5;
155 }
156
157 ndev_ib_mtu = iboe_get_mtu(ndev->mtu);
158
159 dev_put(ndev);
160
161 props->active_mtu = min(props->max_mtu, ndev_ib_mtu);
162
163 props->active_width = IB_WIDTH_4X; /* TODO */
164 props->active_speed = IB_SPEED_QDR; /* TODO */
165
166 return 0;
167}
168
3cca2606
AS		 169static void ib_gid_to_mlx5_roce_addr(const union ib_gid *gid,
170 const struct ib_gid_attr *attr,
171 void *mlx5_addr)
172{
173#define MLX5_SET_RA(p, f, v) MLX5_SET(roce_addr_layout, p, f, v)
174 char *mlx5_addr_l3_addr = MLX5_ADDR_OF(roce_addr_layout, mlx5_addr,
175 source_l3_address);
176 void *mlx5_addr_mac = MLX5_ADDR_OF(roce_addr_layout, mlx5_addr,
177 source_mac_47_32);
178
179 if (!gid)
180 return;
181
182 ether_addr_copy(mlx5_addr_mac, attr->ndev->dev_addr);
183
184 if (is_vlan_dev(attr->ndev)) {
185 MLX5_SET_RA(mlx5_addr, vlan_valid, 1);
186 MLX5_SET_RA(mlx5_addr, vlan_id, vlan_dev_vlan_id(attr->ndev));
187 }
188
189 switch (attr->gid_type) {
190 case IB_GID_TYPE_IB:
191 MLX5_SET_RA(mlx5_addr, roce_version, MLX5_ROCE_VERSION_1);
192 break;
193 case IB_GID_TYPE_ROCE_UDP_ENCAP:
194 MLX5_SET_RA(mlx5_addr, roce_version, MLX5_ROCE_VERSION_2);
195 break;
196
197 default:
198 WARN_ON(true);
199 }
200
201 if (attr->gid_type != IB_GID_TYPE_IB) {
202 if (ipv6_addr_v4mapped((void *)gid))
203 MLX5_SET_RA(mlx5_addr, roce_l3_type,
204 MLX5_ROCE_L3_TYPE_IPV4);
205 else
206 MLX5_SET_RA(mlx5_addr, roce_l3_type,
207 MLX5_ROCE_L3_TYPE_IPV6);
208 }
209
210 if ((attr->gid_type == IB_GID_TYPE_IB) ||
211 !ipv6_addr_v4mapped((void *)gid))
212 memcpy(mlx5_addr_l3_addr, gid, sizeof(*gid));
213 else
214 memcpy(&mlx5_addr_l3_addr[12], &gid->raw[12], 4);
215}
216
217static int set_roce_addr(struct ib_device *device, u8 port_num,
218 unsigned int index,
219 const union ib_gid *gid,
220 const struct ib_gid_attr *attr)
221{
222 struct mlx5_ib_dev *dev = to_mdev(device);
223 u32 in[MLX5_ST_SZ_DW(set_roce_address_in)];
224 u32 out[MLX5_ST_SZ_DW(set_roce_address_out)];
225 void *in_addr = MLX5_ADDR_OF(set_roce_address_in, in, roce_address);
226 enum rdma_link_layer ll = mlx5_ib_port_link_layer(device, port_num);
227
228 if (ll != IB_LINK_LAYER_ETHERNET)
229 return -EINVAL;
230
231 memset(in, 0, sizeof(in));
232
233 ib_gid_to_mlx5_roce_addr(gid, attr, in_addr);
234
235 MLX5_SET(set_roce_address_in, in, roce_address_index, index);
236 MLX5_SET(set_roce_address_in, in, opcode, MLX5_CMD_OP_SET_ROCE_ADDRESS);
237
238 memset(out, 0, sizeof(out));
239 return mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, sizeof(out));
240}
241
242static int mlx5_ib_add_gid(struct ib_device *device, u8 port_num,
243 unsigned int index, const union ib_gid *gid,
244 const struct ib_gid_attr *attr,
245 __always_unused void **context)
246{
247 return set_roce_addr(device, port_num, index, gid, attr);
248}
249
250static int mlx5_ib_del_gid(struct ib_device *device, u8 port_num,
251 unsigned int index, __always_unused void **context)
252{
253 return set_roce_addr(device, port_num, index, NULL, NULL);
254}
255
2811ba51
AS		 256__be16 mlx5_get_roce_udp_sport(struct mlx5_ib_dev *dev, u8 port_num,
257 int index)
258{
259 struct ib_gid_attr attr;
260 union ib_gid gid;
261
262 if (ib_get_cached_gid(&dev->ib_dev, port_num, index, &gid, &attr))
263 return 0;
264
265 if (!attr.ndev)
266 return 0;
267
268 dev_put(attr.ndev);
269
270 if (attr.gid_type != IB_GID_TYPE_ROCE_UDP_ENCAP)
271 return 0;
272
273 return cpu_to_be16(MLX5_CAP_ROCE(dev->mdev, r_roce_min_src_udp_port));
274}
275
1b5daf11
MD		 276static int mlx5_use_mad_ifc(struct mlx5_ib_dev *dev)
277{
278 return !dev->mdev->issi;
279}
280
281enum {
282 MLX5_VPORT_ACCESS_METHOD_MAD,
283 MLX5_VPORT_ACCESS_METHOD_HCA,
284 MLX5_VPORT_ACCESS_METHOD_NIC,
285};
286
287static int mlx5_get_vport_access_method(struct ib_device *ibdev)
288{
289 if (mlx5_use_mad_ifc(to_mdev(ibdev)))
290 return MLX5_VPORT_ACCESS_METHOD_MAD;
291
ebd61f68 292 if (mlx5_ib_port_link_layer(ibdev, 1) ==
1b5daf11
MD		 293 IB_LINK_LAYER_ETHERNET)
294 return MLX5_VPORT_ACCESS_METHOD_NIC;
295
296 return MLX5_VPORT_ACCESS_METHOD_HCA;
297}
298
299static int mlx5_query_system_image_guid(struct ib_device *ibdev,
300 __be64 *sys_image_guid)
301{
302 struct mlx5_ib_dev *dev = to_mdev(ibdev);
303 struct mlx5_core_dev *mdev = dev->mdev;
304 u64 tmp;
305 int err;
306
307 switch (mlx5_get_vport_access_method(ibdev)) {
308 case MLX5_VPORT_ACCESS_METHOD_MAD:
309 return mlx5_query_mad_ifc_system_image_guid(ibdev,
310 sys_image_guid);
311
312 case MLX5_VPORT_ACCESS_METHOD_HCA:
313 err = mlx5_query_hca_vport_system_image_guid(mdev, &tmp);
3f89a643
AS		 314 break;
315
316 case MLX5_VPORT_ACCESS_METHOD_NIC:
317 err = mlx5_query_nic_vport_system_image_guid(mdev, &tmp);
318 break;
1b5daf11
MD		 319
320 default:
321 return -EINVAL;
322 }
3f89a643
AS		 323
324 if (!err)
325 *sys_image_guid = cpu_to_be64(tmp);
326
327 return err;
328
1b5daf11
MD		 329}
330
331static int mlx5_query_max_pkeys(struct ib_device *ibdev,
332 u16 *max_pkeys)
333{
334 struct mlx5_ib_dev *dev = to_mdev(ibdev);
335 struct mlx5_core_dev *mdev = dev->mdev;
336
337 switch (mlx5_get_vport_access_method(ibdev)) {
338 case MLX5_VPORT_ACCESS_METHOD_MAD:
339 return mlx5_query_mad_ifc_max_pkeys(ibdev, max_pkeys);
340
341 case MLX5_VPORT_ACCESS_METHOD_HCA:
342 case MLX5_VPORT_ACCESS_METHOD_NIC:
343 *max_pkeys = mlx5_to_sw_pkey_sz(MLX5_CAP_GEN(mdev,
344 pkey_table_size));
345 return 0;
346
347 default:
348 return -EINVAL;
349 }
350}
351
352static int mlx5_query_vendor_id(struct ib_device *ibdev,
353 u32 *vendor_id)
354{
355 struct mlx5_ib_dev *dev = to_mdev(ibdev);
356
357 switch (mlx5_get_vport_access_method(ibdev)) {
358 case MLX5_VPORT_ACCESS_METHOD_MAD:
359 return mlx5_query_mad_ifc_vendor_id(ibdev, vendor_id);
360
361 case MLX5_VPORT_ACCESS_METHOD_HCA:
362 case MLX5_VPORT_ACCESS_METHOD_NIC:
363 return mlx5_core_query_vendor_id(dev->mdev, vendor_id);
364
365 default:
366 return -EINVAL;
367 }
368}
369
370static int mlx5_query_node_guid(struct mlx5_ib_dev *dev,
371 __be64 *node_guid)
372{
373 u64 tmp;
374 int err;
375
376 switch (mlx5_get_vport_access_method(&dev->ib_dev)) {
377 case MLX5_VPORT_ACCESS_METHOD_MAD:
378 return mlx5_query_mad_ifc_node_guid(dev, node_guid);
379
380 case MLX5_VPORT_ACCESS_METHOD_HCA:
381 err = mlx5_query_hca_vport_node_guid(dev->mdev, &tmp);
3f89a643
AS		 382 break;
383
384 case MLX5_VPORT_ACCESS_METHOD_NIC:
385 err = mlx5_query_nic_vport_node_guid(dev->mdev, &tmp);
386 break;
1b5daf11
MD		 387
388 default:
389 return -EINVAL;
390 }
3f89a643
AS		 391
392 if (!err)
393 *node_guid = cpu_to_be64(tmp);
394
395 return err;
1b5daf11
MD		 396}
397
398struct mlx5_reg_node_desc {
399 u8 desc[64];
400};
401
402static int mlx5_query_node_desc(struct mlx5_ib_dev *dev, char *node_desc)
403{
404 struct mlx5_reg_node_desc in;
405
406 if (mlx5_use_mad_ifc(dev))
407 return mlx5_query_mad_ifc_node_desc(dev, node_desc);
408
409 memset(&in, 0, sizeof(in));
410
411 return mlx5_core_access_reg(dev->mdev, &in, sizeof(in), node_desc,
412 sizeof(struct mlx5_reg_node_desc),
413 MLX5_REG_NODE_DESC, 0, 0);
414}
415
e126ba97 416static int mlx5_ib_query_device(struct ib_device *ibdev,
2528e33e
MB		 417 struct ib_device_attr *props,
418 struct ib_udata *uhw)
e126ba97
EC		 419{
420 struct mlx5_ib_dev *dev = to_mdev(ibdev);
938fe83c 421 struct mlx5_core_dev *mdev = dev->mdev;
e126ba97
EC		 422 int err = -ENOMEM;
423 int max_rq_sg;
424 int max_sq_sg;
e0238a6a 425 u64 min_page_size = 1ull << MLX5_CAP_GEN(mdev, log_pg_sz);
e126ba97 426
2528e33e
MB		 427 if (uhw->inlen || uhw->outlen)
428 return -EINVAL;
429
1b5daf11
MD		 430 memset(props, 0, sizeof(*props));
431 err = mlx5_query_system_image_guid(ibdev,
432 &props->sys_image_guid);
433 if (err)
434 return err;
e126ba97 435
1b5daf11 436 err = mlx5_query_max_pkeys(ibdev, &props->max_pkeys);
e126ba97 437 if (err)
1b5daf11 438 return err;
e126ba97 439
1b5daf11
MD		 440 err = mlx5_query_vendor_id(ibdev, &props->vendor_id);
441 if (err)
442 return err;
e126ba97 443
9603b61d
JM		 444 props->fw_ver = ((u64)fw_rev_maj(dev->mdev) << 32) |
445 (fw_rev_min(dev->mdev) << 16) |
446 fw_rev_sub(dev->mdev);
e126ba97
EC		 447 props->device_cap_flags = IB_DEVICE_CHANGE_PHY_PORT |
448 IB_DEVICE_PORT_ACTIVE_EVENT |
449 IB_DEVICE_SYS_IMAGE_GUID |
1a4c3a3d 450 IB_DEVICE_RC_RNR_NAK_GEN;
938fe83c
SM		 451
452 if (MLX5_CAP_GEN(mdev, pkv))
e126ba97 453 props->device_cap_flags |= IB_DEVICE_BAD_PKEY_CNTR;
938fe83c 454 if (MLX5_CAP_GEN(mdev, qkv))
e126ba97 455 props->device_cap_flags |= IB_DEVICE_BAD_QKEY_CNTR;
938fe83c 456 if (MLX5_CAP_GEN(mdev, apm))
e126ba97 457 props->device_cap_flags |= IB_DEVICE_AUTO_PATH_MIG;
938fe83c 458 if (MLX5_CAP_GEN(mdev, xrc))
e126ba97
EC		 459 props->device_cap_flags |= IB_DEVICE_XRC;
460 props->device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS;
938fe83c 461 if (MLX5_CAP_GEN(mdev, sho)) {
2dea9094
SG		 462 props->device_cap_flags |= IB_DEVICE_SIGNATURE_HANDOVER;
463 /* At this stage no support for signature handover */
464 props->sig_prot_cap = IB_PROT_T10DIF_TYPE_1 |
465 IB_PROT_T10DIF_TYPE_2 |
466 IB_PROT_T10DIF_TYPE_3;
467 props->sig_guard_cap = IB_GUARD_T10DIF_CRC |
468 IB_GUARD_T10DIF_CSUM;
469 }
938fe83c 470 if (MLX5_CAP_GEN(mdev, block_lb_mc))
f360d88a 471 props->device_cap_flags |= IB_DEVICE_BLOCK_MULTICAST_LOOPBACK;
e126ba97 472
1b5daf11
MD		 473 props->vendor_part_id = mdev->pdev->device;
474 props->hw_ver = mdev->pdev->revision;
e126ba97
EC		 475
476 props->max_mr_size = ~0ull;
e0238a6a 477 props->page_size_cap = ~(min_page_size - 1);
938fe83c
SM		 478 props->max_qp = 1 << MLX5_CAP_GEN(mdev, log_max_qp);
479 props->max_qp_wr = 1 << MLX5_CAP_GEN(mdev, log_max_qp_sz);
480 max_rq_sg = MLX5_CAP_GEN(mdev, max_wqe_sz_rq) /
481 sizeof(struct mlx5_wqe_data_seg);
482 max_sq_sg = (MLX5_CAP_GEN(mdev, max_wqe_sz_sq) -
483 sizeof(struct mlx5_wqe_ctrl_seg)) /
484 sizeof(struct mlx5_wqe_data_seg);
e126ba97 485 props->max_sge = min(max_rq_sg, max_sq_sg);
18ebd407 486 props->max_sge_rd = props->max_sge;
938fe83c
SM		 487 props->max_cq = 1 << MLX5_CAP_GEN(mdev, log_max_cq);
488 props->max_cqe = (1 << MLX5_CAP_GEN(mdev, log_max_eq_sz)) - 1;
489 props->max_mr = 1 << MLX5_CAP_GEN(mdev, log_max_mkey);
490 props->max_pd = 1 << MLX5_CAP_GEN(mdev, log_max_pd);
491 props->max_qp_rd_atom = 1 << MLX5_CAP_GEN(mdev, log_max_ra_req_qp);
492 props->max_qp_init_rd_atom = 1 << MLX5_CAP_GEN(mdev, log_max_ra_res_qp);
493 props->max_srq = 1 << MLX5_CAP_GEN(mdev, log_max_srq);
494 props->max_srq_wr = (1 << MLX5_CAP_GEN(mdev, log_max_srq_sz)) - 1;
495 props->local_ca_ack_delay = MLX5_CAP_GEN(mdev, local_ca_ack_delay);
e126ba97 496 props->max_res_rd_atom = props->max_qp_rd_atom * props->max_qp;
e126ba97
EC		 497 props->max_srq_sge = max_rq_sg - 1;
498 props->max_fast_reg_page_list_len = (unsigned int)-1;
81bea28f
EC		 499 props->atomic_cap = IB_ATOMIC_NONE;
500 props->masked_atomic_cap = IB_ATOMIC_NONE;
938fe83c
SM		 501 props->max_mcast_grp = 1 << MLX5_CAP_GEN(mdev, log_max_mcg);
502 props->max_mcast_qp_attach = MLX5_CAP_GEN(mdev, max_qp_mcg);
e126ba97
EC		 503 props->max_total_mcast_qp_attach = props->max_mcast_qp_attach *
504 props->max_mcast_grp;
505 props->max_map_per_fmr = INT_MAX; /* no limit in ConnectIB */
7c60bcbb
MB		 506 props->hca_core_clock = MLX5_CAP_GEN(mdev, device_frequency_khz);
507 props->timestamp_mask = 0x7FFFFFFFFFFFFFFFULL;
e126ba97 508
8cdd312c 509#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
938fe83c 510 if (MLX5_CAP_GEN(mdev, pg))
8cdd312c
HE		 511 props->device_cap_flags |= IB_DEVICE_ON_DEMAND_PAGING;
512 props->odp_caps = dev->odp_caps;
513#endif
514
1b5daf11 515 return 0;
e126ba97
EC		 516}
517
1b5daf11
MD		 518enum mlx5_ib_width {
519 MLX5_IB_WIDTH_1X = 1 << 0,
520 MLX5_IB_WIDTH_2X = 1 << 1,
521 MLX5_IB_WIDTH_4X = 1 << 2,
522 MLX5_IB_WIDTH_8X = 1 << 3,
523 MLX5_IB_WIDTH_12X = 1 << 4
524};
525
526static int translate_active_width(struct ib_device *ibdev, u8 active_width,
527 u8 *ib_width)
e126ba97
EC		 528{
529 struct mlx5_ib_dev *dev = to_mdev(ibdev);
1b5daf11
MD		 530 int err = 0;
531
532 if (active_width & MLX5_IB_WIDTH_1X) {
533 *ib_width = IB_WIDTH_1X;
534 } else if (active_width & MLX5_IB_WIDTH_2X) {
535 mlx5_ib_dbg(dev, "active_width %d is not supported by IB spec\n",
536 (int)active_width);
537 err = -EINVAL;
538 } else if (active_width & MLX5_IB_WIDTH_4X) {
539 *ib_width = IB_WIDTH_4X;
540 } else if (active_width & MLX5_IB_WIDTH_8X) {
541 *ib_width = IB_WIDTH_8X;
542 } else if (active_width & MLX5_IB_WIDTH_12X) {
543 *ib_width = IB_WIDTH_12X;
544 } else {
545 mlx5_ib_dbg(dev, "Invalid active_width %d\n",
546 (int)active_width);
547 err = -EINVAL;
e126ba97
EC		 548 }
549
1b5daf11
MD		 550 return err;
551}
e126ba97 552
1b5daf11
MD		 553static int mlx5_mtu_to_ib_mtu(int mtu)
554{
555 switch (mtu) {
556 case 256: return 1;
557 case 512: return 2;
558 case 1024: return 3;
559 case 2048: return 4;
560 case 4096: return 5;
561 default:
562 pr_warn("invalid mtu\n");
563 return -1;
e126ba97 564 }
1b5daf11 565}
e126ba97 566
1b5daf11
MD		 567enum ib_max_vl_num {
568 __IB_MAX_VL_0 = 1,
569 __IB_MAX_VL_0_1 = 2,
570 __IB_MAX_VL_0_3 = 3,
571 __IB_MAX_VL_0_7 = 4,
572 __IB_MAX_VL_0_14 = 5,
573};
e126ba97 574
1b5daf11
MD		 575enum mlx5_vl_hw_cap {
576 MLX5_VL_HW_0 = 1,
577 MLX5_VL_HW_0_1 = 2,
578 MLX5_VL_HW_0_2 = 3,
579 MLX5_VL_HW_0_3 = 4,
580 MLX5_VL_HW_0_4 = 5,
581 MLX5_VL_HW_0_5 = 6,
582 MLX5_VL_HW_0_6 = 7,
583 MLX5_VL_HW_0_7 = 8,
584 MLX5_VL_HW_0_14 = 15
585};
e126ba97 586
1b5daf11
MD		 587static int translate_max_vl_num(struct ib_device *ibdev, u8 vl_hw_cap,
588 u8 *max_vl_num)
589{
590 switch (vl_hw_cap) {
591 case MLX5_VL_HW_0:
592 *max_vl_num = __IB_MAX_VL_0;
593 break;
594 case MLX5_VL_HW_0_1:
595 *max_vl_num = __IB_MAX_VL_0_1;
596 break;
597 case MLX5_VL_HW_0_3:
598 *max_vl_num = __IB_MAX_VL_0_3;
599 break;
600 case MLX5_VL_HW_0_7:
601 *max_vl_num = __IB_MAX_VL_0_7;
602 break;
603 case MLX5_VL_HW_0_14:
604 *max_vl_num = __IB_MAX_VL_0_14;
605 break;
e126ba97 606
1b5daf11
MD		 607 default:
608 return -EINVAL;
e126ba97 609 }
e126ba97 610
1b5daf11 611 return 0;
e126ba97
EC		 612}
613
1b5daf11
MD		 614static int mlx5_query_hca_port(struct ib_device *ibdev, u8 port,
615 struct ib_port_attr *props)
e126ba97 616{
1b5daf11
MD		 617 struct mlx5_ib_dev *dev = to_mdev(ibdev);
618 struct mlx5_core_dev *mdev = dev->mdev;
619 struct mlx5_hca_vport_context *rep;
620 int max_mtu;
621 int oper_mtu;
622 int err;
623 u8 ib_link_width_oper;
624 u8 vl_hw_cap;
e126ba97 625
1b5daf11
MD		 626 rep = kzalloc(sizeof(*rep), GFP_KERNEL);
627 if (!rep) {
628 err = -ENOMEM;
e126ba97 629 goto out;
e126ba97 630 }
e126ba97 631
1b5daf11 632 memset(props, 0, sizeof(*props));
e126ba97 633
1b5daf11 634 err = mlx5_query_hca_vport_context(mdev, 0, port, 0, rep);
e126ba97
EC		 635 if (err)
636 goto out;
637
1b5daf11
MD		 638 props->lid = rep->lid;
639 props->lmc = rep->lmc;
640 props->sm_lid = rep->sm_lid;
641 props->sm_sl = rep->sm_sl;
642 props->state = rep->vport_state;
643 props->phys_state = rep->port_physical_state;
644 props->port_cap_flags = rep->cap_mask1;
645 props->gid_tbl_len = mlx5_get_gid_table_len(MLX5_CAP_GEN(mdev, gid_table_size));
646 props->max_msg_sz = 1 << MLX5_CAP_GEN(mdev, log_max_msg);
647 props->pkey_tbl_len = mlx5_to_sw_pkey_sz(MLX5_CAP_GEN(mdev, pkey_table_size));
648 props->bad_pkey_cntr = rep->pkey_violation_counter;
649 props->qkey_viol_cntr = rep->qkey_violation_counter;
650 props->subnet_timeout = rep->subnet_timeout;
651 props->init_type_reply = rep->init_type_reply;
e126ba97 652
1b5daf11
MD		 653 err = mlx5_query_port_link_width_oper(mdev, &ib_link_width_oper, port);
654 if (err)
e126ba97 655 goto out;
e126ba97 656
1b5daf11
MD		 657 err = translate_active_width(ibdev, ib_link_width_oper,
658 &props->active_width);
659 if (err)
660 goto out;
661 err = mlx5_query_port_proto_oper(mdev, &props->active_speed, MLX5_PTYS_IB,
662 port);
e126ba97
EC		 663 if (err)
664 goto out;
665
facc9699 666 mlx5_query_port_max_mtu(mdev, &max_mtu, port);
e126ba97 667
1b5daf11 668 props->max_mtu = mlx5_mtu_to_ib_mtu(max_mtu);
e126ba97 669
facc9699 670 mlx5_query_port_oper_mtu(mdev, &oper_mtu, port);
e126ba97 671
1b5daf11 672 props->active_mtu = mlx5_mtu_to_ib_mtu(oper_mtu);
e126ba97 673
1b5daf11
MD		 674 err = mlx5_query_port_vl_hw_cap(mdev, &vl_hw_cap, port);
675 if (err)
676 goto out;
e126ba97 677
1b5daf11
MD		 678 err = translate_max_vl_num(ibdev, vl_hw_cap,
679 &props->max_vl_num);
e126ba97 680out:
1b5daf11 681 kfree(rep);
e126ba97
EC		 682 return err;
683}
684
1b5daf11
MD		 685int mlx5_ib_query_port(struct ib_device *ibdev, u8 port,
686 struct ib_port_attr *props)
e126ba97 687{
1b5daf11
MD		 688 switch (mlx5_get_vport_access_method(ibdev)) {
689 case MLX5_VPORT_ACCESS_METHOD_MAD:
690 return mlx5_query_mad_ifc_port(ibdev, port, props);
e126ba97 691
1b5daf11
MD		 692 case MLX5_VPORT_ACCESS_METHOD_HCA:
693 return mlx5_query_hca_port(ibdev, port, props);
e126ba97 694
3f89a643
AS		 695 case MLX5_VPORT_ACCESS_METHOD_NIC:
696 return mlx5_query_port_roce(ibdev, port, props);
697
1b5daf11
MD		 698 default:
699 return -EINVAL;
700 }
701}
e126ba97 702
1b5daf11
MD		 703static int mlx5_ib_query_gid(struct ib_device *ibdev, u8 port, int index,
704 union ib_gid *gid)
705{
706 struct mlx5_ib_dev *dev = to_mdev(ibdev);
707 struct mlx5_core_dev *mdev = dev->mdev;
e126ba97 708
1b5daf11
MD		 709 switch (mlx5_get_vport_access_method(ibdev)) {
710 case MLX5_VPORT_ACCESS_METHOD_MAD:
711 return mlx5_query_mad_ifc_gids(ibdev, port, index, gid);
e126ba97 712
1b5daf11
MD		 713 case MLX5_VPORT_ACCESS_METHOD_HCA:
714 return mlx5_query_hca_vport_gid(mdev, 0, port, 0, index, gid);
715
716 default:
717 return -EINVAL;
718 }
e126ba97 719
e126ba97
EC		 720}
721
1b5daf11
MD		 722static int mlx5_ib_query_pkey(struct ib_device *ibdev, u8 port, u16 index,
723 u16 *pkey)
724{
725 struct mlx5_ib_dev *dev = to_mdev(ibdev);
726 struct mlx5_core_dev *mdev = dev->mdev;
727
728 switch (mlx5_get_vport_access_method(ibdev)) {
729 case MLX5_VPORT_ACCESS_METHOD_MAD:
730 return mlx5_query_mad_ifc_pkey(ibdev, port, index, pkey);
731
732 case MLX5_VPORT_ACCESS_METHOD_HCA:
733 case MLX5_VPORT_ACCESS_METHOD_NIC:
734 return mlx5_query_hca_vport_pkey(mdev, 0, port, 0, index,
735 pkey);
736 default:
737 return -EINVAL;
738 }
739}
e126ba97
EC		 740
741static int mlx5_ib_modify_device(struct ib_device *ibdev, int mask,
742 struct ib_device_modify *props)
743{
744 struct mlx5_ib_dev *dev = to_mdev(ibdev);
745 struct mlx5_reg_node_desc in;
746 struct mlx5_reg_node_desc out;
747 int err;
748
749 if (mask & ~IB_DEVICE_MODIFY_NODE_DESC)
750 return -EOPNOTSUPP;
751
752 if (!(mask & IB_DEVICE_MODIFY_NODE_DESC))
753 return 0;
754
755 /*
756 * If possible, pass node desc to FW, so it can generate
757 * a 144 trap. If cmd fails, just ignore.
758 */
759 memcpy(&in, props->node_desc, 64);
9603b61d 760 err = mlx5_core_access_reg(dev->mdev, &in, sizeof(in), &out,
e126ba97
EC		 761 sizeof(out), MLX5_REG_NODE_DESC, 0, 1);
762 if (err)
763 return err;
764
765 memcpy(ibdev->node_desc, props->node_desc, 64);
766
767 return err;
768}
769
770static int mlx5_ib_modify_port(struct ib_device *ibdev, u8 port, int mask,
771 struct ib_port_modify *props)
772{
773 struct mlx5_ib_dev *dev = to_mdev(ibdev);
774 struct ib_port_attr attr;
775 u32 tmp;
776 int err;
777
778 mutex_lock(&dev->cap_mask_mutex);
779
780 err = mlx5_ib_query_port(ibdev, port, &attr);
781 if (err)
782 goto out;
783
784 tmp = (attr.port_cap_flags | props->set_port_cap_mask) &
785 ~props->clr_port_cap_mask;
786
9603b61d 787 err = mlx5_set_port_caps(dev->mdev, port, tmp);
e126ba97
EC		 788
789out:
790 mutex_unlock(&dev->cap_mask_mutex);
791 return err;
792}
793
794static struct ib_ucontext *mlx5_ib_alloc_ucontext(struct ib_device *ibdev,
795 struct ib_udata *udata)
796{
797 struct mlx5_ib_dev *dev = to_mdev(ibdev);
b368d7cb
MB		 798 struct mlx5_ib_alloc_ucontext_req_v2 req = {};
799 struct mlx5_ib_alloc_ucontext_resp resp = {};
e126ba97
EC		 800 struct mlx5_ib_ucontext *context;
801 struct mlx5_uuar_info *uuari;
802 struct mlx5_uar *uars;
c1be5232 803 int gross_uuars;
e126ba97 804 int num_uars;
78c0f98c 805 int ver;
e126ba97
EC		 806 int uuarn;
807 int err;
808 int i;
f241e749 809 size_t reqlen;
e126ba97
EC		 810
811 if (!dev->ib_active)
812 return ERR_PTR(-EAGAIN);
813
78c0f98c
EC		 814 reqlen = udata->inlen - sizeof(struct ib_uverbs_cmd_hdr);
815 if (reqlen == sizeof(struct mlx5_ib_alloc_ucontext_req))
816 ver = 0;
b368d7cb 817 else if (reqlen >= sizeof(struct mlx5_ib_alloc_ucontext_req_v2))
78c0f98c
EC		 818 ver = 2;
819 else
820 return ERR_PTR(-EINVAL);
821
b368d7cb 822 err = ib_copy_from_udata(&req, udata, min(reqlen, sizeof(req)));
e126ba97
EC		 823 if (err)
824 return ERR_PTR(err);
825
b368d7cb 826 if (req.flags)
78c0f98c
EC		 827 return ERR_PTR(-EINVAL);
828
e126ba97
EC		 829 if (req.total_num_uuars > MLX5_MAX_UUARS)
830 return ERR_PTR(-ENOMEM);
831
832 if (req.total_num_uuars == 0)
833 return ERR_PTR(-EINVAL);
834
b368d7cb
MB		 835 if (req.comp_mask)
836 return ERR_PTR(-EOPNOTSUPP);
837
838 if (reqlen > sizeof(req) &&
839 !ib_is_udata_cleared(udata, sizeof(req),
840 udata->inlen - sizeof(req)))
841 return ERR_PTR(-EOPNOTSUPP);
842
c1be5232
EC		 843 req.total_num_uuars = ALIGN(req.total_num_uuars,
844 MLX5_NON_FP_BF_REGS_PER_PAGE);
e126ba97
EC		 845 if (req.num_low_latency_uuars > req.total_num_uuars - 1)
846 return ERR_PTR(-EINVAL);
847
c1be5232
EC		 848 num_uars = req.total_num_uuars / MLX5_NON_FP_BF_REGS_PER_PAGE;
849 gross_uuars = num_uars * MLX5_BF_REGS_PER_PAGE;
938fe83c
SM		 850 resp.qp_tab_size = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp);
851 resp.bf_reg_size = 1 << MLX5_CAP_GEN(dev->mdev, log_bf_reg_size);
852 resp.cache_line_size = L1_CACHE_BYTES;
853 resp.max_sq_desc_sz = MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq);
854 resp.max_rq_desc_sz = MLX5_CAP_GEN(dev->mdev, max_wqe_sz_rq);
855 resp.max_send_wqebb = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz);
856 resp.max_recv_wr = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz);
857 resp.max_srq_recv_wr = 1 << MLX5_CAP_GEN(dev->mdev, log_max_srq_sz);
b368d7cb
MB		 858 resp.response_length = min(offsetof(typeof(resp), response_length) +
859 sizeof(resp.response_length), udata->outlen);
e126ba97
EC		 860
861 context = kzalloc(sizeof(*context), GFP_KERNEL);
862 if (!context)
863 return ERR_PTR(-ENOMEM);
864
865 uuari = &context->uuari;
866 mutex_init(&uuari->lock);
867 uars = kcalloc(num_uars, sizeof(*uars), GFP_KERNEL);
868 if (!uars) {
869 err = -ENOMEM;
870 goto out_ctx;
871 }
872
c1be5232 873 uuari->bitmap = kcalloc(BITS_TO_LONGS(gross_uuars),
e126ba97
EC		 874 sizeof(*uuari->bitmap),
875 GFP_KERNEL);
876 if (!uuari->bitmap) {
877 err = -ENOMEM;
878 goto out_uar_ctx;
879 }
880 /*
881 * clear all fast path uuars
882 */
c1be5232 883 for (i = 0; i < gross_uuars; i++) {
e126ba97
EC		 884 uuarn = i & 3;
885 if (uuarn == 2 || uuarn == 3)
886 set_bit(i, uuari->bitmap);
887 }
888
c1be5232 889 uuari->count = kcalloc(gross_uuars, sizeof(*uuari->count), GFP_KERNEL);
e126ba97
EC		 890 if (!uuari->count) {
891 err = -ENOMEM;
892 goto out_bitmap;
893 }
894
895 for (i = 0; i < num_uars; i++) {
9603b61d 896 err = mlx5_cmd_alloc_uar(dev->mdev, &uars[i].index);
e126ba97
EC		 897 if (err)
898 goto out_count;
899 }
900
b4cfe447
HE		 901#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
902 context->ibucontext.invalidate_range = &mlx5_ib_invalidate_range;
903#endif
904
e126ba97
EC		 905 INIT_LIST_HEAD(&context->db_page_list);
906 mutex_init(&context->db_page_mutex);
907
908 resp.tot_uuars = req.total_num_uuars;
938fe83c 909 resp.num_ports = MLX5_CAP_GEN(dev->mdev, num_ports);
b368d7cb
MB		 910
911 if (field_avail(typeof(resp), reserved2, udata->outlen))
912 resp.response_length += sizeof(resp.reserved2);
913
914 if (field_avail(typeof(resp), hca_core_clock_offset, udata->outlen)) {
915 resp.comp_mask |=
916 MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_CORE_CLOCK_OFFSET;
917 resp.hca_core_clock_offset =
918 offsetof(struct mlx5_init_seg, internal_timer_h) %
919 PAGE_SIZE;
920 resp.response_length += sizeof(resp.hca_core_clock_offset);
921 }
922
923 err = ib_copy_to_udata(udata, &resp, resp.response_length);
e126ba97
EC		 924 if (err)
925 goto out_uars;
926
78c0f98c 927 uuari->ver = ver;
e126ba97
EC		 928 uuari->num_low_latency_uuars = req.num_low_latency_uuars;
929 uuari->uars = uars;
930 uuari->num_uars = num_uars;
931 return &context->ibucontext;
932
933out_uars:
934 for (i--; i >= 0; i--)
9603b61d 935 mlx5_cmd_free_uar(dev->mdev, uars[i].index);
e126ba97
EC		 936out_count:
937 kfree(uuari->count);
938
939out_bitmap:
940 kfree(uuari->bitmap);
941
942out_uar_ctx:
943 kfree(uars);
944
945out_ctx:
946 kfree(context);
947 return ERR_PTR(err);
948}
949
950static int mlx5_ib_dealloc_ucontext(struct ib_ucontext *ibcontext)
951{
952 struct mlx5_ib_ucontext *context = to_mucontext(ibcontext);
953 struct mlx5_ib_dev *dev = to_mdev(ibcontext->device);
954 struct mlx5_uuar_info *uuari = &context->uuari;
955 int i;
956
957 for (i = 0; i < uuari->num_uars; i++) {
9603b61d 958 if (mlx5_cmd_free_uar(dev->mdev, uuari->uars[i].index))
e126ba97
EC		 959 mlx5_ib_warn(dev, "failed to free UAR 0x%x\n", uuari->uars[i].index);
960 }
961
962 kfree(uuari->count);
963 kfree(uuari->bitmap);
964 kfree(uuari->uars);
965 kfree(context);
966
967 return 0;
968}
969
970static phys_addr_t uar_index2pfn(struct mlx5_ib_dev *dev, int index)
971{
9603b61d 972 return (pci_resource_start(dev->mdev->pdev, 0) >> PAGE_SHIFT) + index;
e126ba97
EC		 973}
974
975static int get_command(unsigned long offset)
976{
977 return (offset >> MLX5_IB_MMAP_CMD_SHIFT) & MLX5_IB_MMAP_CMD_MASK;
978}
979
980static int get_arg(unsigned long offset)
981{
982 return offset & ((1 << MLX5_IB_MMAP_CMD_SHIFT) - 1);
983}
984
985static int get_index(unsigned long offset)
986{
987 return get_arg(offset);
988}
989
990static int mlx5_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma)
991{
992 struct mlx5_ib_ucontext *context = to_mucontext(ibcontext);
993 struct mlx5_ib_dev *dev = to_mdev(ibcontext->device);
994 struct mlx5_uuar_info *uuari = &context->uuari;
995 unsigned long command;
996 unsigned long idx;
997 phys_addr_t pfn;
998
999 command = get_command(vma->vm_pgoff);
1000 switch (command) {
1001 case MLX5_IB_MMAP_REGULAR_PAGE:
1002 if (vma->vm_end - vma->vm_start != PAGE_SIZE)
1003 return -EINVAL;
1004
1005 idx = get_index(vma->vm_pgoff);
1c3ce90d
EC		 1006 if (idx >= uuari->num_uars)
1007 return -EINVAL;
1008
e126ba97
EC		 1009 pfn = uar_index2pfn(dev, uuari->uars[idx].index);
1010 mlx5_ib_dbg(dev, "uar idx 0x%lx, pfn 0x%llx\n", idx,
1011 (unsigned long long)pfn);
1012
e126ba97
EC		 1013 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
1014 if (io_remap_pfn_range(vma, vma->vm_start, pfn,
1015 PAGE_SIZE, vma->vm_page_prot))
1016 return -EAGAIN;
1017
1018 mlx5_ib_dbg(dev, "mapped WC at 0x%lx, PA 0x%llx\n",
1019 vma->vm_start,
1020 (unsigned long long)pfn << PAGE_SHIFT);
1021 break;
1022
1023 case MLX5_IB_MMAP_GET_CONTIGUOUS_PAGES:
1024 return -ENOSYS;
1025
1026 default:
1027 return -EINVAL;
1028 }
1029
1030 return 0;
1031}
1032
e126ba97
EC		 1033static struct ib_pd *mlx5_ib_alloc_pd(struct ib_device *ibdev,
1034 struct ib_ucontext *context,
1035 struct ib_udata *udata)
1036{
1037 struct mlx5_ib_alloc_pd_resp resp;
1038 struct mlx5_ib_pd *pd;
1039 int err;
1040
1041 pd = kmalloc(sizeof(*pd), GFP_KERNEL);
1042 if (!pd)
1043 return ERR_PTR(-ENOMEM);
1044
9603b61d 1045 err = mlx5_core_alloc_pd(to_mdev(ibdev)->mdev, &pd->pdn);
e126ba97
EC		 1046 if (err) {
1047 kfree(pd);
1048 return ERR_PTR(err);
1049 }
1050
1051 if (context) {
1052 resp.pdn = pd->pdn;
1053 if (ib_copy_to_udata(udata, &resp, sizeof(resp))) {
9603b61d 1054 mlx5_core_dealloc_pd(to_mdev(ibdev)->mdev, pd->pdn);
e126ba97
EC		 1055 kfree(pd);
1056 return ERR_PTR(-EFAULT);
1057 }
e126ba97
EC		 1058 }
1059
1060 return &pd->ibpd;
1061}
1062
1063static int mlx5_ib_dealloc_pd(struct ib_pd *pd)
1064{
1065 struct mlx5_ib_dev *mdev = to_mdev(pd->device);
1066 struct mlx5_ib_pd *mpd = to_mpd(pd);
1067
9603b61d 1068 mlx5_core_dealloc_pd(mdev->mdev, mpd->pdn);
e126ba97
EC		 1069 kfree(mpd);
1070
1071 return 0;
1072}
1073
1074static int mlx5_ib_mcg_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
1075{
1076 struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
1077 int err;
1078
9603b61d 1079 err = mlx5_core_attach_mcg(dev->mdev, gid, ibqp->qp_num);
e126ba97
EC		 1080 if (err)
1081 mlx5_ib_warn(dev, "failed attaching QPN 0x%x, MGID %pI6\n",
1082 ibqp->qp_num, gid->raw);
1083
1084 return err;
1085}
1086
1087static int mlx5_ib_mcg_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
1088{
1089 struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
1090 int err;
1091
9603b61d 1092 err = mlx5_core_detach_mcg(dev->mdev, gid, ibqp->qp_num);
e126ba97
EC		 1093 if (err)
1094 mlx5_ib_warn(dev, "failed detaching QPN 0x%x, MGID %pI6\n",
1095 ibqp->qp_num, gid->raw);
1096
1097 return err;
1098}
1099
1100static int init_node_data(struct mlx5_ib_dev *dev)
1101{
1b5daf11 1102 int err;
e126ba97 1103
1b5daf11 1104 err = mlx5_query_node_desc(dev, dev->ib_dev.node_desc);
e126ba97 1105 if (err)
1b5daf11 1106 return err;
e126ba97 1107
1b5daf11 1108 dev->mdev->rev_id = dev->mdev->pdev->revision;
e126ba97 1109
1b5daf11 1110 return mlx5_query_node_guid(dev, &dev->ib_dev.node_guid);
e126ba97
EC		 1111}
1112
1113static ssize_t show_fw_pages(struct device *device, struct device_attribute *attr,
1114 char *buf)
1115{
1116 struct mlx5_ib_dev *dev =
1117 container_of(device, struct mlx5_ib_dev, ib_dev.dev);
1118
9603b61d 1119 return sprintf(buf, "%d\n", dev->mdev->priv.fw_pages);
e126ba97
EC		 1120}
1121
1122static ssize_t show_reg_pages(struct device *device,
1123 struct device_attribute *attr, char *buf)
1124{
1125 struct mlx5_ib_dev *dev =
1126 container_of(device, struct mlx5_ib_dev, ib_dev.dev);
1127
6aec21f6 1128 return sprintf(buf, "%d\n", atomic_read(&dev->mdev->priv.reg_pages));
e126ba97
EC		 1129}
1130
1131static ssize_t show_hca(struct device *device, struct device_attribute *attr,
1132 char *buf)
1133{
1134 struct mlx5_ib_dev *dev =
1135 container_of(device, struct mlx5_ib_dev, ib_dev.dev);
9603b61d 1136 return sprintf(buf, "MT%d\n", dev->mdev->pdev->device);
e126ba97
EC		 1137}
1138
1139static ssize_t show_fw_ver(struct device *device, struct device_attribute *attr,
1140 char *buf)
1141{
1142 struct mlx5_ib_dev *dev =
1143 container_of(device, struct mlx5_ib_dev, ib_dev.dev);
9603b61d
JM		 1144 return sprintf(buf, "%d.%d.%d\n", fw_rev_maj(dev->mdev),
1145 fw_rev_min(dev->mdev), fw_rev_sub(dev->mdev));
e126ba97
EC		 1146}
1147
1148static ssize_t show_rev(struct device *device, struct device_attribute *attr,
1149 char *buf)
1150{
1151 struct mlx5_ib_dev *dev =
1152 container_of(device, struct mlx5_ib_dev, ib_dev.dev);
9603b61d 1153 return sprintf(buf, "%x\n", dev->mdev->rev_id);
e126ba97
EC		 1154}
1155
1156static ssize_t show_board(struct device *device, struct device_attribute *attr,
1157 char *buf)
1158{
1159 struct mlx5_ib_dev *dev =
1160 container_of(device, struct mlx5_ib_dev, ib_dev.dev);
1161 return sprintf(buf, "%.*s\n", MLX5_BOARD_ID_LEN,
9603b61d 1162 dev->mdev->board_id);
e126ba97
EC		 1163}
1164
1165static DEVICE_ATTR(hw_rev, S_IRUGO, show_rev, NULL);
1166static DEVICE_ATTR(fw_ver, S_IRUGO, show_fw_ver, NULL);
1167static DEVICE_ATTR(hca_type, S_IRUGO, show_hca, NULL);
1168static DEVICE_ATTR(board_id, S_IRUGO, show_board, NULL);
1169static DEVICE_ATTR(fw_pages, S_IRUGO, show_fw_pages, NULL);
1170static DEVICE_ATTR(reg_pages, S_IRUGO, show_reg_pages, NULL);
1171
1172static struct device_attribute *mlx5_class_attributes[] = {
1173 &dev_attr_hw_rev,
1174 &dev_attr_fw_ver,
1175 &dev_attr_hca_type,
1176 &dev_attr_board_id,
1177 &dev_attr_fw_pages,
1178 &dev_attr_reg_pages,
1179};
1180
9603b61d 1181static void mlx5_ib_event(struct mlx5_core_dev *dev, void *context,
4d2f9bbb 1182 enum mlx5_dev_event event, unsigned long param)
e126ba97 1183{
9603b61d 1184 struct mlx5_ib_dev *ibdev = (struct mlx5_ib_dev *)context;
e126ba97 1185 struct ib_event ibev;
9603b61d 1186
e126ba97
EC		 1187 u8 port = 0;
1188
1189 switch (event) {
1190 case MLX5_DEV_EVENT_SYS_ERROR:
1191 ibdev->ib_active = false;
1192 ibev.event = IB_EVENT_DEVICE_FATAL;
1193 break;
1194
1195 case MLX5_DEV_EVENT_PORT_UP:
1196 ibev.event = IB_EVENT_PORT_ACTIVE;
4d2f9bbb 1197 port = (u8)param;
e126ba97
EC		 1198 break;
1199
1200 case MLX5_DEV_EVENT_PORT_DOWN:
1201 ibev.event = IB_EVENT_PORT_ERR;
4d2f9bbb 1202 port = (u8)param;
e126ba97
EC		 1203 break;
1204
1205 case MLX5_DEV_EVENT_PORT_INITIALIZED:
1206 /* not used by ULPs */
1207 return;
1208
1209 case MLX5_DEV_EVENT_LID_CHANGE:
1210 ibev.event = IB_EVENT_LID_CHANGE;
4d2f9bbb 1211 port = (u8)param;
e126ba97
EC		 1212 break;
1213
1214 case MLX5_DEV_EVENT_PKEY_CHANGE:
1215 ibev.event = IB_EVENT_PKEY_CHANGE;
4d2f9bbb 1216 port = (u8)param;
e126ba97
EC		 1217 break;
1218
1219 case MLX5_DEV_EVENT_GUID_CHANGE:
1220 ibev.event = IB_EVENT_GID_CHANGE;
4d2f9bbb 1221 port = (u8)param;
e126ba97
EC		 1222 break;
1223
1224 case MLX5_DEV_EVENT_CLIENT_REREG:
1225 ibev.event = IB_EVENT_CLIENT_REREGISTER;
4d2f9bbb 1226 port = (u8)param;
e126ba97
EC		 1227 break;
1228 }
1229
1230 ibev.device = &ibdev->ib_dev;
1231 ibev.element.port_num = port;
1232
a0c84c32
EC		 1233 if (port < 1 || port > ibdev->num_ports) {
1234 mlx5_ib_warn(ibdev, "warning: event on port %d\n", port);
1235 return;
1236 }
1237
e126ba97
EC		 1238 if (ibdev->ib_active)
1239 ib_dispatch_event(&ibev);
1240}
1241
1242static void get_ext_port_caps(struct mlx5_ib_dev *dev)
1243{
1244 int port;
1245
938fe83c 1246 for (port = 1; port <= MLX5_CAP_GEN(dev->mdev, num_ports); port++)
e126ba97
EC		 1247 mlx5_query_ext_port_caps(dev, port);
1248}
1249
1250static int get_port_caps(struct mlx5_ib_dev *dev)
1251{
1252 struct ib_device_attr *dprops = NULL;
1253 struct ib_port_attr *pprops = NULL;
f614fc15 1254 int err = -ENOMEM;
e126ba97 1255 int port;
2528e33e 1256 struct ib_udata uhw = {.inlen = 0, .outlen = 0};
e126ba97
EC		 1257
1258 pprops = kmalloc(sizeof(*pprops), GFP_KERNEL);
1259 if (!pprops)
1260 goto out;
1261
1262 dprops = kmalloc(sizeof(*dprops), GFP_KERNEL);
1263 if (!dprops)
1264 goto out;
1265
2528e33e 1266 err = mlx5_ib_query_device(&dev->ib_dev, dprops, &uhw);
e126ba97
EC		 1267 if (err) {
1268 mlx5_ib_warn(dev, "query_device failed %d\n", err);
1269 goto out;
1270 }
1271
938fe83c 1272 for (port = 1; port <= MLX5_CAP_GEN(dev->mdev, num_ports); port++) {
e126ba97
EC		 1273 err = mlx5_ib_query_port(&dev->ib_dev, port, pprops);
1274 if (err) {
938fe83c
SM		 1275 mlx5_ib_warn(dev, "query_port %d failed %d\n",
1276 port, err);
e126ba97
EC		 1277 break;
1278 }
938fe83c
SM		 1279 dev->mdev->port_caps[port - 1].pkey_table_len =
1280 dprops->max_pkeys;
1281 dev->mdev->port_caps[port - 1].gid_table_len =
1282 pprops->gid_tbl_len;
e126ba97
EC		 1283 mlx5_ib_dbg(dev, "pkey_table_len %d, gid_table_len %d\n",
1284 dprops->max_pkeys, pprops->gid_tbl_len);
1285 }
1286
1287out:
1288 kfree(pprops);
1289 kfree(dprops);
1290
1291 return err;
1292}
1293
1294static void destroy_umrc_res(struct mlx5_ib_dev *dev)
1295{
1296 int err;
1297
1298 err = mlx5_mr_cache_cleanup(dev);
1299 if (err)
1300 mlx5_ib_warn(dev, "mr cache cleanup failed\n");
1301
1302 mlx5_ib_destroy_qp(dev->umrc.qp);
1303 ib_destroy_cq(dev->umrc.cq);
e126ba97
EC		 1304 ib_dealloc_pd(dev->umrc.pd);
1305}
1306
1307enum {
1308 MAX_UMR_WR = 128,
1309};
1310
1311static int create_umr_res(struct mlx5_ib_dev *dev)
1312{
1313 struct ib_qp_init_attr *init_attr = NULL;
1314 struct ib_qp_attr *attr = NULL;
1315 struct ib_pd *pd;
1316 struct ib_cq *cq;
1317 struct ib_qp *qp;
8e37210b 1318 struct ib_cq_init_attr cq_attr = {};
e126ba97
EC		 1319 int ret;
1320
1321 attr = kzalloc(sizeof(*attr), GFP_KERNEL);
1322 init_attr = kzalloc(sizeof(*init_attr), GFP_KERNEL);
1323 if (!attr || !init_attr) {
1324 ret = -ENOMEM;
1325 goto error_0;
1326 }
1327
1328 pd = ib_alloc_pd(&dev->ib_dev);
1329 if (IS_ERR(pd)) {
1330 mlx5_ib_dbg(dev, "Couldn't create PD for sync UMR QP\n");
1331 ret = PTR_ERR(pd);
1332 goto error_0;
1333 }
1334
8e37210b
MB		 1335 cq_attr.cqe = 128;
1336 cq = ib_create_cq(&dev->ib_dev, mlx5_umr_cq_handler, NULL, NULL,
1337 &cq_attr);
e126ba97
EC		 1338 if (IS_ERR(cq)) {
1339 mlx5_ib_dbg(dev, "Couldn't create CQ for sync UMR QP\n");
1340 ret = PTR_ERR(cq);
1341 goto error_2;
1342 }
1343 ib_req_notify_cq(cq, IB_CQ_NEXT_COMP);
1344
1345 init_attr->send_cq = cq;
1346 init_attr->recv_cq = cq;
1347 init_attr->sq_sig_type = IB_SIGNAL_ALL_WR;
1348 init_attr->cap.max_send_wr = MAX_UMR_WR;
1349 init_attr->cap.max_send_sge = 1;
1350 init_attr->qp_type = MLX5_IB_QPT_REG_UMR;
1351 init_attr->port_num = 1;
1352 qp = mlx5_ib_create_qp(pd, init_attr, NULL);
1353 if (IS_ERR(qp)) {
1354 mlx5_ib_dbg(dev, "Couldn't create sync UMR QP\n");
1355 ret = PTR_ERR(qp);
1356 goto error_3;
1357 }
1358 qp->device = &dev->ib_dev;
1359 qp->real_qp = qp;
1360 qp->uobject = NULL;
1361 qp->qp_type = MLX5_IB_QPT_REG_UMR;
1362
1363 attr->qp_state = IB_QPS_INIT;
1364 attr->port_num = 1;
1365 ret = mlx5_ib_modify_qp(qp, attr, IB_QP_STATE | IB_QP_PKEY_INDEX |
1366 IB_QP_PORT, NULL);
1367 if (ret) {
1368 mlx5_ib_dbg(dev, "Couldn't modify UMR QP\n");
1369 goto error_4;
1370 }
1371
1372 memset(attr, 0, sizeof(*attr));
1373 attr->qp_state = IB_QPS_RTR;
1374 attr->path_mtu = IB_MTU_256;
1375
1376 ret = mlx5_ib_modify_qp(qp, attr, IB_QP_STATE, NULL);
1377 if (ret) {
1378 mlx5_ib_dbg(dev, "Couldn't modify umr QP to rtr\n");
1379 goto error_4;
1380 }
1381
1382 memset(attr, 0, sizeof(*attr));
1383 attr->qp_state = IB_QPS_RTS;
1384 ret = mlx5_ib_modify_qp(qp, attr, IB_QP_STATE, NULL);
1385 if (ret) {
1386 mlx5_ib_dbg(dev, "Couldn't modify umr QP to rts\n");
1387 goto error_4;
1388 }
1389
1390 dev->umrc.qp = qp;
1391 dev->umrc.cq = cq;
e126ba97
EC		 1392 dev->umrc.pd = pd;
1393
1394 sema_init(&dev->umrc.sem, MAX_UMR_WR);
1395 ret = mlx5_mr_cache_init(dev);
1396 if (ret) {
1397 mlx5_ib_warn(dev, "mr cache init failed %d\n", ret);
1398 goto error_4;
1399 }
1400
1401 kfree(attr);
1402 kfree(init_attr);
1403
1404 return 0;
1405
1406error_4:
1407 mlx5_ib_destroy_qp(qp);
1408
1409error_3:
1410 ib_destroy_cq(cq);
1411
1412error_2:
e126ba97
EC		 1413 ib_dealloc_pd(pd);
1414
1415error_0:
1416 kfree(attr);
1417 kfree(init_attr);
1418 return ret;
1419}
1420
1421static int create_dev_resources(struct mlx5_ib_resources *devr)
1422{
1423 struct ib_srq_init_attr attr;
1424 struct mlx5_ib_dev *dev;
bcf4c1ea 1425 struct ib_cq_init_attr cq_attr = {.cqe = 1};
e126ba97
EC		 1426 int ret = 0;
1427
1428 dev = container_of(devr, struct mlx5_ib_dev, devr);
1429
1430 devr->p0 = mlx5_ib_alloc_pd(&dev->ib_dev, NULL, NULL);
1431 if (IS_ERR(devr->p0)) {
1432 ret = PTR_ERR(devr->p0);
1433 goto error0;
1434 }
1435 devr->p0->device = &dev->ib_dev;
1436 devr->p0->uobject = NULL;
1437 atomic_set(&devr->p0->usecnt, 0);
1438
bcf4c1ea 1439 devr->c0 = mlx5_ib_create_cq(&dev->ib_dev, &cq_attr, NULL, NULL);
e126ba97
EC		 1440 if (IS_ERR(devr->c0)) {
1441 ret = PTR_ERR(devr->c0);
1442 goto error1;
1443 }
1444 devr->c0->device = &dev->ib_dev;
1445 devr->c0->uobject = NULL;
1446 devr->c0->comp_handler = NULL;
1447 devr->c0->event_handler = NULL;
1448 devr->c0->cq_context = NULL;
1449 atomic_set(&devr->c0->usecnt, 0);
1450
1451 devr->x0 = mlx5_ib_alloc_xrcd(&dev->ib_dev, NULL, NULL);
1452 if (IS_ERR(devr->x0)) {
1453 ret = PTR_ERR(devr->x0);
1454 goto error2;
1455 }
1456 devr->x0->device = &dev->ib_dev;
1457 devr->x0->inode = NULL;
1458 atomic_set(&devr->x0->usecnt, 0);
1459 mutex_init(&devr->x0->tgt_qp_mutex);
1460 INIT_LIST_HEAD(&devr->x0->tgt_qp_list);
1461
1462 devr->x1 = mlx5_ib_alloc_xrcd(&dev->ib_dev, NULL, NULL);
1463 if (IS_ERR(devr->x1)) {
1464 ret = PTR_ERR(devr->x1);
1465 goto error3;
1466 }
1467 devr->x1->device = &dev->ib_dev;
1468 devr->x1->inode = NULL;
1469 atomic_set(&devr->x1->usecnt, 0);
1470 mutex_init(&devr->x1->tgt_qp_mutex);
1471 INIT_LIST_HEAD(&devr->x1->tgt_qp_list);
1472
1473 memset(&attr, 0, sizeof(attr));
1474 attr.attr.max_sge = 1;
1475 attr.attr.max_wr = 1;
1476 attr.srq_type = IB_SRQT_XRC;
1477 attr.ext.xrc.cq = devr->c0;
1478 attr.ext.xrc.xrcd = devr->x0;
1479
1480 devr->s0 = mlx5_ib_create_srq(devr->p0, &attr, NULL);
1481 if (IS_ERR(devr->s0)) {
1482 ret = PTR_ERR(devr->s0);
1483 goto error4;
1484 }
1485 devr->s0->device = &dev->ib_dev;
1486 devr->s0->pd = devr->p0;
1487 devr->s0->uobject = NULL;
1488 devr->s0->event_handler = NULL;
1489 devr->s0->srq_context = NULL;
1490 devr->s0->srq_type = IB_SRQT_XRC;
1491 devr->s0->ext.xrc.xrcd = devr->x0;
1492 devr->s0->ext.xrc.cq = devr->c0;
1493 atomic_inc(&devr->s0->ext.xrc.xrcd->usecnt);
1494 atomic_inc(&devr->s0->ext.xrc.cq->usecnt);
1495 atomic_inc(&devr->p0->usecnt);
1496 atomic_set(&devr->s0->usecnt, 0);
1497
4aa17b28
HA		 1498 memset(&attr, 0, sizeof(attr));
1499 attr.attr.max_sge = 1;
1500 attr.attr.max_wr = 1;
1501 attr.srq_type = IB_SRQT_BASIC;
1502 devr->s1 = mlx5_ib_create_srq(devr->p0, &attr, NULL);
1503 if (IS_ERR(devr->s1)) {
1504 ret = PTR_ERR(devr->s1);
1505 goto error5;
1506 }
1507 devr->s1->device = &dev->ib_dev;
1508 devr->s1->pd = devr->p0;
1509 devr->s1->uobject = NULL;
1510 devr->s1->event_handler = NULL;
1511 devr->s1->srq_context = NULL;
1512 devr->s1->srq_type = IB_SRQT_BASIC;
1513 devr->s1->ext.xrc.cq = devr->c0;
1514 atomic_inc(&devr->p0->usecnt);
1515 atomic_set(&devr->s0->usecnt, 0);
1516
e126ba97
EC		 1517 return 0;
1518
4aa17b28
HA		 1519error5:
1520 mlx5_ib_destroy_srq(devr->s0);
e126ba97
EC		 1521error4:
1522 mlx5_ib_dealloc_xrcd(devr->x1);
1523error3:
1524 mlx5_ib_dealloc_xrcd(devr->x0);
1525error2:
1526 mlx5_ib_destroy_cq(devr->c0);
1527error1:
1528 mlx5_ib_dealloc_pd(devr->p0);
1529error0:
1530 return ret;
1531}
1532
1533static void destroy_dev_resources(struct mlx5_ib_resources *devr)
1534{
4aa17b28 1535 mlx5_ib_destroy_srq(devr->s1);
e126ba97
EC		 1536 mlx5_ib_destroy_srq(devr->s0);
1537 mlx5_ib_dealloc_xrcd(devr->x0);
1538 mlx5_ib_dealloc_xrcd(devr->x1);
1539 mlx5_ib_destroy_cq(devr->c0);
1540 mlx5_ib_dealloc_pd(devr->p0);
1541}
1542
e53505a8
AS		 1543static u32 get_core_cap_flags(struct ib_device *ibdev)
1544{
1545 struct mlx5_ib_dev *dev = to_mdev(ibdev);
1546 enum rdma_link_layer ll = mlx5_ib_port_link_layer(ibdev, 1);
1547 u8 l3_type_cap = MLX5_CAP_ROCE(dev->mdev, l3_type);
1548 u8 roce_version_cap = MLX5_CAP_ROCE(dev->mdev, roce_version);
1549 u32 ret = 0;
1550
1551 if (ll == IB_LINK_LAYER_INFINIBAND)
1552 return RDMA_CORE_PORT_IBA_IB;
1553
1554 if (!(l3_type_cap & MLX5_ROCE_L3_TYPE_IPV4_CAP))
1555 return 0;
1556
1557 if (!(l3_type_cap & MLX5_ROCE_L3_TYPE_IPV6_CAP))
1558 return 0;
1559
1560 if (roce_version_cap & MLX5_ROCE_VERSION_1_CAP)
1561 ret |= RDMA_CORE_PORT_IBA_ROCE;
1562
1563 if (roce_version_cap & MLX5_ROCE_VERSION_2_CAP)
1564 ret |= RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
1565
1566 return ret;
1567}
1568
7738613e
IW		 1569static int mlx5_port_immutable(struct ib_device *ibdev, u8 port_num,
1570 struct ib_port_immutable *immutable)
1571{
1572 struct ib_port_attr attr;
1573 int err;
1574
1575 err = mlx5_ib_query_port(ibdev, port_num, &attr);
1576 if (err)
1577 return err;
1578
1579 immutable->pkey_tbl_len = attr.pkey_tbl_len;
1580 immutable->gid_tbl_len = attr.gid_tbl_len;
e53505a8 1581 immutable->core_cap_flags = get_core_cap_flags(ibdev);
337877a4 1582 immutable->max_mad_size = IB_MGMT_MAD_SIZE;
7738613e
IW		 1583
1584 return 0;
1585}
1586
fc24fc5e
AS		 1587static int mlx5_enable_roce(struct mlx5_ib_dev *dev)
1588{
e53505a8
AS		 1589 int err;
1590
fc24fc5e 1591 dev->roce.nb.notifier_call = mlx5_netdev_event;
e53505a8
AS		 1592 err = register_netdevice_notifier(&dev->roce.nb);
1593 if (err)
1594 return err;
1595
1596 err = mlx5_nic_vport_enable_roce(dev->mdev);
1597 if (err)
1598 goto err_unregister_netdevice_notifier;
1599
1600 return 0;
1601
1602err_unregister_netdevice_notifier:
1603 unregister_netdevice_notifier(&dev->roce.nb);
1604 return err;
fc24fc5e
AS		 1605}
1606
1607static void mlx5_disable_roce(struct mlx5_ib_dev *dev)
1608{
e53505a8 1609 mlx5_nic_vport_disable_roce(dev->mdev);
fc24fc5e
AS		 1610 unregister_netdevice_notifier(&dev->roce.nb);
1611}
1612
9603b61d 1613static void *mlx5_ib_add(struct mlx5_core_dev *mdev)
e126ba97 1614{
e126ba97 1615 struct mlx5_ib_dev *dev;
ebd61f68
AS		 1616 enum rdma_link_layer ll;
1617 int port_type_cap;
e126ba97
EC		 1618 int err;
1619 int i;
1620
ebd61f68
AS		 1621 port_type_cap = MLX5_CAP_GEN(mdev, port_type);
1622 ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap);
1623
e53505a8 1624 if ((ll == IB_LINK_LAYER_ETHERNET) && !MLX5_CAP_GEN(mdev, roce))
647241ea
MD		 1625 return NULL;
1626
e126ba97
EC		 1627 printk_once(KERN_INFO "%s", mlx5_version);
1628
1629 dev = (struct mlx5_ib_dev *)ib_alloc_device(sizeof(*dev));
1630 if (!dev)
9603b61d 1631 return NULL;
e126ba97 1632
9603b61d 1633 dev->mdev = mdev;
e126ba97 1634
fc24fc5e 1635 rwlock_init(&dev->roce.netdev_lock);
e126ba97
EC		 1636 err = get_port_caps(dev);
1637 if (err)
9603b61d 1638 goto err_dealloc;
e126ba97 1639
1b5daf11
MD		 1640 if (mlx5_use_mad_ifc(dev))
1641 get_ext_port_caps(dev);
e126ba97 1642
e126ba97
EC		 1643 MLX5_INIT_DOORBELL_LOCK(&dev->uar_lock);
1644
1645 strlcpy(dev->ib_dev.name, "mlx5_%d", IB_DEVICE_NAME_MAX);
1646 dev->ib_dev.owner = THIS_MODULE;
1647 dev->ib_dev.node_type = RDMA_NODE_IB_CA;
c6790aa9 1648 dev->ib_dev.local_dma_lkey = 0 /* not supported for now */;
938fe83c 1649 dev->num_ports = MLX5_CAP_GEN(mdev, num_ports);
e126ba97 1650 dev->ib_dev.phys_port_cnt = dev->num_ports;
233d05d2
SM		 1651 dev->ib_dev.num_comp_vectors =
1652 dev->mdev->priv.eq_table.num_comp_vectors;
e126ba97
EC		 1653 dev->ib_dev.dma_device = &mdev->pdev->dev;
1654
1655 dev->ib_dev.uverbs_abi_ver = MLX5_IB_UVERBS_ABI_VERSION;
1656 dev->ib_dev.uverbs_cmd_mask =
1657 (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) |
1658 (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) |
1659 (1ull << IB_USER_VERBS_CMD_QUERY_PORT) |
1660 (1ull << IB_USER_VERBS_CMD_ALLOC_PD) |
1661 (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) |
1662 (1ull << IB_USER_VERBS_CMD_REG_MR) |
1663 (1ull << IB_USER_VERBS_CMD_DEREG_MR) |
1664 (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
1665 (1ull << IB_USER_VERBS_CMD_CREATE_CQ) |
1666 (1ull << IB_USER_VERBS_CMD_RESIZE_CQ) |
1667 (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) |
1668 (1ull << IB_USER_VERBS_CMD_CREATE_QP) |
1669 (1ull << IB_USER_VERBS_CMD_MODIFY_QP) |
1670 (1ull << IB_USER_VERBS_CMD_QUERY_QP) |
1671 (1ull << IB_USER_VERBS_CMD_DESTROY_QP) |
1672 (1ull << IB_USER_VERBS_CMD_ATTACH_MCAST) |
1673 (1ull << IB_USER_VERBS_CMD_DETACH_MCAST) |
1674 (1ull << IB_USER_VERBS_CMD_CREATE_SRQ) |
1675 (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ) |
1676 (1ull << IB_USER_VERBS_CMD_QUERY_SRQ) |
1677 (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ) |
1678 (1ull << IB_USER_VERBS_CMD_CREATE_XSRQ) |
1679 (1ull << IB_USER_VERBS_CMD_OPEN_QP);
1707cb4a
HE		 1680 dev->ib_dev.uverbs_ex_cmd_mask =
1681 (1ull << IB_USER_VERBS_EX_CMD_QUERY_DEVICE);
e126ba97
EC		 1682
1683 dev->ib_dev.query_device = mlx5_ib_query_device;
1684 dev->ib_dev.query_port = mlx5_ib_query_port;
ebd61f68 1685 dev->ib_dev.get_link_layer = mlx5_ib_port_link_layer;
fc24fc5e
AS		 1686 if (ll == IB_LINK_LAYER_ETHERNET)
1687 dev->ib_dev.get_netdev = mlx5_ib_get_netdev;
e126ba97 1688 dev->ib_dev.query_gid = mlx5_ib_query_gid;
3cca2606
AS		 1689 dev->ib_dev.add_gid = mlx5_ib_add_gid;
1690 dev->ib_dev.del_gid = mlx5_ib_del_gid;
e126ba97
EC		 1691 dev->ib_dev.query_pkey = mlx5_ib_query_pkey;
1692 dev->ib_dev.modify_device = mlx5_ib_modify_device;
1693 dev->ib_dev.modify_port = mlx5_ib_modify_port;
1694 dev->ib_dev.alloc_ucontext = mlx5_ib_alloc_ucontext;
1695 dev->ib_dev.dealloc_ucontext = mlx5_ib_dealloc_ucontext;
1696 dev->ib_dev.mmap = mlx5_ib_mmap;
1697 dev->ib_dev.alloc_pd = mlx5_ib_alloc_pd;
1698 dev->ib_dev.dealloc_pd = mlx5_ib_dealloc_pd;
1699 dev->ib_dev.create_ah = mlx5_ib_create_ah;
1700 dev->ib_dev.query_ah = mlx5_ib_query_ah;
1701 dev->ib_dev.destroy_ah = mlx5_ib_destroy_ah;
1702 dev->ib_dev.create_srq = mlx5_ib_create_srq;
1703 dev->ib_dev.modify_srq = mlx5_ib_modify_srq;
1704 dev->ib_dev.query_srq = mlx5_ib_query_srq;
1705 dev->ib_dev.destroy_srq = mlx5_ib_destroy_srq;
1706 dev->ib_dev.post_srq_recv = mlx5_ib_post_srq_recv;
1707 dev->ib_dev.create_qp = mlx5_ib_create_qp;
1708 dev->ib_dev.modify_qp = mlx5_ib_modify_qp;
1709 dev->ib_dev.query_qp = mlx5_ib_query_qp;
1710 dev->ib_dev.destroy_qp = mlx5_ib_destroy_qp;
1711 dev->ib_dev.post_send = mlx5_ib_post_send;
1712 dev->ib_dev.post_recv = mlx5_ib_post_recv;
1713 dev->ib_dev.create_cq = mlx5_ib_create_cq;
1714 dev->ib_dev.modify_cq = mlx5_ib_modify_cq;
1715 dev->ib_dev.resize_cq = mlx5_ib_resize_cq;
1716 dev->ib_dev.destroy_cq = mlx5_ib_destroy_cq;
1717 dev->ib_dev.poll_cq = mlx5_ib_poll_cq;
1718 dev->ib_dev.req_notify_cq = mlx5_ib_arm_cq;
1719 dev->ib_dev.get_dma_mr = mlx5_ib_get_dma_mr;
1720 dev->ib_dev.reg_user_mr = mlx5_ib_reg_user_mr;
1721 dev->ib_dev.dereg_mr = mlx5_ib_dereg_mr;
1722 dev->ib_dev.attach_mcast = mlx5_ib_mcg_attach;
1723 dev->ib_dev.detach_mcast = mlx5_ib_mcg_detach;
1724 dev->ib_dev.process_mad = mlx5_ib_process_mad;
9bee178b 1725 dev->ib_dev.alloc_mr = mlx5_ib_alloc_mr;
8a187ee5 1726 dev->ib_dev.map_mr_sg = mlx5_ib_map_mr_sg;
d5436ba0 1727 dev->ib_dev.check_mr_status = mlx5_ib_check_mr_status;
7738613e 1728 dev->ib_dev.get_port_immutable = mlx5_port_immutable;
e126ba97 1729
938fe83c 1730 mlx5_ib_internal_fill_odp_caps(dev);
8cdd312c 1731
938fe83c 1732 if (MLX5_CAP_GEN(mdev, xrc)) {
e126ba97
EC		 1733 dev->ib_dev.alloc_xrcd = mlx5_ib_alloc_xrcd;
1734 dev->ib_dev.dealloc_xrcd = mlx5_ib_dealloc_xrcd;
1735 dev->ib_dev.uverbs_cmd_mask |=
1736 (1ull << IB_USER_VERBS_CMD_OPEN_XRCD) |
1737 (1ull << IB_USER_VERBS_CMD_CLOSE_XRCD);
1738 }
1739
1740 err = init_node_data(dev);
1741 if (err)
233d05d2 1742 goto err_dealloc;
e126ba97
EC		 1743
1744 mutex_init(&dev->cap_mask_mutex);
e126ba97 1745
fc24fc5e
AS		 1746 if (ll == IB_LINK_LAYER_ETHERNET) {
1747 err = mlx5_enable_roce(dev);
1748 if (err)
1749 goto err_dealloc;
1750 }
1751
e126ba97
EC		 1752 err = create_dev_resources(&dev->devr);
1753 if (err)
fc24fc5e 1754 goto err_disable_roce;
e126ba97 1755
6aec21f6 1756 err = mlx5_ib_odp_init_one(dev);
281d1a92 1757 if (err)
e126ba97
EC		 1758 goto err_rsrc;
1759
6aec21f6
HE		 1760 err = ib_register_device(&dev->ib_dev, NULL);
1761 if (err)
1762 goto err_odp;
1763
e126ba97
EC		 1764 err = create_umr_res(dev);
1765 if (err)
1766 goto err_dev;
1767
1768 for (i = 0; i < ARRAY_SIZE(mlx5_class_attributes); i++) {
281d1a92
WY		 1769 err = device_create_file(&dev->ib_dev.dev,
1770 mlx5_class_attributes[i]);
1771 if (err)
e126ba97
EC		 1772 goto err_umrc;
1773 }
1774
1775 dev->ib_active = true;
1776
9603b61d 1777 return dev;
e126ba97
EC		 1778
1779err_umrc:
1780 destroy_umrc_res(dev);
1781
1782err_dev:
1783 ib_unregister_device(&dev->ib_dev);
1784
6aec21f6
HE		 1785err_odp:
1786 mlx5_ib_odp_remove_one(dev);
1787
e126ba97
EC		 1788err_rsrc:
1789 destroy_dev_resources(&dev->devr);
1790
fc24fc5e
AS		 1791err_disable_roce:
1792 if (ll == IB_LINK_LAYER_ETHERNET)
1793 mlx5_disable_roce(dev);
1794
9603b61d 1795err_dealloc:
e126ba97
EC		 1796 ib_dealloc_device((struct ib_device *)dev);
1797
9603b61d 1798 return NULL;
e126ba97
EC		 1799}
1800
9603b61d 1801static void mlx5_ib_remove(struct mlx5_core_dev *mdev, void *context)
e126ba97 1802{
9603b61d 1803 struct mlx5_ib_dev *dev = context;
fc24fc5e 1804 enum rdma_link_layer ll = mlx5_ib_port_link_layer(&dev->ib_dev, 1);
6aec21f6 1805
e126ba97 1806 ib_unregister_device(&dev->ib_dev);
eefd56e5 1807 destroy_umrc_res(dev);
6aec21f6 1808 mlx5_ib_odp_remove_one(dev);
e126ba97 1809 destroy_dev_resources(&dev->devr);
fc24fc5e
AS		 1810 if (ll == IB_LINK_LAYER_ETHERNET)
1811 mlx5_disable_roce(dev);
e126ba97
EC		 1812 ib_dealloc_device(&dev->ib_dev);
1813}
1814
9603b61d
JM		 1815static struct mlx5_interface mlx5_ib_interface = {
1816 .add = mlx5_ib_add,
1817 .remove = mlx5_ib_remove,
1818 .event = mlx5_ib_event,
64613d94 1819 .protocol = MLX5_INTERFACE_PROTOCOL_IB,
e126ba97
EC		 1820};
1821
1822static int __init mlx5_ib_init(void)
1823{
6aec21f6
HE		 1824 int err;
1825
9603b61d
JM		 1826 if (deprecated_prof_sel != 2)
1827 pr_warn("prof_sel is deprecated for mlx5_ib, set it for mlx5_core\n");
1828
6aec21f6
HE		 1829 err = mlx5_ib_odp_init();
1830 if (err)
1831 return err;
1832
1833 err = mlx5_register_interface(&mlx5_ib_interface);
1834 if (err)
1835 goto clean_odp;
1836
1837 return err;
1838
1839clean_odp:
1840 mlx5_ib_odp_cleanup();
1841 return err;
e126ba97
EC		 1842}
1843
1844static void __exit mlx5_ib_cleanup(void)
1845{
9603b61d 1846 mlx5_unregister_interface(&mlx5_ib_interface);
6aec21f6 1847 mlx5_ib_odp_cleanup();
e126ba97
EC		 1848}
1849
1850module_init(mlx5_ib_init);
1851module_exit(mlx5_ib_cleanup);
Linux 6.x block layer and io_uring tree(s)