projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
mlx5_core: Add support for page faults events and low level handling
[linux-2.6-block.git] / drivers / infiniband / hw / mlx5 / main.c
CommitLineData
e126ba97
EC		 1/*
2 * Copyright (c) 2013, Mellanox Technologies inc. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33#include <asm-generic/kmap_types.h>
34#include <linux/module.h>
35#include <linux/init.h>
36#include <linux/errno.h>
37#include <linux/pci.h>
38#include <linux/dma-mapping.h>
39#include <linux/slab.h>
40#include <linux/io-mapping.h>
41#include <linux/sched.h>
42#include <rdma/ib_user_verbs.h>
43#include <rdma/ib_smi.h>
44#include <rdma/ib_umem.h>
45#include "user.h"
46#include "mlx5_ib.h"
47
48#define DRIVER_NAME "mlx5_ib"
169a1d85
AV		 49#define DRIVER_VERSION "2.2-1"
50#define DRIVER_RELDATE "Feb 2014"
e126ba97
EC		 51
52MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>");
53MODULE_DESCRIPTION("Mellanox Connect-IB HCA IB driver");
54MODULE_LICENSE("Dual BSD/GPL");
55MODULE_VERSION(DRIVER_VERSION);
56
9603b61d
JM		 57static int deprecated_prof_sel = 2;
58module_param_named(prof_sel, deprecated_prof_sel, int, 0444);
59MODULE_PARM_DESC(prof_sel, "profile selector. Deprecated here. Moved to module mlx5_core");
e126ba97
EC		 60
61static char mlx5_version[] =
62 DRIVER_NAME ": Mellanox Connect-IB Infiniband driver v"
63 DRIVER_VERSION " (" DRIVER_RELDATE ")\n";
64
e126ba97
EC		 65int mlx5_vector2eqn(struct mlx5_ib_dev *dev, int vector, int *eqn, int *irqn)
66{
9603b61d 67 struct mlx5_eq_table *table = &dev->mdev->priv.eq_table;
e126ba97
EC		 68 struct mlx5_eq *eq, *n;
69 int err = -ENOENT;
70
71 spin_lock(&table->lock);
72 list_for_each_entry_safe(eq, n, &dev->eqs_list, list) {
73 if (eq->index == vector) {
74 *eqn = eq->eqn;
75 *irqn = eq->irqn;
76 err = 0;
77 break;
78 }
79 }
80 spin_unlock(&table->lock);
81
82 return err;
83}
84
85static int alloc_comp_eqs(struct mlx5_ib_dev *dev)
86{
9603b61d 87 struct mlx5_eq_table *table = &dev->mdev->priv.eq_table;
ada9f5d0 88 char name[MLX5_MAX_EQ_NAME];
e126ba97
EC		 89 struct mlx5_eq *eq, *n;
90 int ncomp_vec;
91 int nent;
92 int err;
93 int i;
94
95 INIT_LIST_HEAD(&dev->eqs_list);
96 ncomp_vec = table->num_comp_vectors;
97 nent = MLX5_COMP_EQ_SIZE;
98 for (i = 0; i < ncomp_vec; i++) {
99 eq = kzalloc(sizeof(*eq), GFP_KERNEL);
100 if (!eq) {
101 err = -ENOMEM;
102 goto clean;
103 }
104
ada9f5d0 105 snprintf(name, MLX5_MAX_EQ_NAME, "mlx5_comp%d", i);
9603b61d 106 err = mlx5_create_map_eq(dev->mdev, eq,
e126ba97 107 i + MLX5_EQ_VEC_COMP_BASE, nent, 0,
9603b61d 108 name, &dev->mdev->priv.uuari.uars[0]);
e126ba97
EC		 109 if (err) {
110 kfree(eq);
111 goto clean;
112 }
113 mlx5_ib_dbg(dev, "allocated completion EQN %d\n", eq->eqn);
114 eq->index = i;
115 spin_lock(&table->lock);
116 list_add_tail(&eq->list, &dev->eqs_list);
117 spin_unlock(&table->lock);
118 }
119
120 dev->num_comp_vectors = ncomp_vec;
121 return 0;
122
123clean:
124 spin_lock(&table->lock);
125 list_for_each_entry_safe(eq, n, &dev->eqs_list, list) {
126 list_del(&eq->list);
127 spin_unlock(&table->lock);
9603b61d 128 if (mlx5_destroy_unmap_eq(dev->mdev, eq))
e126ba97
EC		 129 mlx5_ib_warn(dev, "failed to destroy EQ 0x%x\n", eq->eqn);
130 kfree(eq);
131 spin_lock(&table->lock);
132 }
133 spin_unlock(&table->lock);
134 return err;
135}
136
137static void free_comp_eqs(struct mlx5_ib_dev *dev)
138{
9603b61d 139 struct mlx5_eq_table *table = &dev->mdev->priv.eq_table;
e126ba97
EC		 140 struct mlx5_eq *eq, *n;
141
142 spin_lock(&table->lock);
143 list_for_each_entry_safe(eq, n, &dev->eqs_list, list) {
144 list_del(&eq->list);
145 spin_unlock(&table->lock);
9603b61d 146 if (mlx5_destroy_unmap_eq(dev->mdev, eq))
e126ba97
EC		 147 mlx5_ib_warn(dev, "failed to destroy EQ 0x%x\n", eq->eqn);
148 kfree(eq);
149 spin_lock(&table->lock);
150 }
151 spin_unlock(&table->lock);
152}
153
154static int mlx5_ib_query_device(struct ib_device *ibdev,
155 struct ib_device_attr *props)
156{
157 struct mlx5_ib_dev *dev = to_mdev(ibdev);
158 struct ib_smp *in_mad = NULL;
159 struct ib_smp *out_mad = NULL;
c7a08ac7 160 struct mlx5_general_caps *gen;
e126ba97
EC		 161 int err = -ENOMEM;
162 int max_rq_sg;
163 int max_sq_sg;
164 u64 flags;
165
c7a08ac7 166 gen = &dev->mdev->caps.gen;
e126ba97
EC		 167 in_mad = kzalloc(sizeof(*in_mad), GFP_KERNEL);
168 out_mad = kmalloc(sizeof(*out_mad), GFP_KERNEL);
169 if (!in_mad || !out_mad)
170 goto out;
171
172 init_query_mad(in_mad);
173 in_mad->attr_id = IB_SMP_ATTR_NODE_INFO;
174
175 err = mlx5_MAD_IFC(to_mdev(ibdev), 1, 1, 1, NULL, NULL, in_mad, out_mad);
176 if (err)
177 goto out;
178
179 memset(props, 0, sizeof(*props));
180
9603b61d
JM		 181 props->fw_ver = ((u64)fw_rev_maj(dev->mdev) << 32) |
182 (fw_rev_min(dev->mdev) << 16) |
183 fw_rev_sub(dev->mdev);
e126ba97
EC		 184 props->device_cap_flags = IB_DEVICE_CHANGE_PHY_PORT |
185 IB_DEVICE_PORT_ACTIVE_EVENT |
186 IB_DEVICE_SYS_IMAGE_GUID |
1a4c3a3d 187 IB_DEVICE_RC_RNR_NAK_GEN;
c7a08ac7 188 flags = gen->flags;
e126ba97
EC		 189 if (flags & MLX5_DEV_CAP_FLAG_BAD_PKEY_CNTR)
190 props->device_cap_flags |= IB_DEVICE_BAD_PKEY_CNTR;
191 if (flags & MLX5_DEV_CAP_FLAG_BAD_QKEY_CNTR)
192 props->device_cap_flags |= IB_DEVICE_BAD_QKEY_CNTR;
193 if (flags & MLX5_DEV_CAP_FLAG_APM)
194 props->device_cap_flags |= IB_DEVICE_AUTO_PATH_MIG;
195 props->device_cap_flags |= IB_DEVICE_LOCAL_DMA_LKEY;
196 if (flags & MLX5_DEV_CAP_FLAG_XRC)
197 props->device_cap_flags |= IB_DEVICE_XRC;
198 props->device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS;
2dea9094
SG		 199 if (flags & MLX5_DEV_CAP_FLAG_SIG_HAND_OVER) {
200 props->device_cap_flags |= IB_DEVICE_SIGNATURE_HANDOVER;
201 /* At this stage no support for signature handover */
202 props->sig_prot_cap = IB_PROT_T10DIF_TYPE_1 |
203 IB_PROT_T10DIF_TYPE_2 |
204 IB_PROT_T10DIF_TYPE_3;
205 props->sig_guard_cap = IB_GUARD_T10DIF_CRC |
206 IB_GUARD_T10DIF_CSUM;
207 }
f360d88a
EC		 208 if (flags & MLX5_DEV_CAP_FLAG_BLOCK_MCAST)
209 props->device_cap_flags |= IB_DEVICE_BLOCK_MULTICAST_LOOPBACK;
e126ba97
EC		 210
211 props->vendor_id = be32_to_cpup((__be32 *)(out_mad->data + 36)) &
212 0xffffff;
213 props->vendor_part_id = be16_to_cpup((__be16 *)(out_mad->data + 30));
214 props->hw_ver = be32_to_cpup((__be32 *)(out_mad->data + 32));
215 memcpy(&props->sys_image_guid, out_mad->data + 4, 8);
216
217 props->max_mr_size = ~0ull;
c7a08ac7
EC		 218 props->page_size_cap = gen->min_page_sz;
219 props->max_qp = 1 << gen->log_max_qp;
220 props->max_qp_wr = gen->max_wqes;
221 max_rq_sg = gen->max_rq_desc_sz / sizeof(struct mlx5_wqe_data_seg);
222 max_sq_sg = (gen->max_sq_desc_sz - sizeof(struct mlx5_wqe_ctrl_seg)) /
e126ba97
EC		 223 sizeof(struct mlx5_wqe_data_seg);
224 props->max_sge = min(max_rq_sg, max_sq_sg);
c7a08ac7
EC		 225 props->max_cq = 1 << gen->log_max_cq;
226 props->max_cqe = gen->max_cqes - 1;
227 props->max_mr = 1 << gen->log_max_mkey;
228 props->max_pd = 1 << gen->log_max_pd;
229 props->max_qp_rd_atom = 1 << gen->log_max_ra_req_qp;
230 props->max_qp_init_rd_atom = 1 << gen->log_max_ra_res_qp;
231 props->max_srq = 1 << gen->log_max_srq;
232 props->max_srq_wr = gen->max_srq_wqes - 1;
233 props->local_ca_ack_delay = gen->local_ca_ack_delay;
e126ba97 234 props->max_res_rd_atom = props->max_qp_rd_atom * props->max_qp;
e126ba97
EC		 235 props->max_srq_sge = max_rq_sg - 1;
236 props->max_fast_reg_page_list_len = (unsigned int)-1;
c7a08ac7 237 props->local_ca_ack_delay = gen->local_ca_ack_delay;
81bea28f
EC		 238 props->atomic_cap = IB_ATOMIC_NONE;
239 props->masked_atomic_cap = IB_ATOMIC_NONE;
e126ba97 240 props->max_pkeys = be16_to_cpup((__be16 *)(out_mad->data + 28));
c7a08ac7
EC		 241 props->max_mcast_grp = 1 << gen->log_max_mcg;
242 props->max_mcast_qp_attach = gen->max_qp_mcg;
e126ba97
EC		 243 props->max_total_mcast_qp_attach = props->max_mcast_qp_attach *
244 props->max_mcast_grp;
245 props->max_map_per_fmr = INT_MAX; /* no limit in ConnectIB */
246
247out:
248 kfree(in_mad);
249 kfree(out_mad);
250
251 return err;
252}
253
254int mlx5_ib_query_port(struct ib_device *ibdev, u8 port,
255 struct ib_port_attr *props)
256{
257 struct mlx5_ib_dev *dev = to_mdev(ibdev);
258 struct ib_smp *in_mad = NULL;
259 struct ib_smp *out_mad = NULL;
c7a08ac7 260 struct mlx5_general_caps *gen;
e126ba97
EC		 261 int ext_active_speed;
262 int err = -ENOMEM;
263
c7a08ac7
EC		 264 gen = &dev->mdev->caps.gen;
265 if (port < 1 || port > gen->num_ports) {
e126ba97
EC		 266 mlx5_ib_warn(dev, "invalid port number %d\n", port);
267 return -EINVAL;
268 }
269
270 in_mad = kzalloc(sizeof(*in_mad), GFP_KERNEL);
271 out_mad = kmalloc(sizeof(*out_mad), GFP_KERNEL);
272 if (!in_mad || !out_mad)
273 goto out;
274
275 memset(props, 0, sizeof(*props));
276
277 init_query_mad(in_mad);
278 in_mad->attr_id = IB_SMP_ATTR_PORT_INFO;
279 in_mad->attr_mod = cpu_to_be32(port);
280
281 err = mlx5_MAD_IFC(dev, 1, 1, port, NULL, NULL, in_mad, out_mad);
282 if (err) {
283 mlx5_ib_warn(dev, "err %d\n", err);
284 goto out;
285 }
286
287
288 props->lid = be16_to_cpup((__be16 *)(out_mad->data + 16));
289 props->lmc = out_mad->data[34] & 0x7;
290 props->sm_lid = be16_to_cpup((__be16 *)(out_mad->data + 18));
291 props->sm_sl = out_mad->data[36] & 0xf;
292 props->state = out_mad->data[32] & 0xf;
293 props->phys_state = out_mad->data[33] >> 4;
294 props->port_cap_flags = be32_to_cpup((__be32 *)(out_mad->data + 20));
295 props->gid_tbl_len = out_mad->data[50];
c7a08ac7
EC		 296 props->max_msg_sz = 1 << gen->log_max_msg;
297 props->pkey_tbl_len = gen->port[port - 1].pkey_table_len;
e126ba97
EC		 298 props->bad_pkey_cntr = be16_to_cpup((__be16 *)(out_mad->data + 46));
299 props->qkey_viol_cntr = be16_to_cpup((__be16 *)(out_mad->data + 48));
300 props->active_width = out_mad->data[31] & 0xf;
301 props->active_speed = out_mad->data[35] >> 4;
302 props->max_mtu = out_mad->data[41] & 0xf;
303 props->active_mtu = out_mad->data[36] >> 4;
304 props->subnet_timeout = out_mad->data[51] & 0x1f;
305 props->max_vl_num = out_mad->data[37] >> 4;
306 props->init_type_reply = out_mad->data[41] >> 4;
307
308 /* Check if extended speeds (EDR/FDR/...) are supported */
309 if (props->port_cap_flags & IB_PORT_EXTENDED_SPEEDS_SUP) {
310 ext_active_speed = out_mad->data[62] >> 4;
311
312 switch (ext_active_speed) {
313 case 1:
314 props->active_speed = 16; /* FDR */
315 break;
316 case 2:
317 props->active_speed = 32; /* EDR */
318 break;
319 }
320 }
321
322 /* If reported active speed is QDR, check if is FDR-10 */
323 if (props->active_speed == 4) {
c7a08ac7 324 if (gen->ext_port_cap[port - 1] &
e126ba97
EC		 325 MLX_EXT_PORT_CAP_FLAG_EXTENDED_PORT_INFO) {
326 init_query_mad(in_mad);
327 in_mad->attr_id = MLX5_ATTR_EXTENDED_PORT_INFO;
328 in_mad->attr_mod = cpu_to_be32(port);
329
330 err = mlx5_MAD_IFC(dev, 1, 1, port,
331 NULL, NULL, in_mad, out_mad);
332 if (err)
333 goto out;
334
335 /* Checking LinkSpeedActive for FDR-10 */
336 if (out_mad->data[15] & 0x1)
337 props->active_speed = 8;
338 }
339 }
340
341out:
342 kfree(in_mad);
343 kfree(out_mad);
344
345 return err;
346}
347
348static int mlx5_ib_query_gid(struct ib_device *ibdev, u8 port, int index,
349 union ib_gid *gid)
350{
351 struct ib_smp *in_mad = NULL;
352 struct ib_smp *out_mad = NULL;
353 int err = -ENOMEM;
354
355 in_mad = kzalloc(sizeof(*in_mad), GFP_KERNEL);
356 out_mad = kmalloc(sizeof(*out_mad), GFP_KERNEL);
357 if (!in_mad || !out_mad)
358 goto out;
359
360 init_query_mad(in_mad);
361 in_mad->attr_id = IB_SMP_ATTR_PORT_INFO;
362 in_mad->attr_mod = cpu_to_be32(port);
363
364 err = mlx5_MAD_IFC(to_mdev(ibdev), 1, 1, port, NULL, NULL, in_mad, out_mad);
365 if (err)
366 goto out;
367
368 memcpy(gid->raw, out_mad->data + 8, 8);
369
370 init_query_mad(in_mad);
371 in_mad->attr_id = IB_SMP_ATTR_GUID_INFO;
372 in_mad->attr_mod = cpu_to_be32(index / 8);
373
374 err = mlx5_MAD_IFC(to_mdev(ibdev), 1, 1, port, NULL, NULL, in_mad, out_mad);
375 if (err)
376 goto out;
377
378 memcpy(gid->raw + 8, out_mad->data + (index % 8) * 8, 8);
379
380out:
381 kfree(in_mad);
382 kfree(out_mad);
383 return err;
384}
385
386static int mlx5_ib_query_pkey(struct ib_device *ibdev, u8 port, u16 index,
387 u16 *pkey)
388{
389 struct ib_smp *in_mad = NULL;
390 struct ib_smp *out_mad = NULL;
391 int err = -ENOMEM;
392
393 in_mad = kzalloc(sizeof(*in_mad), GFP_KERNEL);
394 out_mad = kmalloc(sizeof(*out_mad), GFP_KERNEL);
395 if (!in_mad || !out_mad)
396 goto out;
397
398 init_query_mad(in_mad);
399 in_mad->attr_id = IB_SMP_ATTR_PKEY_TABLE;
400 in_mad->attr_mod = cpu_to_be32(index / 32);
401
402 err = mlx5_MAD_IFC(to_mdev(ibdev), 1, 1, port, NULL, NULL, in_mad, out_mad);
403 if (err)
404 goto out;
405
406 *pkey = be16_to_cpu(((__be16 *)out_mad->data)[index % 32]);
407
408out:
409 kfree(in_mad);
410 kfree(out_mad);
411 return err;
412}
413
414struct mlx5_reg_node_desc {
415 u8 desc[64];
416};
417
418static int mlx5_ib_modify_device(struct ib_device *ibdev, int mask,
419 struct ib_device_modify *props)
420{
421 struct mlx5_ib_dev *dev = to_mdev(ibdev);
422 struct mlx5_reg_node_desc in;
423 struct mlx5_reg_node_desc out;
424 int err;
425
426 if (mask & ~IB_DEVICE_MODIFY_NODE_DESC)
427 return -EOPNOTSUPP;
428
429 if (!(mask & IB_DEVICE_MODIFY_NODE_DESC))
430 return 0;
431
432 /*
433 * If possible, pass node desc to FW, so it can generate
434 * a 144 trap. If cmd fails, just ignore.
435 */
436 memcpy(&in, props->node_desc, 64);
9603b61d 437 err = mlx5_core_access_reg(dev->mdev, &in, sizeof(in), &out,
e126ba97
EC		 438 sizeof(out), MLX5_REG_NODE_DESC, 0, 1);
439 if (err)
440 return err;
441
442 memcpy(ibdev->node_desc, props->node_desc, 64);
443
444 return err;
445}
446
447static int mlx5_ib_modify_port(struct ib_device *ibdev, u8 port, int mask,
448 struct ib_port_modify *props)
449{
450 struct mlx5_ib_dev *dev = to_mdev(ibdev);
451 struct ib_port_attr attr;
452 u32 tmp;
453 int err;
454
455 mutex_lock(&dev->cap_mask_mutex);
456
457 err = mlx5_ib_query_port(ibdev, port, &attr);
458 if (err)
459 goto out;
460
461 tmp = (attr.port_cap_flags | props->set_port_cap_mask) &
462 ~props->clr_port_cap_mask;
463
9603b61d 464 err = mlx5_set_port_caps(dev->mdev, port, tmp);
e126ba97
EC		 465
466out:
467 mutex_unlock(&dev->cap_mask_mutex);
468 return err;
469}
470
471static struct ib_ucontext *mlx5_ib_alloc_ucontext(struct ib_device *ibdev,
472 struct ib_udata *udata)
473{
474 struct mlx5_ib_dev *dev = to_mdev(ibdev);
78c0f98c 475 struct mlx5_ib_alloc_ucontext_req_v2 req;
e126ba97
EC		 476 struct mlx5_ib_alloc_ucontext_resp resp;
477 struct mlx5_ib_ucontext *context;
c7a08ac7 478 struct mlx5_general_caps *gen;
e126ba97
EC		 479 struct mlx5_uuar_info *uuari;
480 struct mlx5_uar *uars;
c1be5232 481 int gross_uuars;
e126ba97 482 int num_uars;
78c0f98c 483 int ver;
e126ba97
EC		 484 int uuarn;
485 int err;
486 int i;
f241e749 487 size_t reqlen;
e126ba97 488
c7a08ac7 489 gen = &dev->mdev->caps.gen;
e126ba97
EC		 490 if (!dev->ib_active)
491 return ERR_PTR(-EAGAIN);
492
78c0f98c
EC		 493 memset(&req, 0, sizeof(req));
494 reqlen = udata->inlen - sizeof(struct ib_uverbs_cmd_hdr);
495 if (reqlen == sizeof(struct mlx5_ib_alloc_ucontext_req))
496 ver = 0;
497 else if (reqlen == sizeof(struct mlx5_ib_alloc_ucontext_req_v2))
498 ver = 2;
499 else
500 return ERR_PTR(-EINVAL);
501
502 err = ib_copy_from_udata(&req, udata, reqlen);
e126ba97
EC		 503 if (err)
504 return ERR_PTR(err);
505
78c0f98c
EC		 506 if (req.flags || req.reserved)
507 return ERR_PTR(-EINVAL);
508
e126ba97
EC		 509 if (req.total_num_uuars > MLX5_MAX_UUARS)
510 return ERR_PTR(-ENOMEM);
511
512 if (req.total_num_uuars == 0)
513 return ERR_PTR(-EINVAL);
514
c1be5232
EC		 515 req.total_num_uuars = ALIGN(req.total_num_uuars,
516 MLX5_NON_FP_BF_REGS_PER_PAGE);
e126ba97
EC		 517 if (req.num_low_latency_uuars > req.total_num_uuars - 1)
518 return ERR_PTR(-EINVAL);
519
c1be5232
EC		 520 num_uars = req.total_num_uuars / MLX5_NON_FP_BF_REGS_PER_PAGE;
521 gross_uuars = num_uars * MLX5_BF_REGS_PER_PAGE;
c7a08ac7
EC		 522 resp.qp_tab_size = 1 << gen->log_max_qp;
523 resp.bf_reg_size = gen->bf_reg_size;
e126ba97 524 resp.cache_line_size = L1_CACHE_BYTES;
c7a08ac7
EC		 525 resp.max_sq_desc_sz = gen->max_sq_desc_sz;
526 resp.max_rq_desc_sz = gen->max_rq_desc_sz;
527 resp.max_send_wqebb = gen->max_wqes;
528 resp.max_recv_wr = gen->max_wqes;
529 resp.max_srq_recv_wr = gen->max_srq_wqes;
e126ba97
EC		 530
531 context = kzalloc(sizeof(*context), GFP_KERNEL);
532 if (!context)
533 return ERR_PTR(-ENOMEM);
534
535 uuari = &context->uuari;
536 mutex_init(&uuari->lock);
537 uars = kcalloc(num_uars, sizeof(*uars), GFP_KERNEL);
538 if (!uars) {
539 err = -ENOMEM;
540 goto out_ctx;
541 }
542
c1be5232 543 uuari->bitmap = kcalloc(BITS_TO_LONGS(gross_uuars),
e126ba97
EC		 544 sizeof(*uuari->bitmap),
545 GFP_KERNEL);
546 if (!uuari->bitmap) {
547 err = -ENOMEM;
548 goto out_uar_ctx;
549 }
550 /*
551 * clear all fast path uuars
552 */
c1be5232 553 for (i = 0; i < gross_uuars; i++) {
e126ba97
EC		 554 uuarn = i & 3;
555 if (uuarn == 2 || uuarn == 3)
556 set_bit(i, uuari->bitmap);
557 }
558
c1be5232 559 uuari->count = kcalloc(gross_uuars, sizeof(*uuari->count), GFP_KERNEL);
e126ba97
EC		 560 if (!uuari->count) {
561 err = -ENOMEM;
562 goto out_bitmap;
563 }
564
565 for (i = 0; i < num_uars; i++) {
9603b61d 566 err = mlx5_cmd_alloc_uar(dev->mdev, &uars[i].index);
e126ba97
EC		 567 if (err)
568 goto out_count;
569 }
570
571 INIT_LIST_HEAD(&context->db_page_list);
572 mutex_init(&context->db_page_mutex);
573
574 resp.tot_uuars = req.total_num_uuars;
c7a08ac7 575 resp.num_ports = gen->num_ports;
92b0ca7c
DC		 576 err = ib_copy_to_udata(udata, &resp,
577 sizeof(resp) - sizeof(resp.reserved));
e126ba97
EC		 578 if (err)
579 goto out_uars;
580
78c0f98c 581 uuari->ver = ver;
e126ba97
EC		 582 uuari->num_low_latency_uuars = req.num_low_latency_uuars;
583 uuari->uars = uars;
584 uuari->num_uars = num_uars;
585 return &context->ibucontext;
586
587out_uars:
588 for (i--; i >= 0; i--)
9603b61d 589 mlx5_cmd_free_uar(dev->mdev, uars[i].index);
e126ba97
EC		 590out_count:
591 kfree(uuari->count);
592
593out_bitmap:
594 kfree(uuari->bitmap);
595
596out_uar_ctx:
597 kfree(uars);
598
599out_ctx:
600 kfree(context);
601 return ERR_PTR(err);
602}
603
604static int mlx5_ib_dealloc_ucontext(struct ib_ucontext *ibcontext)
605{
606 struct mlx5_ib_ucontext *context = to_mucontext(ibcontext);
607 struct mlx5_ib_dev *dev = to_mdev(ibcontext->device);
608 struct mlx5_uuar_info *uuari = &context->uuari;
609 int i;
610
611 for (i = 0; i < uuari->num_uars; i++) {
9603b61d 612 if (mlx5_cmd_free_uar(dev->mdev, uuari->uars[i].index))
e126ba97
EC		 613 mlx5_ib_warn(dev, "failed to free UAR 0x%x\n", uuari->uars[i].index);
614 }
615
616 kfree(uuari->count);
617 kfree(uuari->bitmap);
618 kfree(uuari->uars);
619 kfree(context);
620
621 return 0;
622}
623
624static phys_addr_t uar_index2pfn(struct mlx5_ib_dev *dev, int index)
625{
9603b61d 626 return (pci_resource_start(dev->mdev->pdev, 0) >> PAGE_SHIFT) + index;
e126ba97
EC		 627}
628
629static int get_command(unsigned long offset)
630{
631 return (offset >> MLX5_IB_MMAP_CMD_SHIFT) & MLX5_IB_MMAP_CMD_MASK;
632}
633
634static int get_arg(unsigned long offset)
635{
636 return offset & ((1 << MLX5_IB_MMAP_CMD_SHIFT) - 1);
637}
638
639static int get_index(unsigned long offset)
640{
641 return get_arg(offset);
642}
643
644static int mlx5_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma)
645{
646 struct mlx5_ib_ucontext *context = to_mucontext(ibcontext);
647 struct mlx5_ib_dev *dev = to_mdev(ibcontext->device);
648 struct mlx5_uuar_info *uuari = &context->uuari;
649 unsigned long command;
650 unsigned long idx;
651 phys_addr_t pfn;
652
653 command = get_command(vma->vm_pgoff);
654 switch (command) {
655 case MLX5_IB_MMAP_REGULAR_PAGE:
656 if (vma->vm_end - vma->vm_start != PAGE_SIZE)
657 return -EINVAL;
658
659 idx = get_index(vma->vm_pgoff);
1c3ce90d
EC		 660 if (idx >= uuari->num_uars)
661 return -EINVAL;
662
e126ba97
EC		 663 pfn = uar_index2pfn(dev, uuari->uars[idx].index);
664 mlx5_ib_dbg(dev, "uar idx 0x%lx, pfn 0x%llx\n", idx,
665 (unsigned long long)pfn);
666
e126ba97
EC		 667 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
668 if (io_remap_pfn_range(vma, vma->vm_start, pfn,
669 PAGE_SIZE, vma->vm_page_prot))
670 return -EAGAIN;
671
672 mlx5_ib_dbg(dev, "mapped WC at 0x%lx, PA 0x%llx\n",
673 vma->vm_start,
674 (unsigned long long)pfn << PAGE_SHIFT);
675 break;
676
677 case MLX5_IB_MMAP_GET_CONTIGUOUS_PAGES:
678 return -ENOSYS;
679
680 default:
681 return -EINVAL;
682 }
683
684 return 0;
685}
686
687static int alloc_pa_mkey(struct mlx5_ib_dev *dev, u32 *key, u32 pdn)
688{
689 struct mlx5_create_mkey_mbox_in *in;
690 struct mlx5_mkey_seg *seg;
691 struct mlx5_core_mr mr;
692 int err;
693
694 in = kzalloc(sizeof(*in), GFP_KERNEL);
695 if (!in)
696 return -ENOMEM;
697
698 seg = &in->seg;
699 seg->flags = MLX5_PERM_LOCAL_READ | MLX5_ACCESS_MODE_PA;
700 seg->flags_pd = cpu_to_be32(pdn | MLX5_MKEY_LEN64);
701 seg->qpn_mkey7_0 = cpu_to_be32(0xffffff << 8);
702 seg->start_addr = 0;
703
9603b61d 704 err = mlx5_core_create_mkey(dev->mdev, &mr, in, sizeof(*in),
746b5583 705 NULL, NULL, NULL);
e126ba97
EC		 706 if (err) {
707 mlx5_ib_warn(dev, "failed to create mkey, %d\n", err);
708 goto err_in;
709 }
710
711 kfree(in);
712 *key = mr.key;
713
714 return 0;
715
716err_in:
717 kfree(in);
718
719 return err;
720}
721
722static void free_pa_mkey(struct mlx5_ib_dev *dev, u32 key)
723{
724 struct mlx5_core_mr mr;
725 int err;
726
727 memset(&mr, 0, sizeof(mr));
728 mr.key = key;
9603b61d 729 err = mlx5_core_destroy_mkey(dev->mdev, &mr);
e126ba97
EC		 730 if (err)
731 mlx5_ib_warn(dev, "failed to destroy mkey 0x%x\n", key);
732}
733
734static struct ib_pd *mlx5_ib_alloc_pd(struct ib_device *ibdev,
735 struct ib_ucontext *context,
736 struct ib_udata *udata)
737{
738 struct mlx5_ib_alloc_pd_resp resp;
739 struct mlx5_ib_pd *pd;
740 int err;
741
742 pd = kmalloc(sizeof(*pd), GFP_KERNEL);
743 if (!pd)
744 return ERR_PTR(-ENOMEM);
745
9603b61d 746 err = mlx5_core_alloc_pd(to_mdev(ibdev)->mdev, &pd->pdn);
e126ba97
EC		 747 if (err) {
748 kfree(pd);
749 return ERR_PTR(err);
750 }
751
752 if (context) {
753 resp.pdn = pd->pdn;
754 if (ib_copy_to_udata(udata, &resp, sizeof(resp))) {
9603b61d 755 mlx5_core_dealloc_pd(to_mdev(ibdev)->mdev, pd->pdn);
e126ba97
EC		 756 kfree(pd);
757 return ERR_PTR(-EFAULT);
758 }
759 } else {
760 err = alloc_pa_mkey(to_mdev(ibdev), &pd->pa_lkey, pd->pdn);
761 if (err) {
9603b61d 762 mlx5_core_dealloc_pd(to_mdev(ibdev)->mdev, pd->pdn);
e126ba97
EC		 763 kfree(pd);
764 return ERR_PTR(err);
765 }
766 }
767
768 return &pd->ibpd;
769}
770
771static int mlx5_ib_dealloc_pd(struct ib_pd *pd)
772{
773 struct mlx5_ib_dev *mdev = to_mdev(pd->device);
774 struct mlx5_ib_pd *mpd = to_mpd(pd);
775
776 if (!pd->uobject)
777 free_pa_mkey(mdev, mpd->pa_lkey);
778
9603b61d 779 mlx5_core_dealloc_pd(mdev->mdev, mpd->pdn);
e126ba97
EC		 780 kfree(mpd);
781
782 return 0;
783}
784
785static int mlx5_ib_mcg_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
786{
787 struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
788 int err;
789
9603b61d 790 err = mlx5_core_attach_mcg(dev->mdev, gid, ibqp->qp_num);
e126ba97
EC		 791 if (err)
792 mlx5_ib_warn(dev, "failed attaching QPN 0x%x, MGID %pI6\n",
793 ibqp->qp_num, gid->raw);
794
795 return err;
796}
797
798static int mlx5_ib_mcg_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
799{
800 struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
801 int err;
802
9603b61d 803 err = mlx5_core_detach_mcg(dev->mdev, gid, ibqp->qp_num);
e126ba97
EC		 804 if (err)
805 mlx5_ib_warn(dev, "failed detaching QPN 0x%x, MGID %pI6\n",
806 ibqp->qp_num, gid->raw);
807
808 return err;
809}
810
811static int init_node_data(struct mlx5_ib_dev *dev)
812{
813 struct ib_smp *in_mad = NULL;
814 struct ib_smp *out_mad = NULL;
815 int err = -ENOMEM;
816
817 in_mad = kzalloc(sizeof(*in_mad), GFP_KERNEL);
818 out_mad = kmalloc(sizeof(*out_mad), GFP_KERNEL);
819 if (!in_mad || !out_mad)
820 goto out;
821
822 init_query_mad(in_mad);
823 in_mad->attr_id = IB_SMP_ATTR_NODE_DESC;
824
825 err = mlx5_MAD_IFC(dev, 1, 1, 1, NULL, NULL, in_mad, out_mad);
826 if (err)
827 goto out;
828
829 memcpy(dev->ib_dev.node_desc, out_mad->data, 64);
830
831 in_mad->attr_id = IB_SMP_ATTR_NODE_INFO;
832
833 err = mlx5_MAD_IFC(dev, 1, 1, 1, NULL, NULL, in_mad, out_mad);
834 if (err)
835 goto out;
836
9603b61d 837 dev->mdev->rev_id = be32_to_cpup((__be32 *)(out_mad->data + 32));
e126ba97
EC		 838 memcpy(&dev->ib_dev.node_guid, out_mad->data + 12, 8);
839
840out:
841 kfree(in_mad);
842 kfree(out_mad);
843 return err;
844}
845
846static ssize_t show_fw_pages(struct device *device, struct device_attribute *attr,
847 char *buf)
848{
849 struct mlx5_ib_dev *dev =
850 container_of(device, struct mlx5_ib_dev, ib_dev.dev);
851
9603b61d 852 return sprintf(buf, "%d\n", dev->mdev->priv.fw_pages);
e126ba97
EC		 853}
854
855static ssize_t show_reg_pages(struct device *device,
856 struct device_attribute *attr, char *buf)
857{
858 struct mlx5_ib_dev *dev =
859 container_of(device, struct mlx5_ib_dev, ib_dev.dev);
860
9603b61d 861 return sprintf(buf, "%d\n", dev->mdev->priv.reg_pages);
e126ba97
EC		 862}
863
864static ssize_t show_hca(struct device *device, struct device_attribute *attr,
865 char *buf)
866{
867 struct mlx5_ib_dev *dev =
868 container_of(device, struct mlx5_ib_dev, ib_dev.dev);
9603b61d 869 return sprintf(buf, "MT%d\n", dev->mdev->pdev->device);
e126ba97
EC		 870}
871
872static ssize_t show_fw_ver(struct device *device, struct device_attribute *attr,
873 char *buf)
874{
875 struct mlx5_ib_dev *dev =
876 container_of(device, struct mlx5_ib_dev, ib_dev.dev);
9603b61d
JM		 877 return sprintf(buf, "%d.%d.%d\n", fw_rev_maj(dev->mdev),
878 fw_rev_min(dev->mdev), fw_rev_sub(dev->mdev));
e126ba97
EC		 879}
880
881static ssize_t show_rev(struct device *device, struct device_attribute *attr,
882 char *buf)
883{
884 struct mlx5_ib_dev *dev =
885 container_of(device, struct mlx5_ib_dev, ib_dev.dev);
9603b61d 886 return sprintf(buf, "%x\n", dev->mdev->rev_id);
e126ba97
EC		 887}
888
889static ssize_t show_board(struct device *device, struct device_attribute *attr,
890 char *buf)
891{
892 struct mlx5_ib_dev *dev =
893 container_of(device, struct mlx5_ib_dev, ib_dev.dev);
894 return sprintf(buf, "%.*s\n", MLX5_BOARD_ID_LEN,
9603b61d 895 dev->mdev->board_id);
e126ba97
EC		 896}
897
898static DEVICE_ATTR(hw_rev, S_IRUGO, show_rev, NULL);
899static DEVICE_ATTR(fw_ver, S_IRUGO, show_fw_ver, NULL);
900static DEVICE_ATTR(hca_type, S_IRUGO, show_hca, NULL);
901static DEVICE_ATTR(board_id, S_IRUGO, show_board, NULL);
902static DEVICE_ATTR(fw_pages, S_IRUGO, show_fw_pages, NULL);
903static DEVICE_ATTR(reg_pages, S_IRUGO, show_reg_pages, NULL);
904
905static struct device_attribute *mlx5_class_attributes[] = {
906 &dev_attr_hw_rev,
907 &dev_attr_fw_ver,
908 &dev_attr_hca_type,
909 &dev_attr_board_id,
910 &dev_attr_fw_pages,
911 &dev_attr_reg_pages,
912};
913
9603b61d 914static void mlx5_ib_event(struct mlx5_core_dev *dev, void *context,
4d2f9bbb 915 enum mlx5_dev_event event, unsigned long param)
e126ba97 916{
9603b61d 917 struct mlx5_ib_dev *ibdev = (struct mlx5_ib_dev *)context;
e126ba97 918 struct ib_event ibev;
9603b61d 919
e126ba97
EC		 920 u8 port = 0;
921
922 switch (event) {
923 case MLX5_DEV_EVENT_SYS_ERROR:
924 ibdev->ib_active = false;
925 ibev.event = IB_EVENT_DEVICE_FATAL;
926 break;
927
928 case MLX5_DEV_EVENT_PORT_UP:
929 ibev.event = IB_EVENT_PORT_ACTIVE;
4d2f9bbb 930 port = (u8)param;
e126ba97
EC		 931 break;
932
933 case MLX5_DEV_EVENT_PORT_DOWN:
934 ibev.event = IB_EVENT_PORT_ERR;
4d2f9bbb 935 port = (u8)param;
e126ba97
EC		 936 break;
937
938 case MLX5_DEV_EVENT_PORT_INITIALIZED:
939 /* not used by ULPs */
940 return;
941
942 case MLX5_DEV_EVENT_LID_CHANGE:
943 ibev.event = IB_EVENT_LID_CHANGE;
4d2f9bbb 944 port = (u8)param;
e126ba97
EC		 945 break;
946
947 case MLX5_DEV_EVENT_PKEY_CHANGE:
948 ibev.event = IB_EVENT_PKEY_CHANGE;
4d2f9bbb 949 port = (u8)param;
e126ba97
EC		 950 break;
951
952 case MLX5_DEV_EVENT_GUID_CHANGE:
953 ibev.event = IB_EVENT_GID_CHANGE;
4d2f9bbb 954 port = (u8)param;
e126ba97
EC		 955 break;
956
957 case MLX5_DEV_EVENT_CLIENT_REREG:
958 ibev.event = IB_EVENT_CLIENT_REREGISTER;
4d2f9bbb 959 port = (u8)param;
e126ba97
EC		 960 break;
961 }
962
963 ibev.device = &ibdev->ib_dev;
964 ibev.element.port_num = port;
965
a0c84c32
EC		 966 if (port < 1 || port > ibdev->num_ports) {
967 mlx5_ib_warn(ibdev, "warning: event on port %d\n", port);
968 return;
969 }
970
e126ba97
EC		 971 if (ibdev->ib_active)
972 ib_dispatch_event(&ibev);
973}
974
975static void get_ext_port_caps(struct mlx5_ib_dev *dev)
976{
c7a08ac7 977 struct mlx5_general_caps *gen;
e126ba97
EC		 978 int port;
979
c7a08ac7
EC		 980 gen = &dev->mdev->caps.gen;
981 for (port = 1; port <= gen->num_ports; port++)
e126ba97
EC		 982 mlx5_query_ext_port_caps(dev, port);
983}
984
985static int get_port_caps(struct mlx5_ib_dev *dev)
986{
987 struct ib_device_attr *dprops = NULL;
988 struct ib_port_attr *pprops = NULL;
c7a08ac7 989 struct mlx5_general_caps *gen;
e126ba97
EC		 990 int err = 0;
991 int port;
992
c7a08ac7 993 gen = &dev->mdev->caps.gen;
e126ba97
EC		 994 pprops = kmalloc(sizeof(*pprops), GFP_KERNEL);
995 if (!pprops)
996 goto out;
997
998 dprops = kmalloc(sizeof(*dprops), GFP_KERNEL);
999 if (!dprops)
1000 goto out;
1001
1002 err = mlx5_ib_query_device(&dev->ib_dev, dprops);
1003 if (err) {
1004 mlx5_ib_warn(dev, "query_device failed %d\n", err);
1005 goto out;
1006 }
1007
c7a08ac7 1008 for (port = 1; port <= gen->num_ports; port++) {
e126ba97
EC		 1009 err = mlx5_ib_query_port(&dev->ib_dev, port, pprops);
1010 if (err) {
1011 mlx5_ib_warn(dev, "query_port %d failed %d\n", port, err);
1012 break;
1013 }
c7a08ac7
EC		 1014 gen->port[port - 1].pkey_table_len = dprops->max_pkeys;
1015 gen->port[port - 1].gid_table_len = pprops->gid_tbl_len;
e126ba97
EC		 1016 mlx5_ib_dbg(dev, "pkey_table_len %d, gid_table_len %d\n",
1017 dprops->max_pkeys, pprops->gid_tbl_len);
1018 }
1019
1020out:
1021 kfree(pprops);
1022 kfree(dprops);
1023
1024 return err;
1025}
1026
1027static void destroy_umrc_res(struct mlx5_ib_dev *dev)
1028{
1029 int err;
1030
1031 err = mlx5_mr_cache_cleanup(dev);
1032 if (err)
1033 mlx5_ib_warn(dev, "mr cache cleanup failed\n");
1034
1035 mlx5_ib_destroy_qp(dev->umrc.qp);
1036 ib_destroy_cq(dev->umrc.cq);
1037 ib_dereg_mr(dev->umrc.mr);
1038 ib_dealloc_pd(dev->umrc.pd);
1039}
1040
1041enum {
1042 MAX_UMR_WR = 128,
1043};
1044
1045static int create_umr_res(struct mlx5_ib_dev *dev)
1046{
1047 struct ib_qp_init_attr *init_attr = NULL;
1048 struct ib_qp_attr *attr = NULL;
1049 struct ib_pd *pd;
1050 struct ib_cq *cq;
1051 struct ib_qp *qp;
1052 struct ib_mr *mr;
1053 int ret;
1054
1055 attr = kzalloc(sizeof(*attr), GFP_KERNEL);
1056 init_attr = kzalloc(sizeof(*init_attr), GFP_KERNEL);
1057 if (!attr || !init_attr) {
1058 ret = -ENOMEM;
1059 goto error_0;
1060 }
1061
1062 pd = ib_alloc_pd(&dev->ib_dev);
1063 if (IS_ERR(pd)) {
1064 mlx5_ib_dbg(dev, "Couldn't create PD for sync UMR QP\n");
1065 ret = PTR_ERR(pd);
1066 goto error_0;
1067 }
1068
1069 mr = ib_get_dma_mr(pd, IB_ACCESS_LOCAL_WRITE);
1070 if (IS_ERR(mr)) {
1071 mlx5_ib_dbg(dev, "Couldn't create DMA MR for sync UMR QP\n");
1072 ret = PTR_ERR(mr);
1073 goto error_1;
1074 }
1075
1076 cq = ib_create_cq(&dev->ib_dev, mlx5_umr_cq_handler, NULL, NULL, 128,
1077 0);
1078 if (IS_ERR(cq)) {
1079 mlx5_ib_dbg(dev, "Couldn't create CQ for sync UMR QP\n");
1080 ret = PTR_ERR(cq);
1081 goto error_2;
1082 }
1083 ib_req_notify_cq(cq, IB_CQ_NEXT_COMP);
1084
1085 init_attr->send_cq = cq;
1086 init_attr->recv_cq = cq;
1087 init_attr->sq_sig_type = IB_SIGNAL_ALL_WR;
1088 init_attr->cap.max_send_wr = MAX_UMR_WR;
1089 init_attr->cap.max_send_sge = 1;
1090 init_attr->qp_type = MLX5_IB_QPT_REG_UMR;
1091 init_attr->port_num = 1;
1092 qp = mlx5_ib_create_qp(pd, init_attr, NULL);
1093 if (IS_ERR(qp)) {
1094 mlx5_ib_dbg(dev, "Couldn't create sync UMR QP\n");
1095 ret = PTR_ERR(qp);
1096 goto error_3;
1097 }
1098 qp->device = &dev->ib_dev;
1099 qp->real_qp = qp;
1100 qp->uobject = NULL;
1101 qp->qp_type = MLX5_IB_QPT_REG_UMR;
1102
1103 attr->qp_state = IB_QPS_INIT;
1104 attr->port_num = 1;
1105 ret = mlx5_ib_modify_qp(qp, attr, IB_QP_STATE | IB_QP_PKEY_INDEX |
1106 IB_QP_PORT, NULL);
1107 if (ret) {
1108 mlx5_ib_dbg(dev, "Couldn't modify UMR QP\n");
1109 goto error_4;
1110 }
1111
1112 memset(attr, 0, sizeof(*attr));
1113 attr->qp_state = IB_QPS_RTR;
1114 attr->path_mtu = IB_MTU_256;
1115
1116 ret = mlx5_ib_modify_qp(qp, attr, IB_QP_STATE, NULL);
1117 if (ret) {
1118 mlx5_ib_dbg(dev, "Couldn't modify umr QP to rtr\n");
1119 goto error_4;
1120 }
1121
1122 memset(attr, 0, sizeof(*attr));
1123 attr->qp_state = IB_QPS_RTS;
1124 ret = mlx5_ib_modify_qp(qp, attr, IB_QP_STATE, NULL);
1125 if (ret) {
1126 mlx5_ib_dbg(dev, "Couldn't modify umr QP to rts\n");
1127 goto error_4;
1128 }
1129
1130 dev->umrc.qp = qp;
1131 dev->umrc.cq = cq;
1132 dev->umrc.mr = mr;
1133 dev->umrc.pd = pd;
1134
1135 sema_init(&dev->umrc.sem, MAX_UMR_WR);
1136 ret = mlx5_mr_cache_init(dev);
1137 if (ret) {
1138 mlx5_ib_warn(dev, "mr cache init failed %d\n", ret);
1139 goto error_4;
1140 }
1141
1142 kfree(attr);
1143 kfree(init_attr);
1144
1145 return 0;
1146
1147error_4:
1148 mlx5_ib_destroy_qp(qp);
1149
1150error_3:
1151 ib_destroy_cq(cq);
1152
1153error_2:
1154 ib_dereg_mr(mr);
1155
1156error_1:
1157 ib_dealloc_pd(pd);
1158
1159error_0:
1160 kfree(attr);
1161 kfree(init_attr);
1162 return ret;
1163}
1164
1165static int create_dev_resources(struct mlx5_ib_resources *devr)
1166{
1167 struct ib_srq_init_attr attr;
1168 struct mlx5_ib_dev *dev;
1169 int ret = 0;
1170
1171 dev = container_of(devr, struct mlx5_ib_dev, devr);
1172
1173 devr->p0 = mlx5_ib_alloc_pd(&dev->ib_dev, NULL, NULL);
1174 if (IS_ERR(devr->p0)) {
1175 ret = PTR_ERR(devr->p0);
1176 goto error0;
1177 }
1178 devr->p0->device = &dev->ib_dev;
1179 devr->p0->uobject = NULL;
1180 atomic_set(&devr->p0->usecnt, 0);
1181
1182 devr->c0 = mlx5_ib_create_cq(&dev->ib_dev, 1, 0, NULL, NULL);
1183 if (IS_ERR(devr->c0)) {
1184 ret = PTR_ERR(devr->c0);
1185 goto error1;
1186 }
1187 devr->c0->device = &dev->ib_dev;
1188 devr->c0->uobject = NULL;
1189 devr->c0->comp_handler = NULL;
1190 devr->c0->event_handler = NULL;
1191 devr->c0->cq_context = NULL;
1192 atomic_set(&devr->c0->usecnt, 0);
1193
1194 devr->x0 = mlx5_ib_alloc_xrcd(&dev->ib_dev, NULL, NULL);
1195 if (IS_ERR(devr->x0)) {
1196 ret = PTR_ERR(devr->x0);
1197 goto error2;
1198 }
1199 devr->x0->device = &dev->ib_dev;
1200 devr->x0->inode = NULL;
1201 atomic_set(&devr->x0->usecnt, 0);
1202 mutex_init(&devr->x0->tgt_qp_mutex);
1203 INIT_LIST_HEAD(&devr->x0->tgt_qp_list);
1204
1205 devr->x1 = mlx5_ib_alloc_xrcd(&dev->ib_dev, NULL, NULL);
1206 if (IS_ERR(devr->x1)) {
1207 ret = PTR_ERR(devr->x1);
1208 goto error3;
1209 }
1210 devr->x1->device = &dev->ib_dev;
1211 devr->x1->inode = NULL;
1212 atomic_set(&devr->x1->usecnt, 0);
1213 mutex_init(&devr->x1->tgt_qp_mutex);
1214 INIT_LIST_HEAD(&devr->x1->tgt_qp_list);
1215
1216 memset(&attr, 0, sizeof(attr));
1217 attr.attr.max_sge = 1;
1218 attr.attr.max_wr = 1;
1219 attr.srq_type = IB_SRQT_XRC;
1220 attr.ext.xrc.cq = devr->c0;
1221 attr.ext.xrc.xrcd = devr->x0;
1222
1223 devr->s0 = mlx5_ib_create_srq(devr->p0, &attr, NULL);
1224 if (IS_ERR(devr->s0)) {
1225 ret = PTR_ERR(devr->s0);
1226 goto error4;
1227 }
1228 devr->s0->device = &dev->ib_dev;
1229 devr->s0->pd = devr->p0;
1230 devr->s0->uobject = NULL;
1231 devr->s0->event_handler = NULL;
1232 devr->s0->srq_context = NULL;
1233 devr->s0->srq_type = IB_SRQT_XRC;
1234 devr->s0->ext.xrc.xrcd = devr->x0;
1235 devr->s0->ext.xrc.cq = devr->c0;
1236 atomic_inc(&devr->s0->ext.xrc.xrcd->usecnt);
1237 atomic_inc(&devr->s0->ext.xrc.cq->usecnt);
1238 atomic_inc(&devr->p0->usecnt);
1239 atomic_set(&devr->s0->usecnt, 0);
1240
1241 return 0;
1242
1243error4:
1244 mlx5_ib_dealloc_xrcd(devr->x1);
1245error3:
1246 mlx5_ib_dealloc_xrcd(devr->x0);
1247error2:
1248 mlx5_ib_destroy_cq(devr->c0);
1249error1:
1250 mlx5_ib_dealloc_pd(devr->p0);
1251error0:
1252 return ret;
1253}
1254
1255static void destroy_dev_resources(struct mlx5_ib_resources *devr)
1256{
1257 mlx5_ib_destroy_srq(devr->s0);
1258 mlx5_ib_dealloc_xrcd(devr->x0);
1259 mlx5_ib_dealloc_xrcd(devr->x1);
1260 mlx5_ib_destroy_cq(devr->c0);
1261 mlx5_ib_dealloc_pd(devr->p0);
1262}
1263
9603b61d 1264static void *mlx5_ib_add(struct mlx5_core_dev *mdev)
e126ba97 1265{
e126ba97
EC		 1266 struct mlx5_ib_dev *dev;
1267 int err;
1268 int i;
1269
1270 printk_once(KERN_INFO "%s", mlx5_version);
1271
1272 dev = (struct mlx5_ib_dev *)ib_alloc_device(sizeof(*dev));
1273 if (!dev)
9603b61d 1274 return NULL;
e126ba97 1275
9603b61d 1276 dev->mdev = mdev;
e126ba97
EC		 1277
1278 err = get_port_caps(dev);
1279 if (err)
9603b61d 1280 goto err_dealloc;
e126ba97
EC		 1281
1282 get_ext_port_caps(dev);
1283
1284 err = alloc_comp_eqs(dev);
1285 if (err)
9603b61d 1286 goto err_dealloc;
e126ba97
EC		 1287
1288 MLX5_INIT_DOORBELL_LOCK(&dev->uar_lock);
1289
1290 strlcpy(dev->ib_dev.name, "mlx5_%d", IB_DEVICE_NAME_MAX);
1291 dev->ib_dev.owner = THIS_MODULE;
1292 dev->ib_dev.node_type = RDMA_NODE_IB_CA;
c7a08ac7
EC		 1293 dev->ib_dev.local_dma_lkey = mdev->caps.gen.reserved_lkey;
1294 dev->num_ports = mdev->caps.gen.num_ports;
e126ba97
EC		 1295 dev->ib_dev.phys_port_cnt = dev->num_ports;
1296 dev->ib_dev.num_comp_vectors = dev->num_comp_vectors;
1297 dev->ib_dev.dma_device = &mdev->pdev->dev;
1298
1299 dev->ib_dev.uverbs_abi_ver = MLX5_IB_UVERBS_ABI_VERSION;
1300 dev->ib_dev.uverbs_cmd_mask =
1301 (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) |
1302 (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) |
1303 (1ull << IB_USER_VERBS_CMD_QUERY_PORT) |
1304 (1ull << IB_USER_VERBS_CMD_ALLOC_PD) |
1305 (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) |
1306 (1ull << IB_USER_VERBS_CMD_REG_MR) |
1307 (1ull << IB_USER_VERBS_CMD_DEREG_MR) |
1308 (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
1309 (1ull << IB_USER_VERBS_CMD_CREATE_CQ) |
1310 (1ull << IB_USER_VERBS_CMD_RESIZE_CQ) |
1311 (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) |
1312 (1ull << IB_USER_VERBS_CMD_CREATE_QP) |
1313 (1ull << IB_USER_VERBS_CMD_MODIFY_QP) |
1314 (1ull << IB_USER_VERBS_CMD_QUERY_QP) |
1315 (1ull << IB_USER_VERBS_CMD_DESTROY_QP) |
1316 (1ull << IB_USER_VERBS_CMD_ATTACH_MCAST) |
1317 (1ull << IB_USER_VERBS_CMD_DETACH_MCAST) |
1318 (1ull << IB_USER_VERBS_CMD_CREATE_SRQ) |
1319 (1ull << IB_USER_VERBS_CMD_MODIFY_SRQ) |
1320 (1ull << IB_USER_VERBS_CMD_QUERY_SRQ) |
1321 (1ull << IB_USER_VERBS_CMD_DESTROY_SRQ) |
1322 (1ull << IB_USER_VERBS_CMD_CREATE_XSRQ) |
1323 (1ull << IB_USER_VERBS_CMD_OPEN_QP);
1324
1325 dev->ib_dev.query_device = mlx5_ib_query_device;
1326 dev->ib_dev.query_port = mlx5_ib_query_port;
1327 dev->ib_dev.query_gid = mlx5_ib_query_gid;
1328 dev->ib_dev.query_pkey = mlx5_ib_query_pkey;
1329 dev->ib_dev.modify_device = mlx5_ib_modify_device;
1330 dev->ib_dev.modify_port = mlx5_ib_modify_port;
1331 dev->ib_dev.alloc_ucontext = mlx5_ib_alloc_ucontext;
1332 dev->ib_dev.dealloc_ucontext = mlx5_ib_dealloc_ucontext;
1333 dev->ib_dev.mmap = mlx5_ib_mmap;
1334 dev->ib_dev.alloc_pd = mlx5_ib_alloc_pd;
1335 dev->ib_dev.dealloc_pd = mlx5_ib_dealloc_pd;
1336 dev->ib_dev.create_ah = mlx5_ib_create_ah;
1337 dev->ib_dev.query_ah = mlx5_ib_query_ah;
1338 dev->ib_dev.destroy_ah = mlx5_ib_destroy_ah;
1339 dev->ib_dev.create_srq = mlx5_ib_create_srq;
1340 dev->ib_dev.modify_srq = mlx5_ib_modify_srq;
1341 dev->ib_dev.query_srq = mlx5_ib_query_srq;
1342 dev->ib_dev.destroy_srq = mlx5_ib_destroy_srq;
1343 dev->ib_dev.post_srq_recv = mlx5_ib_post_srq_recv;
1344 dev->ib_dev.create_qp = mlx5_ib_create_qp;
1345 dev->ib_dev.modify_qp = mlx5_ib_modify_qp;
1346 dev->ib_dev.query_qp = mlx5_ib_query_qp;
1347 dev->ib_dev.destroy_qp = mlx5_ib_destroy_qp;
1348 dev->ib_dev.post_send = mlx5_ib_post_send;
1349 dev->ib_dev.post_recv = mlx5_ib_post_recv;
1350 dev->ib_dev.create_cq = mlx5_ib_create_cq;
1351 dev->ib_dev.modify_cq = mlx5_ib_modify_cq;
1352 dev->ib_dev.resize_cq = mlx5_ib_resize_cq;
1353 dev->ib_dev.destroy_cq = mlx5_ib_destroy_cq;
1354 dev->ib_dev.poll_cq = mlx5_ib_poll_cq;
1355 dev->ib_dev.req_notify_cq = mlx5_ib_arm_cq;
1356 dev->ib_dev.get_dma_mr = mlx5_ib_get_dma_mr;
1357 dev->ib_dev.reg_user_mr = mlx5_ib_reg_user_mr;
1358 dev->ib_dev.dereg_mr = mlx5_ib_dereg_mr;
3121e3c4 1359 dev->ib_dev.destroy_mr = mlx5_ib_destroy_mr;
e126ba97
EC		 1360 dev->ib_dev.attach_mcast = mlx5_ib_mcg_attach;
1361 dev->ib_dev.detach_mcast = mlx5_ib_mcg_detach;
1362 dev->ib_dev.process_mad = mlx5_ib_process_mad;
3121e3c4 1363 dev->ib_dev.create_mr = mlx5_ib_create_mr;
e126ba97
EC		 1364 dev->ib_dev.alloc_fast_reg_mr = mlx5_ib_alloc_fast_reg_mr;
1365 dev->ib_dev.alloc_fast_reg_page_list = mlx5_ib_alloc_fast_reg_page_list;
1366 dev->ib_dev.free_fast_reg_page_list = mlx5_ib_free_fast_reg_page_list;
d5436ba0 1367 dev->ib_dev.check_mr_status = mlx5_ib_check_mr_status;
e126ba97 1368
c7a08ac7 1369 if (mdev->caps.gen.flags & MLX5_DEV_CAP_FLAG_XRC) {
e126ba97
EC		 1370 dev->ib_dev.alloc_xrcd = mlx5_ib_alloc_xrcd;
1371 dev->ib_dev.dealloc_xrcd = mlx5_ib_dealloc_xrcd;
1372 dev->ib_dev.uverbs_cmd_mask |=
1373 (1ull << IB_USER_VERBS_CMD_OPEN_XRCD) |
1374 (1ull << IB_USER_VERBS_CMD_CLOSE_XRCD);
1375 }
1376
1377 err = init_node_data(dev);
1378 if (err)
1379 goto err_eqs;
1380
1381 mutex_init(&dev->cap_mask_mutex);
1382 spin_lock_init(&dev->mr_lock);
1383
1384 err = create_dev_resources(&dev->devr);
1385 if (err)
1386 goto err_eqs;
1387
281d1a92
WY		 1388 err = ib_register_device(&dev->ib_dev, NULL);
1389 if (err)
e126ba97
EC		 1390 goto err_rsrc;
1391
1392 err = create_umr_res(dev);
1393 if (err)
1394 goto err_dev;
1395
1396 for (i = 0; i < ARRAY_SIZE(mlx5_class_attributes); i++) {
281d1a92
WY		 1397 err = device_create_file(&dev->ib_dev.dev,
1398 mlx5_class_attributes[i]);
1399 if (err)
e126ba97
EC		 1400 goto err_umrc;
1401 }
1402
1403 dev->ib_active = true;
1404
9603b61d 1405 return dev;
e126ba97
EC		 1406
1407err_umrc:
1408 destroy_umrc_res(dev);
1409
1410err_dev:
1411 ib_unregister_device(&dev->ib_dev);
1412
1413err_rsrc:
1414 destroy_dev_resources(&dev->devr);
1415
1416err_eqs:
1417 free_comp_eqs(dev);
1418
9603b61d 1419err_dealloc:
e126ba97
EC		 1420 ib_dealloc_device((struct ib_device *)dev);
1421
9603b61d 1422 return NULL;
e126ba97
EC		 1423}
1424
9603b61d 1425static void mlx5_ib_remove(struct mlx5_core_dev *mdev, void *context)
e126ba97 1426{
9603b61d 1427 struct mlx5_ib_dev *dev = context;
e126ba97 1428 ib_unregister_device(&dev->ib_dev);
eefd56e5 1429 destroy_umrc_res(dev);
e126ba97
EC		 1430 destroy_dev_resources(&dev->devr);
1431 free_comp_eqs(dev);
e126ba97
EC		 1432 ib_dealloc_device(&dev->ib_dev);
1433}
1434
9603b61d
JM		 1435static struct mlx5_interface mlx5_ib_interface = {
1436 .add = mlx5_ib_add,
1437 .remove = mlx5_ib_remove,
1438 .event = mlx5_ib_event,
e126ba97
EC		 1439};
1440
1441static int __init mlx5_ib_init(void)
1442{
9603b61d
JM		 1443 if (deprecated_prof_sel != 2)
1444 pr_warn("prof_sel is deprecated for mlx5_ib, set it for mlx5_core\n");
1445
1446 return mlx5_register_interface(&mlx5_ib_interface);
e126ba97
EC		 1447}
1448
1449static void __exit mlx5_ib_cleanup(void)
1450{
9603b61d 1451 mlx5_unregister_interface(&mlx5_ib_interface);
e126ba97
EC		 1452}
1453
1454module_init(mlx5_ib_init);
1455module_exit(mlx5_ib_cleanup);
Linux 6.x block layer and io_uring tree(s)