projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
net/mlx5: Remove unused CAPs
[linux-2.6-block.git] / drivers / net / ethernet / mellanox / mlx5 / core / main.c
CommitLineData
e126ba97 1/*
302bdf68 2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
e126ba97
EC		 3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
adec640e 33#include <linux/highmem.h>
e126ba97
EC		 34#include <linux/module.h>
35#include <linux/init.h>
36#include <linux/errno.h>
37#include <linux/pci.h>
38#include <linux/dma-mapping.h>
39#include <linux/slab.h>
db058a18 40#include <linux/interrupt.h>
e3297246 41#include <linux/delay.h>
e126ba97
EC		 42#include <linux/mlx5/driver.h>
43#include <linux/mlx5/cq.h>
44#include <linux/mlx5/qp.h>
e126ba97 45#include <linux/debugfs.h>
f66f049f 46#include <linux/kmod.h>
b775516b 47#include <linux/mlx5/mlx5_ifc.h>
c85023e1 48#include <linux/mlx5/vport.h>
907af0f0 49#include <linux/version.h>
feae9087 50#include <net/devlink.h>
e126ba97 51#include "mlx5_core.h"
f2f3df55 52#include "lib/eq.h"
16d76083 53#include "fs_core.h"
eeb66cdb 54#include "lib/mpfs.h"
073bb189 55#include "eswitch.h"
1f28d776 56#include "devlink.h"
38b9f903 57#include "fw_reset.h"
52ec462e 58#include "lib/mlx5.h"
5945e1ad 59#include "lib/tout.h"
e29341fb 60#include "fpga/core.h"
c6e3b421 61#include "en_accel/ipsec.h"
7c39afb3 62#include "lib/clock.h"
358aa5ce 63#include "lib/vxlan.h"
0ccc171e 64#include "lib/geneve.h"
fadd59fc 65#include "lib/devcom.h"
b25bbc2f 66#include "lib/pci_vsc.h"
24406953 67#include "diag/fw_tracer.h"
591905ba 68#include "ecpf.h"
87175120 69#include "lib/hv_vhca.h"
12206b17 70#include "diag/rsc_dump.h"
f3196bb0 71#include "sf/vhca_event.h"
90d010b8 72#include "sf/dev/dev.h"
6a327321 73#include "sf/sf.h"
3b43190b 74#include "mlx5_irq.h"
1f507e80 75#include "hwmon.h"
e126ba97 76
e126ba97 77MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>");
048f3143 78MODULE_DESCRIPTION("Mellanox 5th generation network adapters (ConnectX series) core driver");
e126ba97 79MODULE_LICENSE("Dual BSD/GPL");
e126ba97 80
f663ad98
KH		 81unsigned int mlx5_core_debug_mask;
82module_param_named(debug_mask, mlx5_core_debug_mask, uint, 0644);
e126ba97
EC		 83MODULE_PARM_DESC(debug_mask, "debug mask: 1 = dump cmd data, 2 = dump cmd exec time, 3 = both. Default=0");
84
f663ad98
KH		 85static unsigned int prof_sel = MLX5_DEFAULT_PROF;
86module_param_named(prof_sel, prof_sel, uint, 0444);
9603b61d
JM		 87MODULE_PARM_DESC(prof_sel, "profile selector. Valid range 0 - 2");
88
8737f818 89static u32 sw_owner_id[4];
dc402ccc
YH		 90#define MAX_SW_VHCA_ID (BIT(__mlx5_bit_sz(cmd_hca_cap_2, sw_vhca_id)) - 1)
91static DEFINE_IDA(sw_vhca_ida);
8737f818 92
f91e6d89
EBE		 93enum {
94 MLX5_ATOMIC_REQ_MODE_BE = 0x0,
95 MLX5_ATOMIC_REQ_MODE_HOST_ENDIANNESS = 0x1,
96};
97
f79a609e
MS		 98#define LOG_MAX_SUPPORTED_QPS 0xff
99
9603b61d
JM		 100static struct mlx5_profile profile[] = {
101 [0] = {
102 .mask = 0,
9df839a7 103 .num_cmd_caches = MLX5_NUM_COMMAND_CACHES,
9603b61d
JM		 104 },
105 [1] = {
106 .mask = MLX5_PROF_MASK_QP_SIZE,
107 .log_max_qp = 12,
9df839a7
PP		 108 .num_cmd_caches = MLX5_NUM_COMMAND_CACHES,
109
9603b61d
JM		 110 },
111 [2] = {
112 .mask = MLX5_PROF_MASK_QP_SIZE |
113 MLX5_PROF_MASK_MR_CACHE,
f79a609e 114 .log_max_qp = LOG_MAX_SUPPORTED_QPS,
9df839a7 115 .num_cmd_caches = MLX5_NUM_COMMAND_CACHES,
9603b61d
JM		 116 .mr_cache[0] = {
117 .size = 500,
118 .limit = 250
119 },
120 .mr_cache[1] = {
121 .size = 500,
122 .limit = 250
123 },
124 .mr_cache[2] = {
125 .size = 500,
126 .limit = 250
127 },
128 .mr_cache[3] = {
129 .size = 500,
130 .limit = 250
131 },
132 .mr_cache[4] = {
133 .size = 500,
134 .limit = 250
135 },
136 .mr_cache[5] = {
137 .size = 500,
138 .limit = 250
139 },
140 .mr_cache[6] = {
141 .size = 500,
142 .limit = 250
143 },
144 .mr_cache[7] = {
145 .size = 500,
146 .limit = 250
147 },
148 .mr_cache[8] = {
149 .size = 500,
150 .limit = 250
151 },
152 .mr_cache[9] = {
153 .size = 500,
154 .limit = 250
155 },
156 .mr_cache[10] = {
157 .size = 500,
158 .limit = 250
159 },
160 .mr_cache[11] = {
161 .size = 500,
162 .limit = 250
163 },
164 .mr_cache[12] = {
165 .size = 64,
166 .limit = 32
167 },
168 .mr_cache[13] = {
169 .size = 32,
170 .limit = 16
171 },
172 .mr_cache[14] = {
173 .size = 16,
174 .limit = 8
175 },
176 .mr_cache[15] = {
177 .size = 8,
178 .limit = 4
179 },
180 },
9df839a7
PP		 181 [3] = {
182 .mask = MLX5_PROF_MASK_QP_SIZE,
183 .log_max_qp = LOG_MAX_SUPPORTED_QPS,
184 .num_cmd_caches = 0,
185 },
9603b61d 186};
e126ba97 187
b8a92577
DJ		 188static int wait_fw_init(struct mlx5_core_dev *dev, u32 max_wait_mili,
189 u32 warn_time_mili)
e3297246 190{
b8a92577 191 unsigned long warn = jiffies + msecs_to_jiffies(warn_time_mili);
e3297246 192 unsigned long end = jiffies + msecs_to_jiffies(max_wait_mili);
cdfc6ffb 193 u32 fw_initializing;
e3297246
EC		 194 int err = 0;
195
cdfc6ffb
SD		 196 do {
197 fw_initializing = ioread32be(&dev->iseg->initializing);
198 if (!(fw_initializing >> 31))
199 break;
8324a02c 200 if (time_after(jiffies, end) ||
c05d145a 201 test_bit(MLX5_BREAK_FW_WAIT, &dev->intf_state)) {
e3297246
EC		 202 err = -EBUSY;
203 break;
204 }
b8a92577 205 if (warn_time_mili && time_after(jiffies, warn)) {
cdfc6ffb
SD		 206 mlx5_core_warn(dev, "Waiting for FW initialization, timeout abort in %ds (0x%x)\n",
207 jiffies_to_msecs(end - warn) / 1000, fw_initializing);
b8a92577
DJ		 208 warn = jiffies + msecs_to_jiffies(warn_time_mili);
209 }
5945e1ad 210 msleep(mlx5_tout_ms(dev, FW_PRE_INIT_WAIT));
cdfc6ffb 211 } while (true);
e3297246
EC		 212
213 return err;
214}
215
012e50e1
HN		 216static void mlx5_set_driver_version(struct mlx5_core_dev *dev)
217{
218 int driver_ver_sz = MLX5_FLD_SZ_BYTES(set_driver_version_in,
219 driver_version);
3ac0e69e 220 u8 in[MLX5_ST_SZ_BYTES(set_driver_version_in)] = {};
012e50e1
HN		 221 int remaining_size = driver_ver_sz;
222 char *string;
223
224 if (!MLX5_CAP_GEN(dev, driver_version))
225 return;
226
227 string = MLX5_ADDR_OF(set_driver_version_in, in, driver_version);
228
229 strncpy(string, "Linux", remaining_size);
230
231 remaining_size = max_t(int, 0, driver_ver_sz - strlen(string));
232 strncat(string, ",", remaining_size);
233
234 remaining_size = max_t(int, 0, driver_ver_sz - strlen(string));
17a7612b 235 strncat(string, KBUILD_MODNAME, remaining_size);
012e50e1
HN		 236
237 remaining_size = max_t(int, 0, driver_ver_sz - strlen(string));
238 strncat(string, ",", remaining_size);
239
240 remaining_size = max_t(int, 0, driver_ver_sz - strlen(string));
907af0f0
LR		 241
242 snprintf(string + strlen(string), remaining_size, "%u.%u.%u",
88a68672
SL		 243 LINUX_VERSION_MAJOR, LINUX_VERSION_PATCHLEVEL,
244 LINUX_VERSION_SUBLEVEL);
012e50e1
HN		 245
246 /*Send the command*/
247 MLX5_SET(set_driver_version_in, in, opcode,
248 MLX5_CMD_OP_SET_DRIVER_VERSION);
249
3ac0e69e 250 mlx5_cmd_exec_in(dev, set_driver_version, in);
012e50e1
HN		 251}
252
e126ba97
EC		 253static int set_dma_caps(struct pci_dev *pdev)
254{
255 int err;
256
eb9c5c0d 257 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
e126ba97 258 if (err) {
1a91de28 259 dev_warn(&pdev->dev, "Warning: couldn't set 64-bit PCI DMA mask\n");
eb9c5c0d 260 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
e126ba97 261 if (err) {
1a91de28 262 dev_err(&pdev->dev, "Can't set PCI DMA mask, aborting\n");
e126ba97
EC		 263 return err;
264 }
265 }
266
e126ba97
EC		 267 dma_set_max_seg_size(&pdev->dev, 2u * 1024 * 1024 * 1024);
268 return err;
269}
270
89d44f0a
MD		 271static int mlx5_pci_enable_device(struct mlx5_core_dev *dev)
272{
273 struct pci_dev *pdev = dev->pdev;
274 int err = 0;
275
276 mutex_lock(&dev->pci_status_mutex);
277 if (dev->pci_status == MLX5_PCI_STATUS_DISABLED) {
278 err = pci_enable_device(pdev);
279 if (!err)
280 dev->pci_status = MLX5_PCI_STATUS_ENABLED;
281 }
282 mutex_unlock(&dev->pci_status_mutex);
283
284 return err;
285}
286
287static void mlx5_pci_disable_device(struct mlx5_core_dev *dev)
288{
289 struct pci_dev *pdev = dev->pdev;
290
291 mutex_lock(&dev->pci_status_mutex);
292 if (dev->pci_status == MLX5_PCI_STATUS_ENABLED) {
293 pci_disable_device(pdev);
294 dev->pci_status = MLX5_PCI_STATUS_DISABLED;
295 }
296 mutex_unlock(&dev->pci_status_mutex);
297}
298
e126ba97
EC		 299static int request_bar(struct pci_dev *pdev)
300{
301 int err = 0;
302
303 if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM)) {
1a91de28 304 dev_err(&pdev->dev, "Missing registers BAR, aborting\n");
e126ba97
EC		 305 return -ENODEV;
306 }
307
17a7612b 308 err = pci_request_regions(pdev, KBUILD_MODNAME);
e126ba97
EC		 309 if (err)
310 dev_err(&pdev->dev, "Couldn't get PCI resources, aborting\n");
311
312 return err;
313}
314
315static void release_bar(struct pci_dev *pdev)
316{
317 pci_release_regions(pdev);
318}
319
bd10838a 320struct mlx5_reg_host_endianness {
e126ba97
EC		 321 u8 he;
322 u8 rsvd[15];
323};
324
2974ab6e 325static u16 to_fw_pkey_sz(struct mlx5_core_dev *dev, u32 size)
c7a08ac7
EC		 326{
327 switch (size) {
328 case 128:
329 return 0;
330 case 256:
331 return 1;
332 case 512:
333 return 2;
334 case 1024:
335 return 3;
336 case 2048:
337 return 4;
338 case 4096:
339 return 5;
340 default:
2974ab6e 341 mlx5_core_warn(dev, "invalid pkey table size %d\n", size);
c7a08ac7
EC		 342 return 0;
343 }
344}
345
c7d4e6ab
JP		 346void mlx5_core_uplink_netdev_set(struct mlx5_core_dev *dev, struct net_device *netdev)
347{
348 mutex_lock(&dev->mlx5e_res.uplink_netdev_lock);
349 dev->mlx5e_res.uplink_netdev = netdev;
350 mlx5_blocking_notifier_call_chain(dev, MLX5_DRIVER_EVENT_UPLINK_NETDEV,
351 netdev);
352 mutex_unlock(&dev->mlx5e_res.uplink_netdev_lock);
353}
354
355void mlx5_core_uplink_netdev_event_replay(struct mlx5_core_dev *dev)
356{
357 mutex_lock(&dev->mlx5e_res.uplink_netdev_lock);
358 mlx5_blocking_notifier_call_chain(dev, MLX5_DRIVER_EVENT_UPLINK_NETDEV,
359 dev->mlx5e_res.uplink_netdev);
360 mutex_unlock(&dev->mlx5e_res.uplink_netdev_lock);
361}
362EXPORT_SYMBOL(mlx5_core_uplink_netdev_event_replay);
363
b06e7de8
LR		 364static int mlx5_core_get_caps_mode(struct mlx5_core_dev *dev,
365 enum mlx5_cap_type cap_type,
366 enum mlx5_cap_mode cap_mode)
c7a08ac7 367{
b775516b
EC		 368 u8 in[MLX5_ST_SZ_BYTES(query_hca_cap_in)];
369 int out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out);
938fe83c
SM		 370 void *out, *hca_caps;
371 u16 opmod = (cap_type << 1) | (cap_mode & 0x01);
e126ba97
EC		 372 int err;
373
b775516b
EC		 374 memset(in, 0, sizeof(in));
375 out = kzalloc(out_sz, GFP_KERNEL);
c7a08ac7 376 if (!out)
e126ba97 377 return -ENOMEM;
938fe83c 378
b775516b
EC		 379 MLX5_SET(query_hca_cap_in, in, opcode, MLX5_CMD_OP_QUERY_HCA_CAP);
380 MLX5_SET(query_hca_cap_in, in, op_mod, opmod);
3ac0e69e 381 err = mlx5_cmd_exec_inout(dev, query_hca_cap, in, out);
c7a08ac7 382 if (err) {
938fe83c
SM		 383 mlx5_core_warn(dev,
384 "QUERY_HCA_CAP : type(%x) opmode(%x) Failed(%d)\n",
385 cap_type, cap_mode, err);
e126ba97
EC		 386 goto query_ex;
387 }
c7a08ac7 388
938fe83c
SM		 389 hca_caps = MLX5_ADDR_OF(query_hca_cap_out, out, capability);
390
391 switch (cap_mode) {
392 case HCA_CAP_OPMOD_GET_MAX:
48f02eef 393 memcpy(dev->caps.hca[cap_type]->max, hca_caps,
938fe83c
SM		 394 MLX5_UN_SZ_BYTES(hca_cap_union));
395 break;
396 case HCA_CAP_OPMOD_GET_CUR:
48f02eef 397 memcpy(dev->caps.hca[cap_type]->cur, hca_caps,
938fe83c
SM		 398 MLX5_UN_SZ_BYTES(hca_cap_union));
399 break;
400 default:
401 mlx5_core_warn(dev,
402 "Tried to query dev cap type(%x) with wrong opmode(%x)\n",
403 cap_type, cap_mode);
404 err = -EINVAL;
405 break;
406 }
c7a08ac7
EC		 407query_ex:
408 kfree(out);
409 return err;
410}
411
b06e7de8
LR		 412int mlx5_core_get_caps(struct mlx5_core_dev *dev, enum mlx5_cap_type cap_type)
413{
414 int ret;
415
416 ret = mlx5_core_get_caps_mode(dev, cap_type, HCA_CAP_OPMOD_GET_CUR);
417 if (ret)
418 return ret;
419 return mlx5_core_get_caps_mode(dev, cap_type, HCA_CAP_OPMOD_GET_MAX);
420}
421
a2a322f4 422static int set_caps(struct mlx5_core_dev *dev, void *in, int opmod)
c7a08ac7 423{
b775516b 424 MLX5_SET(set_hca_cap_in, in, opcode, MLX5_CMD_OP_SET_HCA_CAP);
f91e6d89 425 MLX5_SET(set_hca_cap_in, in, op_mod, opmod << 1);
3ac0e69e 426 return mlx5_cmd_exec_in(dev, set_hca_cap, in);
c7a08ac7
EC		 427}
428
a2a322f4 429static int handle_hca_cap_atomic(struct mlx5_core_dev *dev, void *set_ctx)
f91e6d89 430{
f91e6d89 431 void *set_hca_cap;
f91e6d89
EBE		 432 int req_endianness;
433 int err;
434
a2a322f4 435 if (!MLX5_CAP_GEN(dev, atomic))
f91e6d89 436 return 0;
a2a322f4
LR		 437
438 err = mlx5_core_get_caps(dev, MLX5_CAP_ATOMIC);
439 if (err)
440 return err;
f91e6d89
EBE		 441
442 req_endianness =
443 MLX5_CAP_ATOMIC(dev,
bd10838a 444 supported_atomic_req_8B_endianness_mode_1);
f91e6d89
EBE		 445
446 if (req_endianness != MLX5_ATOMIC_REQ_MODE_HOST_ENDIANNESS)
447 return 0;
448
f91e6d89
EBE		 449 set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, set_ctx, capability);
450
451 /* Set requestor to host endianness */
bd10838a 452 MLX5_SET(atomic_caps, set_hca_cap, atomic_req_8B_endianness_mode,
f91e6d89
EBE		 453 MLX5_ATOMIC_REQ_MODE_HOST_ENDIANNESS);
454
a2a322f4 455 return set_caps(dev, set_ctx, MLX5_SET_HCA_CAP_OP_MOD_ATOMIC);
f91e6d89
EBE		 456}
457
a2a322f4 458static int handle_hca_cap_odp(struct mlx5_core_dev *dev, void *set_ctx)
46861e3e 459{
46861e3e 460 void *set_hca_cap;
fca22e7e 461 bool do_set = false;
46861e3e
MS		 462 int err;
463
37b6bb77
LR		 464 if (!IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING) ||
465 !MLX5_CAP_GEN(dev, pg))
46861e3e
MS		 466 return 0;
467
468 err = mlx5_core_get_caps(dev, MLX5_CAP_ODP);
469 if (err)
470 return err;
471
46861e3e 472 set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, set_ctx, capability);
48f02eef 473 memcpy(set_hca_cap, dev->caps.hca[MLX5_CAP_ODP]->cur,
46861e3e
MS		 474 MLX5_ST_SZ_BYTES(odp_cap));
475
fca22e7e
MS		 476#define ODP_CAP_SET_MAX(dev, field) \
477 do { \
478 u32 _res = MLX5_CAP_ODP_MAX(dev, field); \
479 if (_res) { \
480 do_set = true; \
481 MLX5_SET(odp_cap, set_hca_cap, field, _res); \
482 } \
483 } while (0)
484
485 ODP_CAP_SET_MAX(dev, ud_odp_caps.srq_receive);
486 ODP_CAP_SET_MAX(dev, rc_odp_caps.srq_receive);
487 ODP_CAP_SET_MAX(dev, xrc_odp_caps.srq_receive);
488 ODP_CAP_SET_MAX(dev, xrc_odp_caps.send);
489 ODP_CAP_SET_MAX(dev, xrc_odp_caps.receive);
490 ODP_CAP_SET_MAX(dev, xrc_odp_caps.write);
491 ODP_CAP_SET_MAX(dev, xrc_odp_caps.read);
492 ODP_CAP_SET_MAX(dev, xrc_odp_caps.atomic);
00679b63
MG		 493 ODP_CAP_SET_MAX(dev, dc_odp_caps.srq_receive);
494 ODP_CAP_SET_MAX(dev, dc_odp_caps.send);
495 ODP_CAP_SET_MAX(dev, dc_odp_caps.receive);
496 ODP_CAP_SET_MAX(dev, dc_odp_caps.write);
497 ODP_CAP_SET_MAX(dev, dc_odp_caps.read);
498 ODP_CAP_SET_MAX(dev, dc_odp_caps.atomic);
fca22e7e 499
a2a322f4
LR		 500 if (!do_set)
501 return 0;
fca22e7e 502
a2a322f4 503 return set_caps(dev, set_ctx, MLX5_SET_HCA_CAP_OP_MOD_ODP);
46861e3e
MS		 504}
505
8680a60f
SD		 506static int max_uc_list_get_devlink_param(struct mlx5_core_dev *dev)
507{
508 struct devlink *devlink = priv_to_devlink(dev);
509 union devlink_param_value val;
510 int err;
511
075935f0
JP		 512 err = devl_param_driverinit_value_get(devlink,
513 DEVLINK_PARAM_GENERIC_ID_MAX_MACS,
514 &val);
8680a60f
SD		 515 if (!err)
516 return val.vu32;
517 mlx5_core_dbg(dev, "Failed to get param. err = %d\n", err);
518 return err;
519}
520
9ca05b0f
MS		 521bool mlx5_is_roce_on(struct mlx5_core_dev *dev)
522{
523 struct devlink *devlink = priv_to_devlink(dev);
524 union devlink_param_value val;
525 int err;
526
075935f0
JP		 527 err = devl_param_driverinit_value_get(devlink,
528 DEVLINK_PARAM_GENERIC_ID_ENABLE_ROCE,
529 &val);
9ca05b0f
MS		 530
531 if (!err)
532 return val.vbool;
533
534 mlx5_core_dbg(dev, "Failed to get param. err = %d\n", err);
535 return MLX5_CAP_GEN(dev, roce);
536}
537EXPORT_SYMBOL(mlx5_is_roce_on);
538
dc402ccc
YH		 539static int handle_hca_cap_2(struct mlx5_core_dev *dev, void *set_ctx)
540{
541 void *set_hca_cap;
542 int err;
543
544 if (!MLX5_CAP_GEN_MAX(dev, hca_cap_2))
545 return 0;
546
547 err = mlx5_core_get_caps(dev, MLX5_CAP_GENERAL_2);
548 if (err)
549 return err;
550
551 if (!MLX5_CAP_GEN_2_MAX(dev, sw_vhca_id_valid) ||
552 !(dev->priv.sw_vhca_id > 0))
553 return 0;
554
555 set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, set_ctx,
556 capability);
557 memcpy(set_hca_cap, dev->caps.hca[MLX5_CAP_GENERAL_2]->cur,
558 MLX5_ST_SZ_BYTES(cmd_hca_cap_2));
559 MLX5_SET(cmd_hca_cap_2, set_hca_cap, sw_vhca_id_valid, 1);
560
561 return set_caps(dev, set_ctx, MLX5_CAP_GENERAL_2);
562}
563
a2a322f4 564static int handle_hca_cap(struct mlx5_core_dev *dev, void *set_ctx)
c7a08ac7 565{
3410fbcd 566 struct mlx5_profile *prof = &dev->profile;
938fe83c 567 void *set_hca_cap;
8680a60f 568 int max_uc_list;
a2a322f4 569 int err;
e126ba97 570
b06e7de8 571 err = mlx5_core_get_caps(dev, MLX5_CAP_GENERAL);
e126ba97 572 if (err)
a2a322f4 573 return err;
e126ba97 574
938fe83c
SM		 575 set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, set_ctx,
576 capability);
48f02eef 577 memcpy(set_hca_cap, dev->caps.hca[MLX5_CAP_GENERAL]->cur,
938fe83c
SM		 578 MLX5_ST_SZ_BYTES(cmd_hca_cap));
579
580 mlx5_core_dbg(dev, "Current Pkey table size %d Setting new size %d\n",
707c4602 581 mlx5_to_sw_pkey_sz(MLX5_CAP_GEN(dev, pkey_table_size)),
938fe83c 582 128);
c7a08ac7 583 /* we limit the size of the pkey table to 128 entries for now */
938fe83c 584 MLX5_SET(cmd_hca_cap, set_hca_cap, pkey_table_size,
2974ab6e 585 to_fw_pkey_sz(dev, 128));
c7a08ac7 586
883371c4 587 /* Check log_max_qp from HCA caps to set in current profile */
f79a609e 588 if (prof->log_max_qp == LOG_MAX_SUPPORTED_QPS) {
a6e9085d 589 prof->log_max_qp = min_t(u8, 18, MLX5_CAP_GEN_MAX(dev, log_max_qp));
f79a609e 590 } else if (MLX5_CAP_GEN_MAX(dev, log_max_qp) < prof->log_max_qp) {
883371c4 591 mlx5_core_warn(dev, "log_max_qp value in current profile is %d, changing it to HCA capability limit (%d)\n",
3410fbcd 592 prof->log_max_qp,
883371c4 593 MLX5_CAP_GEN_MAX(dev, log_max_qp));
3410fbcd 594 prof->log_max_qp = MLX5_CAP_GEN_MAX(dev, log_max_qp);
883371c4 595 }
c7a08ac7 596 if (prof->mask & MLX5_PROF_MASK_QP_SIZE)
938fe83c
SM		 597 MLX5_SET(cmd_hca_cap, set_hca_cap, log_max_qp,
598 prof->log_max_qp);
c7a08ac7 599
938fe83c
SM		 600 /* disable cmdif checksum */
601 MLX5_SET(cmd_hca_cap, set_hca_cap, cmdif_checksum, 0);
c7a08ac7 602
91828bd8
MD		 603 /* Enable 4K UAR only when HCA supports it and page size is bigger
604 * than 4K.
605 */
606 if (MLX5_CAP_GEN_MAX(dev, uar_4k) && PAGE_SIZE > 4096)
f502d834
EC		 607 MLX5_SET(cmd_hca_cap, set_hca_cap, uar_4k, 1);
608
fe1e1876
CS		 609 MLX5_SET(cmd_hca_cap, set_hca_cap, log_uar_page_sz, PAGE_SHIFT - 12);
610
f32f5bd2
DJ		 611 if (MLX5_CAP_GEN_MAX(dev, cache_line_128byte))
612 MLX5_SET(cmd_hca_cap,
613 set_hca_cap,
614 cache_line_128byte,
c67f100e 615 cache_line_size() >= 128 ? 1 : 0);
f32f5bd2 616
dd44572a
MS		 617 if (MLX5_CAP_GEN_MAX(dev, dct))
618 MLX5_SET(cmd_hca_cap, set_hca_cap, dct, 1);
619
e7f4d0bc
MS		 620 if (MLX5_CAP_GEN_MAX(dev, pci_sync_for_fw_update_event))
621 MLX5_SET(cmd_hca_cap, set_hca_cap, pci_sync_for_fw_update_event, 1);
7a9770f1
MS		 622 if (MLX5_CAP_GEN_MAX(dev, pci_sync_for_fw_update_with_driver_unload))
623 MLX5_SET(cmd_hca_cap, set_hca_cap,
624 pci_sync_for_fw_update_with_driver_unload, 1);
e7f4d0bc 625
c4b76d8d
DJ		 626 if (MLX5_CAP_GEN_MAX(dev, num_vhca_ports))
627 MLX5_SET(cmd_hca_cap,
628 set_hca_cap,
629 num_vhca_ports,
630 MLX5_CAP_GEN_MAX(dev, num_vhca_ports));
631
c6168161
EBE		 632 if (MLX5_CAP_GEN_MAX(dev, release_all_pages))
633 MLX5_SET(cmd_hca_cap, set_hca_cap, release_all_pages, 1);
634
4dca6509
MG		 635 if (MLX5_CAP_GEN_MAX(dev, mkey_by_name))
636 MLX5_SET(cmd_hca_cap, set_hca_cap, mkey_by_name, 1);
637
f3196bb0
PP		 638 mlx5_vhca_state_cap_handle(dev, set_hca_cap);
639
604774ad
LR		 640 if (MLX5_CAP_GEN_MAX(dev, num_total_dynamic_vf_msix))
641 MLX5_SET(cmd_hca_cap, set_hca_cap, num_total_dynamic_vf_msix,
642 MLX5_CAP_GEN_MAX(dev, num_total_dynamic_vf_msix));
643
c4ad5f2b 644 if (MLX5_CAP_GEN(dev, roce_rw_supported) && MLX5_CAP_GEN_MAX(dev, roce))
9ca05b0f
MS		 645 MLX5_SET(cmd_hca_cap, set_hca_cap, roce,
646 mlx5_is_roce_on(dev));
fbfa97b4 647
8680a60f
SD		 648 max_uc_list = max_uc_list_get_devlink_param(dev);
649 if (max_uc_list > 0)
650 MLX5_SET(cmd_hca_cap, set_hca_cap, log_max_current_uc_list,
651 ilog2(max_uc_list));
652
a2a322f4 653 return set_caps(dev, set_ctx, MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE);
e126ba97 654}
c7a08ac7 655
fbfa97b4
SD		 656/* Cached MLX5_CAP_GEN(dev, roce) can be out of sync this early in the
657 * boot process.
658 * In case RoCE cap is writable in FW and user/devlink requested to change the
659 * cap, we are yet to query the final state of the above cap.
660 * Hence, the need for this function.
661 *
662 * Returns
663 * True:
664 * 1) RoCE cap is read only in FW and already disabled
665 * OR:
666 * 2) RoCE cap is writable in FW and user/devlink requested it off.
667 *
668 * In any other case, return False.
669 */
670static bool is_roce_fw_disabled(struct mlx5_core_dev *dev)
671{
9ca05b0f 672 return (MLX5_CAP_GEN(dev, roce_rw_supported) && !mlx5_is_roce_on(dev)) ||
fbfa97b4
SD		 673 (!MLX5_CAP_GEN(dev, roce_rw_supported) && !MLX5_CAP_GEN(dev, roce));
674}
675
59e9e8e4
MZ		 676static int handle_hca_cap_roce(struct mlx5_core_dev *dev, void *set_ctx)
677{
678 void *set_hca_cap;
679 int err;
680
fbfa97b4 681 if (is_roce_fw_disabled(dev))
59e9e8e4
MZ		 682 return 0;
683
684 err = mlx5_core_get_caps(dev, MLX5_CAP_ROCE);
685 if (err)
686 return err;
687
688 if (MLX5_CAP_ROCE(dev, sw_r_roce_src_udp_port) ||
689 !MLX5_CAP_ROCE_MAX(dev, sw_r_roce_src_udp_port))
690 return 0;
691
692 set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, set_ctx, capability);
48f02eef 693 memcpy(set_hca_cap, dev->caps.hca[MLX5_CAP_ROCE]->cur,
59e9e8e4
MZ		 694 MLX5_ST_SZ_BYTES(roce_cap));
695 MLX5_SET(roce_cap, set_hca_cap, sw_r_roce_src_udp_port, 1);
696
f4244e55
OHT		 697 if (MLX5_CAP_ROCE_MAX(dev, qp_ooo_transmit_default))
698 MLX5_SET(roce_cap, set_hca_cap, qp_ooo_transmit_default, 1);
699
59e9e8e4 700 err = set_caps(dev, set_ctx, MLX5_SET_HCA_CAP_OP_MOD_ROCE);
e126ba97
EC		 701 return err;
702}
703
90b1df74
LC		 704static int handle_hca_cap_port_selection(struct mlx5_core_dev *dev,
705 void *set_ctx)
706{
707 void *set_hca_cap;
708 int err;
709
710 if (!MLX5_CAP_GEN(dev, port_selection_cap))
711 return 0;
712
713 err = mlx5_core_get_caps(dev, MLX5_CAP_PORT_SELECTION);
714 if (err)
715 return err;
716
717 if (MLX5_CAP_PORT_SELECTION(dev, port_select_flow_table_bypass) ||
718 !MLX5_CAP_PORT_SELECTION_MAX(dev, port_select_flow_table_bypass))
719 return 0;
720
721 set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, set_ctx, capability);
722 memcpy(set_hca_cap, dev->caps.hca[MLX5_CAP_PORT_SELECTION]->cur,
723 MLX5_ST_SZ_BYTES(port_selection_cap));
724 MLX5_SET(port_selection_cap, set_hca_cap, port_select_flow_table_bypass, 1);
725
f9c895a7 726 err = set_caps(dev, set_ctx, MLX5_SET_HCA_CAP_OP_MOD_PORT_SELECTION);
90b1df74
LC		 727
728 return err;
729}
730
37b6bb77
LR		 731static int set_hca_cap(struct mlx5_core_dev *dev)
732{
a2a322f4
LR		 733 int set_sz = MLX5_ST_SZ_BYTES(set_hca_cap_in);
734 void *set_ctx;
37b6bb77
LR		 735 int err;
736
a2a322f4
LR		 737 set_ctx = kzalloc(set_sz, GFP_KERNEL);
738 if (!set_ctx)
739 return -ENOMEM;
740
741 err = handle_hca_cap(dev, set_ctx);
37b6bb77 742 if (err) {
98a8e6fc 743 mlx5_core_err(dev, "handle_hca_cap failed\n");
37b6bb77
LR		 744 goto out;
745 }
746
a2a322f4
LR		 747 memset(set_ctx, 0, set_sz);
748 err = handle_hca_cap_atomic(dev, set_ctx);
37b6bb77 749 if (err) {
98a8e6fc 750 mlx5_core_err(dev, "handle_hca_cap_atomic failed\n");
37b6bb77
LR		 751 goto out;
752 }
753
a2a322f4
LR		 754 memset(set_ctx, 0, set_sz);
755 err = handle_hca_cap_odp(dev, set_ctx);
37b6bb77 756 if (err) {
98a8e6fc 757 mlx5_core_err(dev, "handle_hca_cap_odp failed\n");
37b6bb77
LR		 758 goto out;
759 }
760
59e9e8e4
MZ		 761 memset(set_ctx, 0, set_sz);
762 err = handle_hca_cap_roce(dev, set_ctx);
763 if (err) {
764 mlx5_core_err(dev, "handle_hca_cap_roce failed\n");
765 goto out;
766 }
767
dc402ccc
YH		 768 memset(set_ctx, 0, set_sz);
769 err = handle_hca_cap_2(dev, set_ctx);
770 if (err) {
771 mlx5_core_err(dev, "handle_hca_cap_2 failed\n");
772 goto out;
773 }
774
90b1df74
LC		 775 memset(set_ctx, 0, set_sz);
776 err = handle_hca_cap_port_selection(dev, set_ctx);
777 if (err) {
778 mlx5_core_err(dev, "handle_hca_cap_port_selection failed\n");
779 goto out;
780 }
781
37b6bb77 782out:
a2a322f4 783 kfree(set_ctx);
37b6bb77
LR		 784 return err;
785}
786
e126ba97
EC		 787static int set_hca_ctrl(struct mlx5_core_dev *dev)
788{
bd10838a
OG		 789 struct mlx5_reg_host_endianness he_in;
790 struct mlx5_reg_host_endianness he_out;
e126ba97
EC		 791 int err;
792
fc50db98
EC		 793 if (!mlx5_core_is_pf(dev))
794 return 0;
795
e126ba97
EC		 796 memset(&he_in, 0, sizeof(he_in));
797 he_in.he = MLX5_SET_HOST_ENDIANNESS;
798 err = mlx5_core_access_reg(dev, &he_in, sizeof(he_in),
799 &he_out, sizeof(he_out),
800 MLX5_REG_HOST_ENDIANNESS, 0, 1);
801 return err;
802}
803
c85023e1
HN		 804static int mlx5_core_set_hca_defaults(struct mlx5_core_dev *dev)
805{
806 int ret = 0;
807
808 /* Disable local_lb by default */
8978cc92 809 if (MLX5_CAP_GEN(dev, port_type) == MLX5_CAP_PORT_TYPE_ETH)
c85023e1
HN		 810 ret = mlx5_nic_vport_update_local_lb(dev, false);
811
812 return ret;
813}
814
0b107106 815int mlx5_core_enable_hca(struct mlx5_core_dev *dev, u16 func_id)
cd23b14b 816{
3ac0e69e 817 u32 in[MLX5_ST_SZ_DW(enable_hca_in)] = {};
cd23b14b 818
0b107106
EC		 819 MLX5_SET(enable_hca_in, in, opcode, MLX5_CMD_OP_ENABLE_HCA);
820 MLX5_SET(enable_hca_in, in, function_id, func_id);
22e939a9
BW		 821 MLX5_SET(enable_hca_in, in, embedded_cpu_function,
822 dev->caps.embedded_cpu);
3ac0e69e 823 return mlx5_cmd_exec_in(dev, enable_hca, in);
cd23b14b
EC		 824}
825
0b107106 826int mlx5_core_disable_hca(struct mlx5_core_dev *dev, u16 func_id)
cd23b14b 827{
3ac0e69e 828 u32 in[MLX5_ST_SZ_DW(disable_hca_in)] = {};
cd23b14b 829
0b107106
EC		 830 MLX5_SET(disable_hca_in, in, opcode, MLX5_CMD_OP_DISABLE_HCA);
831 MLX5_SET(disable_hca_in, in, function_id, func_id);
22e939a9
BW		 832 MLX5_SET(enable_hca_in, in, embedded_cpu_function,
833 dev->caps.embedded_cpu);
3ac0e69e 834 return mlx5_cmd_exec_in(dev, disable_hca, in);
cd23b14b
EC		 835}
836
f62b8bb8
AV		 837static int mlx5_core_set_issi(struct mlx5_core_dev *dev)
838{
3ac0e69e
LR		 839 u32 query_out[MLX5_ST_SZ_DW(query_issi_out)] = {};
840 u32 query_in[MLX5_ST_SZ_DW(query_issi_in)] = {};
f62b8bb8 841 u32 sup_issi;
c4f287c4 842 int err;
f62b8bb8
AV		 843
844 MLX5_SET(query_issi_in, query_in, opcode, MLX5_CMD_OP_QUERY_ISSI);
3ac0e69e 845 err = mlx5_cmd_exec_inout(dev, query_issi, query_in, query_out);
f62b8bb8 846 if (err) {
605bef00
SM		 847 u32 syndrome = MLX5_GET(query_issi_out, query_out, syndrome);
848 u8 status = MLX5_GET(query_issi_out, query_out, status);
c4f287c4 849
f9c14e46
KH		 850 if (!status || syndrome == MLX5_DRIVER_SYND) {
851 mlx5_core_err(dev, "Failed to query ISSI err(%d) status(%d) synd(%d)\n",
852 err, status, syndrome);
853 return err;
f62b8bb8
AV		 854 }
855
f9c14e46
KH		 856 mlx5_core_warn(dev, "Query ISSI is not supported by FW, ISSI is 0\n");
857 dev->issi = 0;
858 return 0;
f62b8bb8
AV		 859 }
860
861 sup_issi = MLX5_GET(query_issi_out, query_out, supported_issi_dw0);
862
863 if (sup_issi & (1 << 1)) {
3ac0e69e 864 u32 set_in[MLX5_ST_SZ_DW(set_issi_in)] = {};
f62b8bb8
AV		 865
866 MLX5_SET(set_issi_in, set_in, opcode, MLX5_CMD_OP_SET_ISSI);
867 MLX5_SET(set_issi_in, set_in, current_issi, 1);
3ac0e69e 868 err = mlx5_cmd_exec_in(dev, set_issi, set_in);
f62b8bb8 869 if (err) {
f9c14e46
KH		 870 mlx5_core_err(dev, "Failed to set ISSI to 1 err(%d)\n",
871 err);
f62b8bb8
AV		 872 return err;
873 }
874
875 dev->issi = 1;
876
877 return 0;
e74a1db0 878 } else if (sup_issi & (1 << 0) || !sup_issi) {
f62b8bb8
AV		 879 return 0;
880 }
881
9eb78923 882 return -EOPNOTSUPP;
f62b8bb8 883}
f62b8bb8 884
11f3b84d
SM		 885static int mlx5_pci_init(struct mlx5_core_dev *dev, struct pci_dev *pdev,
886 const struct pci_device_id *id)
a31208b1 887{
a31208b1 888 int err = 0;
e126ba97 889
d22663ed 890 mutex_init(&dev->pci_status_mutex);
11f3b84d 891 pci_set_drvdata(dev->pdev, dev);
311c7c71 892
aa8106f1 893 dev->bar_addr = pci_resource_start(pdev, 0);
311c7c71 894
89d44f0a 895 err = mlx5_pci_enable_device(dev);
e126ba97 896 if (err) {
98a8e6fc 897 mlx5_core_err(dev, "Cannot enable PCI device, aborting\n");
11f3b84d 898 return err;
e126ba97
EC		 899 }
900
901 err = request_bar(pdev);
902 if (err) {
98a8e6fc 903 mlx5_core_err(dev, "error requesting BARs, aborting\n");
e126ba97
EC		 904 goto err_disable;
905 }
906
907 pci_set_master(pdev);
908
909 err = set_dma_caps(pdev);
910 if (err) {
98a8e6fc 911 mlx5_core_err(dev, "Failed setting DMA capabilities mask, aborting\n");
e126ba97
EC		 912 goto err_clr_master;
913 }
914
ce4eee53
MG		 915 if (pci_enable_atomic_ops_to_root(pdev, PCI_EXP_DEVCAP2_ATOMIC_COMP32) &&
916 pci_enable_atomic_ops_to_root(pdev, PCI_EXP_DEVCAP2_ATOMIC_COMP64) &&
917 pci_enable_atomic_ops_to_root(pdev, PCI_EXP_DEVCAP2_ATOMIC_COMP128))
918 mlx5_core_dbg(dev, "Enabling pci atomics failed\n");
919
aa8106f1 920 dev->iseg_base = dev->bar_addr;
e126ba97
EC		 921 dev->iseg = ioremap(dev->iseg_base, sizeof(*dev->iseg));
922 if (!dev->iseg) {
923 err = -ENOMEM;
98a8e6fc 924 mlx5_core_err(dev, "Failed mapping initialization segment, aborting\n");
e126ba97
EC		 925 goto err_clr_master;
926 }
a31208b1 927
b25bbc2f 928 mlx5_pci_vsc_init(dev);
a31208b1
MD		 929 return 0;
930
931err_clr_master:
a31208b1
MD		 932 release_bar(dev->pdev);
933err_disable:
89d44f0a 934 mlx5_pci_disable_device(dev);
a31208b1
MD		 935 return err;
936}
937
868bc06b 938static void mlx5_pci_close(struct mlx5_core_dev *dev)
a31208b1 939{
42ea9f1b
SD		 940 /* health work might still be active, and it needs pci bar in
941 * order to know the NIC state. Therefore, drain the health WQ
942 * before removing the pci bars
943 */
944 mlx5_drain_health_wq(dev);
a31208b1 945 iounmap(dev->iseg);
a31208b1 946 release_bar(dev->pdev);
89d44f0a 947 mlx5_pci_disable_device(dev);
a31208b1
MD		 948}
949
868bc06b 950static int mlx5_init_once(struct mlx5_core_dev *dev)
59211bd3 951{
59211bd3
MHY		 952 int err;
953
88d162b4
RD		 954 dev->priv.devc = mlx5_devcom_register_device(dev);
955 if (IS_ERR(dev->priv.devc))
956 mlx5_core_warn(dev, "failed to register devcom device %ld\n",
957 PTR_ERR(dev->priv.devc));
fadd59fc 958
59211bd3
MHY		 959 err = mlx5_query_board_id(dev);
960 if (err) {
98a8e6fc 961 mlx5_core_err(dev, "query board id failed\n");
fadd59fc 962 goto err_devcom;
59211bd3
MHY		 963 }
964
561aa15a
YA		 965 err = mlx5_irq_table_init(dev);
966 if (err) {
967 mlx5_core_err(dev, "failed to initialize irq table\n");
968 goto err_devcom;
969 }
970
f2f3df55 971 err = mlx5_eq_table_init(dev);
59211bd3 972 if (err) {
98a8e6fc 973 mlx5_core_err(dev, "failed to initialize eq\n");
561aa15a 974 goto err_irq_cleanup;
59211bd3
MHY		 975 }
976
69c1280b
SM		 977 err = mlx5_events_init(dev);
978 if (err) {
98a8e6fc 979 mlx5_core_err(dev, "failed to initialize events\n");
69c1280b
SM		 980 goto err_eq_cleanup;
981 }
982
38b9f903
MS		 983 err = mlx5_fw_reset_init(dev);
984 if (err) {
985 mlx5_core_err(dev, "failed to initialize fw reset events\n");
986 goto err_events_cleanup;
987 }
988
9f818c8a 989 mlx5_cq_debugfs_init(dev);
59211bd3 990
52ec462e
IT		 991 mlx5_init_reserved_gids(dev);
992
7c39afb3
FD		 993 mlx5_init_clock(dev);
994
358aa5ce 995 dev->vxlan = mlx5_vxlan_create(dev);
0ccc171e 996 dev->geneve = mlx5_geneve_create(dev);
358aa5ce 997
59211bd3
MHY		 998 err = mlx5_init_rl_table(dev);
999 if (err) {
98a8e6fc 1000 mlx5_core_err(dev, "Failed to init rate limiting\n");
59211bd3
MHY		 1001 goto err_tables_cleanup;
1002 }
1003
eeb66cdb
SM		 1004 err = mlx5_mpfs_init(dev);
1005 if (err) {
98a8e6fc 1006 mlx5_core_err(dev, "Failed to init l2 table %d\n", err);
eeb66cdb
SM		 1007 goto err_rl_cleanup;
1008 }
1009
86eec50b 1010 err = mlx5_sriov_init(dev);
c2d6e31a 1011 if (err) {
86eec50b 1012 mlx5_core_err(dev, "Failed to init sriov %d\n", err);
eeb66cdb 1013 goto err_mpfs_cleanup;
c2d6e31a 1014 }
c2d6e31a 1015
86eec50b 1016 err = mlx5_eswitch_init(dev);
c2d6e31a 1017 if (err) {
86eec50b
BW		 1018 mlx5_core_err(dev, "Failed to init eswitch %d\n", err);
1019 goto err_sriov_cleanup;
c2d6e31a
MHY		 1020 }
1021
9410733c
IT		 1022 err = mlx5_fpga_init(dev);
1023 if (err) {
98a8e6fc 1024 mlx5_core_err(dev, "Failed to init fpga device %d\n", err);
86eec50b 1025 goto err_eswitch_cleanup;
9410733c
IT		 1026 }
1027
f3196bb0
PP		 1028 err = mlx5_vhca_event_init(dev);
1029 if (err) {
1030 mlx5_core_err(dev, "Failed to init vhca event notifier %d\n", err);
1031 goto err_fpga_cleanup;
1032 }
1033
8f010541
PP		 1034 err = mlx5_sf_hw_table_init(dev);
1035 if (err) {
1036 mlx5_core_err(dev, "Failed to init SF HW table %d\n", err);
1037 goto err_sf_hw_table_cleanup;
1038 }
1039
1040 err = mlx5_sf_table_init(dev);
1041 if (err) {
1042 mlx5_core_err(dev, "Failed to init SF table %d\n", err);
1043 goto err_sf_table_cleanup;
1044 }
1045
b3388697
SD		 1046 err = mlx5_fs_core_alloc(dev);
1047 if (err) {
1048 mlx5_core_err(dev, "Failed to alloc flow steering\n");
1049 goto err_fs;
1050 }
1051
c9b9dcb4
AL		 1052 dev->dm = mlx5_dm_create(dev);
1053 if (IS_ERR(dev->dm))
a6573514 1054 mlx5_core_warn(dev, "Failed to init device memory %ld\n", PTR_ERR(dev->dm));
c9b9dcb4 1055
24406953 1056 dev->tracer = mlx5_fw_tracer_create(dev);
87175120 1057 dev->hv_vhca = mlx5_hv_vhca_create(dev);
12206b17 1058 dev->rsc_dump = mlx5_rsc_dump_create(dev);
24406953 1059
59211bd3
MHY		 1060 return 0;
1061
b3388697
SD		 1062err_fs:
1063 mlx5_sf_table_cleanup(dev);
8f010541
PP		 1064err_sf_table_cleanup:
1065 mlx5_sf_hw_table_cleanup(dev);
1066err_sf_hw_table_cleanup:
1067 mlx5_vhca_event_cleanup(dev);
f3196bb0
PP		 1068err_fpga_cleanup:
1069 mlx5_fpga_cleanup(dev);
c2d6e31a 1070err_eswitch_cleanup:
c2d6e31a 1071 mlx5_eswitch_cleanup(dev->priv.eswitch);
86eec50b
BW		 1072err_sriov_cleanup:
1073 mlx5_sriov_cleanup(dev);
eeb66cdb 1074err_mpfs_cleanup:
eeb66cdb 1075 mlx5_mpfs_cleanup(dev);
c2d6e31a 1076err_rl_cleanup:
c2d6e31a 1077 mlx5_cleanup_rl_table(dev);
59211bd3 1078err_tables_cleanup:
0ccc171e 1079 mlx5_geneve_destroy(dev->geneve);
358aa5ce 1080 mlx5_vxlan_destroy(dev->vxlan);
2a35b2c2
JP		 1081 mlx5_cleanup_clock(dev);
1082 mlx5_cleanup_reserved_gids(dev);
02d92f79 1083 mlx5_cq_debugfs_cleanup(dev);
38b9f903
MS		 1084 mlx5_fw_reset_cleanup(dev);
1085err_events_cleanup:
69c1280b 1086 mlx5_events_cleanup(dev);
59211bd3 1087err_eq_cleanup:
f2f3df55 1088 mlx5_eq_table_cleanup(dev);
561aa15a
YA		 1089err_irq_cleanup:
1090 mlx5_irq_table_cleanup(dev);
fadd59fc 1091err_devcom:
88d162b4 1092 mlx5_devcom_unregister_device(dev->priv.devc);
59211bd3 1093
59211bd3
MHY		 1094 return err;
1095}
1096
1097static void mlx5_cleanup_once(struct mlx5_core_dev *dev)
1098{
12206b17 1099 mlx5_rsc_dump_destroy(dev);
87175120 1100 mlx5_hv_vhca_destroy(dev->hv_vhca);
24406953 1101 mlx5_fw_tracer_destroy(dev->tracer);
c9b9dcb4 1102 mlx5_dm_cleanup(dev);
b3388697 1103 mlx5_fs_core_free(dev);
8f010541
PP		 1104 mlx5_sf_table_cleanup(dev);
1105 mlx5_sf_hw_table_cleanup(dev);
f3196bb0 1106 mlx5_vhca_event_cleanup(dev);
9410733c 1107 mlx5_fpga_cleanup(dev);
c2d6e31a 1108 mlx5_eswitch_cleanup(dev->priv.eswitch);
86eec50b 1109 mlx5_sriov_cleanup(dev);
eeb66cdb 1110 mlx5_mpfs_cleanup(dev);
59211bd3 1111 mlx5_cleanup_rl_table(dev);
0ccc171e 1112 mlx5_geneve_destroy(dev->geneve);
358aa5ce 1113 mlx5_vxlan_destroy(dev->vxlan);
7c39afb3 1114 mlx5_cleanup_clock(dev);
52ec462e 1115 mlx5_cleanup_reserved_gids(dev);
02d92f79 1116 mlx5_cq_debugfs_cleanup(dev);
38b9f903 1117 mlx5_fw_reset_cleanup(dev);
69c1280b 1118 mlx5_events_cleanup(dev);
f2f3df55 1119 mlx5_eq_table_cleanup(dev);
561aa15a 1120 mlx5_irq_table_cleanup(dev);
88d162b4 1121 mlx5_devcom_unregister_device(dev->priv.devc);
59211bd3
MHY		 1122}
1123
2059cf51 1124static int mlx5_function_enable(struct mlx5_core_dev *dev, bool boot, u64 timeout)
a31208b1 1125{
a31208b1
MD		 1126 int err;
1127
98a8e6fc
HN		 1128 mlx5_core_info(dev, "firmware version: %d.%d.%d\n", fw_rev_maj(dev),
1129 fw_rev_min(dev), fw_rev_sub(dev));
e126ba97 1130
00c6bcb0
TG		 1131 /* Only PFs hold the relevant PCIe information for this query */
1132 if (mlx5_core_is_pf(dev))
1133 pcie_print_link_status(dev->pdev);
1134
6c780a02
EC		 1135 /* wait for firmware to accept initialization segments configurations
1136 */
37ca95e6 1137 err = wait_fw_init(dev, timeout,
5945e1ad 1138 mlx5_tout_ms(dev, FW_PRE_INIT_WARN_MESSAGE_INTERVAL));
6c780a02 1139 if (err) {
5945e1ad 1140 mlx5_core_err(dev, "Firmware over %llu MS in pre-initializing state, aborting\n",
37ca95e6 1141 timeout);
76091b0f 1142 return err;
6c780a02
EC		 1143 }
1144
06cd555f 1145 err = mlx5_cmd_enable(dev);
e126ba97 1146 if (err) {
98a8e6fc 1147 mlx5_core_err(dev, "Failed initializing command interface, aborting\n");
76091b0f 1148 return err;
e126ba97
EC		 1149 }
1150
5945e1ad
AT		 1151 mlx5_tout_query_iseg(dev);
1152
1153 err = wait_fw_init(dev, mlx5_tout_ms(dev, FW_INIT), 0);
e3297246 1154 if (err) {
5945e1ad
AT		 1155 mlx5_core_err(dev, "Firmware over %llu MS in initializing state, aborting\n",
1156 mlx5_tout_ms(dev, FW_INIT));
55378a23 1157 goto err_cmd_cleanup;
e3297246
EC		 1158 }
1159
bbfa4b58 1160 dev->caps.embedded_cpu = mlx5_read_embedded_cpu(dev);
f7936ddd
EBE		 1161 mlx5_cmd_set_state(dev, MLX5_CMDIF_STATE_UP);
1162
9b98d395
MS		 1163 mlx5_start_health_poll(dev);
1164
0b107106 1165 err = mlx5_core_enable_hca(dev, 0);
cd23b14b 1166 if (err) {
98a8e6fc 1167 mlx5_core_err(dev, "enable hca failed\n");
9b98d395 1168 goto stop_health_poll;
cd23b14b
EC		 1169 }
1170
f62b8bb8
AV		 1171 err = mlx5_core_set_issi(dev);
1172 if (err) {
98a8e6fc 1173 mlx5_core_err(dev, "failed to set issi\n");
f62b8bb8
AV		 1174 goto err_disable_hca;
1175 }
f62b8bb8 1176
cd23b14b
EC		 1177 err = mlx5_satisfy_startup_pages(dev, 1);
1178 if (err) {
98a8e6fc 1179 mlx5_core_err(dev, "failed to allocate boot pages\n");
cd23b14b
EC		 1180 goto err_disable_hca;
1181 }
1182
32def412
AT		 1183 err = mlx5_tout_query_dtor(dev);
1184 if (err) {
1185 mlx5_core_err(dev, "failed to read dtor\n");
1186 goto reclaim_boot_pages;
1187 }
1188
2059cf51
SD		 1189 return 0;
1190
1191reclaim_boot_pages:
1192 mlx5_reclaim_startup_pages(dev);
1193err_disable_hca:
1194 mlx5_core_disable_hca(dev, 0);
1195stop_health_poll:
1196 mlx5_stop_health_poll(dev, boot);
1197err_cmd_cleanup:
1198 mlx5_cmd_set_state(dev, MLX5_CMDIF_STATE_DOWN);
06cd555f 1199 mlx5_cmd_disable(dev);
2059cf51
SD		 1200
1201 return err;
1202}
1203
1204static void mlx5_function_disable(struct mlx5_core_dev *dev, bool boot)
1205{
1206 mlx5_reclaim_startup_pages(dev);
1207 mlx5_core_disable_hca(dev, 0);
1208 mlx5_stop_health_poll(dev, boot);
1209 mlx5_cmd_set_state(dev, MLX5_CMDIF_STATE_DOWN);
06cd555f 1210 mlx5_cmd_disable(dev);
2059cf51
SD		 1211}
1212
1213static int mlx5_function_open(struct mlx5_core_dev *dev)
1214{
1215 int err;
1216
e126ba97
EC		 1217 err = set_hca_ctrl(dev);
1218 if (err) {
98a8e6fc 1219 mlx5_core_err(dev, "set_hca_ctrl failed\n");
2059cf51 1220 return err;
e126ba97
EC		 1221 }
1222
37b6bb77 1223 err = set_hca_cap(dev);
f91e6d89 1224 if (err) {
98a8e6fc 1225 mlx5_core_err(dev, "set_hca_cap failed\n");
2059cf51 1226 return err;
46861e3e
MS		 1227 }
1228
cd23b14b 1229 err = mlx5_satisfy_startup_pages(dev, 0);
e126ba97 1230 if (err) {
98a8e6fc 1231 mlx5_core_err(dev, "failed to allocate init pages\n");
2059cf51 1232 return err;
e126ba97
EC		 1233 }
1234
8737f818 1235 err = mlx5_cmd_init_hca(dev, sw_owner_id);
e126ba97 1236 if (err) {
98a8e6fc 1237 mlx5_core_err(dev, "init hca failed\n");
2059cf51 1238 return err;
e126ba97
EC		 1239 }
1240
012e50e1
HN		 1241 mlx5_set_driver_version(dev);
1242
bba1574c
DJ		 1243 err = mlx5_query_hca_caps(dev);
1244 if (err) {
98a8e6fc 1245 mlx5_core_err(dev, "query hca failed\n");
2059cf51 1246 return err;
bba1574c 1247 }
9b98d395 1248 mlx5_start_health_fw_log_up(dev);
e161105e 1249 return 0;
e161105e
SM		 1250}
1251
2059cf51 1252static int mlx5_function_close(struct mlx5_core_dev *dev)
e161105e
SM		 1253{
1254 int err;
1255
e161105e
SM		 1256 err = mlx5_cmd_teardown_hca(dev);
1257 if (err) {
98a8e6fc 1258 mlx5_core_err(dev, "tear_down_hca failed, skip cleanup\n");
e161105e 1259 return err;
e126ba97 1260 }
e161105e
SM		 1261
1262 return 0;
1263}
1264
2059cf51
SD		 1265static int mlx5_function_setup(struct mlx5_core_dev *dev, bool boot, u64 timeout)
1266{
1267 int err;
1268
1269 err = mlx5_function_enable(dev, boot, timeout);
1270 if (err)
1271 return err;
1272
1273 err = mlx5_function_open(dev);
1274 if (err)
1275 mlx5_function_disable(dev, boot);
1276 return err;
1277}
1278
1279static int mlx5_function_teardown(struct mlx5_core_dev *dev, bool boot)
1280{
1281 int err = mlx5_function_close(dev);
1282
1283 if (!err)
1284 mlx5_function_disable(dev, boot);
1285 return err;
1286}
1287
a80d1b68 1288static int mlx5_load(struct mlx5_core_dev *dev)
e161105e 1289{
e161105e 1290 int err;
e126ba97 1291
01187175 1292 dev->priv.uar = mlx5_get_uars_page(dev);
72f36be0 1293 if (IS_ERR(dev->priv.uar)) {
98a8e6fc 1294 mlx5_core_err(dev, "Failed allocating uar, aborting\n");
72f36be0 1295 err = PTR_ERR(dev->priv.uar);
a80d1b68 1296 return err;
e126ba97
EC		 1297 }
1298
69c1280b 1299 mlx5_events_start(dev);
0cf53c12
SM		 1300 mlx5_pagealloc_start(dev);
1301
e1706e62
YA		 1302 err = mlx5_irq_table_create(dev);
1303 if (err) {
1304 mlx5_core_err(dev, "Failed to alloc IRQs\n");
1305 goto err_irq_table;
1306 }
1307
c8e21b3b 1308 err = mlx5_eq_table_create(dev);
e126ba97 1309 if (err) {
98a8e6fc 1310 mlx5_core_err(dev, "Failed to create EQs\n");
c8e21b3b 1311 goto err_eq_table;
e126ba97
EC		 1312 }
1313
24406953
FD		 1314 err = mlx5_fw_tracer_init(dev->tracer);
1315 if (err) {
f62eb932
AL		 1316 mlx5_core_err(dev, "Failed to init FW tracer %d\n", err);
1317 mlx5_fw_tracer_destroy(dev->tracer);
1318 dev->tracer = NULL;
24406953
FD		 1319 }
1320
38b9f903 1321 mlx5_fw_reset_events_start(dev);
87175120
EBE		 1322 mlx5_hv_vhca_init(dev->hv_vhca);
1323
12206b17
AL		 1324 err = mlx5_rsc_dump_init(dev);
1325 if (err) {
f62eb932
AL		 1326 mlx5_core_err(dev, "Failed to init Resource dump %d\n", err);
1327 mlx5_rsc_dump_destroy(dev);
1328 dev->rsc_dump = NULL;
12206b17
AL		 1329 }
1330
04e87170
MB		 1331 err = mlx5_fpga_device_start(dev);
1332 if (err) {
98a8e6fc 1333 mlx5_core_err(dev, "fpga device start failed %d\n", err);
04e87170
MB		 1334 goto err_fpga_start;
1335 }
1336
b3388697 1337 err = mlx5_fs_core_init(dev);
59211bd3 1338 if (err) {
98a8e6fc 1339 mlx5_core_err(dev, "Failed to init flow steering\n");
c85023e1 1340 goto err_fs;
59211bd3 1341 }
e126ba97 1342
c85023e1 1343 err = mlx5_core_set_hca_defaults(dev);
86d722ad 1344 if (err) {
98a8e6fc 1345 mlx5_core_err(dev, "Failed to set hca defaults\n");
94a4b841 1346 goto err_set_hca;
86d722ad 1347 }
1466cc5b 1348
f3196bb0
PP		 1349 mlx5_vhca_event_start(dev);
1350
6a327321
PP		 1351 err = mlx5_sf_hw_table_create(dev);
1352 if (err) {
1353 mlx5_core_err(dev, "sf table create failed %d\n", err);
1354 goto err_vhca;
1355 }
1356
22e939a9
BW		 1357 err = mlx5_ec_init(dev);
1358 if (err) {
98a8e6fc 1359 mlx5_core_err(dev, "Failed to init embedded CPU\n");
22e939a9
BW		 1360 goto err_ec;
1361 }
1362
cac1eb2c 1363 mlx5_lag_add_mdev(dev);
5bef709d
PP		 1364 err = mlx5_sriov_attach(dev);
1365 if (err) {
1366 mlx5_core_err(dev, "sriov init failed %d\n", err);
1367 goto err_sriov;
1368 }
1369
90d010b8
PP		 1370 mlx5_sf_dev_table_create(dev);
1371
71b75f0e
MS		 1372 err = mlx5_devlink_traps_register(priv_to_devlink(dev));
1373 if (err)
1374 goto err_traps_reg;
1375
e126ba97
EC		 1376 return 0;
1377
71b75f0e
MS		 1378err_traps_reg:
1379 mlx5_sf_dev_table_destroy(dev);
1380 mlx5_sriov_detach(dev);
59211bd3 1381err_sriov:
cac1eb2c 1382 mlx5_lag_remove_mdev(dev);
5bef709d
PP		 1383 mlx5_ec_cleanup(dev);
1384err_ec:
6a327321
PP		 1385 mlx5_sf_hw_table_destroy(dev);
1386err_vhca:
f3196bb0 1387 mlx5_vhca_event_stop(dev);
94a4b841 1388err_set_hca:
b3388697 1389 mlx5_fs_core_cleanup(dev);
86d722ad 1390err_fs:
04e87170 1391 mlx5_fpga_device_stop(dev);
04e87170 1392err_fpga_start:
12206b17 1393 mlx5_rsc_dump_cleanup(dev);
87175120 1394 mlx5_hv_vhca_cleanup(dev->hv_vhca);
38b9f903 1395 mlx5_fw_reset_events_stop(dev);
24406953 1396 mlx5_fw_tracer_cleanup(dev->tracer);
c8e21b3b 1397 mlx5_eq_table_destroy(dev);
c8e21b3b 1398err_eq_table:
e1706e62
YA		 1399 mlx5_irq_table_destroy(dev);
1400err_irq_table:
0cf53c12 1401 mlx5_pagealloc_stop(dev);
69c1280b 1402 mlx5_events_stop(dev);
868bc06b 1403 mlx5_put_uars_page(dev, dev->priv.uar);
a80d1b68
SM		 1404 return err;
1405}
e126ba97 1406
a80d1b68
SM		 1407static void mlx5_unload(struct mlx5_core_dev *dev)
1408{
71b75f0e 1409 mlx5_devlink_traps_unregister(priv_to_devlink(dev));
90d010b8 1410 mlx5_sf_dev_table_destroy(dev);
f019679e 1411 mlx5_eswitch_disable(dev->priv.eswitch);
7ba930fc 1412 mlx5_sriov_detach(dev);
cac1eb2c 1413 mlx5_lag_remove_mdev(dev);
5bef709d 1414 mlx5_ec_cleanup(dev);
6a327321 1415 mlx5_sf_hw_table_destroy(dev);
f3196bb0 1416 mlx5_vhca_event_stop(dev);
b3388697 1417 mlx5_fs_core_cleanup(dev);
a80d1b68 1418 mlx5_fpga_device_stop(dev);
12206b17 1419 mlx5_rsc_dump_cleanup(dev);
87175120 1420 mlx5_hv_vhca_cleanup(dev->hv_vhca);
38b9f903 1421 mlx5_fw_reset_events_stop(dev);
a80d1b68
SM		 1422 mlx5_fw_tracer_cleanup(dev->tracer);
1423 mlx5_eq_table_destroy(dev);
e1706e62 1424 mlx5_irq_table_destroy(dev);
a80d1b68
SM		 1425 mlx5_pagealloc_stop(dev);
1426 mlx5_events_stop(dev);
1427 mlx5_put_uars_page(dev, dev->priv.uar);
1428}
59211bd3 1429
e71383fb 1430int mlx5_init_one_devl_locked(struct mlx5_core_dev *dev)
a80d1b68 1431{
e71383fb 1432 bool light_probe = mlx5_dev_is_lightweight(dev);
a80d1b68
SM		 1433 int err = 0;
1434
a80d1b68 1435 mutex_lock(&dev->intf_state_mutex);
a80d1b68 1436 dev->state = MLX5_DEVICE_STATE_UP;
e126ba97 1437
9b98d395 1438 err = mlx5_function_setup(dev, true, mlx5_tout_ms(dev, FW_PRE_INIT_TIMEOUT));
a80d1b68 1439 if (err)
4f7400d5 1440 goto err_function;
e126ba97 1441
6dea2f7e
LR		 1442 err = mlx5_init_once(dev);
1443 if (err) {
1444 mlx5_core_err(dev, "sw objs init failed\n");
1445 goto function_teardown;
a80d1b68 1446 }
cd23b14b 1447
e71383fb
SD		 1448 /* In case of light_probe, mlx5_devlink is already registered.
1449 * Hence, don't register devlink again.
1450 */
1451 if (!light_probe) {
1452 err = mlx5_devlink_params_register(priv_to_devlink(dev));
1453 if (err)
1454 goto err_devlink_params_reg;
1455 }
fe578cbb 1456
a80d1b68
SM		 1457 err = mlx5_load(dev);
1458 if (err)
1459 goto err_load;
e126ba97 1460
98f91c45
PP		 1461 set_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state);
1462
6dea2f7e 1463 err = mlx5_register_device(dev);
a925b5e3
LR		 1464 if (err)
1465 goto err_register;
1466
4162f58b
PP		 1467 mutex_unlock(&dev->intf_state_mutex);
1468 return 0;
a80d1b68 1469
a925b5e3 1470err_register:
98f91c45 1471 clear_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state);
a80d1b68
SM		 1472 mlx5_unload(dev);
1473err_load:
e71383fb
SD		 1474 if (!light_probe)
1475 mlx5_devlink_params_unregister(priv_to_devlink(dev));
fe578cbb 1476err_devlink_params_reg:
6dea2f7e 1477 mlx5_cleanup_once(dev);
e161105e 1478function_teardown:
6dea2f7e 1479 mlx5_function_teardown(dev, true);
4f7400d5 1480err_function:
89d44f0a
MD		 1481 dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR;
1482 mutex_unlock(&dev->intf_state_mutex);
e71383fb
SD		 1483 return err;
1484}
1485
1486int mlx5_init_one(struct mlx5_core_dev *dev)
1487{
1488 struct devlink *devlink = priv_to_devlink(dev);
1489 int err;
1490
1491 devl_lock(devlink);
1492 err = mlx5_init_one_devl_locked(dev);
84a433a4 1493 devl_unlock(devlink);
e126ba97
EC		 1494 return err;
1495}
e126ba97 1496
6dea2f7e 1497void mlx5_uninit_one(struct mlx5_core_dev *dev)
e126ba97 1498{
84a433a4
MS		 1499 struct devlink *devlink = priv_to_devlink(dev);
1500
1501 devl_lock(devlink);
98f91c45
PP		 1502 mutex_lock(&dev->intf_state_mutex);
1503
6dea2f7e 1504 mlx5_unregister_device(dev);
689a248d 1505
b3cb5388 1506 if (!test_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state)) {
98a8e6fc
HN		 1507 mlx5_core_warn(dev, "%s: interface is down, NOP\n",
1508 __func__);
53d737df 1509 mlx5_devlink_params_unregister(priv_to_devlink(dev));
6dea2f7e 1510 mlx5_cleanup_once(dev);
89d44f0a
MD		 1511 goto out;
1512 }
6b6adee3 1513
9ade8c7c 1514 clear_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state);
6dea2f7e 1515 mlx5_unload(dev);
fe578cbb 1516 mlx5_devlink_params_unregister(priv_to_devlink(dev));
6dea2f7e
LR		 1517 mlx5_cleanup_once(dev);
1518 mlx5_function_teardown(dev, true);
1519out:
1520 mutex_unlock(&dev->intf_state_mutex);
84a433a4 1521 devl_unlock(devlink);
6dea2f7e
LR		 1522}
1523
84a433a4 1524int mlx5_load_one_devl_locked(struct mlx5_core_dev *dev, bool recovery)
6dea2f7e
LR		 1525{
1526 int err = 0;
37ca95e6 1527 u64 timeout;
6dea2f7e 1528
84a433a4 1529 devl_assert_locked(priv_to_devlink(dev));
6dea2f7e
LR		 1530 mutex_lock(&dev->intf_state_mutex);
1531 if (test_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state)) {
1532 mlx5_core_warn(dev, "interface is up, NOP\n");
1533 goto out;
1534 }
1535 /* remove any previous indication of internal error */
1536 dev->state = MLX5_DEVICE_STATE_UP;
1537
37ca95e6
GL		 1538 if (recovery)
1539 timeout = mlx5_tout_ms(dev, FW_PRE_INIT_ON_RECOVERY_TIMEOUT);
1540 else
1541 timeout = mlx5_tout_ms(dev, FW_PRE_INIT_TIMEOUT);
9b98d395 1542 err = mlx5_function_setup(dev, false, timeout);
6dea2f7e
LR		 1543 if (err)
1544 goto err_function;
1545
1546 err = mlx5_load(dev);
1547 if (err)
1548 goto err_load;
1549
1550 set_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state);
9ade8c7c 1551
6dea2f7e
LR		 1552 err = mlx5_attach_device(dev);
1553 if (err)
1554 goto err_attach;
1555
1556 mutex_unlock(&dev->intf_state_mutex);
1557 return 0;
1558
1559err_attach:
1560 clear_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state);
a80d1b68 1561 mlx5_unload(dev);
6dea2f7e
LR		 1562err_load:
1563 mlx5_function_teardown(dev, false);
1564err_function:
1565 dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR;
1566out:
1567 mutex_unlock(&dev->intf_state_mutex);
1568 return err;
1569}
a80d1b68 1570
21608a2c 1571int mlx5_load_one(struct mlx5_core_dev *dev, bool recovery)
6dea2f7e 1572{
84a433a4
MS		 1573 struct devlink *devlink = priv_to_devlink(dev);
1574 int ret;
1575
1576 devl_lock(devlink);
21608a2c 1577 ret = mlx5_load_one_devl_locked(dev, recovery);
84a433a4
MS		 1578 devl_unlock(devlink);
1579 return ret;
1580}
1581
72ed5d56 1582void mlx5_unload_one_devl_locked(struct mlx5_core_dev *dev, bool suspend)
84a433a4
MS		 1583{
1584 devl_assert_locked(priv_to_devlink(dev));
6dea2f7e
LR		 1585 mutex_lock(&dev->intf_state_mutex);
1586
72ed5d56 1587 mlx5_detach_device(dev, suspend);
6dea2f7e
LR		 1588
1589 if (!test_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state)) {
1590 mlx5_core_warn(dev, "%s: interface is down, NOP\n",
1591 __func__);
1592 goto out;
1593 }
9603b61d 1594
6dea2f7e
LR		 1595 clear_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state);
1596 mlx5_unload(dev);
1597 mlx5_function_teardown(dev, false);
ac6ea6e8 1598out:
89d44f0a 1599 mutex_unlock(&dev->intf_state_mutex);
9603b61d 1600}
64613d94 1601
72ed5d56 1602void mlx5_unload_one(struct mlx5_core_dev *dev, bool suspend)
84a433a4
MS		 1603{
1604 struct devlink *devlink = priv_to_devlink(dev);
1605
1606 devl_lock(devlink);
72ed5d56 1607 mlx5_unload_one_devl_locked(dev, suspend);
84a433a4
MS		 1608 devl_unlock(devlink);
1609}
1610
e71383fb
SD		 1611/* In case of light probe, we don't need a full query of hca_caps, but only the bellow caps.
1612 * A full query of hca_caps will be done when the device will reload.
1613 */
1614static int mlx5_query_hca_caps_light(struct mlx5_core_dev *dev)
1615{
1616 int err;
1617
1618 err = mlx5_core_get_caps(dev, MLX5_CAP_GENERAL);
1619 if (err)
1620 return err;
1621
1622 if (MLX5_CAP_GEN(dev, eth_net_offloads)) {
1623 err = mlx5_core_get_caps(dev, MLX5_CAP_ETHERNET_OFFLOADS);
1624 if (err)
1625 return err;
1626 }
1627
1628 if (MLX5_CAP_GEN(dev, nic_flow_table) ||
1629 MLX5_CAP_GEN(dev, ipoib_enhanced_offloads)) {
1630 err = mlx5_core_get_caps(dev, MLX5_CAP_FLOW_TABLE);
1631 if (err)
1632 return err;
1633 }
1634
1635 if (MLX5_CAP_GEN_64(dev, general_obj_types) &
1636 MLX5_GENERAL_OBJ_TYPES_CAP_VIRTIO_NET_Q) {
1637 err = mlx5_core_get_caps(dev, MLX5_CAP_VDPA_EMULATION);
1638 if (err)
1639 return err;
1640 }
1641
1642 return 0;
1643}
1644
1645int mlx5_init_one_light(struct mlx5_core_dev *dev)
1646{
1647 struct devlink *devlink = priv_to_devlink(dev);
1648 int err;
1649
1650 dev->state = MLX5_DEVICE_STATE_UP;
1651 err = mlx5_function_enable(dev, true, mlx5_tout_ms(dev, FW_PRE_INIT_TIMEOUT));
1652 if (err) {
1653 mlx5_core_warn(dev, "mlx5_function_enable err=%d\n", err);
1654 goto out;
1655 }
1656
1657 err = mlx5_query_hca_caps_light(dev);
1658 if (err) {
1659 mlx5_core_warn(dev, "mlx5_query_hca_caps_light err=%d\n", err);
1660 goto query_hca_caps_err;
1661 }
1662
1663 devl_lock(devlink);
1664 err = mlx5_devlink_params_register(priv_to_devlink(dev));
1665 devl_unlock(devlink);
1666 if (err) {
1667 mlx5_core_warn(dev, "mlx5_devlink_param_reg err = %d\n", err);
1668 goto query_hca_caps_err;
1669 }
1670
1671 return 0;
1672
1673query_hca_caps_err:
1674 mlx5_function_disable(dev, true);
1675out:
1676 dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR;
1677 return err;
1678}
1679
1680void mlx5_uninit_one_light(struct mlx5_core_dev *dev)
1681{
1682 struct devlink *devlink = priv_to_devlink(dev);
1683
1684 devl_lock(devlink);
1685 mlx5_devlink_params_unregister(priv_to_devlink(dev));
1686 devl_unlock(devlink);
1687 if (dev->state != MLX5_DEVICE_STATE_UP)
1688 return;
1689 mlx5_function_disable(dev, true);
1690}
1691
1692/* xxx_light() function are used in order to configure the device without full
1693 * init (light init). e.g.: There isn't a point in reload a device to light state.
1694 * Hence, mlx5_load_one_light() isn't needed.
1695 */
1696
1697void mlx5_unload_one_light(struct mlx5_core_dev *dev)
1698{
1699 if (dev->state != MLX5_DEVICE_STATE_UP)
1700 return;
1701 mlx5_function_disable(dev, false);
1702 dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR;
1703}
1704
48f02eef
PP		 1705static const int types[] = {
1706 MLX5_CAP_GENERAL,
1707 MLX5_CAP_GENERAL_2,
1708 MLX5_CAP_ETHERNET_OFFLOADS,
1709 MLX5_CAP_IPOIB_ENHANCED_OFFLOADS,
1710 MLX5_CAP_ODP,
1711 MLX5_CAP_ATOMIC,
1712 MLX5_CAP_ROCE,
1713 MLX5_CAP_IPOIB_OFFLOADS,
1714 MLX5_CAP_FLOW_TABLE,
1715 MLX5_CAP_ESWITCH_FLOW_TABLE,
1716 MLX5_CAP_ESWITCH,
48f02eef
PP		 1717 MLX5_CAP_QOS,
1718 MLX5_CAP_DEBUG,
1719 MLX5_CAP_DEV_MEM,
1720 MLX5_CAP_DEV_EVENT,
1721 MLX5_CAP_TLS,
1722 MLX5_CAP_VDPA_EMULATION,
1723 MLX5_CAP_IPSEC,
425a563a 1724 MLX5_CAP_PORT_SELECTION,
8ff0ac5b 1725 MLX5_CAP_MACSEC,
93983863 1726 MLX5_CAP_ADV_VIRTUALIZATION,
fe298bdf 1727 MLX5_CAP_CRYPTO,
48f02eef
PP		 1728};
1729
1730static void mlx5_hca_caps_free(struct mlx5_core_dev *dev)
1731{
1732 int type;
1733 int i;
1734
1735 for (i = 0; i < ARRAY_SIZE(types); i++) {
1736 type = types[i];
1737 kfree(dev->caps.hca[type]);
1738 }
1739}
1740
1741static int mlx5_hca_caps_alloc(struct mlx5_core_dev *dev)
1742{
1743 struct mlx5_hca_cap *cap;
1744 int type;
1745 int i;
1746
1747 for (i = 0; i < ARRAY_SIZE(types); i++) {
1748 cap = kzalloc(sizeof(*cap), GFP_KERNEL);
1749 if (!cap)
1750 goto err;
1751 type = types[i];
1752 dev->caps.hca[type] = cap;
1753 }
1754
1755 return 0;
1756
1757err:
1758 mlx5_hca_caps_free(dev);
1759 return -ENOMEM;
1760}
1761
dd3dd726
EC		 1762static int vhca_id_show(struct seq_file *file, void *priv)
1763{
1764 struct mlx5_core_dev *dev = file->private;
1765
1766 seq_printf(file, "0x%x\n", MLX5_CAP_GEN(dev, vhca_id));
1767 return 0;
1768}
1769
1770DEFINE_SHOW_ATTRIBUTE(vhca_id);
1771
1958fc2f 1772int mlx5_mdev_init(struct mlx5_core_dev *dev, int profile_idx)
9603b61d 1773{
11f3b84d 1774 struct mlx5_priv *priv = &dev->priv;
9603b61d
JM		 1775 int err;
1776
3410fbcd 1777 memcpy(&dev->profile, &profile[profile_idx], sizeof(dev->profile));
d59b73a6 1778 lockdep_register_key(&dev->lock_key);
89d44f0a 1779 mutex_init(&dev->intf_state_mutex);
d59b73a6 1780 lockdep_set_class(&dev->intf_state_mutex, &dev->lock_key);
c7d4e6ab 1781 mutex_init(&dev->mlx5e_res.uplink_netdev_lock);
d9aaed83 1782
01187175
EC		 1783 mutex_init(&priv->bfregs.reg_head.lock);
1784 mutex_init(&priv->bfregs.wc_head.lock);
1785 INIT_LIST_HEAD(&priv->bfregs.reg_head.list);
1786 INIT_LIST_HEAD(&priv->bfregs.wc_head.list);
1787
11f3b84d
SM		 1788 mutex_init(&priv->alloc_mutex);
1789 mutex_init(&priv->pgdir_mutex);
1790 INIT_LIST_HEAD(&priv->pgdir_list);
11f3b84d 1791
44f66ac9 1792 priv->numa_node = dev_to_node(mlx5_core_dma_dev(dev));
66771a1c
MS		 1793 priv->dbg.dbg_root = debugfs_create_dir(dev_name(dev->device),
1794 mlx5_debugfs_root);
dd3dd726 1795 debugfs_create_file("vhca_id", 0400, priv->dbg.dbg_root, dev, &vhca_id_fops);
3d347b1b
AL		 1796 INIT_LIST_HEAD(&priv->traps);
1797
06cd555f
SD		 1798 err = mlx5_cmd_init(dev);
1799 if (err) {
1800 mlx5_core_err(dev, "Failed initializing cmdif SW structs, aborting\n");
1801 goto err_cmd_init;
1802 }
1803
76091b0f
AT		 1804 err = mlx5_tout_init(dev);
1805 if (err) {
1806 mlx5_core_err(dev, "Failed initializing timeouts, aborting\n");
1807 goto err_timeout_init;
1808 }
1809
ac6ea6e8 1810 err = mlx5_health_init(dev);
52c368dc
SM		 1811 if (err)
1812 goto err_health_init;
ac6ea6e8 1813
0cf53c12
SM		 1814 err = mlx5_pagealloc_init(dev);
1815 if (err)
1816 goto err_pagealloc_init;
59211bd3 1817
a925b5e3
LR		 1818 err = mlx5_adev_init(dev);
1819 if (err)
1820 goto err_adev_init;
1821
48f02eef
PP		 1822 err = mlx5_hca_caps_alloc(dev);
1823 if (err)
1824 goto err_hca_caps;
1825
dc402ccc
YH		 1826 /* The conjunction of sw_vhca_id with sw_owner_id will be a global
1827 * unique id per function which uses mlx5_core.
1828 * Those values are supplied to FW as part of the init HCA command to
1829 * be used by both driver and FW when it's applicable.
1830 */
1831 dev->priv.sw_vhca_id = ida_alloc_range(&sw_vhca_ida, 1,
1832 MAX_SW_VHCA_ID,
1833 GFP_KERNEL);
1834 if (dev->priv.sw_vhca_id < 0)
1835 mlx5_core_err(dev, "failed to allocate sw_vhca_id, err=%d\n",
1836 dev->priv.sw_vhca_id);
1837
11f3b84d 1838 return 0;
52c368dc 1839
48f02eef
PP		 1840err_hca_caps:
1841 mlx5_adev_cleanup(dev);
a925b5e3
LR		 1842err_adev_init:
1843 mlx5_pagealloc_cleanup(dev);
52c368dc
SM		 1844err_pagealloc_init:
1845 mlx5_health_cleanup(dev);
1846err_health_init:
76091b0f
AT		 1847 mlx5_tout_cleanup(dev);
1848err_timeout_init:
06cd555f
SD		 1849 mlx5_cmd_cleanup(dev);
1850err_cmd_init:
66771a1c 1851 debugfs_remove(dev->priv.dbg.dbg_root);
810cbb25
PP		 1852 mutex_destroy(&priv->pgdir_mutex);
1853 mutex_destroy(&priv->alloc_mutex);
1854 mutex_destroy(&priv->bfregs.wc_head.lock);
1855 mutex_destroy(&priv->bfregs.reg_head.lock);
1856 mutex_destroy(&dev->intf_state_mutex);
d59b73a6 1857 lockdep_unregister_key(&dev->lock_key);
52c368dc 1858 return err;
11f3b84d
SM		 1859}
1860
1958fc2f 1861void mlx5_mdev_uninit(struct mlx5_core_dev *dev)
11f3b84d 1862{
810cbb25
PP		 1863 struct mlx5_priv *priv = &dev->priv;
1864
dc402ccc
YH		 1865 if (priv->sw_vhca_id > 0)
1866 ida_free(&sw_vhca_ida, dev->priv.sw_vhca_id);
1867
48f02eef 1868 mlx5_hca_caps_free(dev);
a925b5e3 1869 mlx5_adev_cleanup(dev);
52c368dc
SM		 1870 mlx5_pagealloc_cleanup(dev);
1871 mlx5_health_cleanup(dev);
76091b0f 1872 mlx5_tout_cleanup(dev);
06cd555f 1873 mlx5_cmd_cleanup(dev);
66771a1c 1874 debugfs_remove_recursive(dev->priv.dbg.dbg_root);
810cbb25
PP		 1875 mutex_destroy(&priv->pgdir_mutex);
1876 mutex_destroy(&priv->alloc_mutex);
1877 mutex_destroy(&priv->bfregs.wc_head.lock);
1878 mutex_destroy(&priv->bfregs.reg_head.lock);
c7d4e6ab 1879 mutex_destroy(&dev->mlx5e_res.uplink_netdev_lock);
810cbb25 1880 mutex_destroy(&dev->intf_state_mutex);
d59b73a6 1881 lockdep_unregister_key(&dev->lock_key);
11f3b84d
SM		 1882}
1883
6dea2f7e 1884static int probe_one(struct pci_dev *pdev, const struct pci_device_id *id)
9603b61d
JM		 1885{
1886 struct mlx5_core_dev *dev;
feae9087 1887 struct devlink *devlink;
9603b61d
JM		 1888 int err;
1889
919d13a7 1890 devlink = mlx5_devlink_alloc(&pdev->dev);
feae9087 1891 if (!devlink) {
1f28d776 1892 dev_err(&pdev->dev, "devlink alloc failed\n");
9603b61d
JM		 1893 return -ENOMEM;
1894 }
feae9087
OG		 1895
1896 dev = devlink_priv(devlink);
27b942fb
PP		 1897 dev->device = &pdev->dev;
1898 dev->pdev = pdev;
9603b61d 1899
386e75af
HN		 1900 dev->coredev_type = id->driver_data & MLX5_PCI_DEV_IS_VF ?
1901 MLX5_COREDEV_VF : MLX5_COREDEV_PF;
1902
a925b5e3 1903 dev->priv.adev_idx = mlx5_adev_idx_alloc();
4d8be211
LR		 1904 if (dev->priv.adev_idx < 0) {
1905 err = dev->priv.adev_idx;
1906 goto adev_init_err;
1907 }
a925b5e3 1908
27b942fb 1909 err = mlx5_mdev_init(dev, prof_sel);
11f3b84d
SM		 1910 if (err)
1911 goto mdev_init_err;
01187175 1912
11f3b84d 1913 err = mlx5_pci_init(dev, pdev, id);
9603b61d 1914 if (err) {
98a8e6fc
HN		 1915 mlx5_core_err(dev, "mlx5_pci_init failed with error code %d\n",
1916 err);
11f3b84d 1917 goto pci_init_err;
9603b61d
JM		 1918 }
1919
6dea2f7e 1920 err = mlx5_init_one(dev);
9603b61d 1921 if (err) {
6dea2f7e 1922 mlx5_core_err(dev, "mlx5_init_one failed with error code %d\n",
98a8e6fc 1923 err);
6dea2f7e 1924 goto err_init_one;
9603b61d 1925 }
59211bd3 1926
8b9d8baa
AV		 1927 err = mlx5_crdump_enable(dev);
1928 if (err)
1929 dev_err(&pdev->dev, "mlx5_crdump_enable failed with error code %d\n", err);
1930
1f507e80 1931 err = mlx5_hwmon_dev_register(dev);
c1fef618 1932 if (err)
1f507e80 1933 mlx5_core_err(dev, "mlx5_hwmon_dev_register failed with error code %d\n", err);
c1fef618 1934
5d47f6c8 1935 pci_save_state(pdev);
64ea2d0e 1936 devlink_register(devlink);
9603b61d
JM		 1937 return 0;
1938
6dea2f7e 1939err_init_one:
868bc06b 1940 mlx5_pci_close(dev);
11f3b84d
SM		 1941pci_init_err:
1942 mlx5_mdev_uninit(dev);
1943mdev_init_err:
a925b5e3 1944 mlx5_adev_idx_free(dev->priv.adev_idx);
4d8be211 1945adev_init_err:
1f28d776 1946 mlx5_devlink_free(devlink);
a31208b1 1947
9603b61d
JM		 1948 return err;
1949}
a31208b1 1950
9603b61d
JM		 1951static void remove_one(struct pci_dev *pdev)
1952{
1953 struct mlx5_core_dev *dev = pci_get_drvdata(pdev);
feae9087 1954 struct devlink *devlink = priv_to_devlink(dev);
9603b61d 1955
031a163f 1956 set_bit(MLX5_BREAK_FW_WAIT, &dev->intf_state);
824c8dc4
SD		 1957 /* mlx5_drain_fw_reset() and mlx5_drain_health_wq() are using
1958 * devlink notify APIs.
1959 * Hence, we must drain them before unregistering the devlink.
16d42d31
SD		 1960 */
1961 mlx5_drain_fw_reset(dev);
824c8dc4 1962 mlx5_drain_health_wq(dev);
64ea2d0e 1963 devlink_unregister(devlink);
6d98f314 1964 mlx5_sriov_disable(pdev, false);
1f507e80 1965 mlx5_hwmon_dev_unregister(dev);
8b9d8baa 1966 mlx5_crdump_disable(dev);
6dea2f7e 1967 mlx5_uninit_one(dev);
868bc06b 1968 mlx5_pci_close(dev);
11f3b84d 1969 mlx5_mdev_uninit(dev);
a925b5e3 1970 mlx5_adev_idx_free(dev->priv.adev_idx);
1f28d776 1971 mlx5_devlink_free(devlink);
9603b61d
JM		 1972}
1973
fad1783a
SM		 1974#define mlx5_pci_trace(dev, fmt, ...) ({ \
1975 struct mlx5_core_dev *__dev = (dev); \
1976 mlx5_core_info(__dev, "%s Device state = %d health sensors: %d pci_status: %d. " fmt, \
1977 __func__, __dev->state, mlx5_health_check_fatal_sensors(__dev), \
1978 __dev->pci_status, ##__VA_ARGS__); \
1979})
1980
1981static const char *result2str(enum pci_ers_result result)
1982{
1983 return result == PCI_ERS_RESULT_NEED_RESET ? "need reset" :
1984 result == PCI_ERS_RESULT_DISCONNECT ? "disconnect" :
1985 result == PCI_ERS_RESULT_RECOVERED ? "recovered" :
1986 "unknown";
1987}
1988
89d44f0a
MD		 1989static pci_ers_result_t mlx5_pci_err_detected(struct pci_dev *pdev,
1990 pci_channel_state_t state)
1991{
1992 struct mlx5_core_dev *dev = pci_get_drvdata(pdev);
fad1783a 1993 enum pci_ers_result res;
89d44f0a 1994
fad1783a 1995 mlx5_pci_trace(dev, "Enter, pci channel state = %d\n", state);
04c0c1ab 1996
8812c24d 1997 mlx5_enter_error_state(dev, false);
3e5b72ac 1998 mlx5_error_sw_reset(dev);
aab8e1a2 1999 mlx5_unload_one(dev, false);
b3bd076f
MS		 2000 mlx5_drain_health_wq(dev);
2001 mlx5_pci_disable_device(dev);
05ac2c0b 2002
fad1783a 2003 res = state == pci_channel_io_perm_failure ?
89d44f0a 2004 PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET;
fad1783a 2005
394164f9
RN		 2006 mlx5_core_info(dev, "%s Device state = %d pci_status: %d. Exit, result = %d, %s\n",
2007 __func__, dev->state, dev->pci_status, res, result2str(res));
fad1783a 2008 return res;
89d44f0a
MD		 2009}
2010
d57847dc
DJ		 2011/* wait for the device to show vital signs by waiting
2012 * for the health counter to start counting.
89d44f0a 2013 */
d57847dc 2014static int wait_vital(struct pci_dev *pdev)
89d44f0a
MD		 2015{
2016 struct mlx5_core_dev *dev = pci_get_drvdata(pdev);
2017 struct mlx5_core_health *health = &dev->priv.health;
2018 const int niter = 100;
d57847dc 2019 u32 last_count = 0;
89d44f0a 2020 u32 count;
89d44f0a
MD		 2021 int i;
2022
89d44f0a
MD		 2023 for (i = 0; i < niter; i++) {
2024 count = ioread32be(health->health_counter);
2025 if (count && count != 0xffffffff) {
d57847dc 2026 if (last_count && last_count != count) {
98a8e6fc
HN		 2027 mlx5_core_info(dev,
2028 "wait vital counter value 0x%x after %d iterations\n",
2029 count, i);
d57847dc
DJ		 2030 return 0;
2031 }
2032 last_count = count;
89d44f0a
MD		 2033 }
2034 msleep(50);
2035 }
2036
d57847dc 2037 return -ETIMEDOUT;
89d44f0a
MD		 2038}
2039
1061c90f 2040static pci_ers_result_t mlx5_pci_slot_reset(struct pci_dev *pdev)
89d44f0a 2041{
fad1783a 2042 enum pci_ers_result res = PCI_ERS_RESULT_DISCONNECT;
89d44f0a 2043 struct mlx5_core_dev *dev = pci_get_drvdata(pdev);
89d44f0a
MD		 2044 int err;
2045
394164f9
RN		 2046 mlx5_core_info(dev, "%s Device state = %d pci_status: %d. Enter\n",
2047 __func__, dev->state, dev->pci_status);
89d44f0a 2048
1061c90f 2049 err = mlx5_pci_enable_device(dev);
d57847dc 2050 if (err) {
98a8e6fc
HN		 2051 mlx5_core_err(dev, "%s: mlx5_pci_enable_device failed with error code: %d\n",
2052 __func__, err);
fad1783a 2053 goto out;
1061c90f
MHY		 2054 }
2055
2056 pci_set_master(pdev);
2057 pci_restore_state(pdev);
5d47f6c8 2058 pci_save_state(pdev);
1061c90f 2059
fad1783a
SM		 2060 err = wait_vital(pdev);
2061 if (err) {
2062 mlx5_core_err(dev, "%s: wait vital failed with error code: %d\n",
2063 __func__, err);
2064 goto out;
d57847dc 2065 }
89d44f0a 2066
fad1783a
SM		 2067 res = PCI_ERS_RESULT_RECOVERED;
2068out:
394164f9
RN		 2069 mlx5_core_info(dev, "%s Device state = %d pci_status: %d. Exit, err = %d, result = %d, %s\n",
2070 __func__, dev->state, dev->pci_status, err, res, result2str(res));
fad1783a 2071 return res;
1061c90f
MHY		 2072}
2073
1061c90f
MHY		 2074static void mlx5_pci_resume(struct pci_dev *pdev)
2075{
2076 struct mlx5_core_dev *dev = pci_get_drvdata(pdev);
1061c90f
MHY		 2077 int err;
2078
fad1783a 2079 mlx5_pci_trace(dev, "Enter, loading driver..\n");
1061c90f 2080
21608a2c
MS		 2081 err = mlx5_load_one(dev, false);
2082
416ef713
RN		 2083 if (!err)
2084 devlink_health_reporter_state_update(dev->priv.health.fw_fatal_reporter,
2085 DEVLINK_HEALTH_REPORTER_STATE_HEALTHY);
2086
fad1783a
SM		 2087 mlx5_pci_trace(dev, "Done, err = %d, device %s\n", err,
2088 !err ? "recovered" : "Failed");
89d44f0a
MD		 2089}
2090
2091static const struct pci_error_handlers mlx5_err_handler = {
2092 .error_detected = mlx5_pci_err_detected,
2093 .slot_reset = mlx5_pci_slot_reset,
2094 .resume = mlx5_pci_resume
2095};
2096
8812c24d
MD		 2097static int mlx5_try_fast_unload(struct mlx5_core_dev *dev)
2098{
fcd29ad1
FD		 2099 bool fast_teardown = false, force_teardown = false;
2100 int ret = 1;
2101
2102 fast_teardown = MLX5_CAP_GEN(dev, fast_teardown);
2103 force_teardown = MLX5_CAP_GEN(dev, force_teardown);
2104
2105 mlx5_core_dbg(dev, "force teardown firmware support=%d\n", force_teardown);
2106 mlx5_core_dbg(dev, "fast teardown firmware support=%d\n", fast_teardown);
8812c24d 2107
fcd29ad1 2108 if (!fast_teardown && !force_teardown)
8812c24d 2109 return -EOPNOTSUPP;
8812c24d
MD		 2110
2111 if (dev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) {
2112 mlx5_core_dbg(dev, "Device in internal error state, giving up\n");
2113 return -EAGAIN;
2114 }
2115
d2aa060d 2116 /* Panic tear down fw command will stop the PCI bus communication
b0ea505b 2117 * with the HCA, so the health poll is no longer needed.
d2aa060d
HN		 2118 */
2119 mlx5_drain_health_wq(dev);
76d5581c 2120 mlx5_stop_health_poll(dev, false);
d2aa060d 2121
fcd29ad1
FD		 2122 ret = mlx5_cmd_fast_teardown_hca(dev);
2123 if (!ret)
2124 goto succeed;
2125
8812c24d 2126 ret = mlx5_cmd_force_teardown_hca(dev);
fcd29ad1
FD		 2127 if (!ret)
2128 goto succeed;
2129
2130 mlx5_core_dbg(dev, "Firmware couldn't do fast unload error: %d\n", ret);
2131 mlx5_start_health_poll(dev);
2132 return ret;
8812c24d 2133
fcd29ad1 2134succeed:
8812c24d
MD		 2135 mlx5_enter_error_state(dev, true);
2136
1ef903bf
DJ		 2137 /* Some platforms requiring freeing the IRQ's in the shutdown
2138 * flow. If they aren't freed they can't be allocated after
2139 * kexec. There is no need to cleanup the mlx5_core software
2140 * contexts.
2141 */
1ef903bf
DJ		 2142 mlx5_core_eq_free_irqs(dev);
2143
8812c24d
MD		 2144 return 0;
2145}
2146
5fc7197d
MD		 2147static void shutdown(struct pci_dev *pdev)
2148{
2149 struct mlx5_core_dev *dev = pci_get_drvdata(pdev);
8812c24d 2150 int err;
5fc7197d 2151
98a8e6fc 2152 mlx5_core_info(dev, "Shutdown was called\n");
8324a02c 2153 set_bit(MLX5_BREAK_FW_WAIT, &dev->intf_state);
8812c24d
MD		 2154 err = mlx5_try_fast_unload(dev);
2155 if (err)
72ed5d56 2156 mlx5_unload_one(dev, false);
5fc7197d
MD		 2157 mlx5_pci_disable_device(dev);
2158}
2159
8fc3e29b
MB		 2160static int mlx5_suspend(struct pci_dev *pdev, pm_message_t state)
2161{
2162 struct mlx5_core_dev *dev = pci_get_drvdata(pdev);
2163
72ed5d56 2164 mlx5_unload_one(dev, true);
8fc3e29b
MB		 2165
2166 return 0;
2167}
2168
2169static int mlx5_resume(struct pci_dev *pdev)
2170{
2171 struct mlx5_core_dev *dev = pci_get_drvdata(pdev);
2172
21608a2c 2173 return mlx5_load_one(dev, false);
8fc3e29b
MB		 2174}
2175
9603b61d 2176static const struct pci_device_id mlx5_core_pci_table[] = {
bbad7c21 2177 { PCI_VDEVICE(MELLANOX, PCI_DEVICE_ID_MELLANOX_CONNECTIB) },
fc50db98 2178 { PCI_VDEVICE(MELLANOX, 0x1012), MLX5_PCI_DEV_IS_VF}, /* Connect-IB VF */
bbad7c21 2179 { PCI_VDEVICE(MELLANOX, PCI_DEVICE_ID_MELLANOX_CONNECTX4) },
fc50db98 2180 { PCI_VDEVICE(MELLANOX, 0x1014), MLX5_PCI_DEV_IS_VF}, /* ConnectX-4 VF */
bbad7c21 2181 { PCI_VDEVICE(MELLANOX, PCI_DEVICE_ID_MELLANOX_CONNECTX4_LX) },
fc50db98 2182 { PCI_VDEVICE(MELLANOX, 0x1016), MLX5_PCI_DEV_IS_VF}, /* ConnectX-4LX VF */
7092fe86 2183 { PCI_VDEVICE(MELLANOX, 0x1017) }, /* ConnectX-5, PCIe 3.0 */
64dbbdfe 2184 { PCI_VDEVICE(MELLANOX, 0x1018), MLX5_PCI_DEV_IS_VF}, /* ConnectX-5 VF */
d0dd989f
MD		 2185 { PCI_VDEVICE(MELLANOX, 0x1019) }, /* ConnectX-5 Ex */
2186 { PCI_VDEVICE(MELLANOX, 0x101a), MLX5_PCI_DEV_IS_VF}, /* ConnectX-5 Ex VF */
2187 { PCI_VDEVICE(MELLANOX, 0x101b) }, /* ConnectX-6 */
2188 { PCI_VDEVICE(MELLANOX, 0x101c), MLX5_PCI_DEV_IS_VF}, /* ConnectX-6 VF */
85327a9c
EBE		 2189 { PCI_VDEVICE(MELLANOX, 0x101d) }, /* ConnectX-6 Dx */
2190 { PCI_VDEVICE(MELLANOX, 0x101e), MLX5_PCI_DEV_IS_VF}, /* ConnectX Family mlx5Gen Virtual Function */
b7eca940 2191 { PCI_VDEVICE(MELLANOX, 0x101f) }, /* ConnectX-6 LX */
505a7f54 2192 { PCI_VDEVICE(MELLANOX, 0x1021) }, /* ConnectX-7 */
f908a35b 2193 { PCI_VDEVICE(MELLANOX, 0x1023) }, /* ConnectX-8 */
2e9d3e83
NO		 2194 { PCI_VDEVICE(MELLANOX, 0xa2d2) }, /* BlueField integrated ConnectX-5 network controller */
2195 { PCI_VDEVICE(MELLANOX, 0xa2d3), MLX5_PCI_DEV_IS_VF}, /* BlueField integrated ConnectX-5 network controller VF */
d19a79ee 2196 { PCI_VDEVICE(MELLANOX, 0xa2d6) }, /* BlueField-2 integrated ConnectX-6 Dx network controller */
dd8595ea 2197 { PCI_VDEVICE(MELLANOX, 0xa2dc) }, /* BlueField-3 integrated ConnectX-7 network controller */
f908a35b 2198 { PCI_VDEVICE(MELLANOX, 0xa2df) }, /* BlueField-4 integrated ConnectX-8 network controller */
9603b61d
JM		 2199 { 0, }
2200};
2201
2202MODULE_DEVICE_TABLE(pci, mlx5_core_pci_table);
2203
04c0c1ab
MHY		 2204void mlx5_disable_device(struct mlx5_core_dev *dev)
2205{
b3bd076f 2206 mlx5_error_sw_reset(dev);
72ed5d56 2207 mlx5_unload_one_devl_locked(dev, false);
04c0c1ab
MHY		 2208}
2209
fe06992b 2210int mlx5_recover_device(struct mlx5_core_dev *dev)
04c0c1ab 2211{
33de865f
MS		 2212 if (!mlx5_core_is_sf(dev)) {
2213 mlx5_pci_disable_device(dev);
2214 if (mlx5_pci_slot_reset(dev->pdev) != PCI_ERS_RESULT_RECOVERED)
2215 return -EIO;
2216 }
fe06992b 2217
d3dbdc9f 2218 return mlx5_load_one_devl_locked(dev, true);
04c0c1ab
MHY		 2219}
2220
9603b61d 2221static struct pci_driver mlx5_core_driver = {
17a7612b 2222 .name = KBUILD_MODNAME,
9603b61d 2223 .id_table = mlx5_core_pci_table,
6dea2f7e 2224 .probe = probe_one,
89d44f0a 2225 .remove = remove_one,
8fc3e29b
MB		 2226 .suspend = mlx5_suspend,
2227 .resume = mlx5_resume,
5fc7197d 2228 .shutdown = shutdown,
fc50db98
EC		 2229 .err_handler = &mlx5_err_handler,
2230 .sriov_configure = mlx5_core_sriov_configure,
e71b75f7
LR		 2231 .sriov_get_vf_total_msix = mlx5_sriov_get_vf_total_msix,
2232 .sriov_set_msix_vec_count = mlx5_core_sriov_set_msix_vec_count,
9603b61d 2233};
e126ba97 2234
1695b97b
YH		 2235/**
2236 * mlx5_vf_get_core_dev - Get the mlx5 core device from a given VF PCI device if
2237 * mlx5_core is its driver.
2238 * @pdev: The associated PCI device.
2239 *
2240 * Upon return the interface state lock stay held to let caller uses it safely.
2241 * Caller must ensure to use the returned mlx5 device for a narrow window
2242 * and put it back with mlx5_vf_put_core_dev() immediately once usage was over.
2243 *
2244 * Return: Pointer to the associated mlx5_core_dev or NULL.
2245 */
2246struct mlx5_core_dev *mlx5_vf_get_core_dev(struct pci_dev *pdev)
1695b97b
YH		 2247{
2248 struct mlx5_core_dev *mdev;
2249
2250 mdev = pci_iov_get_pf_drvdata(pdev, &mlx5_core_driver);
2251 if (IS_ERR(mdev))
2252 return NULL;
2253
2254 mutex_lock(&mdev->intf_state_mutex);
2255 if (!test_bit(MLX5_INTERFACE_STATE_UP, &mdev->intf_state)) {
2256 mutex_unlock(&mdev->intf_state_mutex);
2257 return NULL;
2258 }
2259
2260 return mdev;
2261}
2262EXPORT_SYMBOL(mlx5_vf_get_core_dev);
2263
2264/**
2265 * mlx5_vf_put_core_dev - Put the mlx5 core device back.
2266 * @mdev: The mlx5 core device.
2267 *
2268 * Upon return the interface state lock is unlocked and caller should not
2269 * access the mdev any more.
2270 */
2271void mlx5_vf_put_core_dev(struct mlx5_core_dev *mdev)
1695b97b
YH		 2272{
2273 mutex_unlock(&mdev->intf_state_mutex);
2274}
2275EXPORT_SYMBOL(mlx5_vf_put_core_dev);
2276
f663ad98
KH		 2277static void mlx5_core_verify_params(void)
2278{
2279 if (prof_sel >= ARRAY_SIZE(profile)) {
2280 pr_warn("mlx5_core: WARNING: Invalid module parameter prof_sel %d, valid range 0-%zu, changing back to default(%d)\n",
2281 prof_sel,
2282 ARRAY_SIZE(profile) - 1,
2283 MLX5_DEFAULT_PROF);
2284 prof_sel = MLX5_DEFAULT_PROF;
2285 }
2286}
2287
2c1e1b94 2288static int __init mlx5_init(void)
e126ba97
EC		 2289{
2290 int err;
2291
17a7612b
LR		 2292 WARN_ONCE(strcmp(MLX5_ADEV_NAME, KBUILD_MODNAME),
2293 "mlx5_core name not in sync with kernel module name");
2294
8737f818
DJ		 2295 get_random_bytes(&sw_owner_id, sizeof(sw_owner_id));
2296
f663ad98 2297 mlx5_core_verify_params();
e126ba97 2298 mlx5_register_debugfs();
e126ba97 2299
8f0d1451 2300 err = mlx5e_init();
9603b61d 2301 if (err)
ac6ea6e8 2302 goto err_debug;
9603b61d 2303
1958fc2f
PP		 2304 err = mlx5_sf_driver_register();
2305 if (err)
2306 goto err_sf;
2307
8f0d1451 2308 err = pci_register_driver(&mlx5_core_driver);
c633e799 2309 if (err)
8f0d1451 2310 goto err_pci;
f62b8bb8 2311
e126ba97
EC		 2312 return 0;
2313
8f0d1451 2314err_pci:
c633e799 2315 mlx5_sf_driver_unregister();
1958fc2f 2316err_sf:
8f0d1451 2317 mlx5e_cleanup();
e126ba97
EC		 2318err_debug:
2319 mlx5_unregister_debugfs();
2320 return err;
2321}
2322
2c1e1b94 2323static void __exit mlx5_cleanup(void)
e126ba97 2324{
9603b61d 2325 pci_unregister_driver(&mlx5_core_driver);
8f0d1451
SD		 2326 mlx5_sf_driver_unregister();
2327 mlx5e_cleanup();
e126ba97
EC		 2328 mlx5_unregister_debugfs();
2329}
2330
2c1e1b94
RD		 2331module_init(mlx5_init);
2332module_exit(mlx5_cleanup);
Linux 6.x block layer and io_uring tree(s)