drivers/net/ethernet/mellanox/mlxsw/core.c

   1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0
   2 /* Copyright (c) 2015-2018 Mellanox Technologies. All rights reserved */
   3
   4 #include <linux/kernel.h>
   5 #include <linux/module.h>
   6 #include <linux/device.h>
   7 #include <linux/export.h>
   8 #include <linux/err.h>
   9 #include <linux/if_link.h>
  10 #include <linux/netdevice.h>
  11 #include <linux/completion.h>
  12 #include <linux/skbuff.h>
  13 #include <linux/etherdevice.h>
  14 #include <linux/types.h>
  15 #include <linux/string.h>
  16 #include <linux/gfp.h>
  17 #include <linux/random.h>
  18 #include <linux/jiffies.h>
  19 #include <linux/mutex.h>
  20 #include <linux/rcupdate.h>
  21 #include <linux/slab.h>
  22 #include <linux/workqueue.h>
  23 #include <asm/byteorder.h>
  24 #include <net/devlink.h>
  25 #include <trace/events/devlink.h>
  26
  27 #include "core.h"
  28 #include "item.h"
  29 #include "cmd.h"
  30 #include "port.h"
  31 #include "trap.h"
  32 #include "emad.h"
  33 #include "reg.h"
  34 #include "resources.h"
  35
  36 static LIST_HEAD(mlxsw_core_driver_list);
  37 static DEFINE_SPINLOCK(mlxsw_core_driver_list_lock);
  38
  39 static const char mlxsw_core_driver_name[] = "mlxsw_core";
  40
  41 static struct workqueue_struct *mlxsw_wq;
  42 static struct workqueue_struct *mlxsw_owq;
  43
  44 struct mlxsw_core_port {
  45         struct devlink_port devlink_port;
  46         void *port_driver_priv;
  47         u8 local_port;
  48 };
  49
  50 void *mlxsw_core_port_driver_priv(struct mlxsw_core_port *mlxsw_core_port)
  51 {
  52         return mlxsw_core_port->port_driver_priv;
  53 }
  54 EXPORT_SYMBOL(mlxsw_core_port_driver_priv);
  55
  56 static bool mlxsw_core_port_check(struct mlxsw_core_port *mlxsw_core_port)
  57 {
  58         return mlxsw_core_port->port_driver_priv != NULL;
  59 }
  60
  61 struct mlxsw_core {
  62         struct mlxsw_driver *driver;
  63         const struct mlxsw_bus *bus;
  64         void *bus_priv;
  65         const struct mlxsw_bus_info *bus_info;
  66         struct workqueue_struct *emad_wq;
  67         struct list_head rx_listener_list;
  68         struct list_head event_listener_list;
  69         struct {
  70                 atomic64_t tid;
  71                 struct list_head trans_list;
  72                 spinlock_t trans_list_lock; /* protects trans_list writes */
  73                 bool use_emad;
  74         } emad;
  75         struct {
  76                 u8 *mapping; /* lag_id+port_index to local_port mapping */
  77         } lag;
  78         struct mlxsw_res res;
  79         struct mlxsw_hwmon *hwmon;
  80         struct mlxsw_thermal *thermal;
  81         struct mlxsw_core_port *ports;
  82         unsigned int max_ports;
  83         bool fw_flash_in_progress;
  84         unsigned long driver_priv[0];
  85         /* driver_priv has to be always the last item */
  86 };
  87
  88 #define MLXSW_PORT_MAX_PORTS_DEFAULT    0x40
  89
  90 static int mlxsw_ports_init(struct mlxsw_core *mlxsw_core)
  91 {
  92         /* Switch ports are numbered from 1 to queried value */
  93         if (MLXSW_CORE_RES_VALID(mlxsw_core, MAX_SYSTEM_PORT))
  94                 mlxsw_core->max_ports = MLXSW_CORE_RES_GET(mlxsw_core,
  95                                                            MAX_SYSTEM_PORT) + 1;
  96         else
  97                 mlxsw_core->max_ports = MLXSW_PORT_MAX_PORTS_DEFAULT + 1;
  98
  99         mlxsw_core->ports = kcalloc(mlxsw_core->max_ports,
 100                                     sizeof(struct mlxsw_core_port), GFP_KERNEL);
 101         if (!mlxsw_core->ports)
 102                 return -ENOMEM;
 103
 104         return 0;
 105 }
 106
 107 static void mlxsw_ports_fini(struct mlxsw_core *mlxsw_core)
 108 {
 109         kfree(mlxsw_core->ports);
 110 }
 111
 112 unsigned int mlxsw_core_max_ports(const struct mlxsw_core *mlxsw_core)
 113 {
 114         return mlxsw_core->max_ports;
 115 }
 116 EXPORT_SYMBOL(mlxsw_core_max_ports);
 117
 118 void *mlxsw_core_driver_priv(struct mlxsw_core *mlxsw_core)
 119 {
 120         return mlxsw_core->driver_priv;
 121 }
 122 EXPORT_SYMBOL(mlxsw_core_driver_priv);
 123
 124 bool mlxsw_core_res_query_enabled(const struct mlxsw_core *mlxsw_core)
 125 {
 126         return mlxsw_core->driver->res_query_enabled;
 127 }
 128 EXPORT_SYMBOL(mlxsw_core_res_query_enabled);
 129
 130 bool
 131 mlxsw_core_fw_rev_minor_subminor_validate(const struct mlxsw_fw_rev *rev,
 132                                           const struct mlxsw_fw_rev *req_rev)
 133 {
 134         return rev->minor > req_rev->minor ||
 135                (rev->minor == req_rev->minor &&
 136                 rev->subminor >= req_rev->subminor);
 137 }
 138 EXPORT_SYMBOL(mlxsw_core_fw_rev_minor_subminor_validate);
 139
 140 struct mlxsw_rx_listener_item {
 141         struct list_head list;
 142         struct mlxsw_rx_listener rxl;
 143         void *priv;
 144 };
 145
 146 struct mlxsw_event_listener_item {
 147         struct list_head list;
 148         struct mlxsw_event_listener el;
 149         void *priv;
 150 };
 151
 152 /******************
 153  * EMAD processing
 154  ******************/
 155
 156 /* emad_eth_hdr_dmac
 157  * Destination MAC in EMAD's Ethernet header.
 158  * Must be set to 01:02:c9:00:00:01
 159  */
 160 MLXSW_ITEM_BUF(emad, eth_hdr, dmac, 0x00, 6);
 161
 162 /* emad_eth_hdr_smac
 163  * Source MAC in EMAD's Ethernet header.
 164  * Must be set to 00:02:c9:01:02:03
 165  */
 166 MLXSW_ITEM_BUF(emad, eth_hdr, smac, 0x06, 6);
 167
 168 /* emad_eth_hdr_ethertype
 169  * Ethertype in EMAD's Ethernet header.
 170  * Must be set to 0x8932
 171  */
 172 MLXSW_ITEM32(emad, eth_hdr, ethertype, 0x0C, 16, 16);
 173
 174 /* emad_eth_hdr_mlx_proto
 175  * Mellanox protocol.
 176  * Must be set to 0x0.
 177  */
 178 MLXSW_ITEM32(emad, eth_hdr, mlx_proto, 0x0C, 8, 8);
 179
 180 /* emad_eth_hdr_ver
 181  * Mellanox protocol version.
 182  * Must be set to 0x0.
 183  */
 184 MLXSW_ITEM32(emad, eth_hdr, ver, 0x0C, 4, 4);
 185
 186 /* emad_op_tlv_type
 187  * Type of the TLV.
 188  * Must be set to 0x1 (operation TLV).
 189  */
 190 MLXSW_ITEM32(emad, op_tlv, type, 0x00, 27, 5);
 191
 192 /* emad_op_tlv_len
 193  * Length of the operation TLV in u32.
 194  * Must be set to 0x4.
 195  */
 196 MLXSW_ITEM32(emad, op_tlv, len, 0x00, 16, 11);
 197
 198 /* emad_op_tlv_dr
 199  * Direct route bit. Setting to 1 indicates the EMAD is a direct route
 200  * EMAD. DR TLV must follow.
 201  *
 202  * Note: Currently not supported and must not be set.
 203  */
 204 MLXSW_ITEM32(emad, op_tlv, dr, 0x00, 15, 1);
 205
 206 /* emad_op_tlv_status
 207  * Returned status in case of EMAD response. Must be set to 0 in case
 208  * of EMAD request.
 209  * 0x0 - success
 210  * 0x1 - device is busy. Requester should retry
 211  * 0x2 - Mellanox protocol version not supported
 212  * 0x3 - unknown TLV
 213  * 0x4 - register not supported
 214  * 0x5 - operation class not supported
 215  * 0x6 - EMAD method not supported
 216  * 0x7 - bad parameter (e.g. port out of range)
 217  * 0x8 - resource not available
 218  * 0x9 - message receipt acknowledgment. Requester should retry
 219  * 0x70 - internal error
 220  */
 221 MLXSW_ITEM32(emad, op_tlv, status, 0x00, 8, 7);
 222
 223 /* emad_op_tlv_register_id
 224  * Register ID of register within register TLV.
 225  */
 226 MLXSW_ITEM32(emad, op_tlv, register_id, 0x04, 16, 16);
 227
 228 /* emad_op_tlv_r
 229  * Response bit. Setting to 1 indicates Response, otherwise request.
 230  */
 231 MLXSW_ITEM32(emad, op_tlv, r, 0x04, 15, 1);
 232
 233 /* emad_op_tlv_method
 234  * EMAD method type.
 235  * 0x1 - query
 236  * 0x2 - write
 237  * 0x3 - send (currently not supported)
 238  * 0x4 - event
 239  */
 240 MLXSW_ITEM32(emad, op_tlv, method, 0x04, 8, 7);
 241
 242 /* emad_op_tlv_class
 243  * EMAD operation class. Must be set to 0x1 (REG_ACCESS).
 244  */
 245 MLXSW_ITEM32(emad, op_tlv, class, 0x04, 0, 8);
 246
 247 /* emad_op_tlv_tid
 248  * EMAD transaction ID. Used for pairing request and response EMADs.
 249  */
 250 MLXSW_ITEM64(emad, op_tlv, tid, 0x08, 0, 64);
 251
 252 /* emad_reg_tlv_type
 253  * Type of the TLV.
 254  * Must be set to 0x3 (register TLV).
 255  */
 256 MLXSW_ITEM32(emad, reg_tlv, type, 0x00, 27, 5);
 257
 258 /* emad_reg_tlv_len
 259  * Length of the operation TLV in u32.
 260  */
 261 MLXSW_ITEM32(emad, reg_tlv, len, 0x00, 16, 11);
 262
 263 /* emad_end_tlv_type
 264  * Type of the TLV.
 265  * Must be set to 0x0 (end TLV).
 266  */
 267 MLXSW_ITEM32(emad, end_tlv, type, 0x00, 27, 5);
 268
 269 /* emad_end_tlv_len
 270  * Length of the end TLV in u32.
 271  * Must be set to 1.
 272  */
 273 MLXSW_ITEM32(emad, end_tlv, len, 0x00, 16, 11);
 274
 275 enum mlxsw_core_reg_access_type {
 276         MLXSW_CORE_REG_ACCESS_TYPE_QUERY,
 277         MLXSW_CORE_REG_ACCESS_TYPE_WRITE,
 278 };
 279
 280 static inline const char *
 281 mlxsw_core_reg_access_type_str(enum mlxsw_core_reg_access_type type)
 282 {
 283         switch (type) {
 284         case MLXSW_CORE_REG_ACCESS_TYPE_QUERY:
 285                 return "query";
 286         case MLXSW_CORE_REG_ACCESS_TYPE_WRITE:
 287                 return "write";
 288         }
 289         BUG();
 290 }
 291
 292 static void mlxsw_emad_pack_end_tlv(char *end_tlv)
 293 {
 294         mlxsw_emad_end_tlv_type_set(end_tlv, MLXSW_EMAD_TLV_TYPE_END);
 295         mlxsw_emad_end_tlv_len_set(end_tlv, MLXSW_EMAD_END_TLV_LEN);
 296 }
 297
 298 static void mlxsw_emad_pack_reg_tlv(char *reg_tlv,
 299                                     const struct mlxsw_reg_info *reg,
 300                                     char *payload)
 301 {
 302         mlxsw_emad_reg_tlv_type_set(reg_tlv, MLXSW_EMAD_TLV_TYPE_REG);
 303         mlxsw_emad_reg_tlv_len_set(reg_tlv, reg->len / sizeof(u32) + 1);
 304         memcpy(reg_tlv + sizeof(u32), payload, reg->len);
 305 }
 306
 307 static void mlxsw_emad_pack_op_tlv(char *op_tlv,
 308                                    const struct mlxsw_reg_info *reg,
 309                                    enum mlxsw_core_reg_access_type type,
 310                                    u64 tid)
 311 {
 312         mlxsw_emad_op_tlv_type_set(op_tlv, MLXSW_EMAD_TLV_TYPE_OP);
 313         mlxsw_emad_op_tlv_len_set(op_tlv, MLXSW_EMAD_OP_TLV_LEN);
 314         mlxsw_emad_op_tlv_dr_set(op_tlv, 0);
 315         mlxsw_emad_op_tlv_status_set(op_tlv, 0);
 316         mlxsw_emad_op_tlv_register_id_set(op_tlv, reg->id);
 317         mlxsw_emad_op_tlv_r_set(op_tlv, MLXSW_EMAD_OP_TLV_REQUEST);
 318         if (type == MLXSW_CORE_REG_ACCESS_TYPE_QUERY)
 319                 mlxsw_emad_op_tlv_method_set(op_tlv,
 320                                              MLXSW_EMAD_OP_TLV_METHOD_QUERY);
 321         else
 322                 mlxsw_emad_op_tlv_method_set(op_tlv,
 323                                              MLXSW_EMAD_OP_TLV_METHOD_WRITE);
 324         mlxsw_emad_op_tlv_class_set(op_tlv,
 325                                     MLXSW_EMAD_OP_TLV_CLASS_REG_ACCESS);
 326         mlxsw_emad_op_tlv_tid_set(op_tlv, tid);
 327 }
 328
 329 static int mlxsw_emad_construct_eth_hdr(struct sk_buff *skb)
 330 {
 331         char *eth_hdr = skb_push(skb, MLXSW_EMAD_ETH_HDR_LEN);
 332
 333         mlxsw_emad_eth_hdr_dmac_memcpy_to(eth_hdr, MLXSW_EMAD_EH_DMAC);
 334         mlxsw_emad_eth_hdr_smac_memcpy_to(eth_hdr, MLXSW_EMAD_EH_SMAC);
 335         mlxsw_emad_eth_hdr_ethertype_set(eth_hdr, MLXSW_EMAD_EH_ETHERTYPE);
 336         mlxsw_emad_eth_hdr_mlx_proto_set(eth_hdr, MLXSW_EMAD_EH_MLX_PROTO);
 337         mlxsw_emad_eth_hdr_ver_set(eth_hdr, MLXSW_EMAD_EH_PROTO_VERSION);
 338
 339         skb_reset_mac_header(skb);
 340
 341         return 0;
 342 }
 343
 344 static void mlxsw_emad_construct(struct sk_buff *skb,
 345                                  const struct mlxsw_reg_info *reg,
 346                                  char *payload,
 347                                  enum mlxsw_core_reg_access_type type,
 348                                  u64 tid)
 349 {
 350         char *buf;
 351
 352         buf = skb_push(skb, MLXSW_EMAD_END_TLV_LEN * sizeof(u32));
 353         mlxsw_emad_pack_end_tlv(buf);
 354
 355         buf = skb_push(skb, reg->len + sizeof(u32));
 356         mlxsw_emad_pack_reg_tlv(buf, reg, payload);
 357
 358         buf = skb_push(skb, MLXSW_EMAD_OP_TLV_LEN * sizeof(u32));
 359         mlxsw_emad_pack_op_tlv(buf, reg, type, tid);
 360
 361         mlxsw_emad_construct_eth_hdr(skb);
 362 }
 363
 364 static char *mlxsw_emad_op_tlv(const struct sk_buff *skb)
 365 {
 366         return ((char *) (skb->data + MLXSW_EMAD_ETH_HDR_LEN));
 367 }
 368
 369 static char *mlxsw_emad_reg_tlv(const struct sk_buff *skb)
 370 {
 371         return ((char *) (skb->data + MLXSW_EMAD_ETH_HDR_LEN +
 372                                       MLXSW_EMAD_OP_TLV_LEN * sizeof(u32)));
 373 }
 374
 375 static char *mlxsw_emad_reg_payload(const char *op_tlv)
 376 {
 377         return ((char *) (op_tlv + (MLXSW_EMAD_OP_TLV_LEN + 1) * sizeof(u32)));
 378 }
 379
 380 static u64 mlxsw_emad_get_tid(const struct sk_buff *skb)
 381 {
 382         char *op_tlv;
 383
 384         op_tlv = mlxsw_emad_op_tlv(skb);
 385         return mlxsw_emad_op_tlv_tid_get(op_tlv);
 386 }
 387
 388 static bool mlxsw_emad_is_resp(const struct sk_buff *skb)
 389 {
 390         char *op_tlv;
 391
 392         op_tlv = mlxsw_emad_op_tlv(skb);
 393         return (mlxsw_emad_op_tlv_r_get(op_tlv) == MLXSW_EMAD_OP_TLV_RESPONSE);
 394 }
 395
 396 static int mlxsw_emad_process_status(char *op_tlv,
 397                                      enum mlxsw_emad_op_tlv_status *p_status)
 398 {
 399         *p_status = mlxsw_emad_op_tlv_status_get(op_tlv);
 400
 401         switch (*p_status) {
 402         case MLXSW_EMAD_OP_TLV_STATUS_SUCCESS:
 403                 return 0;
 404         case MLXSW_EMAD_OP_TLV_STATUS_BUSY:
 405         case MLXSW_EMAD_OP_TLV_STATUS_MESSAGE_RECEIPT_ACK:
 406                 return -EAGAIN;
 407         case MLXSW_EMAD_OP_TLV_STATUS_VERSION_NOT_SUPPORTED:
 408         case MLXSW_EMAD_OP_TLV_STATUS_UNKNOWN_TLV:
 409         case MLXSW_EMAD_OP_TLV_STATUS_REGISTER_NOT_SUPPORTED:
 410         case MLXSW_EMAD_OP_TLV_STATUS_CLASS_NOT_SUPPORTED:
 411         case MLXSW_EMAD_OP_TLV_STATUS_METHOD_NOT_SUPPORTED:
 412         case MLXSW_EMAD_OP_TLV_STATUS_BAD_PARAMETER:
 413         case MLXSW_EMAD_OP_TLV_STATUS_RESOURCE_NOT_AVAILABLE:
 414         case MLXSW_EMAD_OP_TLV_STATUS_INTERNAL_ERROR:
 415         default:
 416                 return -EIO;
 417         }
 418 }
 419
 420 static int
 421 mlxsw_emad_process_status_skb(struct sk_buff *skb,
 422                               enum mlxsw_emad_op_tlv_status *p_status)
 423 {
 424         return mlxsw_emad_process_status(mlxsw_emad_op_tlv(skb), p_status);
 425 }
 426
 427 struct mlxsw_reg_trans {
 428         struct list_head list;
 429         struct list_head bulk_list;
 430         struct mlxsw_core *core;
 431         struct sk_buff *tx_skb;
 432         struct mlxsw_tx_info tx_info;
 433         struct delayed_work timeout_dw;
 434         unsigned int retries;
 435         u64 tid;
 436         struct completion completion;
 437         atomic_t active;
 438         mlxsw_reg_trans_cb_t *cb;
 439         unsigned long cb_priv;
 440         const struct mlxsw_reg_info *reg;
 441         enum mlxsw_core_reg_access_type type;
 442         int err;
 443         enum mlxsw_emad_op_tlv_status emad_status;
 444         struct rcu_head rcu;
 445 };
 446
 447 #define MLXSW_EMAD_TIMEOUT_DURING_FW_FLASH_MS   3000
 448 #define MLXSW_EMAD_TIMEOUT_MS                   200
 449
 450 static void mlxsw_emad_trans_timeout_schedule(struct mlxsw_reg_trans *trans)
 451 {
 452         unsigned long timeout = msecs_to_jiffies(MLXSW_EMAD_TIMEOUT_MS);
 453
 454         if (trans->core->fw_flash_in_progress)
 455                 timeout = msecs_to_jiffies(MLXSW_EMAD_TIMEOUT_DURING_FW_FLASH_MS);
 456
 457         queue_delayed_work(trans->core->emad_wq, &trans->timeout_dw, timeout);
 458 }
 459
 460 static int mlxsw_emad_transmit(struct mlxsw_core *mlxsw_core,
 461                                struct mlxsw_reg_trans *trans)
 462 {
 463         struct sk_buff *skb;
 464         int err;
 465
 466         skb = skb_copy(trans->tx_skb, GFP_KERNEL);
 467         if (!skb)
 468                 return -ENOMEM;
 469
 470         trace_devlink_hwmsg(priv_to_devlink(mlxsw_core), false, 0,
 471                             skb->data + mlxsw_core->driver->txhdr_len,
 472                             skb->len - mlxsw_core->driver->txhdr_len);
 473
 474         atomic_set(&trans->active, 1);
 475         err = mlxsw_core_skb_transmit(mlxsw_core, skb, &trans->tx_info);
 476         if (err) {
 477                 dev_kfree_skb(skb);
 478                 return err;
 479         }
 480         mlxsw_emad_trans_timeout_schedule(trans);
 481         return 0;
 482 }
 483
 484 static void mlxsw_emad_trans_finish(struct mlxsw_reg_trans *trans, int err)
 485 {
 486         struct mlxsw_core *mlxsw_core = trans->core;
 487
 488         dev_kfree_skb(trans->tx_skb);
 489         spin_lock_bh(&mlxsw_core->emad.trans_list_lock);
 490         list_del_rcu(&trans->list);
 491         spin_unlock_bh(&mlxsw_core->emad.trans_list_lock);
 492         trans->err = err;
 493         complete(&trans->completion);
 494 }
 495
 496 static void mlxsw_emad_transmit_retry(struct mlxsw_core *mlxsw_core,
 497                                       struct mlxsw_reg_trans *trans)
 498 {
 499         int err;
 500
 501         if (trans->retries < MLXSW_EMAD_MAX_RETRY) {
 502                 trans->retries++;
 503                 err = mlxsw_emad_transmit(trans->core, trans);
 504                 if (err == 0)
 505                         return;
 506         } else {
 507                 err = -EIO;
 508         }
 509         mlxsw_emad_trans_finish(trans, err);
 510 }
 511
 512 static void mlxsw_emad_trans_timeout_work(struct work_struct *work)
 513 {
 514         struct mlxsw_reg_trans *trans = container_of(work,
 515                                                      struct mlxsw_reg_trans,
 516                                                      timeout_dw.work);
 517
 518         if (!atomic_dec_and_test(&trans->active))
 519                 return;
 520
 521         mlxsw_emad_transmit_retry(trans->core, trans);
 522 }
 523
 524 static void mlxsw_emad_process_response(struct mlxsw_core *mlxsw_core,
 525                                         struct mlxsw_reg_trans *trans,
 526                                         struct sk_buff *skb)
 527 {
 528         int err;
 529
 530         if (!atomic_dec_and_test(&trans->active))
 531                 return;
 532
 533         err = mlxsw_emad_process_status_skb(skb, &trans->emad_status);
 534         if (err == -EAGAIN) {
 535                 mlxsw_emad_transmit_retry(mlxsw_core, trans);
 536         } else {
 537                 if (err == 0) {
 538                         char *op_tlv = mlxsw_emad_op_tlv(skb);
 539
 540                         if (trans->cb)
 541                                 trans->cb(mlxsw_core,
 542                                           mlxsw_emad_reg_payload(op_tlv),
 543                                           trans->reg->len, trans->cb_priv);
 544                 }
 545                 mlxsw_emad_trans_finish(trans, err);
 546         }
 547 }
 548
 549 /* called with rcu read lock held */
 550 static void mlxsw_emad_rx_listener_func(struct sk_buff *skb, u8 local_port,
 551                                         void *priv)
 552 {
 553         struct mlxsw_core *mlxsw_core = priv;
 554         struct mlxsw_reg_trans *trans;
 555
 556         trace_devlink_hwmsg(priv_to_devlink(mlxsw_core), true, 0,
 557                             skb->data, skb->len);
 558
 559         if (!mlxsw_emad_is_resp(skb))
 560                 goto free_skb;
 561
 562         list_for_each_entry_rcu(trans, &mlxsw_core->emad.trans_list, list) {
 563                 if (mlxsw_emad_get_tid(skb) == trans->tid) {
 564                         mlxsw_emad_process_response(mlxsw_core, trans, skb);
 565                         break;
 566                 }
 567         }
 568
 569 free_skb:
 570         dev_kfree_skb(skb);
 571 }
 572
 573 static const struct mlxsw_listener mlxsw_emad_rx_listener =
 574         MLXSW_RXL(mlxsw_emad_rx_listener_func, ETHEMAD, TRAP_TO_CPU, false,
 575                   EMAD, DISCARD);
 576
 577 static int mlxsw_emad_init(struct mlxsw_core *mlxsw_core)
 578 {
 579         struct workqueue_struct *emad_wq;
 580         u64 tid;
 581         int err;
 582
 583         if (!(mlxsw_core->bus->features & MLXSW_BUS_F_TXRX))
 584                 return 0;
 585
 586         emad_wq = alloc_workqueue("mlxsw_core_emad", 0, 0);
 587         if (!emad_wq)
 588                 return -ENOMEM;
 589         mlxsw_core->emad_wq = emad_wq;
 590
 591         /* Set the upper 32 bits of the transaction ID field to a random
 592          * number. This allows us to discard EMADs addressed to other
 593          * devices.
 594          */
 595         get_random_bytes(&tid, 4);
 596         tid <<= 32;
 597         atomic64_set(&mlxsw_core->emad.tid, tid);
 598
 599         INIT_LIST_HEAD(&mlxsw_core->emad.trans_list);
 600         spin_lock_init(&mlxsw_core->emad.trans_list_lock);
 601
 602         err = mlxsw_core_trap_register(mlxsw_core, &mlxsw_emad_rx_listener,
 603                                        mlxsw_core);
 604         if (err)
 605                 return err;
 606
 607         err = mlxsw_core->driver->basic_trap_groups_set(mlxsw_core);
 608         if (err)
 609                 goto err_emad_trap_set;
 610         mlxsw_core->emad.use_emad = true;
 611
 612         return 0;
 613
 614 err_emad_trap_set:
 615         mlxsw_core_trap_unregister(mlxsw_core, &mlxsw_emad_rx_listener,
 616                                    mlxsw_core);
 617         destroy_workqueue(mlxsw_core->emad_wq);
 618         return err;
 619 }
 620
 621 static void mlxsw_emad_fini(struct mlxsw_core *mlxsw_core)
 622 {
 623
 624         if (!(mlxsw_core->bus->features & MLXSW_BUS_F_TXRX))
 625                 return;
 626
 627         mlxsw_core->emad.use_emad = false;
 628         mlxsw_core_trap_unregister(mlxsw_core, &mlxsw_emad_rx_listener,
 629                                    mlxsw_core);
 630         destroy_workqueue(mlxsw_core->emad_wq);
 631 }
 632
 633 static struct sk_buff *mlxsw_emad_alloc(const struct mlxsw_core *mlxsw_core,
 634                                         u16 reg_len)
 635 {
 636         struct sk_buff *skb;
 637         u16 emad_len;
 638
 639         emad_len = (reg_len + sizeof(u32) + MLXSW_EMAD_ETH_HDR_LEN +
 640                     (MLXSW_EMAD_OP_TLV_LEN + MLXSW_EMAD_END_TLV_LEN) *
 641                     sizeof(u32) + mlxsw_core->driver->txhdr_len);
 642         if (emad_len > MLXSW_EMAD_MAX_FRAME_LEN)
 643                 return NULL;
 644
 645         skb = netdev_alloc_skb(NULL, emad_len);
 646         if (!skb)
 647                 return NULL;
 648         memset(skb->data, 0, emad_len);
 649         skb_reserve(skb, emad_len);
 650
 651         return skb;
 652 }
 653
 654 static int mlxsw_emad_reg_access(struct mlxsw_core *mlxsw_core,
 655                                  const struct mlxsw_reg_info *reg,
 656                                  char *payload,
 657                                  enum mlxsw_core_reg_access_type type,
 658                                  struct mlxsw_reg_trans *trans,
 659                                  struct list_head *bulk_list,
 660                                  mlxsw_reg_trans_cb_t *cb,
 661                                  unsigned long cb_priv, u64 tid)
 662 {
 663         struct sk_buff *skb;
 664         int err;
 665
 666         dev_dbg(mlxsw_core->bus_info->dev, "EMAD reg access (tid=%llx,reg_id=%x(%s),type=%s)\n",
 667                 tid, reg->id, mlxsw_reg_id_str(reg->id),
 668                 mlxsw_core_reg_access_type_str(type));
 669
 670         skb = mlxsw_emad_alloc(mlxsw_core, reg->len);
 671         if (!skb)
 672                 return -ENOMEM;
 673
 674         list_add_tail(&trans->bulk_list, bulk_list);
 675         trans->core = mlxsw_core;
 676         trans->tx_skb = skb;
 677         trans->tx_info.local_port = MLXSW_PORT_CPU_PORT;
 678         trans->tx_info.is_emad = true;
 679         INIT_DELAYED_WORK(&trans->timeout_dw, mlxsw_emad_trans_timeout_work);
 680         trans->tid = tid;
 681         init_completion(&trans->completion);
 682         trans->cb = cb;
 683         trans->cb_priv = cb_priv;
 684         trans->reg = reg;
 685         trans->type = type;
 686
 687         mlxsw_emad_construct(skb, reg, payload, type, trans->tid);
 688         mlxsw_core->driver->txhdr_construct(skb, &trans->tx_info);
 689
 690         spin_lock_bh(&mlxsw_core->emad.trans_list_lock);
 691         list_add_tail_rcu(&trans->list, &mlxsw_core->emad.trans_list);
 692         spin_unlock_bh(&mlxsw_core->emad.trans_list_lock);
 693         err = mlxsw_emad_transmit(mlxsw_core, trans);
 694         if (err)
 695                 goto err_out;
 696         return 0;
 697
 698 err_out:
 699         spin_lock_bh(&mlxsw_core->emad.trans_list_lock);
 700         list_del_rcu(&trans->list);
 701         spin_unlock_bh(&mlxsw_core->emad.trans_list_lock);
 702         list_del(&trans->bulk_list);
 703         dev_kfree_skb(trans->tx_skb);
 704         return err;
 705 }
 706
 707 /*****************
 708  * Core functions
 709  *****************/
 710
 711 int mlxsw_core_driver_register(struct mlxsw_driver *mlxsw_driver)
 712 {
 713         spin_lock(&mlxsw_core_driver_list_lock);
 714         list_add_tail(&mlxsw_driver->list, &mlxsw_core_driver_list);
 715         spin_unlock(&mlxsw_core_driver_list_lock);
 716         return 0;
 717 }
 718 EXPORT_SYMBOL(mlxsw_core_driver_register);
 719
 720 void mlxsw_core_driver_unregister(struct mlxsw_driver *mlxsw_driver)
 721 {
 722         spin_lock(&mlxsw_core_driver_list_lock);
 723         list_del(&mlxsw_driver->list);
 724         spin_unlock(&mlxsw_core_driver_list_lock);
 725 }
 726 EXPORT_SYMBOL(mlxsw_core_driver_unregister);
 727
 728 static struct mlxsw_driver *__driver_find(const char *kind)
 729 {
 730         struct mlxsw_driver *mlxsw_driver;
 731
 732         list_for_each_entry(mlxsw_driver, &mlxsw_core_driver_list, list) {
 733                 if (strcmp(mlxsw_driver->kind, kind) == 0)
 734                         return mlxsw_driver;
 735         }
 736         return NULL;
 737 }
 738
 739 static struct mlxsw_driver *mlxsw_core_driver_get(const char *kind)
 740 {
 741         struct mlxsw_driver *mlxsw_driver;
 742
 743         spin_lock(&mlxsw_core_driver_list_lock);
 744         mlxsw_driver = __driver_find(kind);
 745         spin_unlock(&mlxsw_core_driver_list_lock);
 746         return mlxsw_driver;
 747 }
 748
 749 static int mlxsw_devlink_port_split(struct devlink *devlink,
 750                                     unsigned int port_index,
 751                                     unsigned int count,
 752                                     struct netlink_ext_ack *extack)
 753 {
 754         struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
 755
 756         if (port_index >= mlxsw_core->max_ports) {
 757                 NL_SET_ERR_MSG_MOD(extack, "Port index exceeds maximum number of ports");
 758                 return -EINVAL;
 759         }
 760         if (!mlxsw_core->driver->port_split)
 761                 return -EOPNOTSUPP;
 762         return mlxsw_core->driver->port_split(mlxsw_core, port_index, count,
 763                                               extack);
 764 }
 765
 766 static int mlxsw_devlink_port_unsplit(struct devlink *devlink,
 767                                       unsigned int port_index,
 768                                       struct netlink_ext_ack *extack)
 769 {
 770         struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
 771
 772         if (port_index >= mlxsw_core->max_ports) {
 773                 NL_SET_ERR_MSG_MOD(extack, "Port index exceeds maximum number of ports");
 774                 return -EINVAL;
 775         }
 776         if (!mlxsw_core->driver->port_unsplit)
 777                 return -EOPNOTSUPP;
 778         return mlxsw_core->driver->port_unsplit(mlxsw_core, port_index,
 779                                                 extack);
 780 }
 781
 782 static int
 783 mlxsw_devlink_sb_pool_get(struct devlink *devlink,
 784                           unsigned int sb_index, u16 pool_index,
 785                           struct devlink_sb_pool_info *pool_info)
 786 {
 787         struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
 788         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
 789
 790         if (!mlxsw_driver->sb_pool_get)
 791                 return -EOPNOTSUPP;
 792         return mlxsw_driver->sb_pool_get(mlxsw_core, sb_index,
 793                                          pool_index, pool_info);
 794 }
 795
 796 static int
 797 mlxsw_devlink_sb_pool_set(struct devlink *devlink,
 798                           unsigned int sb_index, u16 pool_index, u32 size,
 799                           enum devlink_sb_threshold_type threshold_type,
 800                           struct netlink_ext_ack *extack)
 801 {
 802         struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
 803         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
 804
 805         if (!mlxsw_driver->sb_pool_set)
 806                 return -EOPNOTSUPP;
 807         return mlxsw_driver->sb_pool_set(mlxsw_core, sb_index,
 808                                          pool_index, size, threshold_type,
 809                                          extack);
 810 }
 811
 812 static void *__dl_port(struct devlink_port *devlink_port)
 813 {
 814         return container_of(devlink_port, struct mlxsw_core_port, devlink_port);
 815 }
 816
 817 static int mlxsw_devlink_port_type_set(struct devlink_port *devlink_port,
 818                                        enum devlink_port_type port_type)
 819 {
 820         struct mlxsw_core *mlxsw_core = devlink_priv(devlink_port->devlink);
 821         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
 822         struct mlxsw_core_port *mlxsw_core_port = __dl_port(devlink_port);
 823
 824         if (!mlxsw_driver->port_type_set)
 825                 return -EOPNOTSUPP;
 826
 827         return mlxsw_driver->port_type_set(mlxsw_core,
 828                                            mlxsw_core_port->local_port,
 829                                            port_type);
 830 }
 831
 832 static int mlxsw_devlink_sb_port_pool_get(struct devlink_port *devlink_port,
 833                                           unsigned int sb_index, u16 pool_index,
 834                                           u32 *p_threshold)
 835 {
 836         struct mlxsw_core *mlxsw_core = devlink_priv(devlink_port->devlink);
 837         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
 838         struct mlxsw_core_port *mlxsw_core_port = __dl_port(devlink_port);
 839
 840         if (!mlxsw_driver->sb_port_pool_get ||
 841             !mlxsw_core_port_check(mlxsw_core_port))
 842                 return -EOPNOTSUPP;
 843         return mlxsw_driver->sb_port_pool_get(mlxsw_core_port, sb_index,
 844                                               pool_index, p_threshold);
 845 }
 846
 847 static int mlxsw_devlink_sb_port_pool_set(struct devlink_port *devlink_port,
 848                                           unsigned int sb_index, u16 pool_index,
 849                                           u32 threshold,
 850                                           struct netlink_ext_ack *extack)
 851 {
 852         struct mlxsw_core *mlxsw_core = devlink_priv(devlink_port->devlink);
 853         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
 854         struct mlxsw_core_port *mlxsw_core_port = __dl_port(devlink_port);
 855
 856         if (!mlxsw_driver->sb_port_pool_set ||
 857             !mlxsw_core_port_check(mlxsw_core_port))
 858                 return -EOPNOTSUPP;
 859         return mlxsw_driver->sb_port_pool_set(mlxsw_core_port, sb_index,
 860                                               pool_index, threshold, extack);
 861 }
 862
 863 static int
 864 mlxsw_devlink_sb_tc_pool_bind_get(struct devlink_port *devlink_port,
 865                                   unsigned int sb_index, u16 tc_index,
 866                                   enum devlink_sb_pool_type pool_type,
 867                                   u16 *p_pool_index, u32 *p_threshold)
 868 {
 869         struct mlxsw_core *mlxsw_core = devlink_priv(devlink_port->devlink);
 870         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
 871         struct mlxsw_core_port *mlxsw_core_port = __dl_port(devlink_port);
 872
 873         if (!mlxsw_driver->sb_tc_pool_bind_get ||
 874             !mlxsw_core_port_check(mlxsw_core_port))
 875                 return -EOPNOTSUPP;
 876         return mlxsw_driver->sb_tc_pool_bind_get(mlxsw_core_port, sb_index,
 877                                                  tc_index, pool_type,
 878                                                  p_pool_index, p_threshold);
 879 }
 880
 881 static int
 882 mlxsw_devlink_sb_tc_pool_bind_set(struct devlink_port *devlink_port,
 883                                   unsigned int sb_index, u16 tc_index,
 884                                   enum devlink_sb_pool_type pool_type,
 885                                   u16 pool_index, u32 threshold,
 886                                   struct netlink_ext_ack *extack)
 887 {
 888         struct mlxsw_core *mlxsw_core = devlink_priv(devlink_port->devlink);
 889         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
 890         struct mlxsw_core_port *mlxsw_core_port = __dl_port(devlink_port);
 891
 892         if (!mlxsw_driver->sb_tc_pool_bind_set ||
 893             !mlxsw_core_port_check(mlxsw_core_port))
 894                 return -EOPNOTSUPP;
 895         return mlxsw_driver->sb_tc_pool_bind_set(mlxsw_core_port, sb_index,
 896                                                  tc_index, pool_type,
 897                                                  pool_index, threshold, extack);
 898 }
 899
 900 static int mlxsw_devlink_sb_occ_snapshot(struct devlink *devlink,
 901                                          unsigned int sb_index)
 902 {
 903         struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
 904         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
 905
 906         if (!mlxsw_driver->sb_occ_snapshot)
 907                 return -EOPNOTSUPP;
 908         return mlxsw_driver->sb_occ_snapshot(mlxsw_core, sb_index);
 909 }
 910
 911 static int mlxsw_devlink_sb_occ_max_clear(struct devlink *devlink,
 912                                           unsigned int sb_index)
 913 {
 914         struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
 915         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
 916
 917         if (!mlxsw_driver->sb_occ_max_clear)
 918                 return -EOPNOTSUPP;
 919         return mlxsw_driver->sb_occ_max_clear(mlxsw_core, sb_index);
 920 }
 921
 922 static int
 923 mlxsw_devlink_sb_occ_port_pool_get(struct devlink_port *devlink_port,
 924                                    unsigned int sb_index, u16 pool_index,
 925                                    u32 *p_cur, u32 *p_max)
 926 {
 927         struct mlxsw_core *mlxsw_core = devlink_priv(devlink_port->devlink);
 928         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
 929         struct mlxsw_core_port *mlxsw_core_port = __dl_port(devlink_port);
 930
 931         if (!mlxsw_driver->sb_occ_port_pool_get ||
 932             !mlxsw_core_port_check(mlxsw_core_port))
 933                 return -EOPNOTSUPP;
 934         return mlxsw_driver->sb_occ_port_pool_get(mlxsw_core_port, sb_index,
 935                                                   pool_index, p_cur, p_max);
 936 }
 937
 938 static int
 939 mlxsw_devlink_sb_occ_tc_port_bind_get(struct devlink_port *devlink_port,
 940                                       unsigned int sb_index, u16 tc_index,
 941                                       enum devlink_sb_pool_type pool_type,
 942                                       u32 *p_cur, u32 *p_max)
 943 {
 944         struct mlxsw_core *mlxsw_core = devlink_priv(devlink_port->devlink);
 945         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
 946         struct mlxsw_core_port *mlxsw_core_port = __dl_port(devlink_port);
 947
 948         if (!mlxsw_driver->sb_occ_tc_port_bind_get ||
 949             !mlxsw_core_port_check(mlxsw_core_port))
 950                 return -EOPNOTSUPP;
 951         return mlxsw_driver->sb_occ_tc_port_bind_get(mlxsw_core_port,
 952                                                      sb_index, tc_index,
 953                                                      pool_type, p_cur, p_max);
 954 }
 955
 956 static int
 957 mlxsw_devlink_info_get(struct devlink *devlink, struct devlink_info_req *req,
 958                        struct netlink_ext_ack *extack)
 959 {
 960         struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
 961         char fw_info_psid[MLXSW_REG_MGIR_FW_INFO_PSID_SIZE];
 962         u32 hw_rev, fw_major, fw_minor, fw_sub_minor;
 963         char mgir_pl[MLXSW_REG_MGIR_LEN];
 964         char buf[32];
 965         int err;
 966
 967         err = devlink_info_driver_name_put(req,
 968                                            mlxsw_core->bus_info->device_kind);
 969         if (err)
 970                 return err;
 971
 972         mlxsw_reg_mgir_pack(mgir_pl);
 973         err = mlxsw_reg_query(mlxsw_core, MLXSW_REG(mgir), mgir_pl);
 974         if (err)
 975                 return err;
 976         mlxsw_reg_mgir_unpack(mgir_pl, &hw_rev, fw_info_psid, &fw_major,
 977                               &fw_minor, &fw_sub_minor);
 978
 979         sprintf(buf, "%X", hw_rev);
 980         err = devlink_info_version_fixed_put(req, "hw.revision", buf);
 981         if (err)
 982                 return err;
 983
 984         err = devlink_info_version_fixed_put(req, "fw.psid", fw_info_psid);
 985         if (err)
 986                 return err;
 987
 988         sprintf(buf, "%d.%d.%d", fw_major, fw_minor, fw_sub_minor);
 989         err = devlink_info_version_running_put(req, "fw.version", buf);
 990         if (err)
 991                 return err;
 992
 993         return 0;
 994 }
 995
 996 static int
 997 mlxsw_devlink_core_bus_device_reload_down(struct devlink *devlink,
 998                                           bool netns_change,
 999                                           struct netlink_ext_ack *extack)
1000 {
1001         struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1002
1003         if (!(mlxsw_core->bus->features & MLXSW_BUS_F_RESET))
1004                 return -EOPNOTSUPP;
1005
1006         mlxsw_core_bus_device_unregister(mlxsw_core, true);
1007         return 0;
1008 }
1009
1010 static int
1011 mlxsw_devlink_core_bus_device_reload_up(struct devlink *devlink,
1012                                         struct netlink_ext_ack *extack)
1013 {
1014         struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1015
1016         return mlxsw_core_bus_device_register(mlxsw_core->bus_info,
1017                                               mlxsw_core->bus,
1018                                               mlxsw_core->bus_priv, true,
1019                                               devlink, extack);
1020 }
1021
1022 static int mlxsw_devlink_flash_update(struct devlink *devlink,
1023                                       const char *file_name,
1024                                       const char *component,
1025                                       struct netlink_ext_ack *extack)
1026 {
1027         struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1028         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1029
1030         if (!mlxsw_driver->flash_update)
1031                 return -EOPNOTSUPP;
1032         return mlxsw_driver->flash_update(mlxsw_core, file_name,
1033                                           component, extack);
1034 }
1035
1036 static int mlxsw_devlink_trap_init(struct devlink *devlink,
1037                                    const struct devlink_trap *trap,
1038                                    void *trap_ctx)
1039 {
1040         struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1041         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1042
1043         if (!mlxsw_driver->trap_init)
1044                 return -EOPNOTSUPP;
1045         return mlxsw_driver->trap_init(mlxsw_core, trap, trap_ctx);
1046 }
1047
1048 static void mlxsw_devlink_trap_fini(struct devlink *devlink,
1049                                     const struct devlink_trap *trap,
1050                                     void *trap_ctx)
1051 {
1052         struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1053         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1054
1055         if (!mlxsw_driver->trap_fini)
1056                 return;
1057         mlxsw_driver->trap_fini(mlxsw_core, trap, trap_ctx);
1058 }
1059
1060 static int mlxsw_devlink_trap_action_set(struct devlink *devlink,
1061                                          const struct devlink_trap *trap,
1062                                          enum devlink_trap_action action)
1063 {
1064         struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1065         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1066
1067         if (!mlxsw_driver->trap_action_set)
1068                 return -EOPNOTSUPP;
1069         return mlxsw_driver->trap_action_set(mlxsw_core, trap, action);
1070 }
1071
1072 static int
1073 mlxsw_devlink_trap_group_init(struct devlink *devlink,
1074                               const struct devlink_trap_group *group)
1075 {
1076         struct mlxsw_core *mlxsw_core = devlink_priv(devlink);
1077         struct mlxsw_driver *mlxsw_driver = mlxsw_core->driver;
1078
1079         if (!mlxsw_driver->trap_group_init)
1080                 return -EOPNOTSUPP;
1081         return mlxsw_driver->trap_group_init(mlxsw_core, group);
1082 }
1083
1084 static const struct devlink_ops mlxsw_devlink_ops = {
1085         .reload_down            = mlxsw_devlink_core_bus_device_reload_down,
1086         .reload_up              = mlxsw_devlink_core_bus_device_reload_up,
1087         .port_type_set                  = mlxsw_devlink_port_type_set,
1088         .port_split                     = mlxsw_devlink_port_split,
1089         .port_unsplit                   = mlxsw_devlink_port_unsplit,
1090         .sb_pool_get                    = mlxsw_devlink_sb_pool_get,
1091         .sb_pool_set                    = mlxsw_devlink_sb_pool_set,
1092         .sb_port_pool_get               = mlxsw_devlink_sb_port_pool_get,
1093         .sb_port_pool_set               = mlxsw_devlink_sb_port_pool_set,
1094         .sb_tc_pool_bind_get            = mlxsw_devlink_sb_tc_pool_bind_get,
1095         .sb_tc_pool_bind_set            = mlxsw_devlink_sb_tc_pool_bind_set,
1096         .sb_occ_snapshot                = mlxsw_devlink_sb_occ_snapshot,
1097         .sb_occ_max_clear               = mlxsw_devlink_sb_occ_max_clear,
1098         .sb_occ_port_pool_get           = mlxsw_devlink_sb_occ_port_pool_get,
1099         .sb_occ_tc_port_bind_get        = mlxsw_devlink_sb_occ_tc_port_bind_get,
1100         .info_get                       = mlxsw_devlink_info_get,
1101         .flash_update                   = mlxsw_devlink_flash_update,
1102         .trap_init                      = mlxsw_devlink_trap_init,
1103         .trap_fini                      = mlxsw_devlink_trap_fini,
1104         .trap_action_set                = mlxsw_devlink_trap_action_set,
1105         .trap_group_init                = mlxsw_devlink_trap_group_init,
1106 };
1107
1108 static int
1109 __mlxsw_core_bus_device_register(const struct mlxsw_bus_info *mlxsw_bus_info,
1110                                  const struct mlxsw_bus *mlxsw_bus,
1111                                  void *bus_priv, bool reload,
1112                                  struct devlink *devlink,
1113                                  struct netlink_ext_ack *extack)
1114 {
1115         const char *device_kind = mlxsw_bus_info->device_kind;
1116         struct mlxsw_core *mlxsw_core;
1117         struct mlxsw_driver *mlxsw_driver;
1118         struct mlxsw_res *res;
1119         size_t alloc_size;
1120         int err;
1121
1122         mlxsw_driver = mlxsw_core_driver_get(device_kind);
1123         if (!mlxsw_driver)
1124                 return -EINVAL;
1125
1126         if (!reload) {
1127                 alloc_size = sizeof(*mlxsw_core) + mlxsw_driver->priv_size;
1128                 devlink = devlink_alloc(&mlxsw_devlink_ops, alloc_size);
1129                 if (!devlink) {
1130                         err = -ENOMEM;
1131                         goto err_devlink_alloc;
1132                 }
1133         }
1134
1135         mlxsw_core = devlink_priv(devlink);
1136         INIT_LIST_HEAD(&mlxsw_core->rx_listener_list);
1137         INIT_LIST_HEAD(&mlxsw_core->event_listener_list);
1138         mlxsw_core->driver = mlxsw_driver;
1139         mlxsw_core->bus = mlxsw_bus;
1140         mlxsw_core->bus_priv = bus_priv;
1141         mlxsw_core->bus_info = mlxsw_bus_info;
1142
1143         res = mlxsw_driver->res_query_enabled ? &mlxsw_core->res : NULL;
1144         err = mlxsw_bus->init(bus_priv, mlxsw_core, mlxsw_driver->profile, res);
1145         if (err)
1146                 goto err_bus_init;
1147
1148         if (mlxsw_driver->resources_register && !reload) {
1149                 err = mlxsw_driver->resources_register(mlxsw_core);
1150                 if (err)
1151                         goto err_register_resources;
1152         }
1153
1154         err = mlxsw_ports_init(mlxsw_core);
1155         if (err)
1156                 goto err_ports_init;
1157
1158         if (MLXSW_CORE_RES_VALID(mlxsw_core, MAX_LAG) &&
1159             MLXSW_CORE_RES_VALID(mlxsw_core, MAX_LAG_MEMBERS)) {
1160                 alloc_size = sizeof(u8) *
1161                         MLXSW_CORE_RES_GET(mlxsw_core, MAX_LAG) *
1162                         MLXSW_CORE_RES_GET(mlxsw_core, MAX_LAG_MEMBERS);
1163                 mlxsw_core->lag.mapping = kzalloc(alloc_size, GFP_KERNEL);
1164                 if (!mlxsw_core->lag.mapping) {
1165                         err = -ENOMEM;
1166                         goto err_alloc_lag_mapping;
1167                 }
1168         }
1169
1170         err = mlxsw_emad_init(mlxsw_core);
1171         if (err)
1172                 goto err_emad_init;
1173
1174         if (!reload) {
1175                 err = devlink_register(devlink, mlxsw_bus_info->dev);
1176                 if (err)
1177                         goto err_devlink_register;
1178         }
1179
1180         if (mlxsw_driver->params_register && !reload) {
1181                 err = mlxsw_driver->params_register(mlxsw_core);
1182                 if (err)
1183                         goto err_register_params;
1184         }
1185
1186         if (mlxsw_driver->init) {
1187                 err = mlxsw_driver->init(mlxsw_core, mlxsw_bus_info, extack);
1188                 if (err)
1189                         goto err_driver_init;
1190         }
1191
1192         err = mlxsw_hwmon_init(mlxsw_core, mlxsw_bus_info, &mlxsw_core->hwmon);
1193         if (err)
1194                 goto err_hwmon_init;
1195
1196         err = mlxsw_thermal_init(mlxsw_core, mlxsw_bus_info,
1197                                  &mlxsw_core->thermal);
1198         if (err)
1199                 goto err_thermal_init;
1200
1201         if (mlxsw_driver->params_register) {
1202                 devlink_params_publish(devlink);
1203                 devlink_reload_enable(devlink);
1204         }
1205
1206         return 0;
1207
1208 err_thermal_init:
1209         mlxsw_hwmon_fini(mlxsw_core->hwmon);
1210 err_hwmon_init:
1211         if (mlxsw_core->driver->fini)
1212                 mlxsw_core->driver->fini(mlxsw_core);
1213 err_driver_init:
1214         if (mlxsw_driver->params_unregister && !reload)
1215                 mlxsw_driver->params_unregister(mlxsw_core);
1216 err_register_params:
1217         if (!reload)
1218                 devlink_unregister(devlink);
1219 err_devlink_register:
1220         mlxsw_emad_fini(mlxsw_core);
1221 err_emad_init:
1222         kfree(mlxsw_core->lag.mapping);
1223 err_alloc_lag_mapping:
1224         mlxsw_ports_fini(mlxsw_core);
1225 err_ports_init:
1226         if (!reload)
1227                 devlink_resources_unregister(devlink, NULL);
1228 err_register_resources:
1229         mlxsw_bus->fini(bus_priv);
1230 err_bus_init:
1231         if (!reload)
1232                 devlink_free(devlink);
1233 err_devlink_alloc:
1234         return err;
1235 }
1236
1237 int mlxsw_core_bus_device_register(const struct mlxsw_bus_info *mlxsw_bus_info,
1238                                    const struct mlxsw_bus *mlxsw_bus,
1239                                    void *bus_priv, bool reload,
1240                                    struct devlink *devlink,
1241                                    struct netlink_ext_ack *extack)
1242 {
1243         bool called_again = false;
1244         int err;
1245
1246 again:
1247         err = __mlxsw_core_bus_device_register(mlxsw_bus_info, mlxsw_bus,
1248                                                bus_priv, reload,
1249                                                devlink, extack);
1250         /* -EAGAIN is returned in case the FW was updated. FW needs
1251          * a reset, so lets try to call __mlxsw_core_bus_device_register()
1252          * again.
1253          */
1254         if (err == -EAGAIN && !called_again) {
1255                 called_again = true;
1256                 goto again;
1257         }
1258
1259         return err;
1260 }
1261 EXPORT_SYMBOL(mlxsw_core_bus_device_register);
1262
1263 void mlxsw_core_bus_device_unregister(struct mlxsw_core *mlxsw_core,
1264                                       bool reload)
1265 {
1266         struct devlink *devlink = priv_to_devlink(mlxsw_core);
1267
1268         if (!reload)
1269                 devlink_reload_disable(devlink);
1270         if (devlink_is_reload_failed(devlink)) {
1271                 if (!reload)
1272                         /* Only the parts that were not de-initialized in the
1273                          * failed reload attempt need to be de-initialized.
1274                          */
1275                         goto reload_fail_deinit;
1276                 else
1277                         return;
1278         }
1279
1280         if (mlxsw_core->driver->params_unregister)
1281                 devlink_params_unpublish(devlink);
1282         mlxsw_thermal_fini(mlxsw_core->thermal);
1283         mlxsw_hwmon_fini(mlxsw_core->hwmon);
1284         if (mlxsw_core->driver->fini)
1285                 mlxsw_core->driver->fini(mlxsw_core);
1286         if (mlxsw_core->driver->params_unregister && !reload)
1287                 mlxsw_core->driver->params_unregister(mlxsw_core);
1288         if (!reload)
1289                 devlink_unregister(devlink);
1290         mlxsw_emad_fini(mlxsw_core);
1291         kfree(mlxsw_core->lag.mapping);
1292         mlxsw_ports_fini(mlxsw_core);
1293         if (!reload)
1294                 devlink_resources_unregister(devlink, NULL);
1295         mlxsw_core->bus->fini(mlxsw_core->bus_priv);
1296
1297         return;
1298
1299 reload_fail_deinit:
1300         if (mlxsw_core->driver->params_unregister)
1301                 mlxsw_core->driver->params_unregister(mlxsw_core);
1302         devlink_unregister(devlink);
1303         devlink_resources_unregister(devlink, NULL);
1304         devlink_free(devlink);
1305 }
1306 EXPORT_SYMBOL(mlxsw_core_bus_device_unregister);
1307
1308 bool mlxsw_core_skb_transmit_busy(struct mlxsw_core *mlxsw_core,
1309                                   const struct mlxsw_tx_info *tx_info)
1310 {
1311         return mlxsw_core->bus->skb_transmit_busy(mlxsw_core->bus_priv,
1312                                                   tx_info);
1313 }
1314 EXPORT_SYMBOL(mlxsw_core_skb_transmit_busy);
1315
1316 int mlxsw_core_skb_transmit(struct mlxsw_core *mlxsw_core, struct sk_buff *skb,
1317                             const struct mlxsw_tx_info *tx_info)
1318 {
1319         return mlxsw_core->bus->skb_transmit(mlxsw_core->bus_priv, skb,
1320                                              tx_info);
1321 }
1322 EXPORT_SYMBOL(mlxsw_core_skb_transmit);
1323
1324 void mlxsw_core_ptp_transmitted(struct mlxsw_core *mlxsw_core,
1325                                 struct sk_buff *skb, u8 local_port)
1326 {
1327         if (mlxsw_core->driver->ptp_transmitted)
1328                 mlxsw_core->driver->ptp_transmitted(mlxsw_core, skb,
1329                                                     local_port);
1330 }
1331 EXPORT_SYMBOL(mlxsw_core_ptp_transmitted);
1332
1333 static bool __is_rx_listener_equal(const struct mlxsw_rx_listener *rxl_a,
1334                                    const struct mlxsw_rx_listener *rxl_b)
1335 {
1336         return (rxl_a->func == rxl_b->func &&
1337                 rxl_a->local_port == rxl_b->local_port &&
1338                 rxl_a->trap_id == rxl_b->trap_id);
1339 }
1340
1341 static struct mlxsw_rx_listener_item *
1342 __find_rx_listener_item(struct mlxsw_core *mlxsw_core,
1343                         const struct mlxsw_rx_listener *rxl,
1344                         void *priv)
1345 {
1346         struct mlxsw_rx_listener_item *rxl_item;
1347
1348         list_for_each_entry(rxl_item, &mlxsw_core->rx_listener_list, list) {
1349                 if (__is_rx_listener_equal(&rxl_item->rxl, rxl) &&
1350                     rxl_item->priv == priv)
1351                         return rxl_item;
1352         }
1353         return NULL;
1354 }
1355
1356 int mlxsw_core_rx_listener_register(struct mlxsw_core *mlxsw_core,
1357                                     const struct mlxsw_rx_listener *rxl,
1358                                     void *priv)
1359 {
1360         struct mlxsw_rx_listener_item *rxl_item;
1361
1362         rxl_item = __find_rx_listener_item(mlxsw_core, rxl, priv);
1363         if (rxl_item)
1364                 return -EEXIST;
1365         rxl_item = kmalloc(sizeof(*rxl_item), GFP_KERNEL);
1366         if (!rxl_item)
1367                 return -ENOMEM;
1368         rxl_item->rxl = *rxl;
1369         rxl_item->priv = priv;
1370
1371         list_add_rcu(&rxl_item->list, &mlxsw_core->rx_listener_list);
1372         return 0;
1373 }
1374 EXPORT_SYMBOL(mlxsw_core_rx_listener_register);
1375
1376 void mlxsw_core_rx_listener_unregister(struct mlxsw_core *mlxsw_core,
1377                                        const struct mlxsw_rx_listener *rxl,
1378                                        void *priv)
1379 {
1380         struct mlxsw_rx_listener_item *rxl_item;
1381
1382         rxl_item = __find_rx_listener_item(mlxsw_core, rxl, priv);
1383         if (!rxl_item)
1384                 return;
1385         list_del_rcu(&rxl_item->list);
1386         synchronize_rcu();
1387         kfree(rxl_item);
1388 }
1389 EXPORT_SYMBOL(mlxsw_core_rx_listener_unregister);
1390
1391 static void mlxsw_core_event_listener_func(struct sk_buff *skb, u8 local_port,
1392                                            void *priv)
1393 {
1394         struct mlxsw_event_listener_item *event_listener_item = priv;
1395         struct mlxsw_reg_info reg;
1396         char *payload;
1397         char *op_tlv = mlxsw_emad_op_tlv(skb);
1398         char *reg_tlv = mlxsw_emad_reg_tlv(skb);
1399
1400         reg.id = mlxsw_emad_op_tlv_register_id_get(op_tlv);
1401         reg.len = (mlxsw_emad_reg_tlv_len_get(reg_tlv) - 1) * sizeof(u32);
1402         payload = mlxsw_emad_reg_payload(op_tlv);
1403         event_listener_item->el.func(&reg, payload, event_listener_item->priv);
1404         dev_kfree_skb(skb);
1405 }
1406
1407 static bool __is_event_listener_equal(const struct mlxsw_event_listener *el_a,
1408                                       const struct mlxsw_event_listener *el_b)
1409 {
1410         return (el_a->func == el_b->func &&
1411                 el_a->trap_id == el_b->trap_id);
1412 }
1413
1414 static struct mlxsw_event_listener_item *
1415 __find_event_listener_item(struct mlxsw_core *mlxsw_core,
1416                            const struct mlxsw_event_listener *el,
1417                            void *priv)
1418 {
1419         struct mlxsw_event_listener_item *el_item;
1420
1421         list_for_each_entry(el_item, &mlxsw_core->event_listener_list, list) {
1422                 if (__is_event_listener_equal(&el_item->el, el) &&
1423                     el_item->priv == priv)
1424                         return el_item;
1425         }
1426         return NULL;
1427 }
1428
1429 int mlxsw_core_event_listener_register(struct mlxsw_core *mlxsw_core,
1430                                        const struct mlxsw_event_listener *el,
1431                                        void *priv)
1432 {
1433         int err;
1434         struct mlxsw_event_listener_item *el_item;
1435         const struct mlxsw_rx_listener rxl = {
1436                 .func = mlxsw_core_event_listener_func,
1437                 .local_port = MLXSW_PORT_DONT_CARE,
1438                 .trap_id = el->trap_id,
1439         };
1440
1441         el_item = __find_event_listener_item(mlxsw_core, el, priv);
1442         if (el_item)
1443                 return -EEXIST;
1444         el_item = kmalloc(sizeof(*el_item), GFP_KERNEL);
1445         if (!el_item)
1446                 return -ENOMEM;
1447         el_item->el = *el;
1448         el_item->priv = priv;
1449
1450         err = mlxsw_core_rx_listener_register(mlxsw_core, &rxl, el_item);
1451         if (err)
1452                 goto err_rx_listener_register;
1453
1454         /* No reason to save item if we did not manage to register an RX
1455          * listener for it.
1456          */
1457         list_add_rcu(&el_item->list, &mlxsw_core->event_listener_list);
1458
1459         return 0;
1460
1461 err_rx_listener_register:
1462         kfree(el_item);
1463         return err;
1464 }
1465 EXPORT_SYMBOL(mlxsw_core_event_listener_register);
1466
1467 void mlxsw_core_event_listener_unregister(struct mlxsw_core *mlxsw_core,
1468                                           const struct mlxsw_event_listener *el,
1469                                           void *priv)
1470 {
1471         struct mlxsw_event_listener_item *el_item;
1472         const struct mlxsw_rx_listener rxl = {
1473                 .func = mlxsw_core_event_listener_func,
1474                 .local_port = MLXSW_PORT_DONT_CARE,
1475                 .trap_id = el->trap_id,
1476         };
1477
1478         el_item = __find_event_listener_item(mlxsw_core, el, priv);
1479         if (!el_item)
1480                 return;
1481         mlxsw_core_rx_listener_unregister(mlxsw_core, &rxl, el_item);
1482         list_del(&el_item->list);
1483         kfree(el_item);
1484 }
1485 EXPORT_SYMBOL(mlxsw_core_event_listener_unregister);
1486
1487 static int mlxsw_core_listener_register(struct mlxsw_core *mlxsw_core,
1488                                         const struct mlxsw_listener *listener,
1489                                         void *priv)
1490 {
1491         if (listener->is_event)
1492                 return mlxsw_core_event_listener_register(mlxsw_core,
1493                                                 &listener->u.event_listener,
1494                                                 priv);
1495         else
1496                 return mlxsw_core_rx_listener_register(mlxsw_core,
1497                                                 &listener->u.rx_listener,
1498                                                 priv);
1499 }
1500
1501 static void mlxsw_core_listener_unregister(struct mlxsw_core *mlxsw_core,
1502                                       const struct mlxsw_listener *listener,
1503                                       void *priv)
1504 {
1505         if (listener->is_event)
1506                 mlxsw_core_event_listener_unregister(mlxsw_core,
1507                                                      &listener->u.event_listener,
1508                                                      priv);
1509         else
1510                 mlxsw_core_rx_listener_unregister(mlxsw_core,
1511                                                   &listener->u.rx_listener,
1512                                                   priv);
1513 }
1514
1515 int mlxsw_core_trap_register(struct mlxsw_core *mlxsw_core,
1516                              const struct mlxsw_listener *listener, void *priv)
1517 {
1518         char hpkt_pl[MLXSW_REG_HPKT_LEN];
1519         int err;
1520
1521         err = mlxsw_core_listener_register(mlxsw_core, listener, priv);
1522         if (err)
1523                 return err;
1524
1525         mlxsw_reg_hpkt_pack(hpkt_pl, listener->action, listener->trap_id,
1526                             listener->trap_group, listener->is_ctrl);
1527         err = mlxsw_reg_write(mlxsw_core,  MLXSW_REG(hpkt), hpkt_pl);
1528         if (err)
1529                 goto err_trap_set;
1530
1531         return 0;
1532
1533 err_trap_set:
1534         mlxsw_core_listener_unregister(mlxsw_core, listener, priv);
1535         return err;
1536 }
1537 EXPORT_SYMBOL(mlxsw_core_trap_register);
1538
1539 void mlxsw_core_trap_unregister(struct mlxsw_core *mlxsw_core,
1540                                 const struct mlxsw_listener *listener,
1541                                 void *priv)
1542 {
1543         char hpkt_pl[MLXSW_REG_HPKT_LEN];
1544
1545         if (!listener->is_event) {
1546                 mlxsw_reg_hpkt_pack(hpkt_pl, listener->unreg_action,
1547                                     listener->trap_id, listener->trap_group,
1548                                     listener->is_ctrl);
1549                 mlxsw_reg_write(mlxsw_core, MLXSW_REG(hpkt), hpkt_pl);
1550         }
1551
1552         mlxsw_core_listener_unregister(mlxsw_core, listener, priv);
1553 }
1554 EXPORT_SYMBOL(mlxsw_core_trap_unregister);
1555
1556 int mlxsw_core_trap_action_set(struct mlxsw_core *mlxsw_core,
1557                                const struct mlxsw_listener *listener,
1558                                enum mlxsw_reg_hpkt_action action)
1559 {
1560         char hpkt_pl[MLXSW_REG_HPKT_LEN];
1561
1562         mlxsw_reg_hpkt_pack(hpkt_pl, action, listener->trap_id,
1563                             listener->trap_group, listener->is_ctrl);
1564         return mlxsw_reg_write(mlxsw_core, MLXSW_REG(hpkt), hpkt_pl);
1565 }
1566 EXPORT_SYMBOL(mlxsw_core_trap_action_set);
1567
1568 static u64 mlxsw_core_tid_get(struct mlxsw_core *mlxsw_core)
1569 {
1570         return atomic64_inc_return(&mlxsw_core->emad.tid);
1571 }
1572
1573 static int mlxsw_core_reg_access_emad(struct mlxsw_core *mlxsw_core,
1574                                       const struct mlxsw_reg_info *reg,
1575                                       char *payload,
1576                                       enum mlxsw_core_reg_access_type type,
1577                                       struct list_head *bulk_list,
1578                                       mlxsw_reg_trans_cb_t *cb,
1579                                       unsigned long cb_priv)
1580 {
1581         u64 tid = mlxsw_core_tid_get(mlxsw_core);
1582         struct mlxsw_reg_trans *trans;
1583         int err;
1584
1585         trans = kzalloc(sizeof(*trans), GFP_KERNEL);
1586         if (!trans)
1587                 return -ENOMEM;
1588
1589         err = mlxsw_emad_reg_access(mlxsw_core, reg, payload, type, trans,
1590                                     bulk_list, cb, cb_priv, tid);
1591         if (err) {
1592                 kfree(trans);
1593                 return err;
1594         }
1595         return 0;
1596 }
1597
1598 int mlxsw_reg_trans_query(struct mlxsw_core *mlxsw_core,
1599                           const struct mlxsw_reg_info *reg, char *payload,
1600                           struct list_head *bulk_list,
1601                           mlxsw_reg_trans_cb_t *cb, unsigned long cb_priv)
1602 {
1603         return mlxsw_core_reg_access_emad(mlxsw_core, reg, payload,
1604                                           MLXSW_CORE_REG_ACCESS_TYPE_QUERY,
1605                                           bulk_list, cb, cb_priv);
1606 }
1607 EXPORT_SYMBOL(mlxsw_reg_trans_query);
1608
1609 int mlxsw_reg_trans_write(struct mlxsw_core *mlxsw_core,
1610                           const struct mlxsw_reg_info *reg, char *payload,
1611                           struct list_head *bulk_list,
1612                           mlxsw_reg_trans_cb_t *cb, unsigned long cb_priv)
1613 {
1614         return mlxsw_core_reg_access_emad(mlxsw_core, reg, payload,
1615                                           MLXSW_CORE_REG_ACCESS_TYPE_WRITE,
1616                                           bulk_list, cb, cb_priv);
1617 }
1618 EXPORT_SYMBOL(mlxsw_reg_trans_write);
1619
1620 static int mlxsw_reg_trans_wait(struct mlxsw_reg_trans *trans)
1621 {
1622         struct mlxsw_core *mlxsw_core = trans->core;
1623         int err;
1624
1625         wait_for_completion(&trans->completion);
1626         cancel_delayed_work_sync(&trans->timeout_dw);
1627         err = trans->err;
1628
1629         if (trans->retries)
1630                 dev_warn(mlxsw_core->bus_info->dev, "EMAD retries (%d/%d) (tid=%llx)\n",
1631                          trans->retries, MLXSW_EMAD_MAX_RETRY, trans->tid);
1632         if (err) {
1633                 dev_err(mlxsw_core->bus_info->dev, "EMAD reg access failed (tid=%llx,reg_id=%x(%s),type=%s,status=%x(%s))\n",
1634                         trans->tid, trans->reg->id,
1635                         mlxsw_reg_id_str(trans->reg->id),
1636                         mlxsw_core_reg_access_type_str(trans->type),
1637                         trans->emad_status,
1638                         mlxsw_emad_op_tlv_status_str(trans->emad_status));
1639                 trace_devlink_hwerr(priv_to_devlink(mlxsw_core),
1640                                     trans->emad_status,
1641                                     mlxsw_emad_op_tlv_status_str(trans->emad_status));
1642         }
1643
1644         list_del(&trans->bulk_list);
1645         kfree_rcu(trans, rcu);
1646         return err;
1647 }
1648
1649 int mlxsw_reg_trans_bulk_wait(struct list_head *bulk_list)
1650 {
1651         struct mlxsw_reg_trans *trans;
1652         struct mlxsw_reg_trans *tmp;
1653         int sum_err = 0;
1654         int err;
1655
1656         list_for_each_entry_safe(trans, tmp, bulk_list, bulk_list) {
1657                 err = mlxsw_reg_trans_wait(trans);
1658                 if (err && sum_err == 0)
1659                         sum_err = err; /* first error to be returned */
1660         }
1661         return sum_err;
1662 }
1663 EXPORT_SYMBOL(mlxsw_reg_trans_bulk_wait);
1664
1665 static int mlxsw_core_reg_access_cmd(struct mlxsw_core *mlxsw_core,
1666                                      const struct mlxsw_reg_info *reg,
1667                                      char *payload,
1668                                      enum mlxsw_core_reg_access_type type)
1669 {
1670         enum mlxsw_emad_op_tlv_status status;
1671         int err, n_retry;
1672         bool reset_ok;
1673         char *in_mbox, *out_mbox, *tmp;
1674
1675         dev_dbg(mlxsw_core->bus_info->dev, "Reg cmd access (reg_id=%x(%s),type=%s)\n",
1676                 reg->id, mlxsw_reg_id_str(reg->id),
1677                 mlxsw_core_reg_access_type_str(type));
1678
1679         in_mbox = mlxsw_cmd_mbox_alloc();
1680         if (!in_mbox)
1681                 return -ENOMEM;
1682
1683         out_mbox = mlxsw_cmd_mbox_alloc();
1684         if (!out_mbox) {
1685                 err = -ENOMEM;
1686                 goto free_in_mbox;
1687         }
1688
1689         mlxsw_emad_pack_op_tlv(in_mbox, reg, type,
1690                                mlxsw_core_tid_get(mlxsw_core));
1691         tmp = in_mbox + MLXSW_EMAD_OP_TLV_LEN * sizeof(u32);
1692         mlxsw_emad_pack_reg_tlv(tmp, reg, payload);
1693
1694         /* There is a special treatment needed for MRSR (reset) register.
1695          * The command interface will return error after the command
1696          * is executed, so tell the lower layer to expect it
1697          * and cope accordingly.
1698          */
1699         reset_ok = reg->id == MLXSW_REG_MRSR_ID;
1700
1701         n_retry = 0;
1702 retry:
1703         err = mlxsw_cmd_access_reg(mlxsw_core, reset_ok, in_mbox, out_mbox);
1704         if (!err) {
1705                 err = mlxsw_emad_process_status(out_mbox, &status);
1706                 if (err) {
1707                         if (err == -EAGAIN && n_retry++ < MLXSW_EMAD_MAX_RETRY)
1708                                 goto retry;
1709                         dev_err(mlxsw_core->bus_info->dev, "Reg cmd access status failed (status=%x(%s))\n",
1710                                 status, mlxsw_emad_op_tlv_status_str(status));
1711                 }
1712         }
1713
1714         if (!err)
1715                 memcpy(payload, mlxsw_emad_reg_payload(out_mbox),
1716                        reg->len);
1717
1718         mlxsw_cmd_mbox_free(out_mbox);
1719 free_in_mbox:
1720         mlxsw_cmd_mbox_free(in_mbox);
1721         if (err)
1722                 dev_err(mlxsw_core->bus_info->dev, "Reg cmd access failed (reg_id=%x(%s),type=%s)\n",
1723                         reg->id, mlxsw_reg_id_str(reg->id),
1724                         mlxsw_core_reg_access_type_str(type));
1725         return err;
1726 }
1727
1728 static void mlxsw_core_reg_access_cb(struct mlxsw_core *mlxsw_core,
1729                                      char *payload, size_t payload_len,
1730                                      unsigned long cb_priv)
1731 {
1732         char *orig_payload = (char *) cb_priv;
1733
1734         memcpy(orig_payload, payload, payload_len);
1735 }
1736
1737 static int mlxsw_core_reg_access(struct mlxsw_core *mlxsw_core,
1738                                  const struct mlxsw_reg_info *reg,
1739                                  char *payload,
1740                                  enum mlxsw_core_reg_access_type type)
1741 {
1742         LIST_HEAD(bulk_list);
1743         int err;
1744
1745         /* During initialization EMAD interface is not available to us,
1746          * so we default to command interface. We switch to EMAD interface
1747          * after setting the appropriate traps.
1748          */
1749         if (!mlxsw_core->emad.use_emad)
1750                 return mlxsw_core_reg_access_cmd(mlxsw_core, reg,
1751                                                  payload, type);
1752
1753         err = mlxsw_core_reg_access_emad(mlxsw_core, reg,
1754                                          payload, type, &bulk_list,
1755                                          mlxsw_core_reg_access_cb,
1756                                          (unsigned long) payload);
1757         if (err)
1758                 return err;
1759         return mlxsw_reg_trans_bulk_wait(&bulk_list);
1760 }
1761
1762 int mlxsw_reg_query(struct mlxsw_core *mlxsw_core,
1763                     const struct mlxsw_reg_info *reg, char *payload)
1764 {
1765         return mlxsw_core_reg_access(mlxsw_core, reg, payload,
1766                                      MLXSW_CORE_REG_ACCESS_TYPE_QUERY);
1767 }
1768 EXPORT_SYMBOL(mlxsw_reg_query);
1769
1770 int mlxsw_reg_write(struct mlxsw_core *mlxsw_core,
1771                     const struct mlxsw_reg_info *reg, char *payload)
1772 {
1773         return mlxsw_core_reg_access(mlxsw_core, reg, payload,
1774                                      MLXSW_CORE_REG_ACCESS_TYPE_WRITE);
1775 }
1776 EXPORT_SYMBOL(mlxsw_reg_write);
1777
1778 void mlxsw_core_skb_receive(struct mlxsw_core *mlxsw_core, struct sk_buff *skb,
1779                             struct mlxsw_rx_info *rx_info)
1780 {
1781         struct mlxsw_rx_listener_item *rxl_item;
1782         const struct mlxsw_rx_listener *rxl;
1783         u8 local_port;
1784         bool found = false;
1785
1786         if (rx_info->is_lag) {
1787                 dev_dbg_ratelimited(mlxsw_core->bus_info->dev, "%s: lag_id = %d, lag_port_index = 0x%x\n",
1788                                     __func__, rx_info->u.lag_id,
1789                                     rx_info->trap_id);
1790                 /* Upper layer does not care if the skb came from LAG or not,
1791                  * so just get the local_port for the lag port and push it up.
1792                  */
1793                 local_port = mlxsw_core_lag_mapping_get(mlxsw_core,
1794                                                         rx_info->u.lag_id,
1795                                                         rx_info->lag_port_index);
1796         } else {
1797                 local_port = rx_info->u.sys_port;
1798         }
1799
1800         dev_dbg_ratelimited(mlxsw_core->bus_info->dev, "%s: local_port = %d, trap_id = 0x%x\n",
1801                             __func__, local_port, rx_info->trap_id);
1802
1803         if ((rx_info->trap_id >= MLXSW_TRAP_ID_MAX) ||
1804             (local_port >= mlxsw_core->max_ports))
1805                 goto drop;
1806
1807         rcu_read_lock();
1808         list_for_each_entry_rcu(rxl_item, &mlxsw_core->rx_listener_list, list) {
1809                 rxl = &rxl_item->rxl;
1810                 if ((rxl->local_port == MLXSW_PORT_DONT_CARE ||
1811                      rxl->local_port == local_port) &&
1812                     rxl->trap_id == rx_info->trap_id) {
1813                         found = true;
1814                         break;
1815                 }
1816         }
1817         rcu_read_unlock();
1818         if (!found)
1819                 goto drop;
1820
1821         rxl->func(skb, local_port, rxl_item->priv);
1822         return;
1823
1824 drop:
1825         dev_kfree_skb(skb);
1826 }
1827 EXPORT_SYMBOL(mlxsw_core_skb_receive);
1828
1829 static int mlxsw_core_lag_mapping_index(struct mlxsw_core *mlxsw_core,
1830                                         u16 lag_id, u8 port_index)
1831 {
1832         return MLXSW_CORE_RES_GET(mlxsw_core, MAX_LAG_MEMBERS) * lag_id +
1833                port_index;
1834 }
1835
1836 void mlxsw_core_lag_mapping_set(struct mlxsw_core *mlxsw_core,
1837                                 u16 lag_id, u8 port_index, u8 local_port)
1838 {
1839         int index = mlxsw_core_lag_mapping_index(mlxsw_core,
1840                                                  lag_id, port_index);
1841
1842         mlxsw_core->lag.mapping[index] = local_port;
1843 }
1844 EXPORT_SYMBOL(mlxsw_core_lag_mapping_set);
1845
1846 u8 mlxsw_core_lag_mapping_get(struct mlxsw_core *mlxsw_core,
1847                               u16 lag_id, u8 port_index)
1848 {
1849         int index = mlxsw_core_lag_mapping_index(mlxsw_core,
1850                                                  lag_id, port_index);
1851
1852         return mlxsw_core->lag.mapping[index];
1853 }
1854 EXPORT_SYMBOL(mlxsw_core_lag_mapping_get);
1855
1856 void mlxsw_core_lag_mapping_clear(struct mlxsw_core *mlxsw_core,
1857                                   u16 lag_id, u8 local_port)
1858 {
1859         int i;
1860
1861         for (i = 0; i < MLXSW_CORE_RES_GET(mlxsw_core, MAX_LAG_MEMBERS); i++) {
1862                 int index = mlxsw_core_lag_mapping_index(mlxsw_core,
1863                                                          lag_id, i);
1864
1865                 if (mlxsw_core->lag.mapping[index] == local_port)
1866                         mlxsw_core->lag.mapping[index] = 0;
1867         }
1868 }
1869 EXPORT_SYMBOL(mlxsw_core_lag_mapping_clear);
1870
1871 bool mlxsw_core_res_valid(struct mlxsw_core *mlxsw_core,
1872                           enum mlxsw_res_id res_id)
1873 {
1874         return mlxsw_res_valid(&mlxsw_core->res, res_id);
1875 }
1876 EXPORT_SYMBOL(mlxsw_core_res_valid);
1877
1878 u64 mlxsw_core_res_get(struct mlxsw_core *mlxsw_core,
1879                        enum mlxsw_res_id res_id)
1880 {
1881         return mlxsw_res_get(&mlxsw_core->res, res_id);
1882 }
1883 EXPORT_SYMBOL(mlxsw_core_res_get);
1884
1885 static int __mlxsw_core_port_init(struct mlxsw_core *mlxsw_core, u8 local_port,
1886                                   enum devlink_port_flavour flavour,
1887                                   u32 port_number, bool split,
1888                                   u32 split_port_subnumber,
1889                                   const unsigned char *switch_id,
1890                                   unsigned char switch_id_len)
1891 {
1892         struct devlink *devlink = priv_to_devlink(mlxsw_core);
1893         struct mlxsw_core_port *mlxsw_core_port =
1894                                         &mlxsw_core->ports[local_port];
1895         struct devlink_port *devlink_port = &mlxsw_core_port->devlink_port;
1896         int err;
1897
1898         mlxsw_core_port->local_port = local_port;
1899         devlink_port_attrs_set(devlink_port, flavour, port_number,
1900                                split, split_port_subnumber,
1901                                switch_id, switch_id_len);
1902         err = devlink_port_register(devlink, devlink_port, local_port);
1903         if (err)
1904                 memset(mlxsw_core_port, 0, sizeof(*mlxsw_core_port));
1905         return err;
1906 }
1907
1908 static void __mlxsw_core_port_fini(struct mlxsw_core *mlxsw_core, u8 local_port)
1909 {
1910         struct mlxsw_core_port *mlxsw_core_port =
1911                                         &mlxsw_core->ports[local_port];
1912         struct devlink_port *devlink_port = &mlxsw_core_port->devlink_port;
1913
1914         devlink_port_unregister(devlink_port);
1915         memset(mlxsw_core_port, 0, sizeof(*mlxsw_core_port));
1916 }
1917
1918 int mlxsw_core_port_init(struct mlxsw_core *mlxsw_core, u8 local_port,
1919                          u32 port_number, bool split,
1920                          u32 split_port_subnumber,
1921                          const unsigned char *switch_id,
1922                          unsigned char switch_id_len)
1923 {
1924         return __mlxsw_core_port_init(mlxsw_core, local_port,
1925                                       DEVLINK_PORT_FLAVOUR_PHYSICAL,
1926                                       port_number, split, split_port_subnumber,
1927                                       switch_id, switch_id_len);
1928 }
1929 EXPORT_SYMBOL(mlxsw_core_port_init);
1930
1931 void mlxsw_core_port_fini(struct mlxsw_core *mlxsw_core, u8 local_port)
1932 {
1933         __mlxsw_core_port_fini(mlxsw_core, local_port);
1934 }
1935 EXPORT_SYMBOL(mlxsw_core_port_fini);
1936
1937 int mlxsw_core_cpu_port_init(struct mlxsw_core *mlxsw_core,
1938                              void *port_driver_priv,
1939                              const unsigned char *switch_id,
1940                              unsigned char switch_id_len)
1941 {
1942         struct mlxsw_core_port *mlxsw_core_port =
1943                                 &mlxsw_core->ports[MLXSW_PORT_CPU_PORT];
1944         int err;
1945
1946         err = __mlxsw_core_port_init(mlxsw_core, MLXSW_PORT_CPU_PORT,
1947                                      DEVLINK_PORT_FLAVOUR_CPU,
1948                                      0, false, 0,
1949                                      switch_id, switch_id_len);
1950         if (err)
1951                 return err;
1952
1953         mlxsw_core_port->port_driver_priv = port_driver_priv;
1954         return 0;
1955 }
1956 EXPORT_SYMBOL(mlxsw_core_cpu_port_init);
1957
1958 void mlxsw_core_cpu_port_fini(struct mlxsw_core *mlxsw_core)
1959 {
1960         __mlxsw_core_port_fini(mlxsw_core, MLXSW_PORT_CPU_PORT);
1961 }
1962 EXPORT_SYMBOL(mlxsw_core_cpu_port_fini);
1963
1964 void mlxsw_core_port_eth_set(struct mlxsw_core *mlxsw_core, u8 local_port,
1965                              void *port_driver_priv, struct net_device *dev)
1966 {
1967         struct mlxsw_core_port *mlxsw_core_port =
1968                                         &mlxsw_core->ports[local_port];
1969         struct devlink_port *devlink_port = &mlxsw_core_port->devlink_port;
1970
1971         mlxsw_core_port->port_driver_priv = port_driver_priv;
1972         devlink_port_type_eth_set(devlink_port, dev);
1973 }
1974 EXPORT_SYMBOL(mlxsw_core_port_eth_set);
1975
1976 void mlxsw_core_port_ib_set(struct mlxsw_core *mlxsw_core, u8 local_port,
1977                             void *port_driver_priv)
1978 {
1979         struct mlxsw_core_port *mlxsw_core_port =
1980                                         &mlxsw_core->ports[local_port];
1981         struct devlink_port *devlink_port = &mlxsw_core_port->devlink_port;
1982
1983         mlxsw_core_port->port_driver_priv = port_driver_priv;
1984         devlink_port_type_ib_set(devlink_port, NULL);
1985 }
1986 EXPORT_SYMBOL(mlxsw_core_port_ib_set);
1987
1988 void mlxsw_core_port_clear(struct mlxsw_core *mlxsw_core, u8 local_port,
1989                            void *port_driver_priv)
1990 {
1991         struct mlxsw_core_port *mlxsw_core_port =
1992                                         &mlxsw_core->ports[local_port];
1993         struct devlink_port *devlink_port = &mlxsw_core_port->devlink_port;
1994
1995         mlxsw_core_port->port_driver_priv = port_driver_priv;
1996         devlink_port_type_clear(devlink_port);
1997 }
1998 EXPORT_SYMBOL(mlxsw_core_port_clear);
1999
2000 enum devlink_port_type mlxsw_core_port_type_get(struct mlxsw_core *mlxsw_core,
2001                                                 u8 local_port)
2002 {
2003         struct mlxsw_core_port *mlxsw_core_port =
2004                                         &mlxsw_core->ports[local_port];
2005         struct devlink_port *devlink_port = &mlxsw_core_port->devlink_port;
2006
2007         return devlink_port->type;
2008 }
2009 EXPORT_SYMBOL(mlxsw_core_port_type_get);
2010
2011
2012 struct devlink_port *
2013 mlxsw_core_port_devlink_port_get(struct mlxsw_core *mlxsw_core,
2014                                  u8 local_port)
2015 {
2016         struct mlxsw_core_port *mlxsw_core_port =
2017                                         &mlxsw_core->ports[local_port];
2018         struct devlink_port *devlink_port = &mlxsw_core_port->devlink_port;
2019
2020         return devlink_port;
2021 }
2022 EXPORT_SYMBOL(mlxsw_core_port_devlink_port_get);
2023
2024 int mlxsw_core_module_max_width(struct mlxsw_core *mlxsw_core, u8 module)
2025 {
2026         enum mlxsw_reg_pmtm_module_type module_type;
2027         char pmtm_pl[MLXSW_REG_PMTM_LEN];
2028         int err;
2029
2030         mlxsw_reg_pmtm_pack(pmtm_pl, module);
2031         err = mlxsw_reg_query(mlxsw_core, MLXSW_REG(pmtm), pmtm_pl);
2032         if (err)
2033                 return err;
2034         mlxsw_reg_pmtm_unpack(pmtm_pl, &module_type);
2035
2036         /* Here we need to get the module width according to the module type. */
2037
2038         switch (module_type) {
2039         case MLXSW_REG_PMTM_MODULE_TYPE_BP_4X: /* fall through */
2040         case MLXSW_REG_PMTM_MODULE_TYPE_BP_QSFP:
2041                 return 4;
2042         case MLXSW_REG_PMTM_MODULE_TYPE_BP_2X:
2043                 return 2;
2044         case MLXSW_REG_PMTM_MODULE_TYPE_BP_SFP: /* fall through */
2045         case MLXSW_REG_PMTM_MODULE_TYPE_BP_1X:
2046                 return 1;
2047         default:
2048                 return -EINVAL;
2049         }
2050 }
2051 EXPORT_SYMBOL(mlxsw_core_module_max_width);
2052
2053 static void mlxsw_core_buf_dump_dbg(struct mlxsw_core *mlxsw_core,
2054                                     const char *buf, size_t size)
2055 {
2056         __be32 *m = (__be32 *) buf;
2057         int i;
2058         int count = size / sizeof(__be32);
2059
2060         for (i = count - 1; i >= 0; i--)
2061                 if (m[i])
2062                         break;
2063         i++;
2064         count = i ? i : 1;
2065         for (i = 0; i < count; i += 4)
2066                 dev_dbg(mlxsw_core->bus_info->dev, "%04x - %08x %08x %08x %08x\n",
2067                         i * 4, be32_to_cpu(m[i]), be32_to_cpu(m[i + 1]),
2068                         be32_to_cpu(m[i + 2]), be32_to_cpu(m[i + 3]));
2069 }
2070
2071 int mlxsw_cmd_exec(struct mlxsw_core *mlxsw_core, u16 opcode, u8 opcode_mod,
2072                    u32 in_mod, bool out_mbox_direct, bool reset_ok,
2073                    char *in_mbox, size_t in_mbox_size,
2074                    char *out_mbox, size_t out_mbox_size)
2075 {
2076         u8 status;
2077         int err;
2078
2079         BUG_ON(in_mbox_size % sizeof(u32) || out_mbox_size % sizeof(u32));
2080         if (!mlxsw_core->bus->cmd_exec)
2081                 return -EOPNOTSUPP;
2082
2083         dev_dbg(mlxsw_core->bus_info->dev, "Cmd exec (opcode=%x(%s),opcode_mod=%x,in_mod=%x)\n",
2084                 opcode, mlxsw_cmd_opcode_str(opcode), opcode_mod, in_mod);
2085         if (in_mbox) {
2086                 dev_dbg(mlxsw_core->bus_info->dev, "Input mailbox:\n");
2087                 mlxsw_core_buf_dump_dbg(mlxsw_core, in_mbox, in_mbox_size);
2088         }
2089
2090         err = mlxsw_core->bus->cmd_exec(mlxsw_core->bus_priv, opcode,
2091                                         opcode_mod, in_mod, out_mbox_direct,
2092                                         in_mbox, in_mbox_size,
2093                                         out_mbox, out_mbox_size, &status);
2094
2095         if (!err && out_mbox) {
2096                 dev_dbg(mlxsw_core->bus_info->dev, "Output mailbox:\n");
2097                 mlxsw_core_buf_dump_dbg(mlxsw_core, out_mbox, out_mbox_size);
2098         }
2099
2100         if (reset_ok && err == -EIO &&
2101             status == MLXSW_CMD_STATUS_RUNNING_RESET) {
2102                 err = 0;
2103         } else if (err == -EIO && status != MLXSW_CMD_STATUS_OK) {
2104                 dev_err(mlxsw_core->bus_info->dev, "Cmd exec failed (opcode=%x(%s),opcode_mod=%x,in_mod=%x,status=%x(%s))\n",
2105                         opcode, mlxsw_cmd_opcode_str(opcode), opcode_mod,
2106                         in_mod, status, mlxsw_cmd_status_str(status));
2107         } else if (err == -ETIMEDOUT) {
2108                 dev_err(mlxsw_core->bus_info->dev, "Cmd exec timed-out (opcode=%x(%s),opcode_mod=%x,in_mod=%x)\n",
2109                         opcode, mlxsw_cmd_opcode_str(opcode), opcode_mod,
2110                         in_mod);
2111         }
2112
2113         return err;
2114 }
2115 EXPORT_SYMBOL(mlxsw_cmd_exec);
2116
2117 int mlxsw_core_schedule_dw(struct delayed_work *dwork, unsigned long delay)
2118 {
2119         return queue_delayed_work(mlxsw_wq, dwork, delay);
2120 }
2121 EXPORT_SYMBOL(mlxsw_core_schedule_dw);
2122
2123 bool mlxsw_core_schedule_work(struct work_struct *work)
2124 {
2125         return queue_work(mlxsw_owq, work);
2126 }
2127 EXPORT_SYMBOL(mlxsw_core_schedule_work);
2128
2129 void mlxsw_core_flush_owq(void)
2130 {
2131         flush_workqueue(mlxsw_owq);
2132 }
2133 EXPORT_SYMBOL(mlxsw_core_flush_owq);
2134
2135 int mlxsw_core_kvd_sizes_get(struct mlxsw_core *mlxsw_core,
2136                              const struct mlxsw_config_profile *profile,
2137                              u64 *p_single_size, u64 *p_double_size,
2138                              u64 *p_linear_size)
2139 {
2140         struct mlxsw_driver *driver = mlxsw_core->driver;
2141
2142         if (!driver->kvd_sizes_get)
2143                 return -EINVAL;
2144
2145         return driver->kvd_sizes_get(mlxsw_core, profile,
2146                                      p_single_size, p_double_size,
2147                                      p_linear_size);
2148 }
2149 EXPORT_SYMBOL(mlxsw_core_kvd_sizes_get);
2150
2151 void mlxsw_core_fw_flash_start(struct mlxsw_core *mlxsw_core)
2152 {
2153         mlxsw_core->fw_flash_in_progress = true;
2154 }
2155 EXPORT_SYMBOL(mlxsw_core_fw_flash_start);
2156
2157 void mlxsw_core_fw_flash_end(struct mlxsw_core *mlxsw_core)
2158 {
2159         mlxsw_core->fw_flash_in_progress = false;
2160 }
2161 EXPORT_SYMBOL(mlxsw_core_fw_flash_end);
2162
2163 int mlxsw_core_resources_query(struct mlxsw_core *mlxsw_core, char *mbox,
2164                                struct mlxsw_res *res)
2165 {
2166         int index, i;
2167         u64 data;
2168         u16 id;
2169         int err;
2170
2171         if (!res)
2172                 return 0;
2173
2174         mlxsw_cmd_mbox_zero(mbox);
2175
2176         for (index = 0; index < MLXSW_CMD_QUERY_RESOURCES_MAX_QUERIES;
2177              index++) {
2178                 err = mlxsw_cmd_query_resources(mlxsw_core, mbox, index);
2179                 if (err)
2180                         return err;
2181
2182                 for (i = 0; i < MLXSW_CMD_QUERY_RESOURCES_PER_QUERY; i++) {
2183                         id = mlxsw_cmd_mbox_query_resource_id_get(mbox, i);
2184                         data = mlxsw_cmd_mbox_query_resource_data_get(mbox, i);
2185
2186                         if (id == MLXSW_CMD_QUERY_RESOURCES_TABLE_END_ID)
2187                                 return 0;
2188
2189                         mlxsw_res_parse(res, id, data);
2190                 }
2191         }
2192
2193         /* If after MLXSW_RESOURCES_QUERY_MAX_QUERIES we still didn't get
2194          * MLXSW_RESOURCES_TABLE_END_ID, something went bad in the FW.
2195          */
2196         return -EIO;
2197 }
2198 EXPORT_SYMBOL(mlxsw_core_resources_query);
2199
2200 u32 mlxsw_core_read_frc_h(struct mlxsw_core *mlxsw_core)
2201 {
2202         return mlxsw_core->bus->read_frc_h(mlxsw_core->bus_priv);
2203 }
2204 EXPORT_SYMBOL(mlxsw_core_read_frc_h);
2205
2206 u32 mlxsw_core_read_frc_l(struct mlxsw_core *mlxsw_core)
2207 {
2208         return mlxsw_core->bus->read_frc_l(mlxsw_core->bus_priv);
2209 }
2210 EXPORT_SYMBOL(mlxsw_core_read_frc_l);
2211
2212 static int __init mlxsw_core_module_init(void)
2213 {
2214         int err;
2215
2216         mlxsw_wq = alloc_workqueue(mlxsw_core_driver_name, 0, 0);
2217         if (!mlxsw_wq)
2218                 return -ENOMEM;
2219         mlxsw_owq = alloc_ordered_workqueue("%s_ordered", 0,
2220                                             mlxsw_core_driver_name);
2221         if (!mlxsw_owq) {
2222                 err = -ENOMEM;
2223                 goto err_alloc_ordered_workqueue;
2224         }
2225         return 0;
2226
2227 err_alloc_ordered_workqueue:
2228         destroy_workqueue(mlxsw_wq);
2229         return err;
2230 }
2231
2232 static void __exit mlxsw_core_module_exit(void)
2233 {
2234         destroy_workqueue(mlxsw_owq);
2235         destroy_workqueue(mlxsw_wq);
2236 }
2237
2238 module_init(mlxsw_core_module_init);
2239 module_exit(mlxsw_core_module_exit);
2240
2241 MODULE_LICENSE("Dual BSD/GPL");
2242 MODULE_AUTHOR("Jiri Pirko <jiri@mellanox.com>");
2243 MODULE_DESCRIPTION("Mellanox switch device core driver");