From a45af9a96740873db9a4b5bb493ce2ad81ccb4d5 Mon Sep 17 00:00:00 2001 From: Tariq Toukan Date: Sun, 10 Dec 2023 16:24:36 +0200 Subject: [PATCH] net/mlx5: SD, Implement devcom communication and primary election Use devcom to communicate between the different devices. Add a new devcom component type for this. Each device registers itself to the devcom component . Once all devices of a component are registered, the component becomes ready, and a primary device is elected. In principle, any of the devices can act as a primary, they are all capable, and a random election would've worked. However, we aim to achieve predictability and consistency, hence each group always choses the same device, with the lowest PCI BUS number, as primary. Signed-off-by: Tariq Toukan Signed-off-by: Saeed Mahameed --- .../ethernet/mellanox/mlx5/core/lib/devcom.h | 1 + .../net/ethernet/mellanox/mlx5/core/lib/sd.c | 122 +++++++++++++++++- 2 files changed, 121 insertions(+), 2 deletions(-) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.h index ec32b686f586..d58032dd0df7 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/devcom.h @@ -10,6 +10,7 @@ enum mlx5_devcom_component { MLX5_DEVCOM_ESW_OFFLOADS, MLX5_DEVCOM_MPV, MLX5_DEVCOM_HCA_PORTS, + MLX5_DEVCOM_SD_GROUP, MLX5_DEVCOM_NUM_COMPONENTS, }; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/sd.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/sd.c index 9d8b1bb0c0a6..19e674dd1af7 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/sd.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/sd.c @@ -14,6 +14,16 @@ struct mlx5_sd { u32 group_id; u8 host_buses; + struct mlx5_devcom_comp_dev *devcom; + bool primary; + union { + struct { /* primary */ + struct mlx5_core_dev *secondaries[MLX5_SD_MAX_GROUP_SZ - 1]; + }; + struct { /* secondary */ + struct mlx5_core_dev *primary_dev; + }; + }; }; static int mlx5_sd_get_host_buses(struct mlx5_core_dev *dev) @@ -26,13 +36,29 @@ static int mlx5_sd_get_host_buses(struct mlx5_core_dev *dev) return sd->host_buses; } +static struct mlx5_core_dev *mlx5_sd_get_primary(struct mlx5_core_dev *dev) +{ + struct mlx5_sd *sd = mlx5_get_sd(dev); + + if (!sd) + return dev; + + return sd->primary ? dev : sd->primary_dev; +} + struct mlx5_core_dev * mlx5_sd_primary_get_peer(struct mlx5_core_dev *primary, int idx) { + struct mlx5_sd *sd; + if (idx == 0) return primary; - return NULL; + if (idx >= mlx5_sd_get_host_buses(primary)) + return NULL; + + sd = mlx5_get_sd(primary); + return sd->secondaries[idx - 1]; } int mlx5_sd_ch_ix_get_dev_ix(struct mlx5_core_dev *dev, int ch_ix) @@ -136,15 +162,93 @@ static void sd_cleanup(struct mlx5_core_dev *dev) kfree(sd); } +static int sd_register(struct mlx5_core_dev *dev) +{ + struct mlx5_devcom_comp_dev *devcom, *pos; + struct mlx5_core_dev *peer, *primary; + struct mlx5_sd *sd, *primary_sd; + int err, i; + + sd = mlx5_get_sd(dev); + devcom = mlx5_devcom_register_component(dev->priv.devc, MLX5_DEVCOM_SD_GROUP, + sd->group_id, NULL, dev); + if (!devcom) + return -ENOMEM; + + sd->devcom = devcom; + + if (mlx5_devcom_comp_get_size(devcom) != sd->host_buses) + return 0; + + mlx5_devcom_comp_lock(devcom); + mlx5_devcom_comp_set_ready(devcom, true); + mlx5_devcom_comp_unlock(devcom); + + if (!mlx5_devcom_for_each_peer_begin(devcom)) { + err = -ENODEV; + goto err_devcom_unreg; + } + + primary = dev; + mlx5_devcom_for_each_peer_entry(devcom, peer, pos) + if (peer->pdev->bus->number < primary->pdev->bus->number) + primary = peer; + + primary_sd = mlx5_get_sd(primary); + primary_sd->primary = true; + i = 0; + /* loop the secondaries */ + mlx5_devcom_for_each_peer_entry(primary_sd->devcom, peer, pos) { + struct mlx5_sd *peer_sd = mlx5_get_sd(peer); + + primary_sd->secondaries[i++] = peer; + peer_sd->primary = false; + peer_sd->primary_dev = primary; + } + + mlx5_devcom_for_each_peer_end(devcom); + return 0; + +err_devcom_unreg: + mlx5_devcom_comp_lock(sd->devcom); + mlx5_devcom_comp_set_ready(sd->devcom, false); + mlx5_devcom_comp_unlock(sd->devcom); + mlx5_devcom_unregister_component(sd->devcom); + return err; +} + +static void sd_unregister(struct mlx5_core_dev *dev) +{ + struct mlx5_sd *sd = mlx5_get_sd(dev); + + mlx5_devcom_comp_lock(sd->devcom); + mlx5_devcom_comp_set_ready(sd->devcom, false); + mlx5_devcom_comp_unlock(sd->devcom); + mlx5_devcom_unregister_component(sd->devcom); +} + int mlx5_sd_init(struct mlx5_core_dev *dev) { + struct mlx5_sd *sd = mlx5_get_sd(dev); int err; err = sd_init(dev); if (err) return err; + sd = mlx5_get_sd(dev); + if (!sd) + return 0; + + err = sd_register(dev); + if (err) + goto err_sd_cleanup; + return 0; + +err_sd_cleanup: + sd_cleanup(dev); + return err; } void mlx5_sd_cleanup(struct mlx5_core_dev *dev) @@ -154,6 +258,7 @@ void mlx5_sd_cleanup(struct mlx5_core_dev *dev) if (!sd) return; + sd_unregister(dev); sd_cleanup(dev); } @@ -161,5 +266,18 @@ struct auxiliary_device *mlx5_sd_get_adev(struct mlx5_core_dev *dev, struct auxiliary_device *adev, int idx) { - return adev; + struct mlx5_sd *sd = mlx5_get_sd(dev); + struct mlx5_core_dev *primary; + + if (!sd) + return adev; + + if (!mlx5_devcom_comp_is_ready(sd->devcom)) + return NULL; + + primary = mlx5_sd_get_primary(dev); + if (dev == primary) + return adev; + + return &primary->priv.adev[idx]->adev; } -- 2.25.1