[linux-block.git] / drivers / net / ethernet / mellanox / mlx5 / core / lib / aso.c

// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
// Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.

#include <linux/mlx5/device.h>
#include <linux/mlx5/transobj.h>
#include "clock.h"
#include "aso.h"
#include "wq.h"

struct mlx5_aso_cq {
	/* data path - accessed per cqe */
	struct mlx5_cqwq           wq;

	/* data path - accessed per napi poll */
	struct mlx5_core_cq        mcq;

	/* control */
	struct mlx5_core_dev      *mdev;
	struct mlx5_wq_ctrl        wq_ctrl;
} ____cacheline_aligned_in_smp;

struct mlx5_aso {
	/* data path */
	u16                        cc;
	u16                        pc;

	struct mlx5_wqe_ctrl_seg  *doorbell_cseg;
	struct mlx5_aso_cq         cq;

	/* read only */
	struct mlx5_wq_cyc         wq;
	void __iomem              *uar_map;
	u32                        sqn;

	/* control path */
	struct mlx5_wq_ctrl        wq_ctrl;

} ____cacheline_aligned_in_smp;

static void mlx5_aso_free_cq(struct mlx5_aso_cq *cq)
{
	mlx5_wq_destroy(&cq->wq_ctrl);
}

static int mlx5_aso_alloc_cq(struct mlx5_core_dev *mdev, int numa_node,
			     void *cqc_data, struct mlx5_aso_cq *cq)
{
	struct mlx5_core_cq *mcq = &cq->mcq;
	struct mlx5_wq_param param;
	int err;
	u32 i;

	param.buf_numa_node = numa_node;
	param.db_numa_node = numa_node;

	err = mlx5_cqwq_create(mdev, &param, cqc_data, &cq->wq, &cq->wq_ctrl);
	if (err)
		return err;

	mcq->cqe_sz     = 64;
	mcq->set_ci_db  = cq->wq_ctrl.db.db;
	mcq->arm_db     = cq->wq_ctrl.db.db + 1;

	for (i = 0; i < mlx5_cqwq_get_size(&cq->wq); i++) {
		struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(&cq->wq, i);

		cqe->op_own = 0xf1;
	}

	cq->mdev = mdev;

	return 0;
}

static int create_aso_cq(struct mlx5_aso_cq *cq, void *cqc_data)
{
	u32 out[MLX5_ST_SZ_DW(create_cq_out)];
	struct mlx5_core_dev *mdev = cq->mdev;
	struct mlx5_core_cq *mcq = &cq->mcq;
	void *in, *cqc;
	int inlen, eqn;
	int err;

	err = mlx5_comp_eqn_get(mdev, 0, &eqn);
	if (err)
		return err;

	inlen = MLX5_ST_SZ_BYTES(create_cq_in) +
		sizeof(u64) * cq->wq_ctrl.buf.npages;
	in = kvzalloc(inlen, GFP_KERNEL);
	if (!in)
		return -ENOMEM;

	cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context);

	memcpy(cqc, cqc_data, MLX5_ST_SZ_BYTES(cqc));

	mlx5_fill_page_frag_array(&cq->wq_ctrl.buf,
				  (__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas));

	MLX5_SET(cqc,   cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
	MLX5_SET(cqc,   cqc, c_eqn_or_apu_element, eqn);
	MLX5_SET(cqc,   cqc, uar_page,      mdev->priv.uar->index);
	MLX5_SET(cqc,   cqc, log_page_size, cq->wq_ctrl.buf.page_shift -
					    MLX5_ADAPTER_PAGE_SHIFT);
	MLX5_SET64(cqc, cqc, dbr_addr,      cq->wq_ctrl.db.dma);

	err = mlx5_core_create_cq(mdev, mcq, in, inlen, out, sizeof(out));

	kvfree(in);

	return err;
}

static void mlx5_aso_destroy_cq(struct mlx5_aso_cq *cq)
{
	mlx5_core_destroy_cq(cq->mdev, &cq->mcq);
	mlx5_wq_destroy(&cq->wq_ctrl);
}

static int mlx5_aso_create_cq(struct mlx5_core_dev *mdev, int numa_node,
			      struct mlx5_aso_cq *cq)
{
	void *cqc_data;
	int err;

	cqc_data = kvzalloc(MLX5_ST_SZ_BYTES(cqc), GFP_KERNEL);
	if (!cqc_data)
		return -ENOMEM;

	MLX5_SET(cqc, cqc_data, log_cq_size, 1);
	MLX5_SET(cqc, cqc_data, uar_page, mdev->priv.uar->index);
	if (MLX5_CAP_GEN(mdev, cqe_128_always) && cache_line_size() >= 128)
		MLX5_SET(cqc, cqc_data, cqe_sz, CQE_STRIDE_128_PAD);

	err = mlx5_aso_alloc_cq(mdev, numa_node, cqc_data, cq);
	if (err) {
		mlx5_core_err(mdev, "Failed to alloc aso wq cq, err=%d\n", err);
		goto err_out;
	}

	err = create_aso_cq(cq, cqc_data);
	if (err) {
		mlx5_core_err(mdev, "Failed to create aso wq cq, err=%d\n", err);
		goto err_free_cq;
	}

	kvfree(cqc_data);
	return 0;

err_free_cq:
	mlx5_aso_free_cq(cq);
err_out:
	kvfree(cqc_data);
	return err;
}

static int mlx5_aso_alloc_sq(struct mlx5_core_dev *mdev, int numa_node,
			     void *sqc_data, struct mlx5_aso *sq)
{
	void *sqc_wq = MLX5_ADDR_OF(sqc, sqc_data, wq);
	struct mlx5_wq_cyc *wq = &sq->wq;
	struct mlx5_wq_param param;
	int err;

	sq->uar_map = mdev->mlx5e_res.hw_objs.bfreg.map;

	param.db_numa_node = numa_node;
	param.buf_numa_node = numa_node;
	err = mlx5_wq_cyc_create(mdev, &param, sqc_wq, wq, &sq->wq_ctrl);
	if (err)
		return err;
	wq->db = &wq->db[MLX5_SND_DBR];

	return 0;
}

static int create_aso_sq(struct mlx5_core_dev *mdev, int pdn,
			 void *sqc_data, struct mlx5_aso *sq)
{
	void *in, *sqc, *wq;
	int inlen, err;
	u8 ts_format;

	inlen = MLX5_ST_SZ_BYTES(create_sq_in) +
		sizeof(u64) * sq->wq_ctrl.buf.npages;
	in = kvzalloc(inlen, GFP_KERNEL);
	if (!in)
		return -ENOMEM;

	sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
	wq = MLX5_ADDR_OF(sqc, sqc, wq);

	memcpy(sqc, sqc_data, MLX5_ST_SZ_BYTES(sqc));
	MLX5_SET(sqc,  sqc, cqn, sq->cq.mcq.cqn);

	MLX5_SET(sqc,  sqc, state, MLX5_SQC_STATE_RST);
	MLX5_SET(sqc,  sqc, flush_in_error_en, 1);

	ts_format = mlx5_is_real_time_sq(mdev) ?
			MLX5_TIMESTAMP_FORMAT_REAL_TIME :
			MLX5_TIMESTAMP_FORMAT_FREE_RUNNING;
	MLX5_SET(sqc, sqc, ts_format, ts_format);

	MLX5_SET(wq,   wq, wq_type,       MLX5_WQ_TYPE_CYCLIC);
	MLX5_SET(wq,   wq, uar_page,      mdev->mlx5e_res.hw_objs.bfreg.index);
	MLX5_SET(wq,   wq, log_wq_pg_sz,  sq->wq_ctrl.buf.page_shift -
					  MLX5_ADAPTER_PAGE_SHIFT);
	MLX5_SET64(wq, wq, dbr_addr,      sq->wq_ctrl.db.dma);

	mlx5_fill_page_frag_array(&sq->wq_ctrl.buf,
				  (__be64 *)MLX5_ADDR_OF(wq, wq, pas));

	err = mlx5_core_create_sq(mdev, in, inlen, &sq->sqn);

	kvfree(in);

	return err;
}

static int mlx5_aso_set_sq_rdy(struct mlx5_core_dev *mdev, u32 sqn)
{
	void *in, *sqc;
	int inlen, err;

	inlen = MLX5_ST_SZ_BYTES(modify_sq_in);
	in = kvzalloc(inlen, GFP_KERNEL);
	if (!in)
		return -ENOMEM;

	MLX5_SET(modify_sq_in, in, sq_state, MLX5_SQC_STATE_RST);
	sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx);
	MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RDY);

	err = mlx5_core_modify_sq(mdev, sqn, in);

	kvfree(in);

	return err;
}

static int mlx5_aso_create_sq_rdy(struct mlx5_core_dev *mdev, u32 pdn,
				  void *sqc_data, struct mlx5_aso *sq)
{
	int err;

	err = create_aso_sq(mdev, pdn, sqc_data, sq);
	if (err)
		return err;

	err = mlx5_aso_set_sq_rdy(mdev, sq->sqn);
	if (err)
		mlx5_core_destroy_sq(mdev, sq->sqn);

	return err;
}

static void mlx5_aso_free_sq(struct mlx5_aso *sq)
{
	mlx5_wq_destroy(&sq->wq_ctrl);
}

static void mlx5_aso_destroy_sq(struct mlx5_aso *sq)
{
	mlx5_core_destroy_sq(sq->cq.mdev, sq->sqn);
	mlx5_aso_free_sq(sq);
}

static int mlx5_aso_create_sq(struct mlx5_core_dev *mdev, int numa_node,
			      u32 pdn, struct mlx5_aso *sq)
{
	void *sqc_data, *wq;
	int err;

	sqc_data = kvzalloc(MLX5_ST_SZ_BYTES(sqc), GFP_KERNEL);
	if (!sqc_data)
		return -ENOMEM;

	wq = MLX5_ADDR_OF(sqc, sqc_data, wq);
	MLX5_SET(wq, wq, log_wq_stride, ilog2(MLX5_SEND_WQE_BB));
	MLX5_SET(wq, wq, pd, pdn);
	MLX5_SET(wq, wq, log_wq_sz, 1);

	err = mlx5_aso_alloc_sq(mdev, numa_node, sqc_data, sq);
	if (err) {
		mlx5_core_err(mdev, "Failed to alloc aso wq sq, err=%d\n", err);
		goto err_out;
	}

	err = mlx5_aso_create_sq_rdy(mdev, pdn, sqc_data, sq);
	if (err) {
		mlx5_core_err(mdev, "Failed to open aso wq sq, err=%d\n", err);
		goto err_free_asosq;
	}

	mlx5_core_dbg(mdev, "aso sq->sqn = 0x%x\n", sq->sqn);

	kvfree(sqc_data);
	return 0;

err_free_asosq:
	mlx5_aso_free_sq(sq);
err_out:
	kvfree(sqc_data);
	return err;
}

struct mlx5_aso *mlx5_aso_create(struct mlx5_core_dev *mdev, u32 pdn)
{
	int numa_node = dev_to_node(mlx5_core_dma_dev(mdev));
	struct mlx5_aso *aso;
	int err;

	aso = kzalloc(sizeof(*aso), GFP_KERNEL);
	if (!aso)
		return ERR_PTR(-ENOMEM);

	err = mlx5_aso_create_cq(mdev, numa_node, &aso->cq);
	if (err)
		goto err_cq;

	err = mlx5_aso_create_sq(mdev, numa_node, pdn, aso);
	if (err)
		goto err_sq;

	return aso;

err_sq:
	mlx5_aso_destroy_cq(&aso->cq);
err_cq:
	kfree(aso);
	return ERR_PTR(err);
}

void mlx5_aso_destroy(struct mlx5_aso *aso)
{
	mlx5_aso_destroy_sq(aso);
	mlx5_aso_destroy_cq(&aso->cq);
	kfree(aso);
}

void mlx5_aso_build_wqe(struct mlx5_aso *aso, u8 ds_cnt,
			struct mlx5_aso_wqe *aso_wqe,
			u32 obj_id, u32 opc_mode)
{
	struct mlx5_wqe_ctrl_seg *cseg = &aso_wqe->ctrl;

	cseg->opmod_idx_opcode = cpu_to_be32((opc_mode << MLX5_WQE_CTRL_WQE_OPC_MOD_SHIFT) |
					     (aso->pc << MLX5_WQE_CTRL_WQE_INDEX_SHIFT) |
					     MLX5_OPCODE_ACCESS_ASO);
	cseg->qpn_ds     = cpu_to_be32((aso->sqn << MLX5_WQE_CTRL_QPN_SHIFT) | ds_cnt);
	cseg->fm_ce_se   = MLX5_WQE_CTRL_CQ_UPDATE;
	cseg->general_id = cpu_to_be32(obj_id);
}

struct mlx5_aso_wqe *mlx5_aso_get_wqe(struct mlx5_aso *aso)
{
	struct mlx5_aso_wqe *wqe;
	u16 pi;

	pi = mlx5_wq_cyc_ctr2ix(&aso->wq, aso->pc);
	wqe = mlx5_wq_cyc_get_wqe(&aso->wq, pi);
	memset(wqe, 0, sizeof(*wqe));
	return wqe;
}

void mlx5_aso_post_wqe(struct mlx5_aso *aso, bool with_data,
		       struct mlx5_wqe_ctrl_seg *doorbell_cseg)
{
	doorbell_cseg->fm_ce_se |= MLX5_WQE_CTRL_CQ_UPDATE;
	/* ensure wqe is visible to device before updating doorbell record */
	dma_wmb();

	if (with_data)
		aso->pc += MLX5_ASO_WQEBBS_DATA;
	else
		aso->pc += MLX5_ASO_WQEBBS;
	*aso->wq.db = cpu_to_be32(aso->pc);

	/* ensure doorbell record is visible to device before ringing the
	 * doorbell
	 */
	wmb();

	mlx5_write64((__be32 *)doorbell_cseg, aso->uar_map);

	/* Ensure doorbell is written on uar_page before poll_cq */
	WRITE_ONCE(doorbell_cseg, NULL);
}

int mlx5_aso_poll_cq(struct mlx5_aso *aso, bool with_data)
{
	struct mlx5_aso_cq *cq = &aso->cq;
	struct mlx5_cqe64 *cqe;

	cqe = mlx5_cqwq_get_cqe(&cq->wq);
	if (!cqe)
		return -ETIMEDOUT;

	/* sq->cc must be updated only after mlx5_cqwq_update_db_record(),
	 * otherwise a cq overrun may occur
	 */
	mlx5_cqwq_pop(&cq->wq);

	if (unlikely(get_cqe_opcode(cqe) != MLX5_CQE_REQ)) {
		struct mlx5_err_cqe *err_cqe;

		mlx5_core_err(cq->mdev, "Bad OP in ASOSQ CQE: 0x%x\n",
			      get_cqe_opcode(cqe));

		err_cqe = (struct mlx5_err_cqe *)cqe;
		mlx5_core_err(cq->mdev, "vendor_err_synd=%x\n",
			      err_cqe->vendor_err_synd);
		mlx5_core_err(cq->mdev, "syndrome=%x\n",
			      err_cqe->syndrome);
		print_hex_dump(KERN_WARNING, "", DUMP_PREFIX_OFFSET,
			       16, 1, err_cqe,
			       sizeof(*err_cqe), false);
	}

	mlx5_cqwq_update_db_record(&cq->wq);

	/* ensure cq space is freed before enabling more cqes */
	wmb();

	if (with_data)
		aso->cc += MLX5_ASO_WQEBBS_DATA;
	else
		aso->cc += MLX5_ASO_WQEBBS;

	return 0;
}
Commit	Line	Data
cdd04f4d JL	1	// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
	2	// Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
	3
	4	#include <linux/mlx5/device.h>
	5	#include <linux/mlx5/transobj.h>
0f3caaa2	6	#include "clock.h"
cdd04f4d JL	7	#include "aso.h"
	8	#include "wq.h"
	9
	10	struct mlx5_aso_cq {
	11	/* data path - accessed per cqe */
	12	struct mlx5_cqwq wq;
	13
	14	/* data path - accessed per napi poll */
	15	struct mlx5_core_cq mcq;
	16
	17	/* control */
	18	struct mlx5_core_dev *mdev;
	19	struct mlx5_wq_ctrl wq_ctrl;
	20	} ____cacheline_aligned_in_smp;
	21
	22	struct mlx5_aso {
	23	/* data path */
	24	u16 cc;
	25	u16 pc;
	26
	27	struct mlx5_wqe_ctrl_seg *doorbell_cseg;
	28	struct mlx5_aso_cq cq;
	29
	30	/* read only */
	31	struct mlx5_wq_cyc wq;
	32	void __iomem *uar_map;
	33	u32 sqn;
	34
	35	/* control path */
	36	struct mlx5_wq_ctrl wq_ctrl;
	37
	38	} ____cacheline_aligned_in_smp;
	39
	40	static void mlx5_aso_free_cq(struct mlx5_aso_cq *cq)
	41	{
	42	mlx5_wq_destroy(&cq->wq_ctrl);
	43	}
	44
	45	static int mlx5_aso_alloc_cq(struct mlx5_core_dev *mdev, int numa_node,
	46	void cqc_data, struct mlx5_aso_cq cq)
	47	{
	48	struct mlx5_core_cq *mcq = &cq->mcq;
	49	struct mlx5_wq_param param;
	50	int err;
	51	u32 i;
	52
	53	param.buf_numa_node = numa_node;
	54	param.db_numa_node = numa_node;
	55
	56	err = mlx5_cqwq_create(mdev, &param, cqc_data, &cq->wq, &cq->wq_ctrl);
	57	if (err)
	58	return err;
	59
	60	mcq->cqe_sz = 64;
	61	mcq->set_ci_db = cq->wq_ctrl.db.db;
	62	mcq->arm_db = cq->wq_ctrl.db.db + 1;
	63
	64	for (i = 0; i < mlx5_cqwq_get_size(&cq->wq); i++) {
	65	struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(&cq->wq, i);
	66
	67	cqe->op_own = 0xf1;
	68	}
	69
	70	cq->mdev = mdev;
71
72	return 0;
73	}
74
75	static int create_aso_cq(struct mlx5_aso_cq cq, void cqc_data)
76	{
77	u32 out[MLX5_ST_SZ_DW(create_cq_out)];
78	struct mlx5_core_dev *mdev = cq->mdev;
79	struct mlx5_core_cq *mcq = &cq->mcq;
80	void in, cqc;
81	int inlen, eqn;
82	int err;
83
f14c1a14	84	err = mlx5_comp_eqn_get(mdev, 0, &eqn);
cdd04f4d JL	85	if (err)
	86	return err;
	87
	88	inlen = MLX5_ST_SZ_BYTES(create_cq_in) +
	89	sizeof(u64) * cq->wq_ctrl.buf.npages;
	90	in = kvzalloc(inlen, GFP_KERNEL);
	91	if (!in)
	92	return -ENOMEM;
	93
	94	cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context);
	95
	96	memcpy(cqc, cqc_data, MLX5_ST_SZ_BYTES(cqc));
	97
	98	mlx5_fill_page_frag_array(&cq->wq_ctrl.buf,
	99	(__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas));
	100
20cbf8cb	101	MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
cdd04f4d JL	102	MLX5_SET(cqc, cqc, c_eqn_or_apu_element, eqn);
	103	MLX5_SET(cqc, cqc, uar_page, mdev->priv.uar->index);
	104	MLX5_SET(cqc, cqc, log_page_size, cq->wq_ctrl.buf.page_shift -
	105	MLX5_ADAPTER_PAGE_SHIFT);
	106	MLX5_SET64(cqc, cqc, dbr_addr, cq->wq_ctrl.db.dma);
	107
	108	err = mlx5_core_create_cq(mdev, mcq, in, inlen, out, sizeof(out));
	109
	110	kvfree(in);
	111
	112	return err;
	113	}
	114
	115	static void mlx5_aso_destroy_cq(struct mlx5_aso_cq *cq)
	116	{
	117	mlx5_core_destroy_cq(cq->mdev, &cq->mcq);
	118	mlx5_wq_destroy(&cq->wq_ctrl);
	119	}
	120
	121	static int mlx5_aso_create_cq(struct mlx5_core_dev *mdev, int numa_node,
	122	struct mlx5_aso_cq *cq)
	123	{
	124	void *cqc_data;
	125	int err;
	126
	127	cqc_data = kvzalloc(MLX5_ST_SZ_BYTES(cqc), GFP_KERNEL);
	128	if (!cqc_data)
	129	return -ENOMEM;
	130
	131	MLX5_SET(cqc, cqc_data, log_cq_size, 1);
	132	MLX5_SET(cqc, cqc_data, uar_page, mdev->priv.uar->index);
	133	if (MLX5_CAP_GEN(mdev, cqe_128_always) && cache_line_size() >= 128)
	134	MLX5_SET(cqc, cqc_data, cqe_sz, CQE_STRIDE_128_PAD);
	135
	136	err = mlx5_aso_alloc_cq(mdev, numa_node, cqc_data, cq);
	137	if (err) {
	138	mlx5_core_err(mdev, "Failed to alloc aso wq cq, err=%d\n", err);
	139	goto err_out;
	140	}
	141
	142	err = create_aso_cq(cq, cqc_data);
	143	if (err) {
	144	mlx5_core_err(mdev, "Failed to create aso wq cq, err=%d\n", err);
	145	goto err_free_cq;
	146	}
	147
	148	kvfree(cqc_data);
	149	return 0;
	150
	151	err_free_cq:
	152	mlx5_aso_free_cq(cq);
	153	err_out:
	154	kvfree(cqc_data);
	155	return err;
	156	}
	157
	158	static int mlx5_aso_alloc_sq(struct mlx5_core_dev *mdev, int numa_node,
	159	void sqc_data, struct mlx5_aso sq)
	160	{
	161	void *sqc_wq = MLX5_ADDR_OF(sqc, sqc_data, wq);
	162	struct mlx5_wq_cyc *wq = &sq->wq;
	163	struct mlx5_wq_param param;
	164	int err;
	165
166	sq->uar_map = mdev->mlx5e_res.hw_objs.bfreg.map;
167
168	param.db_numa_node = numa_node;
169	param.buf_numa_node = numa_node;
170	err = mlx5_wq_cyc_create(mdev, &param, sqc_wq, wq, &sq->wq_ctrl);
171	if (err)
172	return err;
173	wq->db = &wq->db[MLX5_SND_DBR];
174
175	return 0;
176	}
177
178	static int create_aso_sq(struct mlx5_core_dev *mdev, int pdn,
179	void sqc_data, struct mlx5_aso sq)
180	{
181	void in, sqc, *wq;
182	int inlen, err;
0f3caaa2	183	u8 ts_format;
cdd04f4d JL	184
	185	inlen = MLX5_ST_SZ_BYTES(create_sq_in) +
	186	sizeof(u64) * sq->wq_ctrl.buf.npages;
	187	in = kvzalloc(inlen, GFP_KERNEL);
	188	if (!in)
	189	return -ENOMEM;
	190
	191	sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
	192	wq = MLX5_ADDR_OF(sqc, sqc, wq);
	193
	194	memcpy(sqc, sqc_data, MLX5_ST_SZ_BYTES(sqc));
	195	MLX5_SET(sqc, sqc, cqn, sq->cq.mcq.cqn);
	196
	197	MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RST);
	198	MLX5_SET(sqc, sqc, flush_in_error_en, 1);
	199
0f3caaa2 SM	200	ts_format = mlx5_is_real_time_sq(mdev) ?
	201	MLX5_TIMESTAMP_FORMAT_REAL_TIME :
	202	MLX5_TIMESTAMP_FORMAT_FREE_RUNNING;
	203	MLX5_SET(sqc, sqc, ts_format, ts_format);
	204
cdd04f4d JL	205	MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC);
	206	MLX5_SET(wq, wq, uar_page, mdev->mlx5e_res.hw_objs.bfreg.index);
	207	MLX5_SET(wq, wq, log_wq_pg_sz, sq->wq_ctrl.buf.page_shift -
	208	MLX5_ADAPTER_PAGE_SHIFT);
	209	MLX5_SET64(wq, wq, dbr_addr, sq->wq_ctrl.db.dma);
	210
	211	mlx5_fill_page_frag_array(&sq->wq_ctrl.buf,
	212	(__be64 *)MLX5_ADDR_OF(wq, wq, pas));
	213
	214	err = mlx5_core_create_sq(mdev, in, inlen, &sq->sqn);
	215
	216	kvfree(in);
	217
	218	return err;
	219	}
	220
	221	static int mlx5_aso_set_sq_rdy(struct mlx5_core_dev *mdev, u32 sqn)
	222	{
	223	void in, sqc;
	224	int inlen, err;
	225
	226	inlen = MLX5_ST_SZ_BYTES(modify_sq_in);
	227	in = kvzalloc(inlen, GFP_KERNEL);
	228	if (!in)
	229	return -ENOMEM;
	230
	231	MLX5_SET(modify_sq_in, in, sq_state, MLX5_SQC_STATE_RST);
	232	sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx);
	233	MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RDY);
	234
	235	err = mlx5_core_modify_sq(mdev, sqn, in);
	236
	237	kvfree(in);
	238
	239	return err;
	240	}
	241
	242	static int mlx5_aso_create_sq_rdy(struct mlx5_core_dev *mdev, u32 pdn,
	243	void sqc_data, struct mlx5_aso sq)
	244	{
	245	int err;
	246
	247	err = create_aso_sq(mdev, pdn, sqc_data, sq);
	248	if (err)
	249	return err;
	250
	251	err = mlx5_aso_set_sq_rdy(mdev, sq->sqn);
	252	if (err)
	253	mlx5_core_destroy_sq(mdev, sq->sqn);
	254
	255	return err;
	256	}
	257
	258	static void mlx5_aso_free_sq(struct mlx5_aso *sq)
	259	{
	260	mlx5_wq_destroy(&sq->wq_ctrl);
	261	}
	262
	263	static void mlx5_aso_destroy_sq(struct mlx5_aso *sq)
	264	{
	265	mlx5_core_destroy_sq(sq->cq.mdev, sq->sqn);
	266	mlx5_aso_free_sq(sq);
	267	}
	268
269	static int mlx5_aso_create_sq(struct mlx5_core_dev *mdev, int numa_node,
270	u32 pdn, struct mlx5_aso *sq)
271	{
272	void sqc_data, wq;
273	int err;
274
275	sqc_data = kvzalloc(MLX5_ST_SZ_BYTES(sqc), GFP_KERNEL);
276	if (!sqc_data)
277	return -ENOMEM;
278
279	wq = MLX5_ADDR_OF(sqc, sqc_data, wq);
280	MLX5_SET(wq, wq, log_wq_stride, ilog2(MLX5_SEND_WQE_BB));
281	MLX5_SET(wq, wq, pd, pdn);
282	MLX5_SET(wq, wq, log_wq_sz, 1);
283
284	err = mlx5_aso_alloc_sq(mdev, numa_node, sqc_data, sq);
285	if (err) {
286	mlx5_core_err(mdev, "Failed to alloc aso wq sq, err=%d\n", err);
287	goto err_out;
288	}
289
290	err = mlx5_aso_create_sq_rdy(mdev, pdn, sqc_data, sq);
291	if (err) {
292	mlx5_core_err(mdev, "Failed to open aso wq sq, err=%d\n", err);
293	goto err_free_asosq;
294	}
295
296	mlx5_core_dbg(mdev, "aso sq->sqn = 0x%x\n", sq->sqn);
297
298	kvfree(sqc_data);
299	return 0;
300
301	err_free_asosq:
302	mlx5_aso_free_sq(sq);
303	err_out:
304	kvfree(sqc_data);
305	return err;
306	}
307
308	struct mlx5_aso mlx5_aso_create(struct mlx5_core_dev mdev, u32 pdn)
309	{
310	int numa_node = dev_to_node(mlx5_core_dma_dev(mdev));
311	struct mlx5_aso *aso;
312	int err;
313
314	aso = kzalloc(sizeof(*aso), GFP_KERNEL);
315	if (!aso)
316	return ERR_PTR(-ENOMEM);
317
318	err = mlx5_aso_create_cq(mdev, numa_node, &aso->cq);
319	if (err)
320	goto err_cq;
321
322	err = mlx5_aso_create_sq(mdev, numa_node, pdn, aso);
323	if (err)
324	goto err_sq;
325
326	return aso;
327
328	err_sq:
329	mlx5_aso_destroy_cq(&aso->cq);
330	err_cq:
331	kfree(aso);
332	return ERR_PTR(err);
333	}
334
335	void mlx5_aso_destroy(struct mlx5_aso *aso)
336	{
cdd04f4d JL	337	mlx5_aso_destroy_sq(aso);
	338	mlx5_aso_destroy_cq(&aso->cq);
	339	kfree(aso);
	340	}
c491ded0 JL	341
	342	void mlx5_aso_build_wqe(struct mlx5_aso *aso, u8 ds_cnt,
	343	struct mlx5_aso_wqe *aso_wqe,
	344	u32 obj_id, u32 opc_mode)
	345	{
	346	struct mlx5_wqe_ctrl_seg *cseg = &aso_wqe->ctrl;
	347
	348	cseg->opmod_idx_opcode = cpu_to_be32((opc_mode << MLX5_WQE_CTRL_WQE_OPC_MOD_SHIFT) \|
	349	(aso->pc << MLX5_WQE_CTRL_WQE_INDEX_SHIFT) \|
	350	MLX5_OPCODE_ACCESS_ASO);
	351	cseg->qpn_ds = cpu_to_be32((aso->sqn << MLX5_WQE_CTRL_QPN_SHIFT) \| ds_cnt);
	352	cseg->fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE;
	353	cseg->general_id = cpu_to_be32(obj_id);
	354	}
	355
e77bbde7	356	struct mlx5_aso_wqe mlx5_aso_get_wqe(struct mlx5_aso aso)
c491ded0	357	{
e77bbde7	358	struct mlx5_aso_wqe *wqe;
c491ded0 JL	359	u16 pi;
	360
	361	pi = mlx5_wq_cyc_ctr2ix(&aso->wq, aso->pc);
e77bbde7 LR	362	wqe = mlx5_wq_cyc_get_wqe(&aso->wq, pi);
	363	memset(wqe, 0, sizeof(*wqe));
	364	return wqe;
c491ded0 JL	365	}
	366
	367	void mlx5_aso_post_wqe(struct mlx5_aso *aso, bool with_data,
	368	struct mlx5_wqe_ctrl_seg *doorbell_cseg)
	369	{
	370	doorbell_cseg->fm_ce_se \|= MLX5_WQE_CTRL_CQ_UPDATE;
	371	/* ensure wqe is visible to device before updating doorbell record */
	372	dma_wmb();
	373
	374	if (with_data)
	375	aso->pc += MLX5_ASO_WQEBBS_DATA;
	376	else
	377	aso->pc += MLX5_ASO_WQEBBS;
	378	*aso->wq.db = cpu_to_be32(aso->pc);
	379
	380	/* ensure doorbell record is visible to device before ringing the
	381	* doorbell
	382	*/
	383	wmb();
	384
	385	mlx5_write64((__be32 *)doorbell_cseg, aso->uar_map);
	386
	387	/* Ensure doorbell is written on uar_page before poll_cq */
	388	WRITE_ONCE(doorbell_cseg, NULL);
	389	}
	390
739cfa34	391	int mlx5_aso_poll_cq(struct mlx5_aso *aso, bool with_data)
c491ded0 JL	392	{
	393	struct mlx5_aso_cq *cq = &aso->cq;
	394	struct mlx5_cqe64 *cqe;
c491ded0 JL	395
c491ded0 JL	396	cqe = mlx5_cqwq_get_cqe(&cq->wq);
c491ded0 JL	397	if (!cqe)
	398	return -ETIMEDOUT;
	399
	400	/* sq->cc must be updated only after mlx5_cqwq_update_db_record(),
	401	* otherwise a cq overrun may occur
	402	*/
	403	mlx5_cqwq_pop(&cq->wq);
	404
	405	if (unlikely(get_cqe_opcode(cqe) != MLX5_CQE_REQ)) {
	406	struct mlx5_err_cqe *err_cqe;
	407
	408	mlx5_core_err(cq->mdev, "Bad OP in ASOSQ CQE: 0x%x\n",
	409	get_cqe_opcode(cqe));
	410
	411	err_cqe = (struct mlx5_err_cqe *)cqe;
	412	mlx5_core_err(cq->mdev, "vendor_err_synd=%x\n",
	413	err_cqe->vendor_err_synd);
	414	mlx5_core_err(cq->mdev, "syndrome=%x\n",
	415	err_cqe->syndrome);
	416	print_hex_dump(KERN_WARNING, "", DUMP_PREFIX_OFFSET,
	417	16, 1, err_cqe,
	418	sizeof(*err_cqe), false);
	419	}
	420
	421	mlx5_cqwq_update_db_record(&cq->wq);
	422
	423	/* ensure cq space is freed before enabling more cqes */
	424	wmb();
	425
	426	if (with_data)
	427	aso->cc += MLX5_ASO_WQEBBS_DATA;
	428	else
	429	aso->cc += MLX5_ASO_WQEBBS;
	430
	431	return 0;
	432	}