[linux-2.6-block.git] / drivers / net / ethernet / mellanox / mlx5 / core / eq.c

/*
 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * OpenIB.org BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#include <linux/interrupt.h>
#include <linux/module.h>
#include <linux/mlx5/driver.h>
#include <linux/mlx5/cmd.h>
#include "mlx5_core.h"
#ifdef CONFIG_MLX5_CORE_EN
#include "eswitch.h"
#endif

enum {
	MLX5_EQE_SIZE		= sizeof(struct mlx5_eqe),
	MLX5_EQE_OWNER_INIT_VAL	= 0x1,
};

enum {
	MLX5_EQ_STATE_ARMED		= 0x9,
	MLX5_EQ_STATE_FIRED		= 0xa,
	MLX5_EQ_STATE_ALWAYS_ARMED	= 0xb,
};

enum {
	MLX5_NUM_SPARE_EQE	= 0x80,
	MLX5_NUM_ASYNC_EQE	= 0x100,
	MLX5_NUM_CMD_EQE	= 32,
};

enum {
	MLX5_EQ_DOORBEL_OFFSET	= 0x40,
};

#define MLX5_ASYNC_EVENT_MASK ((1ull << MLX5_EVENT_TYPE_PATH_MIG)	    | \
			       (1ull << MLX5_EVENT_TYPE_COMM_EST)	    | \
			       (1ull << MLX5_EVENT_TYPE_SQ_DRAINED)	    | \
			       (1ull << MLX5_EVENT_TYPE_CQ_ERROR)	    | \
			       (1ull << MLX5_EVENT_TYPE_WQ_CATAS_ERROR)	    | \
			       (1ull << MLX5_EVENT_TYPE_PATH_MIG_FAILED)    | \
			       (1ull << MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR) | \
			       (1ull << MLX5_EVENT_TYPE_WQ_ACCESS_ERROR)    | \
			       (1ull << MLX5_EVENT_TYPE_PORT_CHANGE)	    | \
			       (1ull << MLX5_EVENT_TYPE_SRQ_CATAS_ERROR)    | \
			       (1ull << MLX5_EVENT_TYPE_SRQ_LAST_WQE)	    | \
			       (1ull << MLX5_EVENT_TYPE_SRQ_RQ_LIMIT))

struct map_eq_in {
	u64	mask;
	u32	reserved;
	u32	unmap_eqn;
};

struct cre_des_eq {
	u8	reserved[15];
	u8	eqn;
};

static int mlx5_cmd_destroy_eq(struct mlx5_core_dev *dev, u8 eqn)
{
	u32 out[MLX5_ST_SZ_DW(destroy_eq_out)] = {0};
	u32 in[MLX5_ST_SZ_DW(destroy_eq_in)]   = {0};

	MLX5_SET(destroy_eq_in, in, opcode, MLX5_CMD_OP_DESTROY_EQ);
	MLX5_SET(destroy_eq_in, in, eq_number, eqn);
	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
}

static struct mlx5_eqe *get_eqe(struct mlx5_eq *eq, u32 entry)
{
	return mlx5_buf_offset(&eq->buf, entry * MLX5_EQE_SIZE);
}

static struct mlx5_eqe *next_eqe_sw(struct mlx5_eq *eq)
{
	struct mlx5_eqe *eqe = get_eqe(eq, eq->cons_index & (eq->nent - 1));

	return ((eqe->owner & 1) ^ !!(eq->cons_index & eq->nent)) ? NULL : eqe;
}

static const char *eqe_type_str(u8 type)
{
	switch (type) {
	case MLX5_EVENT_TYPE_COMP:
		return "MLX5_EVENT_TYPE_COMP";
	case MLX5_EVENT_TYPE_PATH_MIG:
		return "MLX5_EVENT_TYPE_PATH_MIG";
	case MLX5_EVENT_TYPE_COMM_EST:
		return "MLX5_EVENT_TYPE_COMM_EST";
	case MLX5_EVENT_TYPE_SQ_DRAINED:
		return "MLX5_EVENT_TYPE_SQ_DRAINED";
	case MLX5_EVENT_TYPE_SRQ_LAST_WQE:
		return "MLX5_EVENT_TYPE_SRQ_LAST_WQE";
	case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT:
		return "MLX5_EVENT_TYPE_SRQ_RQ_LIMIT";
	case MLX5_EVENT_TYPE_CQ_ERROR:
		return "MLX5_EVENT_TYPE_CQ_ERROR";
	case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
		return "MLX5_EVENT_TYPE_WQ_CATAS_ERROR";
	case MLX5_EVENT_TYPE_PATH_MIG_FAILED:
		return "MLX5_EVENT_TYPE_PATH_MIG_FAILED";
	case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
		return "MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR";
	case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
		return "MLX5_EVENT_TYPE_WQ_ACCESS_ERROR";
	case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR:
		return "MLX5_EVENT_TYPE_SRQ_CATAS_ERROR";
	case MLX5_EVENT_TYPE_INTERNAL_ERROR:
		return "MLX5_EVENT_TYPE_INTERNAL_ERROR";
	case MLX5_EVENT_TYPE_PORT_CHANGE:
		return "MLX5_EVENT_TYPE_PORT_CHANGE";
	case MLX5_EVENT_TYPE_GPIO_EVENT:
		return "MLX5_EVENT_TYPE_GPIO_EVENT";
	case MLX5_EVENT_TYPE_PORT_MODULE_EVENT:
		return "MLX5_EVENT_TYPE_PORT_MODULE_EVENT";
	case MLX5_EVENT_TYPE_REMOTE_CONFIG:
		return "MLX5_EVENT_TYPE_REMOTE_CONFIG";
	case MLX5_EVENT_TYPE_DB_BF_CONGESTION:
		return "MLX5_EVENT_TYPE_DB_BF_CONGESTION";
	case MLX5_EVENT_TYPE_STALL_EVENT:
		return "MLX5_EVENT_TYPE_STALL_EVENT";
	case MLX5_EVENT_TYPE_CMD:
		return "MLX5_EVENT_TYPE_CMD";
	case MLX5_EVENT_TYPE_PAGE_REQUEST:
		return "MLX5_EVENT_TYPE_PAGE_REQUEST";
	case MLX5_EVENT_TYPE_PAGE_FAULT:
		return "MLX5_EVENT_TYPE_PAGE_FAULT";
	default:
		return "Unrecognized event";
	}
}

static enum mlx5_dev_event port_subtype_event(u8 subtype)
{
	switch (subtype) {
	case MLX5_PORT_CHANGE_SUBTYPE_DOWN:
		return MLX5_DEV_EVENT_PORT_DOWN;
	case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE:
		return MLX5_DEV_EVENT_PORT_UP;
	case MLX5_PORT_CHANGE_SUBTYPE_INITIALIZED:
		return MLX5_DEV_EVENT_PORT_INITIALIZED;
	case MLX5_PORT_CHANGE_SUBTYPE_LID:
		return MLX5_DEV_EVENT_LID_CHANGE;
	case MLX5_PORT_CHANGE_SUBTYPE_PKEY:
		return MLX5_DEV_EVENT_PKEY_CHANGE;
	case MLX5_PORT_CHANGE_SUBTYPE_GUID:
		return MLX5_DEV_EVENT_GUID_CHANGE;
	case MLX5_PORT_CHANGE_SUBTYPE_CLIENT_REREG:
		return MLX5_DEV_EVENT_CLIENT_REREG;
	}
	return -1;
}

static void eq_update_ci(struct mlx5_eq *eq, int arm)
{
	__be32 __iomem *addr = eq->doorbell + (arm ? 0 : 2);
	u32 val = (eq->cons_index & 0xffffff) | (eq->eqn << 24);
	__raw_writel((__force u32) cpu_to_be32(val), addr);
	/* We still want ordering, just not swabbing, so add a barrier */
	mb();
}

static int mlx5_eq_int(struct mlx5_core_dev *dev, struct mlx5_eq *eq)
{
	struct mlx5_eqe *eqe;
	int eqes_found = 0;
	int set_ci = 0;
	u32 cqn = -1;
	u32 rsn;
	u8 port;

	while ((eqe = next_eqe_sw(eq))) {
		/*
		 * Make sure we read EQ entry contents after we've
		 * checked the ownership bit.
		 */
		dma_rmb();

		mlx5_core_dbg(eq->dev, "eqn %d, eqe type %s\n",
			      eq->eqn, eqe_type_str(eqe->type));
		switch (eqe->type) {
		case MLX5_EVENT_TYPE_COMP:
			cqn = be32_to_cpu(eqe->data.comp.cqn) & 0xffffff;
			mlx5_cq_completion(dev, cqn);
			break;

		case MLX5_EVENT_TYPE_PATH_MIG:
		case MLX5_EVENT_TYPE_COMM_EST:
		case MLX5_EVENT_TYPE_SQ_DRAINED:
		case MLX5_EVENT_TYPE_SRQ_LAST_WQE:
		case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
		case MLX5_EVENT_TYPE_PATH_MIG_FAILED:
		case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
		case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
			rsn = be32_to_cpu(eqe->data.qp_srq.qp_srq_n) & 0xffffff;
			rsn |= (eqe->data.qp_srq.type << MLX5_USER_INDEX_LEN);
			mlx5_core_dbg(dev, "event %s(%d) arrived on resource 0x%x\n",
				      eqe_type_str(eqe->type), eqe->type, rsn);
			mlx5_rsc_event(dev, rsn, eqe->type);
			break;

		case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT:
		case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR:
			rsn = be32_to_cpu(eqe->data.qp_srq.qp_srq_n) & 0xffffff;
			mlx5_core_dbg(dev, "SRQ event %s(%d): srqn 0x%x\n",
				      eqe_type_str(eqe->type), eqe->type, rsn);
			mlx5_srq_event(dev, rsn, eqe->type);
			break;

		case MLX5_EVENT_TYPE_CMD:
			mlx5_cmd_comp_handler(dev, be32_to_cpu(eqe->data.cmd.vector));
			break;

		case MLX5_EVENT_TYPE_PORT_CHANGE:
			port = (eqe->data.port.port >> 4) & 0xf;
			switch (eqe->sub_type) {
			case MLX5_PORT_CHANGE_SUBTYPE_DOWN:
			case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE:
			case MLX5_PORT_CHANGE_SUBTYPE_LID:
			case MLX5_PORT_CHANGE_SUBTYPE_PKEY:
			case MLX5_PORT_CHANGE_SUBTYPE_GUID:
			case MLX5_PORT_CHANGE_SUBTYPE_CLIENT_REREG:
			case MLX5_PORT_CHANGE_SUBTYPE_INITIALIZED:
				if (dev->event)
					dev->event(dev, port_subtype_event(eqe->sub_type),
						   (unsigned long)port);
				break;
			default:
				mlx5_core_warn(dev, "Port event with unrecognized subtype: port %d, sub_type %d\n",
					       port, eqe->sub_type);
			}
			break;
		case MLX5_EVENT_TYPE_CQ_ERROR:
			cqn = be32_to_cpu(eqe->data.cq_err.cqn) & 0xffffff;
			mlx5_core_warn(dev, "CQ error on CQN 0x%x, syndrom 0x%x\n",
				       cqn, eqe->data.cq_err.syndrome);
			mlx5_cq_event(dev, cqn, eqe->type);
			break;

		case MLX5_EVENT_TYPE_PAGE_REQUEST:
			{
				u16 func_id = be16_to_cpu(eqe->data.req_pages.func_id);
				s32 npages = be32_to_cpu(eqe->data.req_pages.num_pages);

				mlx5_core_dbg(dev, "page request for func 0x%x, npages %d\n",
					      func_id, npages);
				mlx5_core_req_pages_handler(dev, func_id, npages);
			}
			break;

#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
		case MLX5_EVENT_TYPE_PAGE_FAULT:
			mlx5_eq_pagefault(dev, eqe);
			break;
#endif

#ifdef CONFIG_MLX5_CORE_EN
		case MLX5_EVENT_TYPE_NIC_VPORT_CHANGE:
			mlx5_eswitch_vport_event(dev->priv.eswitch, eqe);
			break;
#endif

		case MLX5_EVENT_TYPE_PORT_MODULE_EVENT:
			mlx5_port_module_event(dev, eqe);
			break;

		default:
			mlx5_core_warn(dev, "Unhandled event 0x%x on EQ 0x%x\n",
				       eqe->type, eq->eqn);
			break;
		}

		++eq->cons_index;
		eqes_found = 1;
		++set_ci;

		/* The HCA will think the queue has overflowed if we
		 * don't tell it we've been processing events.  We
		 * create our EQs with MLX5_NUM_SPARE_EQE extra
		 * entries, so we must update our consumer index at
		 * least that often.
		 */
		if (unlikely(set_ci >= MLX5_NUM_SPARE_EQE)) {
			eq_update_ci(eq, 0);
			set_ci = 0;
		}
	}

	eq_update_ci(eq, 1);

	if (cqn != -1)
		tasklet_schedule(&eq->tasklet_ctx.task);

	return eqes_found;
}

static irqreturn_t mlx5_msix_handler(int irq, void *eq_ptr)
{
	struct mlx5_eq *eq = eq_ptr;
	struct mlx5_core_dev *dev = eq->dev;

	mlx5_eq_int(dev, eq);

	/* MSI-X vectors always belong to us */
	return IRQ_HANDLED;
}

static void init_eq_buf(struct mlx5_eq *eq)
{
	struct mlx5_eqe *eqe;
	int i;

	for (i = 0; i < eq->nent; i++) {
		eqe = get_eqe(eq, i);
		eqe->owner = MLX5_EQE_OWNER_INIT_VAL;
	}
}

int mlx5_create_map_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq, u8 vecidx,
		       int nent, u64 mask, const char *name, struct mlx5_uar *uar)
{
	u32 out[MLX5_ST_SZ_DW(create_eq_out)] = {0};
	struct mlx5_priv *priv = &dev->priv;
	__be64 *pas;
	void *eqc;
	int inlen;
	u32 *in;
	int err;

	eq->nent = roundup_pow_of_two(nent + MLX5_NUM_SPARE_EQE);
	eq->cons_index = 0;
	err = mlx5_buf_alloc(dev, eq->nent * MLX5_EQE_SIZE, &eq->buf);
	if (err)
		return err;

	init_eq_buf(eq);

	inlen = MLX5_ST_SZ_BYTES(create_eq_in) +
		MLX5_FLD_SZ_BYTES(create_eq_in, pas[0]) * eq->buf.npages;

	in = mlx5_vzalloc(inlen);
	if (!in) {
		err = -ENOMEM;
		goto err_buf;
	}

	pas = (__be64 *)MLX5_ADDR_OF(create_eq_in, in, pas);
	mlx5_fill_page_array(&eq->buf, pas);

	MLX5_SET(create_eq_in, in, opcode, MLX5_CMD_OP_CREATE_EQ);
	MLX5_SET64(create_eq_in, in, event_bitmask, mask);

	eqc = MLX5_ADDR_OF(create_eq_in, in, eq_context_entry);
	MLX5_SET(eqc, eqc, log_eq_size, ilog2(eq->nent));
	MLX5_SET(eqc, eqc, uar_page, uar->index);
	MLX5_SET(eqc, eqc, intr, vecidx);
	MLX5_SET(eqc, eqc, log_page_size,
		 eq->buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT);

	err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
	if (err)
		goto err_in;

	snprintf(priv->irq_info[vecidx].name, MLX5_MAX_IRQ_NAME, "%s@pci:%s",
		 name, pci_name(dev->pdev));

	eq->eqn = MLX5_GET(create_eq_out, out, eq_number);
	eq->irqn = priv->msix_arr[vecidx].vector;
	eq->dev = dev;
	eq->doorbell = uar->map + MLX5_EQ_DOORBEL_OFFSET;
	err = request_irq(eq->irqn, mlx5_msix_handler, 0,
			  priv->irq_info[vecidx].name, eq);
	if (err)
		goto err_eq;

	err = mlx5_debug_eq_add(dev, eq);
	if (err)
		goto err_irq;

	INIT_LIST_HEAD(&eq->tasklet_ctx.list);
	INIT_LIST_HEAD(&eq->tasklet_ctx.process_list);
	spin_lock_init(&eq->tasklet_ctx.lock);
	tasklet_init(&eq->tasklet_ctx.task, mlx5_cq_tasklet_cb,
		     (unsigned long)&eq->tasklet_ctx);

	/* EQs are created in ARMED state
	 */
	eq_update_ci(eq, 1);

	kvfree(in);
	return 0;

err_irq:
	free_irq(priv->msix_arr[vecidx].vector, eq);

err_eq:
	mlx5_cmd_destroy_eq(dev, eq->eqn);

err_in:
	kvfree(in);

err_buf:
	mlx5_buf_free(dev, &eq->buf);
	return err;
}
EXPORT_SYMBOL_GPL(mlx5_create_map_eq);

int mlx5_destroy_unmap_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq)
{
	int err;

	mlx5_debug_eq_remove(dev, eq);
	free_irq(eq->irqn, eq);
	err = mlx5_cmd_destroy_eq(dev, eq->eqn);
	if (err)
		mlx5_core_warn(dev, "failed to destroy a previously created eq: eqn %d\n",
			       eq->eqn);
	synchronize_irq(eq->irqn);
	tasklet_disable(&eq->tasklet_ctx.task);
	mlx5_buf_free(dev, &eq->buf);

	return err;
}
EXPORT_SYMBOL_GPL(mlx5_destroy_unmap_eq);

u32 mlx5_get_msix_vec(struct mlx5_core_dev *dev, int vecidx)
{
	return dev->priv.msix_arr[MLX5_EQ_VEC_ASYNC].vector;
}

int mlx5_eq_init(struct mlx5_core_dev *dev)
{
	int err;

	spin_lock_init(&dev->priv.eq_table.lock);

	err = mlx5_eq_debugfs_init(dev);

	return err;
}


void mlx5_eq_cleanup(struct mlx5_core_dev *dev)
{
	mlx5_eq_debugfs_cleanup(dev);
}

int mlx5_start_eqs(struct mlx5_core_dev *dev)
{
	struct mlx5_eq_table *table = &dev->priv.eq_table;
	u64 async_event_mask = MLX5_ASYNC_EVENT_MASK;
	int err;

	if (MLX5_CAP_GEN(dev, pg))
		async_event_mask |= (1ull << MLX5_EVENT_TYPE_PAGE_FAULT);

	if (MLX5_CAP_GEN(dev, port_type) == MLX5_CAP_PORT_TYPE_ETH &&
	    MLX5_CAP_GEN(dev, vport_group_manager) &&
	    mlx5_core_is_pf(dev))
		async_event_mask |= (1ull << MLX5_EVENT_TYPE_NIC_VPORT_CHANGE);

	if (MLX5_CAP_GEN(dev, port_module_event))
		async_event_mask |= (1ull << MLX5_EVENT_TYPE_PORT_MODULE_EVENT);
	else
		mlx5_core_dbg(dev, "port_module_event is not set\n");

	err = mlx5_create_map_eq(dev, &table->cmd_eq, MLX5_EQ_VEC_CMD,
				 MLX5_NUM_CMD_EQE, 1ull << MLX5_EVENT_TYPE_CMD,
				 "mlx5_cmd_eq", &dev->priv.uuari.uars[0]);
	if (err) {
		mlx5_core_warn(dev, "failed to create cmd EQ %d\n", err);
		return err;
	}

	mlx5_cmd_use_events(dev);

	err = mlx5_create_map_eq(dev, &table->async_eq, MLX5_EQ_VEC_ASYNC,
				 MLX5_NUM_ASYNC_EQE, async_event_mask,
				 "mlx5_async_eq", &dev->priv.uuari.uars[0]);
	if (err) {
		mlx5_core_warn(dev, "failed to create async EQ %d\n", err);
		goto err1;
	}

	err = mlx5_create_map_eq(dev, &table->pages_eq,
				 MLX5_EQ_VEC_PAGES,
				 /* TODO: sriov max_vf + */ 1,
				 1 << MLX5_EVENT_TYPE_PAGE_REQUEST, "mlx5_pages_eq",
				 &dev->priv.uuari.uars[0]);
	if (err) {
		mlx5_core_warn(dev, "failed to create pages EQ %d\n", err);
		goto err2;
	}

	return err;

err2:
	mlx5_destroy_unmap_eq(dev, &table->async_eq);

err1:
	mlx5_cmd_use_polling(dev);
	mlx5_destroy_unmap_eq(dev, &table->cmd_eq);
	return err;
}

int mlx5_stop_eqs(struct mlx5_core_dev *dev)
{
	struct mlx5_eq_table *table = &dev->priv.eq_table;
	int err;

	err = mlx5_destroy_unmap_eq(dev, &table->pages_eq);
	if (err)
		return err;

	mlx5_destroy_unmap_eq(dev, &table->async_eq);
	mlx5_cmd_use_polling(dev);

	err = mlx5_destroy_unmap_eq(dev, &table->cmd_eq);
	if (err)
		mlx5_cmd_use_events(dev);

	return err;
}

int mlx5_core_eq_query(struct mlx5_core_dev *dev, struct mlx5_eq *eq,
		       u32 *out, int outlen)
{
	u32 in[MLX5_ST_SZ_DW(query_eq_in)] = {0};

	MLX5_SET(query_eq_in, in, opcode, MLX5_CMD_OP_QUERY_EQ);
	MLX5_SET(query_eq_in, in, eq_number, eq->eqn);
	return mlx5_cmd_exec(dev, in, sizeof(in), out, outlen);
}
EXPORT_SYMBOL_GPL(mlx5_core_eq_query);
Commit	Line	Data
e126ba97	1	/*
302bdf68	2	* Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
e126ba97 EC	3	*
	4	* This software is available to you under a choice of one of two
	5	* licenses. You may choose to be licensed under the terms of the GNU
	6	* General Public License (GPL) Version 2, available from the file
	7	* COPYING in the main directory of this source tree, or the
	8	* OpenIB.org BSD license below:
	9	*
	10	* Redistribution and use in source and binary forms, with or
	11	* without modification, are permitted provided that the following
	12	* conditions are met:
	13	*
	14	* - Redistributions of source code must retain the above
	15	* copyright notice, this list of conditions and the following
	16	* disclaimer.
	17	*
	18	* - Redistributions in binary form must reproduce the above
	19	* copyright notice, this list of conditions and the following
	20	* disclaimer in the documentation and/or other materials
	21	* provided with the distribution.
	22	*
	23	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	24	* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	25	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
	26	* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
	27	* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
	28	* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
	29	* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	30	* SOFTWARE.
	31	*/
	32
	33	#include <linux/interrupt.h>
	34	#include <linux/module.h>
	35	#include <linux/mlx5/driver.h>
	36	#include <linux/mlx5/cmd.h>
	37	#include "mlx5_core.h"
073bb189 SM	38	#ifdef CONFIG_MLX5_CORE_EN
	39	#include "eswitch.h"
	40	#endif
e126ba97 EC	41
	42	enum {
	43	MLX5_EQE_SIZE = sizeof(struct mlx5_eqe),
	44	MLX5_EQE_OWNER_INIT_VAL = 0x1,
	45	};
	46
	47	enum {
	48	MLX5_EQ_STATE_ARMED = 0x9,
	49	MLX5_EQ_STATE_FIRED = 0xa,
	50	MLX5_EQ_STATE_ALWAYS_ARMED = 0xb,
	51	};
	52
	53	enum {
	54	MLX5_NUM_SPARE_EQE = 0x80,
	55	MLX5_NUM_ASYNC_EQE = 0x100,
	56	MLX5_NUM_CMD_EQE = 32,
	57	};
	58
	59	enum {
	60	MLX5_EQ_DOORBEL_OFFSET = 0x40,
	61	};
	62
	63	#define MLX5_ASYNC_EVENT_MASK ((1ull << MLX5_EVENT_TYPE_PATH_MIG) \| \
	64	(1ull << MLX5_EVENT_TYPE_COMM_EST) \| \
	65	(1ull << MLX5_EVENT_TYPE_SQ_DRAINED) \| \
	66	(1ull << MLX5_EVENT_TYPE_CQ_ERROR) \| \
	67	(1ull << MLX5_EVENT_TYPE_WQ_CATAS_ERROR) \| \
	68	(1ull << MLX5_EVENT_TYPE_PATH_MIG_FAILED) \| \
	69	(1ull << MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR) \| \
	70	(1ull << MLX5_EVENT_TYPE_WQ_ACCESS_ERROR) \| \
	71	(1ull << MLX5_EVENT_TYPE_PORT_CHANGE) \| \
	72	(1ull << MLX5_EVENT_TYPE_SRQ_CATAS_ERROR) \| \
	73	(1ull << MLX5_EVENT_TYPE_SRQ_LAST_WQE) \| \
	74	(1ull << MLX5_EVENT_TYPE_SRQ_RQ_LIMIT))
	75
	76	struct map_eq_in {
	77	u64 mask;
	78	u32 reserved;
	79	u32 unmap_eqn;
	80	};
	81
	82	struct cre_des_eq {
	83	u8 reserved[15];
	84	u8 eqn;
	85	};
	86
	87	static int mlx5_cmd_destroy_eq(struct mlx5_core_dev *dev, u8 eqn)
	88	{
73b626c1 SM	89	u32 out[MLX5_ST_SZ_DW(destroy_eq_out)] = {0};
73b626c1 SM	90	u32 in[MLX5_ST_SZ_DW(destroy_eq_in)] = {0};
e126ba97	91
73b626c1 SM	92	MLX5_SET(destroy_eq_in, in, opcode, MLX5_CMD_OP_DESTROY_EQ);
73b626c1 SM	93	MLX5_SET(destroy_eq_in, in, eq_number, eqn);
c4f287c4	94	return mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out));
e126ba97 EC	95	}
	96
	97	static struct mlx5_eqe get_eqe(struct mlx5_eq eq, u32 entry)
	98	{
	99	return mlx5_buf_offset(&eq->buf, entry * MLX5_EQE_SIZE);
	100	}
	101
	102	static struct mlx5_eqe next_eqe_sw(struct mlx5_eq eq)
	103	{
	104	struct mlx5_eqe *eqe = get_eqe(eq, eq->cons_index & (eq->nent - 1));
	105
	106	return ((eqe->owner & 1) ^ !!(eq->cons_index & eq->nent)) ? NULL : eqe;
	107	}
	108
	109	static const char *eqe_type_str(u8 type)
	110	{
	111	switch (type) {
	112	case MLX5_EVENT_TYPE_COMP:
	113	return "MLX5_EVENT_TYPE_COMP";
	114	case MLX5_EVENT_TYPE_PATH_MIG:
	115	return "MLX5_EVENT_TYPE_PATH_MIG";
	116	case MLX5_EVENT_TYPE_COMM_EST:
	117	return "MLX5_EVENT_TYPE_COMM_EST";
	118	case MLX5_EVENT_TYPE_SQ_DRAINED:
	119	return "MLX5_EVENT_TYPE_SQ_DRAINED";
	120	case MLX5_EVENT_TYPE_SRQ_LAST_WQE:
	121	return "MLX5_EVENT_TYPE_SRQ_LAST_WQE";
	122	case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT:
	123	return "MLX5_EVENT_TYPE_SRQ_RQ_LIMIT";
	124	case MLX5_EVENT_TYPE_CQ_ERROR:
	125	return "MLX5_EVENT_TYPE_CQ_ERROR";
	126	case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
	127	return "MLX5_EVENT_TYPE_WQ_CATAS_ERROR";
	128	case MLX5_EVENT_TYPE_PATH_MIG_FAILED:
	129	return "MLX5_EVENT_TYPE_PATH_MIG_FAILED";
	130	case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
	131	return "MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR";
	132	case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
	133	return "MLX5_EVENT_TYPE_WQ_ACCESS_ERROR";
	134	case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR:
	135	return "MLX5_EVENT_TYPE_SRQ_CATAS_ERROR";
	136	case MLX5_EVENT_TYPE_INTERNAL_ERROR:
	137	return "MLX5_EVENT_TYPE_INTERNAL_ERROR";
	138	case MLX5_EVENT_TYPE_PORT_CHANGE:
	139	return "MLX5_EVENT_TYPE_PORT_CHANGE";
	140	case MLX5_EVENT_TYPE_GPIO_EVENT:
	141	return "MLX5_EVENT_TYPE_GPIO_EVENT";
d4eb4cd7 HN	142	case MLX5_EVENT_TYPE_PORT_MODULE_EVENT:
d4eb4cd7 HN	143	return "MLX5_EVENT_TYPE_PORT_MODULE_EVENT";
e126ba97 EC	144	case MLX5_EVENT_TYPE_REMOTE_CONFIG:
	145	return "MLX5_EVENT_TYPE_REMOTE_CONFIG";
	146	case MLX5_EVENT_TYPE_DB_BF_CONGESTION:
	147	return "MLX5_EVENT_TYPE_DB_BF_CONGESTION";
	148	case MLX5_EVENT_TYPE_STALL_EVENT:
	149	return "MLX5_EVENT_TYPE_STALL_EVENT";
	150	case MLX5_EVENT_TYPE_CMD:
	151	return "MLX5_EVENT_TYPE_CMD";
	152	case MLX5_EVENT_TYPE_PAGE_REQUEST:
	153	return "MLX5_EVENT_TYPE_PAGE_REQUEST";
e420f0c0 HE	154	case MLX5_EVENT_TYPE_PAGE_FAULT:
e420f0c0 HE	155	return "MLX5_EVENT_TYPE_PAGE_FAULT";
e126ba97 EC	156	default:
	157	return "Unrecognized event";
	158	}
	159	}
	160
	161	static enum mlx5_dev_event port_subtype_event(u8 subtype)
	162	{
	163	switch (subtype) {
	164	case MLX5_PORT_CHANGE_SUBTYPE_DOWN:
	165	return MLX5_DEV_EVENT_PORT_DOWN;
	166	case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE:
	167	return MLX5_DEV_EVENT_PORT_UP;
	168	case MLX5_PORT_CHANGE_SUBTYPE_INITIALIZED:
	169	return MLX5_DEV_EVENT_PORT_INITIALIZED;
	170	case MLX5_PORT_CHANGE_SUBTYPE_LID:
	171	return MLX5_DEV_EVENT_LID_CHANGE;
	172	case MLX5_PORT_CHANGE_SUBTYPE_PKEY:
	173	return MLX5_DEV_EVENT_PKEY_CHANGE;
	174	case MLX5_PORT_CHANGE_SUBTYPE_GUID:
	175	return MLX5_DEV_EVENT_GUID_CHANGE;
	176	case MLX5_PORT_CHANGE_SUBTYPE_CLIENT_REREG:
	177	return MLX5_DEV_EVENT_CLIENT_REREG;
	178	}
	179	return -1;
	180	}
	181
	182	static void eq_update_ci(struct mlx5_eq *eq, int arm)
	183	{
	184	__be32 __iomem *addr = eq->doorbell + (arm ? 0 : 2);
	185	u32 val = (eq->cons_index & 0xffffff) \| (eq->eqn << 24);
	186	__raw_writel((__force u32) cpu_to_be32(val), addr);
	187	/* We still want ordering, just not swabbing, so add a barrier */
	188	mb();
	189	}
	190
	191	static int mlx5_eq_int(struct mlx5_core_dev dev, struct mlx5_eq eq)
	192	{
	193	struct mlx5_eqe *eqe;
	194	int eqes_found = 0;
	195	int set_ci = 0;
94c6825e	196	u32 cqn = -1;
5903325a	197	u32 rsn;
e126ba97 EC	198	u8 port;
	199
	200	while ((eqe = next_eqe_sw(eq))) {
	201	/*
	202	* Make sure we read EQ entry contents after we've
	203	* checked the ownership bit.
	204	*/
12b3375f	205	dma_rmb();
e126ba97	206
1a91de28 JP	207	mlx5_core_dbg(eq->dev, "eqn %d, eqe type %s\n",
1a91de28 JP	208	eq->eqn, eqe_type_str(eqe->type));
e126ba97 EC	209	switch (eqe->type) {
	210	case MLX5_EVENT_TYPE_COMP:
	211	cqn = be32_to_cpu(eqe->data.comp.cqn) & 0xffffff;
	212	mlx5_cq_completion(dev, cqn);
	213	break;
	214
	215	case MLX5_EVENT_TYPE_PATH_MIG:
	216	case MLX5_EVENT_TYPE_COMM_EST:
	217	case MLX5_EVENT_TYPE_SQ_DRAINED:
	218	case MLX5_EVENT_TYPE_SRQ_LAST_WQE:
	219	case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
	220	case MLX5_EVENT_TYPE_PATH_MIG_FAILED:
	221	case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
	222	case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
5903325a	223	rsn = be32_to_cpu(eqe->data.qp_srq.qp_srq_n) & 0xffffff;
e2013b21	224	rsn \|= (eqe->data.qp_srq.type << MLX5_USER_INDEX_LEN);
ab62924e EC	225	mlx5_core_dbg(dev, "event %s(%d) arrived on resource 0x%x\n",
ab62924e EC	226	eqe_type_str(eqe->type), eqe->type, rsn);
5903325a	227	mlx5_rsc_event(dev, rsn, eqe->type);
e126ba97 EC	228	break;
	229
	230	case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT:
	231	case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR:
5903325a	232	rsn = be32_to_cpu(eqe->data.qp_srq.qp_srq_n) & 0xffffff;
e126ba97	233	mlx5_core_dbg(dev, "SRQ event %s(%d): srqn 0x%x\n",
5903325a EC	234	eqe_type_str(eqe->type), eqe->type, rsn);
5903325a EC	235	mlx5_srq_event(dev, rsn, eqe->type);
e126ba97 EC	236	break;
	237
	238	case MLX5_EVENT_TYPE_CMD:
	239	mlx5_cmd_comp_handler(dev, be32_to_cpu(eqe->data.cmd.vector));
	240	break;
	241
	242	case MLX5_EVENT_TYPE_PORT_CHANGE:
	243	port = (eqe->data.port.port >> 4) & 0xf;
	244	switch (eqe->sub_type) {
	245	case MLX5_PORT_CHANGE_SUBTYPE_DOWN:
	246	case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE:
	247	case MLX5_PORT_CHANGE_SUBTYPE_LID:
	248	case MLX5_PORT_CHANGE_SUBTYPE_PKEY:
	249	case MLX5_PORT_CHANGE_SUBTYPE_GUID:
	250	case MLX5_PORT_CHANGE_SUBTYPE_CLIENT_REREG:
	251	case MLX5_PORT_CHANGE_SUBTYPE_INITIALIZED:
f241e749	252	if (dev->event)
4d2f9bbb JM	253	dev->event(dev, port_subtype_event(eqe->sub_type),
4d2f9bbb JM	254	(unsigned long)port);
e126ba97 EC	255	break;
	256	default:
	257	mlx5_core_warn(dev, "Port event with unrecognized subtype: port %d, sub_type %d\n",
	258	port, eqe->sub_type);
	259	}
	260	break;
	261	case MLX5_EVENT_TYPE_CQ_ERROR:
	262	cqn = be32_to_cpu(eqe->data.cq_err.cqn) & 0xffffff;
	263	mlx5_core_warn(dev, "CQ error on CQN 0x%x, syndrom 0x%x\n",
	264	cqn, eqe->data.cq_err.syndrome);
	265	mlx5_cq_event(dev, cqn, eqe->type);
	266	break;
	267
	268	case MLX5_EVENT_TYPE_PAGE_REQUEST:
	269	{
	270	u16 func_id = be16_to_cpu(eqe->data.req_pages.func_id);
0a324f31	271	s32 npages = be32_to_cpu(eqe->data.req_pages.num_pages);
e126ba97	272
1a91de28 JP	273	mlx5_core_dbg(dev, "page request for func 0x%x, npages %d\n",
1a91de28 JP	274	func_id, npages);
e126ba97 EC	275	mlx5_core_req_pages_handler(dev, func_id, npages);
	276	}
	277	break;
	278
e420f0c0 HE	279	#ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
	280	case MLX5_EVENT_TYPE_PAGE_FAULT:
	281	mlx5_eq_pagefault(dev, eqe);
	282	break;
	283	#endif
e126ba97	284
073bb189 SM	285	#ifdef CONFIG_MLX5_CORE_EN
	286	case MLX5_EVENT_TYPE_NIC_VPORT_CHANGE:
	287	mlx5_eswitch_vport_event(dev->priv.eswitch, eqe);
	288	break;
	289	#endif
d4eb4cd7 HN	290
	291	case MLX5_EVENT_TYPE_PORT_MODULE_EVENT:
	292	mlx5_port_module_event(dev, eqe);
	293	break;
	294
e126ba97	295	default:
1a91de28 JP	296	mlx5_core_warn(dev, "Unhandled event 0x%x on EQ 0x%x\n",
1a91de28 JP	297	eqe->type, eq->eqn);
e126ba97 EC	298	break;
	299	}
	300
	301	++eq->cons_index;
	302	eqes_found = 1;
	303	++set_ci;
	304
	305	/* The HCA will think the queue has overflowed if we
	306	* don't tell it we've been processing events. We
	307	* create our EQs with MLX5_NUM_SPARE_EQE extra
	308	* entries, so we must update our consumer index at
	309	* least that often.
	310	*/
	311	if (unlikely(set_ci >= MLX5_NUM_SPARE_EQE)) {
	312	eq_update_ci(eq, 0);
	313	set_ci = 0;
	314	}
	315	}
	316
	317	eq_update_ci(eq, 1);
	318
94c6825e MB	319	if (cqn != -1)
	320	tasklet_schedule(&eq->tasklet_ctx.task);
	321
e126ba97 EC	322	return eqes_found;
	323	}
	324
	325	static irqreturn_t mlx5_msix_handler(int irq, void *eq_ptr)
	326	{
	327	struct mlx5_eq *eq = eq_ptr;
	328	struct mlx5_core_dev *dev = eq->dev;
	329
	330	mlx5_eq_int(dev, eq);
	331
	332	/* MSI-X vectors always belong to us */
	333	return IRQ_HANDLED;
	334	}
	335
	336	static void init_eq_buf(struct mlx5_eq *eq)
	337	{
	338	struct mlx5_eqe *eqe;
	339	int i;
	340
	341	for (i = 0; i < eq->nent; i++) {
	342	eqe = get_eqe(eq, i);
	343	eqe->owner = MLX5_EQE_OWNER_INIT_VAL;
	344	}
	345	}
	346
	347	int mlx5_create_map_eq(struct mlx5_core_dev dev, struct mlx5_eq eq, u8 vecidx,
	348	int nent, u64 mask, const char name, struct mlx5_uar uar)
	349	{
73b626c1	350	u32 out[MLX5_ST_SZ_DW(create_eq_out)] = {0};
db058a18	351	struct mlx5_priv *priv = &dev->priv;
73b626c1 SM	352	__be64 *pas;
73b626c1 SM	353	void *eqc;
e126ba97	354	int inlen;
73b626c1 SM	355	u32 *in;
73b626c1 SM	356	int err;
e126ba97 EC	357
e126ba97 EC	358	eq->nent = roundup_pow_of_two(nent + MLX5_NUM_SPARE_EQE);
a31208b1	359	eq->cons_index = 0;
64ffaa21	360	err = mlx5_buf_alloc(dev, eq->nent * MLX5_EQE_SIZE, &eq->buf);
e126ba97 EC	361	if (err)
	362	return err;
	363
	364	init_eq_buf(eq);
	365
73b626c1 SM	366	inlen = MLX5_ST_SZ_BYTES(create_eq_in) +
	367	MLX5_FLD_SZ_BYTES(create_eq_in, pas[0]) * eq->buf.npages;
	368
e126ba97 EC	369	in = mlx5_vzalloc(inlen);
	370	if (!in) {
	371	err = -ENOMEM;
	372	goto err_buf;
	373	}
e126ba97	374
73b626c1 SM	375	pas = (__be64 *)MLX5_ADDR_OF(create_eq_in, in, pas);
73b626c1 SM	376	mlx5_fill_page_array(&eq->buf, pas);
e126ba97	377
73b626c1 SM	378	MLX5_SET(create_eq_in, in, opcode, MLX5_CMD_OP_CREATE_EQ);
73b626c1 SM	379	MLX5_SET64(create_eq_in, in, event_bitmask, mask);
e126ba97	380
73b626c1 SM	381	eqc = MLX5_ADDR_OF(create_eq_in, in, eq_context_entry);
	382	MLX5_SET(eqc, eqc, log_eq_size, ilog2(eq->nent));
	383	MLX5_SET(eqc, eqc, uar_page, uar->index);
	384	MLX5_SET(eqc, eqc, intr, vecidx);
	385	MLX5_SET(eqc, eqc, log_page_size,
	386	eq->buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT);
e126ba97	387
73b626c1	388	err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
73b626c1	389	if (err)
e126ba97	390	goto err_in;
e126ba97	391
db058a18	392	snprintf(priv->irq_info[vecidx].name, MLX5_MAX_IRQ_NAME, "%s@pci:%s",
ada9f5d0	393	name, pci_name(dev->pdev));
db058a18	394
73b626c1	395	eq->eqn = MLX5_GET(create_eq_out, out, eq_number);
61d0e73e	396	eq->irqn = priv->msix_arr[vecidx].vector;
a158906d EC	397	eq->dev = dev;
a158906d EC	398	eq->doorbell = uar->map + MLX5_EQ_DOORBEL_OFFSET;
61d0e73e	399	err = request_irq(eq->irqn, mlx5_msix_handler, 0,
db058a18	400	priv->irq_info[vecidx].name, eq);
e126ba97 EC	401	if (err)
	402	goto err_eq;
	403
e126ba97 EC	404	err = mlx5_debug_eq_add(dev, eq);
	405	if (err)
	406	goto err_irq;
	407
94c6825e MB	408	INIT_LIST_HEAD(&eq->tasklet_ctx.list);
	409	INIT_LIST_HEAD(&eq->tasklet_ctx.process_list);
	410	spin_lock_init(&eq->tasklet_ctx.lock);
	411	tasklet_init(&eq->tasklet_ctx.task, mlx5_cq_tasklet_cb,
	412	(unsigned long)&eq->tasklet_ctx);
	413
e126ba97 EC	414	/* EQs are created in ARMED state
	415	*/
	416	eq_update_ci(eq, 1);
	417
479163f4	418	kvfree(in);
e126ba97 EC	419	return 0;
	420
	421	err_irq:
db058a18	422	free_irq(priv->msix_arr[vecidx].vector, eq);
e126ba97 EC	423
	424	err_eq:
	425	mlx5_cmd_destroy_eq(dev, eq->eqn);
	426
	427	err_in:
479163f4	428	kvfree(in);
e126ba97 EC	429
	430	err_buf:
	431	mlx5_buf_free(dev, &eq->buf);
	432	return err;
	433	}
	434	EXPORT_SYMBOL_GPL(mlx5_create_map_eq);
	435
	436	int mlx5_destroy_unmap_eq(struct mlx5_core_dev dev, struct mlx5_eq eq)
	437	{
e126ba97 EC	438	int err;
	439
	440	mlx5_debug_eq_remove(dev, eq);
61d0e73e	441	free_irq(eq->irqn, eq);
e126ba97 EC	442	err = mlx5_cmd_destroy_eq(dev, eq->eqn);
	443	if (err)
	444	mlx5_core_warn(dev, "failed to destroy a previously created eq: eqn %d\n",
	445	eq->eqn);
61d0e73e	446	synchronize_irq(eq->irqn);
94c6825e	447	tasklet_disable(&eq->tasklet_ctx.task);
e126ba97 EC	448	mlx5_buf_free(dev, &eq->buf);
	449
	450	return err;
	451	}
	452	EXPORT_SYMBOL_GPL(mlx5_destroy_unmap_eq);
	453
daa21560 TT	454	u32 mlx5_get_msix_vec(struct mlx5_core_dev *dev, int vecidx)
	455	{
	456	return dev->priv.msix_arr[MLX5_EQ_VEC_ASYNC].vector;
	457	}
	458
e126ba97 EC	459	int mlx5_eq_init(struct mlx5_core_dev *dev)
	460	{
	461	int err;
	462
	463	spin_lock_init(&dev->priv.eq_table.lock);
	464
	465	err = mlx5_eq_debugfs_init(dev);
	466
	467	return err;
	468	}
	469
	470
	471	void mlx5_eq_cleanup(struct mlx5_core_dev *dev)
	472	{
	473	mlx5_eq_debugfs_cleanup(dev);
	474	}
	475
	476	int mlx5_start_eqs(struct mlx5_core_dev *dev)
	477	{
	478	struct mlx5_eq_table *table = &dev->priv.eq_table;
6887a825	479	u64 async_event_mask = MLX5_ASYNC_EVENT_MASK;
e126ba97 EC	480	int err;
e126ba97 EC	481
938fe83c	482	if (MLX5_CAP_GEN(dev, pg))
e420f0c0 HE	483	async_event_mask \|= (1ull << MLX5_EVENT_TYPE_PAGE_FAULT);
e420f0c0 HE	484
073bb189 SM	485	if (MLX5_CAP_GEN(dev, port_type) == MLX5_CAP_PORT_TYPE_ETH &&
	486	MLX5_CAP_GEN(dev, vport_group_manager) &&
	487	mlx5_core_is_pf(dev))
	488	async_event_mask \|= (1ull << MLX5_EVENT_TYPE_NIC_VPORT_CHANGE);
	489
d4eb4cd7 HN	490	if (MLX5_CAP_GEN(dev, port_module_event))
	491	async_event_mask \|= (1ull << MLX5_EVENT_TYPE_PORT_MODULE_EVENT);
	492	else
	493	mlx5_core_dbg(dev, "port_module_event is not set\n");
	494
e126ba97 EC	495	err = mlx5_create_map_eq(dev, &table->cmd_eq, MLX5_EQ_VEC_CMD,
	496	MLX5_NUM_CMD_EQE, 1ull << MLX5_EVENT_TYPE_CMD,
	497	"mlx5_cmd_eq", &dev->priv.uuari.uars[0]);
	498	if (err) {
	499	mlx5_core_warn(dev, "failed to create cmd EQ %d\n", err);
	500	return err;
	501	}
	502
	503	mlx5_cmd_use_events(dev);
	504
	505	err = mlx5_create_map_eq(dev, &table->async_eq, MLX5_EQ_VEC_ASYNC,
e420f0c0	506	MLX5_NUM_ASYNC_EQE, async_event_mask,
e126ba97 EC	507	"mlx5_async_eq", &dev->priv.uuari.uars[0]);
	508	if (err) {
	509	mlx5_core_warn(dev, "failed to create async EQ %d\n", err);
	510	goto err1;
	511	}
	512
	513	err = mlx5_create_map_eq(dev, &table->pages_eq,
	514	MLX5_EQ_VEC_PAGES,
938fe83c	515	/* TODO: sriov max_vf + */ 1,
e126ba97 EC	516	1 << MLX5_EVENT_TYPE_PAGE_REQUEST, "mlx5_pages_eq",
	517	&dev->priv.uuari.uars[0]);
	518	if (err) {
	519	mlx5_core_warn(dev, "failed to create pages EQ %d\n", err);
	520	goto err2;
	521	}
	522
	523	return err;
	524
	525	err2:
	526	mlx5_destroy_unmap_eq(dev, &table->async_eq);
	527
	528	err1:
	529	mlx5_cmd_use_polling(dev);
	530	mlx5_destroy_unmap_eq(dev, &table->cmd_eq);
	531	return err;
	532	}
	533
	534	int mlx5_stop_eqs(struct mlx5_core_dev *dev)
	535	{
	536	struct mlx5_eq_table *table = &dev->priv.eq_table;
	537	int err;
	538
	539	err = mlx5_destroy_unmap_eq(dev, &table->pages_eq);
	540	if (err)
	541	return err;
	542
	543	mlx5_destroy_unmap_eq(dev, &table->async_eq);
	544	mlx5_cmd_use_polling(dev);
	545
	546	err = mlx5_destroy_unmap_eq(dev, &table->cmd_eq);
	547	if (err)
	548	mlx5_cmd_use_events(dev);
	549
	550	return err;
	551	}
	552
	553	int mlx5_core_eq_query(struct mlx5_core_dev dev, struct mlx5_eq eq,
73b626c1	554	u32 *out, int outlen)
e126ba97	555	{
73b626c1	556	u32 in[MLX5_ST_SZ_DW(query_eq_in)] = {0};
e126ba97	557
73b626c1 SM	558	MLX5_SET(query_eq_in, in, opcode, MLX5_CMD_OP_QUERY_EQ);
73b626c1 SM	559	MLX5_SET(query_eq_in, in, eq_number, eq->eqn);
c4f287c4	560	return mlx5_cmd_exec(dev, in, sizeof(in), out, outlen);
e126ba97 EC	561	}
e126ba97 EC	562	EXPORT_SYMBOL_GPL(mlx5_core_eq_query);