[linux-block.git] / drivers / scsi / qla2xxx / qla_inline.h

/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * QLogic Fibre Channel HBA Driver
 * Copyright (c)  2003-2014 QLogic Corporation
 */

#include "qla_target.h"
/**
 * qla24xx_calc_iocbs() - Determine number of Command Type 3 and
 * Continuation Type 1 IOCBs to allocate.
 *
 * @vha: HA context
 * @dsds: number of data segment descriptors needed
 *
 * Returns the number of IOCB entries needed to store @dsds.
 */
static inline uint16_t
qla24xx_calc_iocbs(scsi_qla_host_t *vha, uint16_t dsds)
{
	uint16_t iocbs;

	iocbs = 1;
	if (dsds > 1) {
		iocbs += (dsds - 1) / 5;
		if ((dsds - 1) % 5)
			iocbs++;
	}
	return iocbs;
}

/*
 * qla2x00_debounce_register
 *      Debounce register.
 *
 * Input:
 *      port = register address.
 *
 * Returns:
 *      register value.
 */
static __inline__ uint16_t
qla2x00_debounce_register(volatile __le16 __iomem *addr)
{
	volatile uint16_t first;
	volatile uint16_t second;

	do {
		first = rd_reg_word(addr);
		barrier();
		cpu_relax();
		second = rd_reg_word(addr);
	} while (first != second);

	return (first);
}

static inline void
qla2x00_poll(struct rsp_que *rsp)
{
	struct qla_hw_data *ha = rsp->hw;

	if (IS_P3P_TYPE(ha))
		qla82xx_poll(0, rsp);
	else
		ha->isp_ops->intr_handler(0, rsp);
}

static inline uint8_t *
host_to_fcp_swap(uint8_t *fcp, uint32_t bsize)
{
       uint32_t *ifcp = (uint32_t *) fcp;
       uint32_t *ofcp = (uint32_t *) fcp;
       uint32_t iter = bsize >> 2;

       for (; iter ; iter--)
               *ofcp++ = swab32(*ifcp++);

       return fcp;
}

static inline void
host_to_adap(uint8_t *src, uint8_t *dst, uint32_t bsize)
{
	uint32_t *isrc = (uint32_t *) src;
	__le32 *odest = (__le32 *) dst;
	uint32_t iter = bsize >> 2;

	for ( ; iter--; isrc++)
		*odest++ = cpu_to_le32(*isrc);
}

static inline void
qla2x00_clean_dsd_pool(struct qla_hw_data *ha, struct crc_context *ctx)
{
	struct dsd_dma *dsd, *tdsd;

	/* clean up allocated prev pool */
	list_for_each_entry_safe(dsd, tdsd, &ctx->dsd_list, list) {
		dma_pool_free(ha->dl_dma_pool, dsd->dsd_addr,
		    dsd->dsd_list_dma);
		list_del(&dsd->list);
		kfree(dsd);
	}
	INIT_LIST_HEAD(&ctx->dsd_list);
}

static inline void
qla2x00_set_fcport_disc_state(fc_port_t *fcport, int state)
{
	int old_val;
	uint8_t shiftbits, mask;
	uint8_t port_dstate_str_sz;

	/* This will have to change when the max no. of states > 16 */
	shiftbits = 4;
	mask = (1 << shiftbits) - 1;

	port_dstate_str_sz = sizeof(port_dstate_str) / sizeof(char *);
	fcport->disc_state = state;
	while (1) {
		old_val = atomic_read(&fcport->shadow_disc_state);
		if (old_val == atomic_cmpxchg(&fcport->shadow_disc_state,
		    old_val, (old_val << shiftbits) | state)) {
			ql_dbg(ql_dbg_disc, fcport->vha, 0x2134,
			    "FCPort %8phC disc_state transition: %s to %s - portid=%06x.\n",
			    fcport->port_name, (old_val & mask) < port_dstate_str_sz ?
				    port_dstate_str[old_val & mask] : "Unknown",
			    port_dstate_str[state], fcport->d_id.b24);
			return;
		}
	}
}

static inline int
qla2x00_hba_err_chk_enabled(srb_t *sp)
{
	/*
	 * Uncomment when corresponding SCSI changes are done.
	 *
	if (!sp->cmd->prot_chk)
		return 0;
	 *
	 */
	switch (scsi_get_prot_op(GET_CMD_SP(sp))) {
	case SCSI_PROT_READ_STRIP:
	case SCSI_PROT_WRITE_INSERT:
		if (ql2xenablehba_err_chk >= 1)
			return 1;
		break;
	case SCSI_PROT_READ_PASS:
	case SCSI_PROT_WRITE_PASS:
		if (ql2xenablehba_err_chk >= 2)
			return 1;
		break;
	case SCSI_PROT_READ_INSERT:
	case SCSI_PROT_WRITE_STRIP:
		return 1;
	}
	return 0;
}

static inline int
qla2x00_reset_active(scsi_qla_host_t *vha)
{
	scsi_qla_host_t *base_vha = pci_get_drvdata(vha->hw->pdev);

	/* Test appropriate base-vha and vha flags. */
	return test_bit(ISP_ABORT_NEEDED, &base_vha->dpc_flags) ||
	    test_bit(ABORT_ISP_ACTIVE, &base_vha->dpc_flags) ||
	    test_bit(ISP_ABORT_RETRY, &base_vha->dpc_flags) ||
	    test_bit(ISP_ABORT_NEEDED, &vha->dpc_flags) ||
	    test_bit(ABORT_ISP_ACTIVE, &vha->dpc_flags);
}

static inline int
qla2x00_chip_is_down(scsi_qla_host_t *vha)
{
	return (qla2x00_reset_active(vha) || !vha->hw->flags.fw_started);
}

static void qla2xxx_init_sp(srb_t *sp, scsi_qla_host_t *vha,
			    struct qla_qpair *qpair, fc_port_t *fcport)
{
	memset(sp, 0, sizeof(*sp));
	sp->fcport = fcport;
	sp->iocbs = 1;
	sp->vha = vha;
	sp->qpair = qpair;
	sp->cmd_type = TYPE_SRB;
	/* ref : INIT - normal flow */
	kref_init(&sp->cmd_kref);
	INIT_LIST_HEAD(&sp->elem);
}

static inline srb_t *
qla2xxx_get_qpair_sp(scsi_qla_host_t *vha, struct qla_qpair *qpair,
    fc_port_t *fcport, gfp_t flag)
{
	srb_t *sp = NULL;
	uint8_t bail;

	QLA_QPAIR_MARK_BUSY(qpair, bail);
	if (unlikely(bail))
		return NULL;

	sp = mempool_alloc(qpair->srb_mempool, flag);
	if (sp)
		qla2xxx_init_sp(sp, vha, qpair, fcport);
	else
		QLA_QPAIR_MARK_NOT_BUSY(qpair);
	return sp;
}

void qla2xxx_rel_done_warning(srb_t *sp, int res);
void qla2xxx_rel_free_warning(srb_t *sp);

static inline void
qla2xxx_rel_qpair_sp(struct qla_qpair *qpair, srb_t *sp)
{
	sp->qpair = NULL;
	sp->done = qla2xxx_rel_done_warning;
	sp->free = qla2xxx_rel_free_warning;
	mempool_free(sp, qpair->srb_mempool);
	QLA_QPAIR_MARK_NOT_BUSY(qpair);
}

static inline srb_t *
qla2x00_get_sp(scsi_qla_host_t *vha, fc_port_t *fcport, gfp_t flag)
{
	srb_t *sp = NULL;
	struct qla_qpair *qpair;

	if (unlikely(qla_vha_mark_busy(vha)))
		return NULL;

	qpair = vha->hw->base_qpair;
	sp = qla2xxx_get_qpair_sp(vha, qpair, fcport, flag);
	if (!sp)
		goto done;

	sp->vha = vha;
done:
	if (!sp)
		QLA_VHA_MARK_NOT_BUSY(vha);
	return sp;
}

static inline void
qla2x00_rel_sp(srb_t *sp)
{
	QLA_VHA_MARK_NOT_BUSY(sp->vha);
	qla2xxx_rel_qpair_sp(sp->qpair, sp);
}

static inline int
qla2x00_gid_list_size(struct qla_hw_data *ha)
{
	if (IS_QLAFX00(ha))
		return sizeof(uint32_t) * 32;
	else
		return sizeof(struct gid_list_info) * ha->max_fibre_devices;
}

static inline void
qla2x00_handle_mbx_completion(struct qla_hw_data *ha, int status)
{
	if (test_bit(MBX_INTR_WAIT, &ha->mbx_cmd_flags) &&
	    (status & MBX_INTERRUPT) && ha->flags.mbox_int) {
		set_bit(MBX_INTERRUPT, &ha->mbx_cmd_flags);
		clear_bit(MBX_INTR_WAIT, &ha->mbx_cmd_flags);
		complete(&ha->mbx_intr_comp);
	}
}

static inline void
qla2x00_set_retry_delay_timestamp(fc_port_t *fcport, uint16_t sts_qual)
{
	u8 scope;
	u16 qual;
#define SQ_SCOPE_MASK		0xc000 /* SAM-6 rev5 5.3.2 */
#define SQ_SCOPE_SHIFT		14
#define SQ_QUAL_MASK		0x3fff

#define SQ_MAX_WAIT_SEC		60 /* Max I/O hold off time in seconds. */
#define SQ_MAX_WAIT_TIME	(SQ_MAX_WAIT_SEC * 10) /* in 100ms. */

	if (!sts_qual) /* Common case. */
		return;

	scope = (sts_qual & SQ_SCOPE_MASK) >> SQ_SCOPE_SHIFT;
	/* Handle only scope 1 or 2, which is for I-T nexus. */
	if (scope != 1 && scope != 2)
		return;

	/* Skip processing, if retry delay timer is already in effect. */
	if (fcport->retry_delay_timestamp &&
	    time_before(jiffies, fcport->retry_delay_timestamp))
		return;

	qual = sts_qual & SQ_QUAL_MASK;
	if (qual < 1 || qual > 0x3fef)
		return;
	qual = min(qual, (u16)SQ_MAX_WAIT_TIME);

	/* qual is expressed in 100ms increments. */
	fcport->retry_delay_timestamp = jiffies + (qual * HZ / 10);

	ql_log(ql_log_warn, fcport->vha, 0x5101,
	       "%8phC: I/O throttling requested (status qualifier = %04xh), holding off I/Os for %ums.\n",
	       fcport->port_name, sts_qual, qual * 100);
}

static inline bool
qla_is_exch_offld_enabled(struct scsi_qla_host *vha)
{
	if (qla_ini_mode_enabled(vha) &&
	    (vha->ql2xiniexchg > FW_DEF_EXCHANGES_CNT))
		return true;
	else if (qla_tgt_mode_enabled(vha) &&
	    (vha->ql2xexchoffld > FW_DEF_EXCHANGES_CNT))
		return true;
	else if (qla_dual_mode_enabled(vha) &&
	    ((vha->ql2xiniexchg + vha->ql2xexchoffld) > FW_DEF_EXCHANGES_CNT))
		return true;
	else
		return false;
}

static inline void
qla_cpu_update(struct qla_qpair *qpair, uint16_t cpuid)
{
	qpair->cpuid = cpuid;

	if (!list_empty(&qpair->hints_list)) {
		struct qla_qpair_hint *h;

		list_for_each_entry(h, &qpair->hints_list, hint_elem)
			h->cpuid = qpair->cpuid;
	}
}

static inline struct qla_qpair_hint *
qla_qpair_to_hint(struct qla_tgt *tgt, struct qla_qpair *qpair)
{
	struct qla_qpair_hint *h;
	u16 i;

	for (i = 0; i < tgt->ha->max_qpairs + 1; i++) {
		h = &tgt->qphints[i];
		if (h->qpair == qpair)
			return h;
	}

	return NULL;
}

static inline void
qla_83xx_start_iocbs(struct qla_qpair *qpair)
{
	struct req_que *req = qpair->req;

	req->ring_index++;
	if (req->ring_index == req->length) {
		req->ring_index = 0;
		req->ring_ptr = req->ring;
	} else
		req->ring_ptr++;

	wrt_reg_dword(req->req_q_in, req->ring_index);
}

static inline int
qla2xxx_get_fc4_priority(struct scsi_qla_host *vha)
{
	uint32_t data;

	data =
	    ((uint8_t *)vha->hw->nvram)[NVRAM_DUAL_FCP_NVME_FLAG_OFFSET];


	return (data >> 6) & BIT_0 ? FC4_PRIORITY_FCP : FC4_PRIORITY_NVME;
}

enum {
	RESOURCE_NONE,
	RESOURCE_IOCB = BIT_0,
	RESOURCE_EXCH = BIT_1,  /* exchange */
	RESOURCE_FORCE = BIT_2,
	RESOURCE_HA = BIT_3,
};

static inline int
qla_get_fw_resources(struct qla_qpair *qp, struct iocb_resource *iores)
{
	u16 iocbs_used, i;
	u16 exch_used;
	struct qla_hw_data *ha = qp->hw;

	if (!ql2xenforce_iocb_limit) {
		iores->res_type = RESOURCE_NONE;
		return 0;
	}
	if (iores->res_type & RESOURCE_FORCE)
		goto force;

	if ((iores->iocb_cnt + qp->fwres.iocbs_used) >= qp->fwres.iocbs_qp_limit) {
		/* no need to acquire qpair lock. It's just rough calculation */
		iocbs_used = ha->base_qpair->fwres.iocbs_used;
		for (i = 0; i < ha->max_qpairs; i++) {
			if (ha->queue_pair_map[i])
				iocbs_used += ha->queue_pair_map[i]->fwres.iocbs_used;
		}

		if ((iores->iocb_cnt + iocbs_used) >= qp->fwres.iocbs_limit) {
			iores->res_type = RESOURCE_NONE;
			return -ENOSPC;
		}
	}

	if (iores->res_type & RESOURCE_EXCH) {
		exch_used = ha->base_qpair->fwres.exch_used;
		for (i = 0; i < ha->max_qpairs; i++) {
			if (ha->queue_pair_map[i])
				exch_used += ha->queue_pair_map[i]->fwres.exch_used;
		}

		if ((exch_used + iores->exch_cnt) >= qp->fwres.exch_limit) {
			iores->res_type = RESOURCE_NONE;
			return -ENOSPC;
		}
	}

	if (ql2xenforce_iocb_limit == 2) {
		if ((iores->iocb_cnt + atomic_read(&ha->fwres.iocb_used)) >=
		    ha->fwres.iocb_limit) {
			iores->res_type = RESOURCE_NONE;
			return -ENOSPC;
		}

		if (iores->res_type & RESOURCE_EXCH) {
			if ((iores->exch_cnt + atomic_read(&ha->fwres.exch_used)) >=
			    ha->fwres.exch_limit) {
				iores->res_type = RESOURCE_NONE;
				return -ENOSPC;
			}
		}
	}

force:
	qp->fwres.iocbs_used += iores->iocb_cnt;
	qp->fwres.exch_used += iores->exch_cnt;
	if (ql2xenforce_iocb_limit == 2) {
		atomic_add(iores->iocb_cnt, &ha->fwres.iocb_used);
		atomic_add(iores->exch_cnt, &ha->fwres.exch_used);
		iores->res_type |= RESOURCE_HA;
	}
	return 0;
}

/*
 * decrement to zero.  This routine will not decrement below zero
 * @v:  pointer of type atomic_t
 * @amount: amount to decrement from v
 */
static void qla_atomic_dtz(atomic_t *v, int amount)
{
	int c, old, dec;

	c = atomic_read(v);
	for (;;) {
		dec = c - amount;
		if (unlikely(dec < 0))
			dec = 0;

		old = atomic_cmpxchg((v), c, dec);
		if (likely(old == c))
			break;
		c = old;
	}
}

static inline void
qla_put_fw_resources(struct qla_qpair *qp, struct iocb_resource *iores)
{
	struct qla_hw_data *ha = qp->hw;

	if (iores->res_type & RESOURCE_HA) {
		if (iores->res_type & RESOURCE_IOCB)
			qla_atomic_dtz(&ha->fwres.iocb_used, iores->iocb_cnt);

		if (iores->res_type & RESOURCE_EXCH)
			qla_atomic_dtz(&ha->fwres.exch_used, iores->exch_cnt);
	}

	if (iores->res_type & RESOURCE_IOCB) {
		if (qp->fwres.iocbs_used >= iores->iocb_cnt) {
			qp->fwres.iocbs_used -= iores->iocb_cnt;
		} else {
			/* should not happen */
			qp->fwres.iocbs_used = 0;
		}
	}

	if (iores->res_type & RESOURCE_EXCH) {
		if (qp->fwres.exch_used >= iores->exch_cnt) {
			qp->fwres.exch_used -= iores->exch_cnt;
		} else {
			/* should not happen */
			qp->fwres.exch_used = 0;
		}
	}
	iores->res_type = RESOURCE_NONE;
}

#define ISP_REG_DISCONNECT 0xffffffffU
/**************************************************************************
 * qla2x00_isp_reg_stat
 *
 * Description:
 *        Read the host status register of ISP before aborting the command.
 *
 * Input:
 *       ha = pointer to host adapter structure.
 *
 *
 * Returns:
 *       Either true or false.
 *
 * Note: Return true if there is register disconnect.
 **************************************************************************/
static inline
uint32_t qla2x00_isp_reg_stat(struct qla_hw_data *ha)
{
	struct device_reg_24xx __iomem *reg = &ha->iobase->isp24;
	struct device_reg_82xx __iomem *reg82 = &ha->iobase->isp82;

	if (IS_P3P_TYPE(ha))
		return ((rd_reg_dword(&reg82->host_int)) == ISP_REG_DISCONNECT);
	else
		return ((rd_reg_dword(&reg->host_status)) ==
			ISP_REG_DISCONNECT);
}

static inline
bool qla_pci_disconnected(struct scsi_qla_host *vha,
			  struct device_reg_24xx __iomem *reg)
{
	uint32_t stat;
	bool ret = false;

	stat = rd_reg_dword(&reg->host_status);
	if (stat == 0xffffffff) {
		ql_log(ql_log_info, vha, 0x8041,
		       "detected PCI disconnect.\n");
		qla_schedule_eeh_work(vha);
		ret = true;
	}
	return ret;
}

static inline bool
fcport_is_smaller(fc_port_t *fcport)
{
	if (wwn_to_u64(fcport->port_name) <
		wwn_to_u64(fcport->vha->port_name))
		return true;
	else
		return false;
}

static inline bool
fcport_is_bigger(fc_port_t *fcport)
{
	return !fcport_is_smaller(fcport);
}

static inline struct qla_qpair *
qla_mapq_nvme_select_qpair(struct qla_hw_data *ha, struct qla_qpair *qpair)
{
	int cpuid = raw_smp_processor_id();

	if (qpair->cpuid != cpuid &&
	    ha->qp_cpu_map[cpuid]) {
		qpair = ha->qp_cpu_map[cpuid];
	}
	return qpair;
}

static inline void
qla_mapq_init_qp_cpu_map(struct qla_hw_data *ha,
			 struct qla_msix_entry *msix,
			 struct qla_qpair *qpair)
{
	const struct cpumask *mask;
	unsigned int cpu;

	if (!ha->qp_cpu_map)
		return;
	mask = pci_irq_get_affinity(ha->pdev, msix->vector_base0);
	if (!mask)
		return;
	qpair->cpuid = cpumask_first(mask);
	for_each_cpu(cpu, mask) {
		ha->qp_cpu_map[cpu] = qpair;
	}
	msix->cpuid = qpair->cpuid;
	qpair->cpu_mapped = true;
}

static inline void
qla_mapq_free_qp_cpu_map(struct qla_hw_data *ha)
{
	if (ha->qp_cpu_map) {
		kfree(ha->qp_cpu_map);
		ha->qp_cpu_map = NULL;
	}
}

static inline int qla_mapq_alloc_qp_cpu_map(struct qla_hw_data *ha)
{
	scsi_qla_host_t *vha = pci_get_drvdata(ha->pdev);

	if (!ha->qp_cpu_map) {
		ha->qp_cpu_map = kcalloc(NR_CPUS, sizeof(struct qla_qpair *),
					 GFP_KERNEL);
		if (!ha->qp_cpu_map) {
			ql_log(ql_log_fatal, vha, 0x0180,
			       "Unable to allocate memory for qp_cpu_map ptrs.\n");
			return -1;
		}
	}
	return 0;
}
Commit	Line	Data
77adf3f0	1	/* SPDX-License-Identifier: GPL-2.0-only */
1da177e4	2	/*
fa90c54f	3	* QLogic Fibre Channel HBA Driver
bd21eaf9	4	* Copyright (c) 2003-2014 QLogic Corporation
1da177e4 LT	5	*/
1da177e4 LT	6
f83adb61	7	#include "qla_target.h"
8ae6d9c7 GM	8	/**
	9	* qla24xx_calc_iocbs() - Determine number of Command Type 3 and
	10	* Continuation Type 1 IOCBs to allocate.
	11	*
2db6228d	12	* @vha: HA context
0a19a725	13	* @dsds: number of data segment descriptors needed
8ae6d9c7 GM	14	*
	15	* Returns the number of IOCB entries needed to store @dsds.
	16	*/
	17	static inline uint16_t
	18	qla24xx_calc_iocbs(scsi_qla_host_t *vha, uint16_t dsds)
	19	{
	20	uint16_t iocbs;
	21
	22	iocbs = 1;
	23	if (dsds > 1) {
	24	iocbs += (dsds - 1) / 5;
	25	if ((dsds - 1) % 5)
	26	iocbs++;
	27	}
	28	return iocbs;
	29	}
	30
1da177e4 LT	31	/*
	32	* qla2x00_debounce_register
	33	* Debounce register.
	34	*
	35	* Input:
	36	* port = register address.
	37	*
	38	* Returns:
	39	* register value.
	40	*/
	41	static __inline__ uint16_t
21038b09	42	qla2x00_debounce_register(volatile __le16 __iomem *addr)
1da177e4 LT	43	{
	44	volatile uint16_t first;
	45	volatile uint16_t second;
	46
	47	do {
04474d3a	48	first = rd_reg_word(addr);
1da177e4 LT	49	barrier();
1da177e4 LT	50	cpu_relax();
04474d3a	51	second = rd_reg_word(addr);
1da177e4 LT	52	} while (first != second);
	53
	54	return (first);
	55	}
	56
fa2a1ce5	57	static inline void
e315cd28	58	qla2x00_poll(struct rsp_que *rsp)
1da177e4	59	{
e315cd28	60	struct qla_hw_data *ha = rsp->hw;
b3a8aa90	61
7ec0effd	62	if (IS_P3P_TYPE(ha))
a9083016 GM	63	qla82xx_poll(0, rsp);
	64	else
	65	ha->isp_ops->intr_handler(0, rsp);
1da177e4 LT	66	}
1da177e4 LT	67
2b6c0cee AV	68	static inline uint8_t *
	69	host_to_fcp_swap(uint8_t *fcp, uint32_t bsize)
	70	{
	71	uint32_t ifcp = (uint32_t ) fcp;
	72	uint32_t ofcp = (uint32_t ) fcp;
	73	uint32_t iter = bsize >> 2;
	74
	75	for (; iter ; iter--)
	76	ofcp++ = swab32(ifcp++);
	77
	78	return fcp;
	79	}
3d71644c	80
8ae6d9c7 GM	81	static inline void
	82	host_to_adap(uint8_t src, uint8_t dst, uint32_t bsize)
	83	{
	84	uint32_t isrc = (uint32_t ) src;
1f8deefe	85	__le32 odest = (__le32 ) dst;
8ae6d9c7 GM	86	uint32_t iter = bsize >> 2;
8ae6d9c7 GM	87
da08ef5c JC	88	for ( ; iter--; isrc++)
da08ef5c JC	89	odest++ = cpu_to_le32(isrc);
8ae6d9c7 GM	90	}
8ae6d9c7 GM	91
bad75002	92	static inline void
d5ff0eed	93	qla2x00_clean_dsd_pool(struct qla_hw_data ha, struct crc_context ctx)
bad75002	94	{
d5ff0eed	95	struct dsd_dma dsd, tdsd;
bad75002 AE	96
bad75002 AE	97	/* clean up allocated prev pool */
d5ff0eed JC	98	list_for_each_entry_safe(dsd, tdsd, &ctx->dsd_list, list) {
	99	dma_pool_free(ha->dl_dma_pool, dsd->dsd_addr,
	100	dsd->dsd_list_dma);
	101	list_del(&dsd->list);
	102	kfree(dsd);
bad75002	103	}
9ba56b95	104	INIT_LIST_HEAD(&ctx->dsd_list);
bad75002	105	}
ec426e10	106
27258a57 SS	107	static inline void
	108	qla2x00_set_fcport_disc_state(fc_port_t *fcport, int state)
	109	{
	110	int old_val;
	111	uint8_t shiftbits, mask;
b1b9d382	112	uint8_t port_dstate_str_sz;
27258a57 SS	113
	114	/* This will have to change when the max no. of states > 16 */
	115	shiftbits = 4;
	116	mask = (1 << shiftbits) - 1;
	117
b1b9d382	118	port_dstate_str_sz = sizeof(port_dstate_str) / sizeof(char *);
27258a57 SS	119	fcport->disc_state = state;
	120	while (1) {
	121	old_val = atomic_read(&fcport->shadow_disc_state);
	122	if (old_val == atomic_cmpxchg(&fcport->shadow_disc_state,
	123	old_val, (old_val << shiftbits) \| state)) {
	124	ql_dbg(ql_dbg_disc, fcport->vha, 0x2134,
	125	"FCPort %8phC disc_state transition: %s to %s - portid=%06x.\n",
b1b9d382 BH	126	fcport->port_name, (old_val & mask) < port_dstate_str_sz ?
b1b9d382 BH	127	port_dstate_str[old_val & mask] : "Unknown",
27258a57 SS	128	port_dstate_str[state], fcport->d_id.b24);
	129	return;
	130	}
	131	}
	132	}
	133
8cb2049c	134	static inline int
e02587d7	135	qla2x00_hba_err_chk_enabled(srb_t *sp)
8cb2049c	136	{
e02587d7 AE	137	/*
	138	* Uncomment when corresponding SCSI changes are done.
	139	*
	140	if (!sp->cmd->prot_chk)
	141	return 0;
	142	*
	143	*/
9ba56b95	144	switch (scsi_get_prot_op(GET_CMD_SP(sp))) {
8cb2049c AE	145	case SCSI_PROT_READ_STRIP:
	146	case SCSI_PROT_WRITE_INSERT:
	147	if (ql2xenablehba_err_chk >= 1)
	148	return 1;
	149	break;
	150	case SCSI_PROT_READ_PASS:
	151	case SCSI_PROT_WRITE_PASS:
	152	if (ql2xenablehba_err_chk >= 2)
	153	return 1;
	154	break;
	155	case SCSI_PROT_READ_INSERT:
	156	case SCSI_PROT_WRITE_STRIP:
	157	return 1;
	158	}
	159	return 0;
	160	}
d051a5aa AV	161
	162	static inline int
	163	qla2x00_reset_active(scsi_qla_host_t *vha)
	164	{
	165	scsi_qla_host_t *base_vha = pci_get_drvdata(vha->hw->pdev);
	166
	167	/* Test appropriate base-vha and vha flags. */
	168	return test_bit(ISP_ABORT_NEEDED, &base_vha->dpc_flags) \|\|
	169	test_bit(ABORT_ISP_ACTIVE, &base_vha->dpc_flags) \|\|
	170	test_bit(ISP_ABORT_RETRY, &base_vha->dpc_flags) \|\|
	171	test_bit(ISP_ABORT_NEEDED, &vha->dpc_flags) \|\|
	172	test_bit(ABORT_ISP_ACTIVE, &vha->dpc_flags);
	173	}
9ba56b95	174
22ebde16 QT	175	static inline int
	176	qla2x00_chip_is_down(scsi_qla_host_t *vha)
	177	{
	178	return (qla2x00_reset_active(vha) \|\| !vha->hw->flags.fw_started);
	179	}
	180
bdb61b9b BVA	181	static void qla2xxx_init_sp(srb_t sp, scsi_qla_host_t vha,
	182	struct qla_qpair qpair, fc_port_t fcport)
	183	{
	184	memset(sp, 0, sizeof(*sp));
	185	sp->fcport = fcport;
	186	sp->iocbs = 1;
	187	sp->vha = vha;
	188	sp->qpair = qpair;
	189	sp->cmd_type = TYPE_SRB;
31e6cdbe SK	190	/* ref : INIT - normal flow */
31e6cdbe SK	191	kref_init(&sp->cmd_kref);
bdb61b9b BVA	192	INIT_LIST_HEAD(&sp->elem);
	193	}
	194
d7459527	195	static inline srb_t *
6a629468 QT	196	qla2xxx_get_qpair_sp(scsi_qla_host_t vha, struct qla_qpair qpair,
6a629468 QT	197	fc_port_t *fcport, gfp_t flag)
d7459527 MH	198	{
	199	srb_t *sp = NULL;
	200	uint8_t bail;
	201
	202	QLA_QPAIR_MARK_BUSY(qpair, bail);
	203	if (unlikely(bail))
	204	return NULL;
	205
	206	sp = mempool_alloc(qpair->srb_mempool, flag);
bdb61b9b BVA	207	if (sp)
	208	qla2xxx_init_sp(sp, vha, qpair, fcport);
	209	else
d7459527 MH	210	QLA_QPAIR_MARK_NOT_BUSY(qpair);
	211	return sp;
	212	}
	213
c0014f94 DW	214	void qla2xxx_rel_done_warning(srb_t *sp, int res);
	215	void qla2xxx_rel_free_warning(srb_t *sp);
	216
d7459527 MH	217	static inline void
	218	qla2xxx_rel_qpair_sp(struct qla_qpair qpair, srb_t sp)
	219	{
ae6ccb0f	220	sp->qpair = NULL;
c0014f94 DW	221	sp->done = qla2xxx_rel_done_warning;
c0014f94 DW	222	sp->free = qla2xxx_rel_free_warning;
d7459527 MH	223	mempool_free(sp, qpair->srb_mempool);
	224	QLA_QPAIR_MARK_NOT_BUSY(qpair);
	225	}
	226
9ba56b95 GM	227	static inline srb_t *
	228	qla2x00_get_sp(scsi_qla_host_t vha, fc_port_t fcport, gfp_t flag)
	229	{
	230	srb_t *sp = NULL;
6a629468	231	struct qla_qpair *qpair;
9ba56b95	232
4fb2169d	233	if (unlikely(qla_vha_mark_busy(vha)))
9ba56b95 GM	234	return NULL;
9ba56b95 GM	235
6a629468 QT	236	qpair = vha->hw->base_qpair;
6a629468 QT	237	sp = qla2xxx_get_qpair_sp(vha, qpair, fcport, flag);
9ba56b95 GM	238	if (!sp)
	239	goto done;
	240
726b8548	241	sp->vha = vha;
9ba56b95 GM	242	done:
	243	if (!sp)
	244	QLA_VHA_MARK_NOT_BUSY(vha);
	245	return sp;
	246	}
	247
b00ee7d7	248	static inline void
25ff6af1	249	qla2x00_rel_sp(srb_t *sp)
b00ee7d7	250	{
25ff6af1	251	QLA_VHA_MARK_NOT_BUSY(sp->vha);
6a629468	252	qla2xxx_rel_qpair_sp(sp->qpair, sp);
b00ee7d7 CD	253	}
b00ee7d7 CD	254
642ef983 CD	255	static inline int
	256	qla2x00_gid_list_size(struct qla_hw_data *ha)
	257	{
8ae6d9c7 GM	258	if (IS_QLAFX00(ha))
	259	return sizeof(uint32_t) * 32;
	260	else
	261	return sizeof(struct gid_list_info) * ha->max_fibre_devices;
642ef983	262	}
3c290d0b	263
36439832	264	static inline void
	265	qla2x00_handle_mbx_completion(struct qla_hw_data *ha, int status)
	266	{
	267	if (test_bit(MBX_INTR_WAIT, &ha->mbx_cmd_flags) &&
	268	(status & MBX_INTERRUPT) && ha->flags.mbox_int) {
	269	set_bit(MBX_INTERRUPT, &ha->mbx_cmd_flags);
	270	clear_bit(MBX_INTR_WAIT, &ha->mbx_cmd_flags);
	271	complete(&ha->mbx_intr_comp);
	272	}
	273	}
e05fe292 CD	274
e05fe292 CD	275	static inline void
3aac0c0f	276	qla2x00_set_retry_delay_timestamp(fc_port_t *fcport, uint16_t sts_qual)
e05fe292	277	{
3aac0c0f AE	278	u8 scope;
	279	u16 qual;
	280	#define SQ_SCOPE_MASK 0xc000 /* SAM-6 rev5 5.3.2 */
	281	#define SQ_SCOPE_SHIFT 14
	282	#define SQ_QUAL_MASK 0x3fff
	283
	284	#define SQ_MAX_WAIT_SEC 60 /* Max I/O hold off time in seconds. */
	285	#define SQ_MAX_WAIT_TIME (SQ_MAX_WAIT_SEC * 10) /* in 100ms. */
	286
	287	if (!sts_qual) /* Common case. */
	288	return;
	289
	290	scope = (sts_qual & SQ_SCOPE_MASK) >> SQ_SCOPE_SHIFT;
	291	/* Handle only scope 1 or 2, which is for I-T nexus. */
	292	if (scope != 1 && scope != 2)
	293	return;
	294
	295	/* Skip processing, if retry delay timer is already in effect. */
	296	if (fcport->retry_delay_timestamp &&
	297	time_before(jiffies, fcport->retry_delay_timestamp))
	298	return;
	299
	300	qual = sts_qual & SQ_QUAL_MASK;
	301	if (qual < 1 \|\| qual > 0x3fef)
	302	return;
	303	qual = min(qual, (u16)SQ_MAX_WAIT_TIME);
	304
	305	/* qual is expressed in 100ms increments. */
	306	fcport->retry_delay_timestamp = jiffies + (qual * HZ / 10);
	307
	308	ql_log(ql_log_warn, fcport->vha, 0x5101,
	309	"%8phC: I/O throttling requested (status qualifier = %04xh), holding off I/Os for %ums.\n",
	310	fcport->port_name, sts_qual, qual * 100);
e05fe292	311	}
99e1b683 QT	312
	313	static inline bool
	314	qla_is_exch_offld_enabled(struct scsi_qla_host *vha)
	315	{
	316	if (qla_ini_mode_enabled(vha) &&
0645cb83	317	(vha->ql2xiniexchg > FW_DEF_EXCHANGES_CNT))
99e1b683 QT	318	return true;
99e1b683 QT	319	else if (qla_tgt_mode_enabled(vha) &&
0645cb83	320	(vha->ql2xexchoffld > FW_DEF_EXCHANGES_CNT))
99e1b683 QT	321	return true;
99e1b683 QT	322	else if (qla_dual_mode_enabled(vha) &&
0645cb83	323	((vha->ql2xiniexchg + vha->ql2xexchoffld) > FW_DEF_EXCHANGES_CNT))
99e1b683 QT	324	return true;
	325	else
	326	return false;
	327	}
e326d22a QT	328
	329	static inline void
	330	qla_cpu_update(struct qla_qpair *qpair, uint16_t cpuid)
	331	{
	332	qpair->cpuid = cpuid;
	333
	334	if (!list_empty(&qpair->hints_list)) {
	335	struct qla_qpair_hint *h;
	336
	337	list_for_each_entry(h, &qpair->hints_list, hint_elem)
	338	h->cpuid = qpair->cpuid;
	339	}
	340	}
	341
	342	static inline struct qla_qpair_hint *
	343	qla_qpair_to_hint(struct qla_tgt tgt, struct qla_qpair qpair)
	344	{
	345	struct qla_qpair_hint *h;
	346	u16 i;
	347
	348	for (i = 0; i < tgt->ha->max_qpairs + 1; i++) {
	349	h = &tgt->qphints[i];
	350	if (h->qpair == qpair)
	351	return h;
	352	}
	353
	354	return NULL;
	355	}
8abfa9e2 QT	356
	357	static inline void
	358	qla_83xx_start_iocbs(struct qla_qpair *qpair)
	359	{
	360	struct req_que *req = qpair->req;
	361
	362	req->ring_index++;
	363	if (req->ring_index == req->length) {
	364	req->ring_index = 0;
	365	req->ring_ptr = req->ring;
	366	} else
	367	req->ring_ptr++;
	368
04474d3a	369	wrt_reg_dword(req->req_q_in, req->ring_index);
8abfa9e2	370	}
84ed362a MH	371
	372	static inline int
	373	qla2xxx_get_fc4_priority(struct scsi_qla_host *vha)
	374	{
	375	uint32_t data;
	376
	377	data =
	378	((uint8_t *)vha->hw->nvram)[NVRAM_DUAL_FCP_NVME_FLAG_OFFSET];
	379
	380
a10c8803	381	return (data >> 6) & BIT_0 ? FC4_PRIORITY_FCP : FC4_PRIORITY_NVME;
84ed362a	382	}
89c72f42 QT	383
	384	enum {
	385	RESOURCE_NONE,
5f63a163	386	RESOURCE_IOCB = BIT_0,
41e5afe5 QT	387	RESOURCE_EXCH = BIT_1, /* exchange */
41e5afe5 QT	388	RESOURCE_FORCE = BIT_2,
e370b64c	389	RESOURCE_HA = BIT_3,
89c72f42 QT	390	};
	391
	392	static inline int
41e5afe5	393	qla_get_fw_resources(struct qla_qpair qp, struct iocb_resource iores)
89c72f42 QT	394	{
89c72f42 QT	395	u16 iocbs_used, i;
41e5afe5	396	u16 exch_used;
e370b64c	397	struct qla_hw_data *ha = qp->hw;
89c72f42 QT	398
	399	if (!ql2xenforce_iocb_limit) {
	400	iores->res_type = RESOURCE_NONE;
	401	return 0;
	402	}
5f63a163 QT	403	if (iores->res_type & RESOURCE_FORCE)
5f63a163 QT	404	goto force;
89c72f42	405
41e5afe5	406	if ((iores->iocb_cnt + qp->fwres.iocbs_used) >= qp->fwres.iocbs_qp_limit) {
89c72f42 QT	407	/* no need to acquire qpair lock. It's just rough calculation */
	408	iocbs_used = ha->base_qpair->fwres.iocbs_used;
	409	for (i = 0; i < ha->max_qpairs; i++) {
	410	if (ha->queue_pair_map[i])
	411	iocbs_used += ha->queue_pair_map[i]->fwres.iocbs_used;
	412	}
	413
41e5afe5 QT	414	if ((iores->iocb_cnt + iocbs_used) >= qp->fwres.iocbs_limit) {
	415	iores->res_type = RESOURCE_NONE;
	416	return -ENOSPC;
	417	}
	418	}
	419
	420	if (iores->res_type & RESOURCE_EXCH) {
	421	exch_used = ha->base_qpair->fwres.exch_used;
	422	for (i = 0; i < ha->max_qpairs; i++) {
	423	if (ha->queue_pair_map[i])
	424	exch_used += ha->queue_pair_map[i]->fwres.exch_used;
	425	}
	426
	427	if ((exch_used + iores->exch_cnt) >= qp->fwres.exch_limit) {
89c72f42 QT	428	iores->res_type = RESOURCE_NONE;
	429	return -ENOSPC;
	430	}
	431	}
e370b64c QT	432
	433	if (ql2xenforce_iocb_limit == 2) {
	434	if ((iores->iocb_cnt + atomic_read(&ha->fwres.iocb_used)) >=
	435	ha->fwres.iocb_limit) {
	436	iores->res_type = RESOURCE_NONE;
	437	return -ENOSPC;
	438	}
	439
	440	if (iores->res_type & RESOURCE_EXCH) {
	441	if ((iores->exch_cnt + atomic_read(&ha->fwres.exch_used)) >=
	442	ha->fwres.exch_limit) {
	443	iores->res_type = RESOURCE_NONE;
	444	return -ENOSPC;
	445	}
	446	}
	447	}
	448
5f63a163	449	force:
41e5afe5 QT	450	qp->fwres.iocbs_used += iores->iocb_cnt;
41e5afe5 QT	451	qp->fwres.exch_used += iores->exch_cnt;
e370b64c QT	452	if (ql2xenforce_iocb_limit == 2) {
	453	atomic_add(iores->iocb_cnt, &ha->fwres.iocb_used);
	454	atomic_add(iores->exch_cnt, &ha->fwres.exch_used);
	455	iores->res_type \|= RESOURCE_HA;
	456	}
41e5afe5	457	return 0;
89c72f42 QT	458	}
89c72f42 QT	459
e370b64c QT	460	/*
	461	* decrement to zero. This routine will not decrement below zero
	462	* @v: pointer of type atomic_t
	463	* @amount: amount to decrement from v
	464	*/
	465	static void qla_atomic_dtz(atomic_t *v, int amount)
	466	{
	467	int c, old, dec;
	468
	469	c = atomic_read(v);
	470	for (;;) {
	471	dec = c - amount;
	472	if (unlikely(dec < 0))
	473	dec = 0;
	474
	475	old = atomic_cmpxchg((v), c, dec);
	476	if (likely(old == c))
	477	break;
	478	c = old;
	479	}
	480	}
	481
89c72f42	482	static inline void
41e5afe5	483	qla_put_fw_resources(struct qla_qpair qp, struct iocb_resource iores)
89c72f42	484	{
e370b64c QT	485	struct qla_hw_data *ha = qp->hw;
	486
	487	if (iores->res_type & RESOURCE_HA) {
	488	if (iores->res_type & RESOURCE_IOCB)
	489	qla_atomic_dtz(&ha->fwres.iocb_used, iores->iocb_cnt);
	490
	491	if (iores->res_type & RESOURCE_EXCH)
	492	qla_atomic_dtz(&ha->fwres.exch_used, iores->exch_cnt);
	493	}
	494
41e5afe5	495	if (iores->res_type & RESOURCE_IOCB) {
89c72f42 QT	496	if (qp->fwres.iocbs_used >= iores->iocb_cnt) {
	497	qp->fwres.iocbs_used -= iores->iocb_cnt;
	498	} else {
41e5afe5	499	/* should not happen */
89c72f42 QT	500	qp->fwres.iocbs_used = 0;
89c72f42 QT	501	}
41e5afe5 QT	502	}
	503
	504	if (iores->res_type & RESOURCE_EXCH) {
	505	if (qp->fwres.exch_used >= iores->exch_cnt) {
	506	qp->fwres.exch_used -= iores->exch_cnt;
	507	} else {
	508	/* should not happen */
	509	qp->fwres.exch_used = 0;
	510	}
89c72f42 QT	511	}
	512	iores->res_type = RESOURCE_NONE;
	513	}
f7a0ed47 QT	514
	515	#define ISP_REG_DISCONNECT 0xffffffffU
	516	/**************************************************************************
	517	* qla2x00_isp_reg_stat
	518	*
	519	* Description:
	520	* Read the host status register of ISP before aborting the command.
	521	*
	522	* Input:
	523	* ha = pointer to host adapter structure.
	524	*
	525	*
	526	* Returns:
	527	* Either true or false.
	528	*
	529	* Note: Return true if there is register disconnect.
	530	**************************************************************************/
	531	static inline
	532	uint32_t qla2x00_isp_reg_stat(struct qla_hw_data *ha)
	533	{
	534	struct device_reg_24xx __iomem *reg = &ha->iobase->isp24;
	535	struct device_reg_82xx __iomem *reg82 = &ha->iobase->isp82;
	536
	537	if (IS_P3P_TYPE(ha))
	538	return ((rd_reg_dword(&reg82->host_int)) == ISP_REG_DISCONNECT);
	539	else
	540	return ((rd_reg_dword(&reg->host_status)) ==
	541	ISP_REG_DISCONNECT);
	542	}
	543
	544	static inline
	545	bool qla_pci_disconnected(struct scsi_qla_host *vha,
	546	struct device_reg_24xx __iomem *reg)
	547	{
	548	uint32_t stat;
	549	bool ret = false;
	550
	551	stat = rd_reg_dword(&reg->host_status);
	552	if (stat == 0xffffffff) {
	553	ql_log(ql_log_info, vha, 0x8041,
	554	"detected PCI disconnect.\n");
	555	qla_schedule_eeh_work(vha);
	556	ret = true;
	557	}
	558	return ret;
	559	}
4de067e5 QT	560
	561	static inline bool
	562	fcport_is_smaller(fc_port_t *fcport)
	563	{
	564	if (wwn_to_u64(fcport->port_name) <
	565	wwn_to_u64(fcport->vha->port_name))
	566	return true;
	567	else
	568	return false;
	569	}
	570
	571	static inline bool
	572	fcport_is_bigger(fc_port_t *fcport)
	573	{
	574	return !fcport_is_smaller(fcport);
	575	}
1d201c81 SD	576
	577	static inline struct qla_qpair *
	578	qla_mapq_nvme_select_qpair(struct qla_hw_data ha, struct qla_qpair qpair)
	579	{
59f10a05	580	int cpuid = raw_smp_processor_id();
1d201c81 SD	581
	582	if (qpair->cpuid != cpuid &&
	583	ha->qp_cpu_map[cpuid]) {
	584	qpair = ha->qp_cpu_map[cpuid];
	585	}
	586	return qpair;
	587	}
	588
	589	static inline void
	590	qla_mapq_init_qp_cpu_map(struct qla_hw_data *ha,
	591	struct qla_msix_entry *msix,
	592	struct qla_qpair *qpair)
	593	{
	594	const struct cpumask *mask;
	595	unsigned int cpu;
	596
	597	if (!ha->qp_cpu_map)
	598	return;
	599	mask = pci_irq_get_affinity(ha->pdev, msix->vector_base0);
d54820b2 GC	600	if (!mask)
d54820b2 GC	601	return;
1d201c81 SD	602	qpair->cpuid = cpumask_first(mask);
	603	for_each_cpu(cpu, mask) {
	604	ha->qp_cpu_map[cpu] = qpair;
	605	}
	606	msix->cpuid = qpair->cpuid;
d54820b2	607	qpair->cpu_mapped = true;
1d201c81 SD	608	}
	609
	610	static inline void
	611	qla_mapq_free_qp_cpu_map(struct qla_hw_data *ha)
	612	{
	613	if (ha->qp_cpu_map) {
	614	kfree(ha->qp_cpu_map);
	615	ha->qp_cpu_map = NULL;
	616	}
	617	}
	618
	619	static inline int qla_mapq_alloc_qp_cpu_map(struct qla_hw_data *ha)
	620	{
	621	scsi_qla_host_t *vha = pci_get_drvdata(ha->pdev);
	622
	623	if (!ha->qp_cpu_map) {
	624	ha->qp_cpu_map = kcalloc(NR_CPUS, sizeof(struct qla_qpair *),
	625	GFP_KERNEL);
	626	if (!ha->qp_cpu_map) {
	627	ql_log(ql_log_fatal, vha, 0x0180,
	628	"Unable to allocate memory for qp_cpu_map ptrs.\n");
	629	return -1;
	630	}
	631	}
	632	return 0;
	633	}