Introduce a new lock to protect the list of fc_fcp_pkt structs in libfc
instead of using the host lock. This reduces the contention of this heavily
used lock, and I see up to a 25% performance gain in CPU bound small I/O
tests when scaling out across multiple quad-core CPUs.
The big win is in removing the host lock from the completion path
completely, as it does not need to be held around the call to scsi_done.
Signed-off-by: Chris Leech <christopher.leech@intel.com>
Signed-off-by: Robert Love <robert.w.love@intel.com>
Signed-off-by: James Bottomley <James.Bottomley@suse.de>
/**
* struct fc_fcp_internal - FCP layer internal data
/**
* struct fc_fcp_internal - FCP layer internal data
- * @scsi_pkt_pool: Memory pool to draw FCP packets from
+ * @scsi_pkt_pool: Memory pool to draw FCP packets from
+ * @scsi_queue_lock: Protects the scsi_pkt_queue
* @scsi_pkt_queue: Current FCP packets
* @last_can_queue_ramp_down_time: ramp down time
* @last_can_queue_ramp_up_time: ramp up time
* @max_can_queue: max can_queue size
*/
struct fc_fcp_internal {
* @scsi_pkt_queue: Current FCP packets
* @last_can_queue_ramp_down_time: ramp down time
* @last_can_queue_ramp_up_time: ramp up time
* @max_can_queue: max can_queue size
*/
struct fc_fcp_internal {
- mempool_t *scsi_pkt_pool;
- struct list_head scsi_pkt_queue;
- unsigned long last_can_queue_ramp_down_time;
- unsigned long last_can_queue_ramp_up_time;
- int max_can_queue;
+ mempool_t *scsi_pkt_pool;
+ spinlock_t scsi_queue_lock;
+ struct list_head scsi_pkt_queue;
+ unsigned long last_can_queue_ramp_down_time;
+ unsigned long last_can_queue_ramp_up_time;
+ int max_can_queue;
};
#define fc_get_scsi_internal(x) ((struct fc_fcp_internal *)(x)->scsi_priv)
};
#define fc_get_scsi_internal(x) ((struct fc_fcp_internal *)(x)->scsi_priv)
unsigned long flags;
fp = fc_frame_alloc(lport, len);
unsigned long flags;
fp = fc_frame_alloc(lport, len);
- if (!fp) {
- spin_lock_irqsave(lport->host->host_lock, flags);
- fc_fcp_can_queue_ramp_down(lport);
- spin_unlock_irqrestore(lport->host->host_lock, flags);
- }
- return fp;
+ if (likely(fp))
+ return fp;
+
+ /* error case */
+ spin_lock_irqsave(lport->host->host_lock, flags);
+ fc_fcp_can_queue_ramp_down(lport);
+ spin_unlock_irqrestore(lport->host->host_lock, flags);
+ return NULL;
struct scsi_cmnd *sc_cmd;
unsigned long flags;
struct scsi_cmnd *sc_cmd;
unsigned long flags;
- spin_lock_irqsave(lport->host->host_lock, flags);
+ spin_lock_irqsave(&si->scsi_queue_lock, flags);
restart:
list_for_each_entry(fsp, &si->scsi_pkt_queue, list) {
sc_cmd = fsp->cmd;
restart:
list_for_each_entry(fsp, &si->scsi_pkt_queue, list) {
sc_cmd = fsp->cmd;
continue;
fc_fcp_pkt_hold(fsp);
continue;
fc_fcp_pkt_hold(fsp);
- spin_unlock_irqrestore(lport->host->host_lock, flags);
+ spin_unlock_irqrestore(&si->scsi_queue_lock, flags);
if (!fc_fcp_lock_pkt(fsp)) {
fc_fcp_cleanup_cmd(fsp, error);
if (!fc_fcp_lock_pkt(fsp)) {
fc_fcp_cleanup_cmd(fsp, error);
}
fc_fcp_pkt_release(fsp);
}
fc_fcp_pkt_release(fsp);
- spin_lock_irqsave(lport->host->host_lock, flags);
+ spin_lock_irqsave(&si->scsi_queue_lock, flags);
/*
* while we dropped the lock multiple pkts could
* have been released, so we have to start over.
*/
goto restart;
}
/*
* while we dropped the lock multiple pkts could
* have been released, so we have to start over.
*/
goto restart;
}
- spin_unlock_irqrestore(lport->host->host_lock, flags);
+ spin_unlock_irqrestore(&si->scsi_queue_lock, flags);
* @fsp: The FCP packet to send
*
* Return: Zero for success and -1 for failure
* @fsp: The FCP packet to send
*
* Return: Zero for success and -1 for failure
- * Locks: Called with the host lock and irqs disabled.
+ * Locks: Called without locks held
*/
static int fc_fcp_pkt_send(struct fc_lport *lport, struct fc_fcp_pkt *fsp)
{
struct fc_fcp_internal *si = fc_get_scsi_internal(lport);
*/
static int fc_fcp_pkt_send(struct fc_lport *lport, struct fc_fcp_pkt *fsp)
{
struct fc_fcp_internal *si = fc_get_scsi_internal(lport);
int rc;
fsp->cmd->SCp.ptr = (char *)fsp;
int rc;
fsp->cmd->SCp.ptr = (char *)fsp;
int_to_scsilun(fsp->cmd->device->lun,
(struct scsi_lun *)fsp->cdb_cmd.fc_lun);
memcpy(fsp->cdb_cmd.fc_cdb, fsp->cmd->cmnd, fsp->cmd->cmd_len);
int_to_scsilun(fsp->cmd->device->lun,
(struct scsi_lun *)fsp->cdb_cmd.fc_lun);
memcpy(fsp->cdb_cmd.fc_cdb, fsp->cmd->cmnd, fsp->cmd->cmd_len);
- list_add_tail(&fsp->list, &si->scsi_pkt_queue);
- spin_unlock_irq(lport->host->host_lock);
+ spin_lock_irqsave(&si->scsi_queue_lock, flags);
+ list_add_tail(&fsp->list, &si->scsi_pkt_queue);
+ spin_unlock_irqrestore(&si->scsi_queue_lock, flags);
rc = lport->tt.fcp_cmd_send(lport, fsp, fc_fcp_recv);
rc = lport->tt.fcp_cmd_send(lport, fsp, fc_fcp_recv);
- spin_lock_irq(lport->host->host_lock);
- if (rc)
+ if (unlikely(rc)) {
+ spin_lock_irqsave(&si->scsi_queue_lock, flags);
+ spin_unlock_irqrestore(&si->scsi_queue_lock, flags);
+ }
struct fcoe_dev_stats *stats;
lport = shost_priv(sc_cmd->device->host);
struct fcoe_dev_stats *stats;
lport = shost_priv(sc_cmd->device->host);
+ spin_unlock_irq(lport->host->host_lock);
rval = fc_remote_port_chkready(rport);
if (rval) {
rval = fc_remote_port_chkready(rport);
if (rval) {
rc = SCSI_MLQUEUE_HOST_BUSY;
}
out:
rc = SCSI_MLQUEUE_HOST_BUSY;
}
out:
+ spin_lock_irq(lport->host->host_lock);
return rc;
}
EXPORT_SYMBOL(fc_queuecommand);
return rc;
}
EXPORT_SYMBOL(fc_queuecommand);
lport = fsp->lp;
si = fc_get_scsi_internal(lport);
lport = fsp->lp;
si = fc_get_scsi_internal(lport);
- spin_lock_irqsave(lport->host->host_lock, flags);
- if (!fsp->cmd) {
- spin_unlock_irqrestore(lport->host->host_lock, flags);
/*
* if can_queue ramp down is done then try can_queue ramp up
/*
* if can_queue ramp down is done then try can_queue ramp up
sc_cmd = fsp->cmd;
fsp->cmd = NULL;
sc_cmd = fsp->cmd;
fsp->cmd = NULL;
- if (!sc_cmd->SCp.ptr) {
- spin_unlock_irqrestore(lport->host->host_lock, flags);
CMD_SCSI_STATUS(sc_cmd) = fsp->cdb_status;
switch (fsp->status_code) {
CMD_SCSI_STATUS(sc_cmd) = fsp->cdb_status;
switch (fsp->status_code) {
+ spin_lock_irqsave(&si->scsi_queue_lock, flags);
+ spin_unlock_irqrestore(&si->scsi_queue_lock, flags);
sc_cmd->SCp.ptr = NULL;
sc_cmd->scsi_done(sc_cmd);
sc_cmd->SCp.ptr = NULL;
sc_cmd->scsi_done(sc_cmd);
- spin_unlock_irqrestore(lport->host->host_lock, flags);
/* release ref from initial allocation in queue command */
fc_fcp_pkt_release(fsp);
/* release ref from initial allocation in queue command */
fc_fcp_pkt_release(fsp);
lport->scsi_priv = si;
si->max_can_queue = lport->host->can_queue;
INIT_LIST_HEAD(&si->scsi_pkt_queue);
lport->scsi_priv = si;
si->max_can_queue = lport->host->can_queue;
INIT_LIST_HEAD(&si->scsi_pkt_queue);
+ spin_lock_init(&si->scsi_queue_lock);
si->scsi_pkt_pool = mempool_create_slab_pool(2, scsi_pkt_cachep);
if (!si->scsi_pkt_pool) {
si->scsi_pkt_pool = mempool_create_slab_pool(2, scsi_pkt_cachep);
if (!si->scsi_pkt_pool) {