irq_poll: make blk-iopoll available outside the block layer
authorChristoph Hellwig <hch@lst.de>
Tue, 10 Nov 2015 13:56:14 +0000 (14:56 +0100)
committerChristoph Hellwig <hch@lst.de>
Fri, 11 Dec 2015 19:52:24 +0000 (11:52 -0800)
The new name is irq_poll as iopoll is already taken.  Better suggestions
welcome.

Signed-off-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Bart Van Assche <bart.vanassche@sandisk.com>
19 files changed:
Documentation/kernel-per-CPU-kthreads.txt
block/Makefile
block/blk-iopoll.c [deleted file]
drivers/scsi/Kconfig
drivers/scsi/be2iscsi/Kconfig
drivers/scsi/be2iscsi/be.h
drivers/scsi/be2iscsi/be_iscsi.c
drivers/scsi/be2iscsi/be_main.c
drivers/scsi/ipr.c
drivers/scsi/ipr.h
include/linux/blk-iopoll.h [deleted file]
include/linux/interrupt.h
include/linux/irq_poll.h [new file with mode: 0644]
include/trace/events/irq.h
lib/Kconfig
lib/Makefile
lib/irq_poll.c [new file with mode: 0644]
tools/lib/traceevent/event-parse.c
tools/perf/util/trace-event-parse.c

index f4cbfe0ba1085b4df3067dcc457219699c5c6150..edec3a3e648d12eee550f953b6dec87ab58e34f8 100644 (file)
@@ -90,7 +90,7 @@ BLOCK_SOFTIRQ:  Do all of the following:
        from being initiated from tasks that might run on the CPU to
        be de-jittered.  (It is OK to force this CPU offline and then
        bring it back online before you start your application.)
-BLOCK_IOPOLL_SOFTIRQ:  Do all of the following:
+IRQ_POLL_SOFTIRQ:  Do all of the following:
 1.     Force block-device interrupts onto some other CPU.
 2.     Initiate any block I/O and block-I/O polling on other CPUs.
 3.     Once your application has started, prevent CPU-hotplug operations
index 00ecc97629db783d735798c961579ee1a531c367..e8504748c7cb2625864eb7253784f8e975f3c456 100644 (file)
@@ -5,7 +5,7 @@
 obj-$(CONFIG_BLOCK) := bio.o elevator.o blk-core.o blk-tag.o blk-sysfs.o \
                        blk-flush.o blk-settings.o blk-ioc.o blk-map.o \
                        blk-exec.o blk-merge.o blk-softirq.o blk-timeout.o \
-                       blk-iopoll.o blk-lib.o blk-mq.o blk-mq-tag.o \
+                       blk-lib.o blk-mq.o blk-mq-tag.o \
                        blk-mq-sysfs.o blk-mq-cpu.o blk-mq-cpumap.o ioctl.o \
                        genhd.o scsi_ioctl.o partition-generic.o ioprio.o \
                        partitions/
diff --git a/block/blk-iopoll.c b/block/blk-iopoll.c
deleted file mode 100644 (file)
index 0736729..0000000
+++ /dev/null
@@ -1,224 +0,0 @@
-/*
- * Functions related to interrupt-poll handling in the block layer. This
- * is similar to NAPI for network devices.
- */
-#include <linux/kernel.h>
-#include <linux/module.h>
-#include <linux/init.h>
-#include <linux/bio.h>
-#include <linux/blkdev.h>
-#include <linux/interrupt.h>
-#include <linux/cpu.h>
-#include <linux/blk-iopoll.h>
-#include <linux/delay.h>
-
-#include "blk.h"
-
-static unsigned int blk_iopoll_budget __read_mostly = 256;
-
-static DEFINE_PER_CPU(struct list_head, blk_cpu_iopoll);
-
-/**
- * blk_iopoll_sched - Schedule a run of the iopoll handler
- * @iop:      The parent iopoll structure
- *
- * Description:
- *     Add this blk_iopoll structure to the pending poll list and trigger the
- *     raise of the blk iopoll softirq. The driver must already have gotten a
- *     successful return from blk_iopoll_sched_prep() before calling this.
- **/
-void blk_iopoll_sched(struct blk_iopoll *iop)
-{
-       unsigned long flags;
-
-       local_irq_save(flags);
-       list_add_tail(&iop->list, this_cpu_ptr(&blk_cpu_iopoll));
-       __raise_softirq_irqoff(BLOCK_IOPOLL_SOFTIRQ);
-       local_irq_restore(flags);
-}
-EXPORT_SYMBOL(blk_iopoll_sched);
-
-/**
- * __blk_iopoll_complete - Mark this @iop as un-polled again
- * @iop:      The parent iopoll structure
- *
- * Description:
- *     See blk_iopoll_complete(). This function must be called with interrupts
- *     disabled.
- **/
-void __blk_iopoll_complete(struct blk_iopoll *iop)
-{
-       list_del(&iop->list);
-       smp_mb__before_atomic();
-       clear_bit_unlock(IOPOLL_F_SCHED, &iop->state);
-}
-EXPORT_SYMBOL(__blk_iopoll_complete);
-
-/**
- * blk_iopoll_complete - Mark this @iop as un-polled again
- * @iop:      The parent iopoll structure
- *
- * Description:
- *     If a driver consumes less than the assigned budget in its run of the
- *     iopoll handler, it'll end the polled mode by calling this function. The
- *     iopoll handler will not be invoked again before blk_iopoll_sched_prep()
- *     is called.
- **/
-void blk_iopoll_complete(struct blk_iopoll *iop)
-{
-       unsigned long flags;
-
-       local_irq_save(flags);
-       __blk_iopoll_complete(iop);
-       local_irq_restore(flags);
-}
-EXPORT_SYMBOL(blk_iopoll_complete);
-
-static void blk_iopoll_softirq(struct softirq_action *h)
-{
-       struct list_head *list = this_cpu_ptr(&blk_cpu_iopoll);
-       int rearm = 0, budget = blk_iopoll_budget;
-       unsigned long start_time = jiffies;
-
-       local_irq_disable();
-
-       while (!list_empty(list)) {
-               struct blk_iopoll *iop;
-               int work, weight;
-
-               /*
-                * If softirq window is exhausted then punt.
-                */
-               if (budget <= 0 || time_after(jiffies, start_time)) {
-                       rearm = 1;
-                       break;
-               }
-
-               local_irq_enable();
-
-               /* Even though interrupts have been re-enabled, this
-                * access is safe because interrupts can only add new
-                * entries to the tail of this list, and only ->poll()
-                * calls can remove this head entry from the list.
-                */
-               iop = list_entry(list->next, struct blk_iopoll, list);
-
-               weight = iop->weight;
-               work = 0;
-               if (test_bit(IOPOLL_F_SCHED, &iop->state))
-                       work = iop->poll(iop, weight);
-
-               budget -= work;
-
-               local_irq_disable();
-
-               /*
-                * Drivers must not modify the iopoll state, if they
-                * consume their assigned weight (or more, some drivers can't
-                * easily just stop processing, they have to complete an
-                * entire mask of commands).In such cases this code
-                * still "owns" the iopoll instance and therefore can
-                * move the instance around on the list at-will.
-                */
-               if (work >= weight) {
-                       if (blk_iopoll_disable_pending(iop))
-                               __blk_iopoll_complete(iop);
-                       else
-                               list_move_tail(&iop->list, list);
-               }
-       }
-
-       if (rearm)
-               __raise_softirq_irqoff(BLOCK_IOPOLL_SOFTIRQ);
-
-       local_irq_enable();
-}
-
-/**
- * blk_iopoll_disable - Disable iopoll on this @iop
- * @iop:      The parent iopoll structure
- *
- * Description:
- *     Disable io polling and wait for any pending callbacks to have completed.
- **/
-void blk_iopoll_disable(struct blk_iopoll *iop)
-{
-       set_bit(IOPOLL_F_DISABLE, &iop->state);
-       while (test_and_set_bit(IOPOLL_F_SCHED, &iop->state))
-               msleep(1);
-       clear_bit(IOPOLL_F_DISABLE, &iop->state);
-}
-EXPORT_SYMBOL(blk_iopoll_disable);
-
-/**
- * blk_iopoll_enable - Enable iopoll on this @iop
- * @iop:      The parent iopoll structure
- *
- * Description:
- *     Enable iopoll on this @iop. Note that the handler run will not be
- *     scheduled, it will only mark it as active.
- **/
-void blk_iopoll_enable(struct blk_iopoll *iop)
-{
-       BUG_ON(!test_bit(IOPOLL_F_SCHED, &iop->state));
-       smp_mb__before_atomic();
-       clear_bit_unlock(IOPOLL_F_SCHED, &iop->state);
-}
-EXPORT_SYMBOL(blk_iopoll_enable);
-
-/**
- * blk_iopoll_init - Initialize this @iop
- * @iop:      The parent iopoll structure
- * @weight:   The default weight (or command completion budget)
- * @poll_fn:  The handler to invoke
- *
- * Description:
- *     Initialize this blk_iopoll structure. Before being actively used, the
- *     driver must call blk_iopoll_enable().
- **/
-void blk_iopoll_init(struct blk_iopoll *iop, int weight, blk_iopoll_fn *poll_fn)
-{
-       memset(iop, 0, sizeof(*iop));
-       INIT_LIST_HEAD(&iop->list);
-       iop->weight = weight;
-       iop->poll = poll_fn;
-       set_bit(IOPOLL_F_SCHED, &iop->state);
-}
-EXPORT_SYMBOL(blk_iopoll_init);
-
-static int blk_iopoll_cpu_notify(struct notifier_block *self,
-                                unsigned long action, void *hcpu)
-{
-       /*
-        * If a CPU goes away, splice its entries to the current CPU
-        * and trigger a run of the softirq
-        */
-       if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
-               int cpu = (unsigned long) hcpu;
-
-               local_irq_disable();
-               list_splice_init(&per_cpu(blk_cpu_iopoll, cpu),
-                                this_cpu_ptr(&blk_cpu_iopoll));
-               __raise_softirq_irqoff(BLOCK_IOPOLL_SOFTIRQ);
-               local_irq_enable();
-       }
-
-       return NOTIFY_OK;
-}
-
-static struct notifier_block blk_iopoll_cpu_notifier = {
-       .notifier_call  = blk_iopoll_cpu_notify,
-};
-
-static __init int blk_iopoll_setup(void)
-{
-       int i;
-
-       for_each_possible_cpu(i)
-               INIT_LIST_HEAD(&per_cpu(blk_cpu_iopoll, i));
-
-       open_softirq(BLOCK_IOPOLL_SOFTIRQ, blk_iopoll_softirq);
-       register_hotcpu_notifier(&blk_iopoll_cpu_notifier);
-       return 0;
-}
-subsys_initcall(blk_iopoll_setup);
index 5f692ae4074959e019a74eb0c81405cb2869195c..92099657b046a99e6ebf78389f9dc988dfc43605 100644 (file)
@@ -1102,6 +1102,7 @@ config SCSI_IPR
        tristate "IBM Power Linux RAID adapter support"
        depends on PCI && SCSI && ATA
        select FW_LOADER
+       select IRQ_POLL
        ---help---
          This driver supports the IBM Power Linux family RAID adapters.
          This includes IBM pSeries 5712, 5703, 5709, and 570A, as well
index 4e7cad27246944f6e79cf21a56fb3f25f96e1234..bad5f32e1f670587ddedda3ec97104e1c2d4555a 100644 (file)
@@ -3,6 +3,7 @@ config BE2ISCSI
        depends on PCI && SCSI && NET
        select SCSI_ISCSI_ATTRS
        select ISCSI_BOOT_SYSFS
+       select IRQ_POLL
 
        help
        This driver implements the iSCSI functionality for Emulex
index 77f992e7472629eeeaedf3470ced43d02077d9ca..a41c6432f4446cf9082916a0859e08be3c0a70c7 100644 (file)
@@ -20,7 +20,7 @@
 
 #include <linux/pci.h>
 #include <linux/if_vlan.h>
-#include <linux/blk-iopoll.h>
+#include <linux/irq_poll.h>
 #define FW_VER_LEN     32
 #define MCC_Q_LEN      128
 #define MCC_CQ_LEN     256
@@ -101,7 +101,7 @@ struct be_eq_obj {
        struct beiscsi_hba *phba;
        struct be_queue_info *cq;
        struct work_struct work_cqs; /* Work Item */
-       struct blk_iopoll       iopoll;
+       struct irq_poll iopoll;
 };
 
 struct be_mcc_obj {
index b7087ba69d8df1c2daa9ac7b1b8a9ab314257a8b..022e87b62e401a37e1d6578e065f389ce560a8b4 100644 (file)
@@ -1292,9 +1292,9 @@ static void beiscsi_flush_cq(struct beiscsi_hba *phba)
 
        for (i = 0; i < phba->num_cpus; i++) {
                pbe_eq = &phwi_context->be_eq[i];
-               blk_iopoll_disable(&pbe_eq->iopoll);
+               irq_poll_disable(&pbe_eq->iopoll);
                beiscsi_process_cq(pbe_eq);
-               blk_iopoll_enable(&pbe_eq->iopoll);
+               irq_poll_enable(&pbe_eq->iopoll);
        }
 }
 
index fe0c5143f8e68a6cb3e9618c98258c9e4118e1ae..1d879ef406d8878bd90612eabff81e01898dfceb 100644 (file)
@@ -910,8 +910,8 @@ static irqreturn_t be_isr_msix(int irq, void *dev_id)
        num_eq_processed = 0;
        while (eqe->dw[offsetof(struct amap_eq_entry, valid) / 32]
                                & EQE_VALID_MASK) {
-               if (!blk_iopoll_sched_prep(&pbe_eq->iopoll))
-                       blk_iopoll_sched(&pbe_eq->iopoll);
+               if (!irq_poll_sched_prep(&pbe_eq->iopoll))
+                       irq_poll_sched(&pbe_eq->iopoll);
 
                AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
                queue_tail_inc(eq);
@@ -972,8 +972,8 @@ static irqreturn_t be_isr(int irq, void *dev_id)
                        spin_unlock_irqrestore(&phba->isr_lock, flags);
                        num_mcceq_processed++;
                } else {
-                       if (!blk_iopoll_sched_prep(&pbe_eq->iopoll))
-                               blk_iopoll_sched(&pbe_eq->iopoll);
+                       if (!irq_poll_sched_prep(&pbe_eq->iopoll))
+                               irq_poll_sched(&pbe_eq->iopoll);
                        num_ioeq_processed++;
                }
                AMAP_SET_BITS(struct amap_eq_entry, valid, eqe, 0);
@@ -2295,7 +2295,7 @@ void beiscsi_process_all_cqs(struct work_struct *work)
        hwi_ring_eq_db(phba, pbe_eq->q.id, 0, 0, 1, 1);
 }
 
-static int be_iopoll(struct blk_iopoll *iop, int budget)
+static int be_iopoll(struct irq_poll *iop, int budget)
 {
        unsigned int ret;
        struct beiscsi_hba *phba;
@@ -2306,7 +2306,7 @@ static int be_iopoll(struct blk_iopoll *iop, int budget)
        pbe_eq->cq_count += ret;
        if (ret < budget) {
                phba = pbe_eq->phba;
-               blk_iopoll_complete(iop);
+               irq_poll_complete(iop);
                beiscsi_log(phba, KERN_INFO,
                            BEISCSI_LOG_CONFIG | BEISCSI_LOG_IO,
                            "BM_%d : rearm pbe_eq->q.id =%d\n",
@@ -5293,7 +5293,7 @@ static void beiscsi_quiesce(struct beiscsi_hba *phba,
 
        for (i = 0; i < phba->num_cpus; i++) {
                pbe_eq = &phwi_context->be_eq[i];
-               blk_iopoll_disable(&pbe_eq->iopoll);
+               irq_poll_disable(&pbe_eq->iopoll);
        }
 
        if (unload_state == BEISCSI_CLEAN_UNLOAD) {
@@ -5579,9 +5579,9 @@ static void beiscsi_eeh_resume(struct pci_dev *pdev)
 
        for (i = 0; i < phba->num_cpus; i++) {
                pbe_eq = &phwi_context->be_eq[i];
-               blk_iopoll_init(&pbe_eq->iopoll, be_iopoll_budget,
+               irq_poll_init(&pbe_eq->iopoll, be_iopoll_budget,
                                be_iopoll);
-               blk_iopoll_enable(&pbe_eq->iopoll);
+               irq_poll_enable(&pbe_eq->iopoll);
        }
 
        i = (phba->msix_enabled) ? i : 0;
@@ -5752,9 +5752,9 @@ static int beiscsi_dev_probe(struct pci_dev *pcidev,
 
        for (i = 0; i < phba->num_cpus; i++) {
                pbe_eq = &phwi_context->be_eq[i];
-               blk_iopoll_init(&pbe_eq->iopoll, be_iopoll_budget,
+               irq_poll_init(&pbe_eq->iopoll, be_iopoll_budget,
                                be_iopoll);
-               blk_iopoll_enable(&pbe_eq->iopoll);
+               irq_poll_enable(&pbe_eq->iopoll);
        }
 
        i = (phba->msix_enabled) ? i : 0;
@@ -5795,7 +5795,7 @@ free_blkenbld:
        destroy_workqueue(phba->wq);
        for (i = 0; i < phba->num_cpus; i++) {
                pbe_eq = &phwi_context->be_eq[i];
-               blk_iopoll_disable(&pbe_eq->iopoll);
+               irq_poll_disable(&pbe_eq->iopoll);
        }
 free_twq:
        beiscsi_clean_port(phba);
index 536cd5a8042245f4ad46929a256098e047d745dd..6b9c738cdc18aa4a43f2f43760239107badadb39 100644 (file)
@@ -3638,7 +3638,7 @@ static struct device_attribute ipr_ioa_reset_attr = {
        .store = ipr_store_reset_adapter
 };
 
-static int ipr_iopoll(struct blk_iopoll *iop, int budget);
+static int ipr_iopoll(struct irq_poll *iop, int budget);
  /**
  * ipr_show_iopoll_weight - Show ipr polling mode
  * @dev:       class device struct
@@ -3681,34 +3681,34 @@ static ssize_t ipr_store_iopoll_weight(struct device *dev,
        int i;
 
        if (!ioa_cfg->sis64) {
-               dev_info(&ioa_cfg->pdev->dev, "blk-iopoll not supported on this adapter\n");
+               dev_info(&ioa_cfg->pdev->dev, "irq_poll not supported on this adapter\n");
                return -EINVAL;
        }
        if (kstrtoul(buf, 10, &user_iopoll_weight))
                return -EINVAL;
 
        if (user_iopoll_weight > 256) {
-               dev_info(&ioa_cfg->pdev->dev, "Invalid blk-iopoll weight. It must be less than 256\n");
+               dev_info(&ioa_cfg->pdev->dev, "Invalid irq_poll weight. It must be less than 256\n");
                return -EINVAL;
        }
 
        if (user_iopoll_weight == ioa_cfg->iopoll_weight) {
-               dev_info(&ioa_cfg->pdev->dev, "Current blk-iopoll weight has the same weight\n");
+               dev_info(&ioa_cfg->pdev->dev, "Current irq_poll weight has the same weight\n");
                return strlen(buf);
        }
 
        if (ioa_cfg->iopoll_weight && ioa_cfg->sis64 && ioa_cfg->nvectors > 1) {
                for (i = 1; i < ioa_cfg->hrrq_num; i++)
-                       blk_iopoll_disable(&ioa_cfg->hrrq[i].iopoll);
+                       irq_poll_disable(&ioa_cfg->hrrq[i].iopoll);
        }
 
        spin_lock_irqsave(shost->host_lock, lock_flags);
        ioa_cfg->iopoll_weight = user_iopoll_weight;
        if (ioa_cfg->iopoll_weight && ioa_cfg->sis64 && ioa_cfg->nvectors > 1) {
                for (i = 1; i < ioa_cfg->hrrq_num; i++) {
-                       blk_iopoll_init(&ioa_cfg->hrrq[i].iopoll,
+                       irq_poll_init(&ioa_cfg->hrrq[i].iopoll,
                                        ioa_cfg->iopoll_weight, ipr_iopoll);
-                       blk_iopoll_enable(&ioa_cfg->hrrq[i].iopoll);
+                       irq_poll_enable(&ioa_cfg->hrrq[i].iopoll);
                }
        }
        spin_unlock_irqrestore(shost->host_lock, lock_flags);
@@ -5569,7 +5569,7 @@ static int ipr_process_hrrq(struct ipr_hrr_queue *hrr_queue, int budget,
        return num_hrrq;
 }
 
-static int ipr_iopoll(struct blk_iopoll *iop, int budget)
+static int ipr_iopoll(struct irq_poll *iop, int budget)
 {
        struct ipr_ioa_cfg *ioa_cfg;
        struct ipr_hrr_queue *hrrq;
@@ -5585,7 +5585,7 @@ static int ipr_iopoll(struct blk_iopoll *iop, int budget)
        completed_ops = ipr_process_hrrq(hrrq, budget, &doneq);
 
        if (completed_ops < budget)
-               blk_iopoll_complete(iop);
+               irq_poll_complete(iop);
        spin_unlock_irqrestore(hrrq->lock, hrrq_flags);
 
        list_for_each_entry_safe(ipr_cmd, temp, &doneq, queue) {
@@ -5693,8 +5693,8 @@ static irqreturn_t ipr_isr_mhrrq(int irq, void *devp)
        if (ioa_cfg->iopoll_weight && ioa_cfg->sis64 && ioa_cfg->nvectors > 1) {
                if ((be32_to_cpu(*hrrq->hrrq_curr) & IPR_HRRQ_TOGGLE_BIT) ==
                       hrrq->toggle_bit) {
-                       if (!blk_iopoll_sched_prep(&hrrq->iopoll))
-                               blk_iopoll_sched(&hrrq->iopoll);
+                       if (!irq_poll_sched_prep(&hrrq->iopoll))
+                               irq_poll_sched(&hrrq->iopoll);
                        spin_unlock_irqrestore(hrrq->lock, hrrq_flags);
                        return IRQ_HANDLED;
                }
@@ -10405,9 +10405,9 @@ static int ipr_probe(struct pci_dev *pdev, const struct pci_device_id *dev_id)
 
        if (ioa_cfg->iopoll_weight && ioa_cfg->sis64 && ioa_cfg->nvectors > 1) {
                for (i = 1; i < ioa_cfg->hrrq_num; i++) {
-                       blk_iopoll_init(&ioa_cfg->hrrq[i].iopoll,
+                       irq_poll_init(&ioa_cfg->hrrq[i].iopoll,
                                        ioa_cfg->iopoll_weight, ipr_iopoll);
-                       blk_iopoll_enable(&ioa_cfg->hrrq[i].iopoll);
+                       irq_poll_enable(&ioa_cfg->hrrq[i].iopoll);
                }
        }
 
@@ -10436,7 +10436,7 @@ static void ipr_shutdown(struct pci_dev *pdev)
        if (ioa_cfg->iopoll_weight && ioa_cfg->sis64 && ioa_cfg->nvectors > 1) {
                ioa_cfg->iopoll_weight = 0;
                for (i = 1; i < ioa_cfg->hrrq_num; i++)
-                       blk_iopoll_disable(&ioa_cfg->hrrq[i].iopoll);
+                       irq_poll_disable(&ioa_cfg->hrrq[i].iopoll);
        }
 
        while (ioa_cfg->in_reset_reload) {
index a34c7a5a995e4bcbf61b1d859fb784b60a9cdbee..56c57068300a1fb947221f8bc32f467aa115ae80 100644 (file)
@@ -32,7 +32,7 @@
 #include <linux/libata.h>
 #include <linux/list.h>
 #include <linux/kref.h>
-#include <linux/blk-iopoll.h>
+#include <linux/irq_poll.h>
 #include <scsi/scsi.h>
 #include <scsi/scsi_cmnd.h>
 
@@ -517,7 +517,7 @@ struct ipr_hrr_queue {
        u8 allow_cmds:1;
        u8 removing_ioa:1;
 
-       struct blk_iopoll iopoll;
+       struct irq_poll iopoll;
 };
 
 /* Command packet structure */
diff --git a/include/linux/blk-iopoll.h b/include/linux/blk-iopoll.h
deleted file mode 100644 (file)
index 77ae77c..0000000
+++ /dev/null
@@ -1,46 +0,0 @@
-#ifndef BLK_IOPOLL_H
-#define BLK_IOPOLL_H
-
-struct blk_iopoll;
-typedef int (blk_iopoll_fn)(struct blk_iopoll *, int);
-
-struct blk_iopoll {
-       struct list_head list;
-       unsigned long state;
-       unsigned long data;
-       int weight;
-       int max;
-       blk_iopoll_fn *poll;
-};
-
-enum {
-       IOPOLL_F_SCHED          = 0,
-       IOPOLL_F_DISABLE        = 1,
-};
-
-/*
- * Returns 0 if we successfully set the IOPOLL_F_SCHED bit, indicating
- * that we were the first to acquire this iop for scheduling. If this iop
- * is currently disabled, return "failure".
- */
-static inline int blk_iopoll_sched_prep(struct blk_iopoll *iop)
-{
-       if (!test_bit(IOPOLL_F_DISABLE, &iop->state))
-               return test_and_set_bit(IOPOLL_F_SCHED, &iop->state);
-
-       return 1;
-}
-
-static inline int blk_iopoll_disable_pending(struct blk_iopoll *iop)
-{
-       return test_bit(IOPOLL_F_DISABLE, &iop->state);
-}
-
-extern void blk_iopoll_sched(struct blk_iopoll *);
-extern void blk_iopoll_init(struct blk_iopoll *, int, blk_iopoll_fn *);
-extern void blk_iopoll_complete(struct blk_iopoll *);
-extern void __blk_iopoll_complete(struct blk_iopoll *);
-extern void blk_iopoll_enable(struct blk_iopoll *);
-extern void blk_iopoll_disable(struct blk_iopoll *);
-
-#endif
index ad16809c85961e16ebc804280e4a78d66cf77634..7ff98c23199ac41f2c53d75dbe066ecb21aaa14a 100644 (file)
@@ -412,7 +412,7 @@ enum
        NET_TX_SOFTIRQ,
        NET_RX_SOFTIRQ,
        BLOCK_SOFTIRQ,
-       BLOCK_IOPOLL_SOFTIRQ,
+       IRQ_POLL_SOFTIRQ,
        TASKLET_SOFTIRQ,
        SCHED_SOFTIRQ,
        HRTIMER_SOFTIRQ, /* Unused, but kept as tools rely on the
diff --git a/include/linux/irq_poll.h b/include/linux/irq_poll.h
new file mode 100644 (file)
index 0000000..50c39dc
--- /dev/null
@@ -0,0 +1,46 @@
+#ifndef IRQ_POLL_H
+#define IRQ_POLL_H
+
+struct irq_poll;
+typedef int (irq_poll_fn)(struct irq_poll *, int);
+
+struct irq_poll {
+       struct list_head list;
+       unsigned long state;
+       unsigned long data;
+       int weight;
+       int max;
+       irq_poll_fn *poll;
+};
+
+enum {
+       IRQ_POLL_F_SCHED        = 0,
+       IRQ_POLL_F_DISABLE      = 1,
+};
+
+/*
+ * Returns 0 if we successfully set the IRQ_POLL_F_SCHED bit, indicating
+ * that we were the first to acquire this iop for scheduling. If this iop
+ * is currently disabled, return "failure".
+ */
+static inline int irq_poll_sched_prep(struct irq_poll *iop)
+{
+       if (!test_bit(IRQ_POLL_F_DISABLE, &iop->state))
+               return test_and_set_bit(IRQ_POLL_F_SCHED, &iop->state);
+
+       return 1;
+}
+
+static inline int irq_poll_disable_pending(struct irq_poll *iop)
+{
+       return test_bit(IRQ_POLL_F_DISABLE, &iop->state);
+}
+
+extern void irq_poll_sched(struct irq_poll *);
+extern void irq_poll_init(struct irq_poll *, int, irq_poll_fn *);
+extern void irq_poll_complete(struct irq_poll *);
+extern void __irq_poll_complete(struct irq_poll *);
+extern void irq_poll_enable(struct irq_poll *);
+extern void irq_poll_disable(struct irq_poll *);
+
+#endif
index ff8f6c091a1504e3d18937244a07b6432c880fb2..f95f25e786ef0de423a3a036a1685ad53da496af 100644 (file)
@@ -15,7 +15,7 @@ struct softirq_action;
                         softirq_name(NET_TX)           \
                         softirq_name(NET_RX)           \
                         softirq_name(BLOCK)            \
-                        softirq_name(BLOCK_IOPOLL)     \
+                        softirq_name(IRQ_POLL)         \
                         softirq_name(TASKLET)          \
                         softirq_name(SCHED)            \
                         softirq_name(HRTIMER)          \
index f0df318104e7272ef97641421c938069a047a85e..e00e1960260a66a12cd525c0cef139e8ff71e169 100644 (file)
@@ -475,6 +475,11 @@ config DDR
          information. This data is useful for drivers handling
          DDR SDRAM controllers.
 
+config IRQ_POLL
+       bool "IRQ polling library"
+       help
+         Helper library to poll interrupt mitigation using polling.
+
 config MPILIB
        tristate
        select CLZ_TAB
index 7f1de26613d26ea900e50751d49b266c7ef1272e..1478ae2565616fefde37f8067a4c8ea83b7a0574 100644 (file)
@@ -164,6 +164,7 @@ obj-$(CONFIG_GENERIC_NET_UTILS) += net_utils.o
 
 obj-$(CONFIG_SG_SPLIT) += sg_split.o
 obj-$(CONFIG_STMP_DEVICE) += stmp_device.o
+obj-$(CONFIG_IRQ_POLL) += irq_poll.o
 
 libfdt_files = fdt.o fdt_ro.o fdt_wip.o fdt_rw.o fdt_sw.o fdt_strerror.o \
               fdt_empty_tree.o
diff --git a/lib/irq_poll.c b/lib/irq_poll.c
new file mode 100644 (file)
index 0000000..e6fd1dc
--- /dev/null
@@ -0,0 +1,221 @@
+/*
+ * Functions related to interrupt-poll handling in the block layer. This
+ * is similar to NAPI for network devices.
+ */
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/bio.h>
+#include <linux/interrupt.h>
+#include <linux/cpu.h>
+#include <linux/irq_poll.h>
+#include <linux/delay.h>
+
+static unsigned int irq_poll_budget __read_mostly = 256;
+
+static DEFINE_PER_CPU(struct list_head, blk_cpu_iopoll);
+
+/**
+ * irq_poll_sched - Schedule a run of the iopoll handler
+ * @iop:      The parent iopoll structure
+ *
+ * Description:
+ *     Add this irq_poll structure to the pending poll list and trigger the
+ *     raise of the blk iopoll softirq. The driver must already have gotten a
+ *     successful return from irq_poll_sched_prep() before calling this.
+ **/
+void irq_poll_sched(struct irq_poll *iop)
+{
+       unsigned long flags;
+
+       local_irq_save(flags);
+       list_add_tail(&iop->list, this_cpu_ptr(&blk_cpu_iopoll));
+       __raise_softirq_irqoff(IRQ_POLL_SOFTIRQ);
+       local_irq_restore(flags);
+}
+EXPORT_SYMBOL(irq_poll_sched);
+
+/**
+ * __irq_poll_complete - Mark this @iop as un-polled again
+ * @iop:      The parent iopoll structure
+ *
+ * Description:
+ *     See irq_poll_complete(). This function must be called with interrupts
+ *     disabled.
+ **/
+void __irq_poll_complete(struct irq_poll *iop)
+{
+       list_del(&iop->list);
+       smp_mb__before_atomic();
+       clear_bit_unlock(IRQ_POLL_F_SCHED, &iop->state);
+}
+EXPORT_SYMBOL(__irq_poll_complete);
+
+/**
+ * irq_poll_complete - Mark this @iop as un-polled again
+ * @iop:      The parent iopoll structure
+ *
+ * Description:
+ *     If a driver consumes less than the assigned budget in its run of the
+ *     iopoll handler, it'll end the polled mode by calling this function. The
+ *     iopoll handler will not be invoked again before irq_poll_sched_prep()
+ *     is called.
+ **/
+void irq_poll_complete(struct irq_poll *iop)
+{
+       unsigned long flags;
+
+       local_irq_save(flags);
+       __irq_poll_complete(iop);
+       local_irq_restore(flags);
+}
+EXPORT_SYMBOL(irq_poll_complete);
+
+static void irq_poll_softirq(struct softirq_action *h)
+{
+       struct list_head *list = this_cpu_ptr(&blk_cpu_iopoll);
+       int rearm = 0, budget = irq_poll_budget;
+       unsigned long start_time = jiffies;
+
+       local_irq_disable();
+
+       while (!list_empty(list)) {
+               struct irq_poll *iop;
+               int work, weight;
+
+               /*
+                * If softirq window is exhausted then punt.
+                */
+               if (budget <= 0 || time_after(jiffies, start_time)) {
+                       rearm = 1;
+                       break;
+               }
+
+               local_irq_enable();
+
+               /* Even though interrupts have been re-enabled, this
+                * access is safe because interrupts can only add new
+                * entries to the tail of this list, and only ->poll()
+                * calls can remove this head entry from the list.
+                */
+               iop = list_entry(list->next, struct irq_poll, list);
+
+               weight = iop->weight;
+               work = 0;
+               if (test_bit(IRQ_POLL_F_SCHED, &iop->state))
+                       work = iop->poll(iop, weight);
+
+               budget -= work;
+
+               local_irq_disable();
+
+               /*
+                * Drivers must not modify the iopoll state, if they
+                * consume their assigned weight (or more, some drivers can't
+                * easily just stop processing, they have to complete an
+                * entire mask of commands).In such cases this code
+                * still "owns" the iopoll instance and therefore can
+                * move the instance around on the list at-will.
+                */
+               if (work >= weight) {
+                       if (irq_poll_disable_pending(iop))
+                               __irq_poll_complete(iop);
+                       else
+                               list_move_tail(&iop->list, list);
+               }
+       }
+
+       if (rearm)
+               __raise_softirq_irqoff(IRQ_POLL_SOFTIRQ);
+
+       local_irq_enable();
+}
+
+/**
+ * irq_poll_disable - Disable iopoll on this @iop
+ * @iop:      The parent iopoll structure
+ *
+ * Description:
+ *     Disable io polling and wait for any pending callbacks to have completed.
+ **/
+void irq_poll_disable(struct irq_poll *iop)
+{
+       set_bit(IRQ_POLL_F_DISABLE, &iop->state);
+       while (test_and_set_bit(IRQ_POLL_F_SCHED, &iop->state))
+               msleep(1);
+       clear_bit(IRQ_POLL_F_DISABLE, &iop->state);
+}
+EXPORT_SYMBOL(irq_poll_disable);
+
+/**
+ * irq_poll_enable - Enable iopoll on this @iop
+ * @iop:      The parent iopoll structure
+ *
+ * Description:
+ *     Enable iopoll on this @iop. Note that the handler run will not be
+ *     scheduled, it will only mark it as active.
+ **/
+void irq_poll_enable(struct irq_poll *iop)
+{
+       BUG_ON(!test_bit(IRQ_POLL_F_SCHED, &iop->state));
+       smp_mb__before_atomic();
+       clear_bit_unlock(IRQ_POLL_F_SCHED, &iop->state);
+}
+EXPORT_SYMBOL(irq_poll_enable);
+
+/**
+ * irq_poll_init - Initialize this @iop
+ * @iop:      The parent iopoll structure
+ * @weight:   The default weight (or command completion budget)
+ * @poll_fn:  The handler to invoke
+ *
+ * Description:
+ *     Initialize this irq_poll structure. Before being actively used, the
+ *     driver must call irq_poll_enable().
+ **/
+void irq_poll_init(struct irq_poll *iop, int weight, irq_poll_fn *poll_fn)
+{
+       memset(iop, 0, sizeof(*iop));
+       INIT_LIST_HEAD(&iop->list);
+       iop->weight = weight;
+       iop->poll = poll_fn;
+       set_bit(IRQ_POLL_F_SCHED, &iop->state);
+}
+EXPORT_SYMBOL(irq_poll_init);
+
+static int irq_poll_cpu_notify(struct notifier_block *self,
+                                unsigned long action, void *hcpu)
+{
+       /*
+        * If a CPU goes away, splice its entries to the current CPU
+        * and trigger a run of the softirq
+        */
+       if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
+               int cpu = (unsigned long) hcpu;
+
+               local_irq_disable();
+               list_splice_init(&per_cpu(blk_cpu_iopoll, cpu),
+                                this_cpu_ptr(&blk_cpu_iopoll));
+               __raise_softirq_irqoff(IRQ_POLL_SOFTIRQ);
+               local_irq_enable();
+       }
+
+       return NOTIFY_OK;
+}
+
+static struct notifier_block irq_poll_cpu_notifier = {
+       .notifier_call  = irq_poll_cpu_notify,
+};
+
+static __init int irq_poll_setup(void)
+{
+       int i;
+
+       for_each_possible_cpu(i)
+               INIT_LIST_HEAD(&per_cpu(blk_cpu_iopoll, i));
+
+       open_softirq(IRQ_POLL_SOFTIRQ, irq_poll_softirq);
+       register_hotcpu_notifier(&irq_poll_cpu_notifier);
+       return 0;
+}
+subsys_initcall(irq_poll_setup);
index 2a912df6771bf5c6abd880667fbece1dcf1e1005..af5a31661086aaa661fd8d92f729dbd340f17893 100644 (file)
@@ -3746,7 +3746,7 @@ static const struct flag flags[] = {
        { "NET_TX_SOFTIRQ", 2 },
        { "NET_RX_SOFTIRQ", 3 },
        { "BLOCK_SOFTIRQ", 4 },
-       { "BLOCK_IOPOLL_SOFTIRQ", 5 },
+       { "IRQ_POLL_SOFTIRQ", 5 },
        { "TASKLET_SOFTIRQ", 6 },
        { "SCHED_SOFTIRQ", 7 },
        { "HRTIMER_SOFTIRQ", 8 },
index 8ff7d620d9427490f529cf7ec63471ae1bcb3b17..33b52eaa39db293b0de4d41c7312f88b39a2c4e2 100644 (file)
@@ -209,7 +209,7 @@ static const struct flag flags[] = {
        { "NET_TX_SOFTIRQ", 2 },
        { "NET_RX_SOFTIRQ", 3 },
        { "BLOCK_SOFTIRQ", 4 },
-       { "BLOCK_IOPOLL_SOFTIRQ", 5 },
+       { "IRQ_POLL_SOFTIRQ", 5 },
        { "TASKLET_SOFTIRQ", 6 },
        { "SCHED_SOFTIRQ", 7 },
        { "HRTIMER_SOFTIRQ", 8 },