Merge tag 'for-linus-20190112' of git://git.kernel.dk/linux-block

[linux-2.6-block.git] / drivers / nvme / host / pci.c

diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c
index e8d0942c9c923d2886f13fb352aaebfe5a49b420..deb1a66bf117a8896d2aeb0d13a9b2453657a49f 100644 (file)
--- a/drivers/nvme/host/pci.c
+++ b/drivers/nvme/host/pci.c
@@ -95,6 +95,7 @@ struct nvme_dev;
 struct nvme_queue;
 
 static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown);
+static bool __nvme_disable_io_queues(struct nvme_dev *dev, u8 opcode);
 
 /*
  * Represents an NVM Express device.  Each nvme_dev is a PCI function.
@@ -1019,9 +1020,11 @@ static void nvme_complete_cqes(struct nvme_queue *nvmeq, u16 start, u16 end)
 
 static inline void nvme_update_cq_head(struct nvme_queue *nvmeq)
 {
-       if (++nvmeq->cq_head == nvmeq->q_depth) {
+       if (nvmeq->cq_head == nvmeq->q_depth - 1) {
                nvmeq->cq_head = 0;
                nvmeq->cq_phase = !nvmeq->cq_phase;
+       } else {
+               nvmeq->cq_head++;
        }
 }
 
@@ -1420,6 +1423,14 @@ static int nvme_suspend_queue(struct nvme_queue *nvmeq)
        return 0;
 }
 
+static void nvme_suspend_io_queues(struct nvme_dev *dev)
+{
+       int i;
+
+       for (i = dev->ctrl.queue_count - 1; i > 0; i--)
+               nvme_suspend_queue(&dev->queues[i]);
+}
+
 static void nvme_disable_admin_queue(struct nvme_dev *dev, bool shutdown)
 {
        struct nvme_queue *nvmeq = &dev->queues[0];
@@ -1885,8 +1896,9 @@ static void nvme_free_host_mem(struct nvme_dev *dev)
                struct nvme_host_mem_buf_desc *desc = &dev->host_mem_descs[i];
                size_t size = le32_to_cpu(desc->size) * dev->ctrl.page_size;
 
-               dma_free_coherent(dev->dev, size, dev->host_mem_desc_bufs[i],
-                               le64_to_cpu(desc->addr));
+               dma_free_attrs(dev->dev, size, dev->host_mem_desc_bufs[i],
+                              le64_to_cpu(desc->addr),
+                              DMA_ATTR_NO_KERNEL_MAPPING | DMA_ATTR_NO_WARN);
        }
 
        kfree(dev->host_mem_desc_bufs);
@@ -1952,8 +1964,9 @@ out_free_bufs:
        while (--i >= 0) {
                size_t size = le32_to_cpu(descs[i].size) * dev->ctrl.page_size;
 
-               dma_free_coherent(dev->dev, size, bufs[i],
-                               le64_to_cpu(descs[i].addr));
+               dma_free_attrs(dev->dev, size, bufs[i],
+                              le64_to_cpu(descs[i].addr),
+                              DMA_ATTR_NO_KERNEL_MAPPING | DMA_ATTR_NO_WARN);
        }
 
        kfree(bufs);
@@ -2132,6 +2145,12 @@ static int nvme_setup_irqs(struct nvme_dev *dev, unsigned int nr_io_queues)
        return result;
 }
 
+static void nvme_disable_io_queues(struct nvme_dev *dev)
+{
+       if (__nvme_disable_io_queues(dev, nvme_admin_delete_sq))
+               __nvme_disable_io_queues(dev, nvme_admin_delete_cq);
+}
+
 static int nvme_setup_io_queues(struct nvme_dev *dev)
 {
        struct nvme_queue *adminq = &dev->queues[0];
@@ -2168,6 +2187,7 @@ static int nvme_setup_io_queues(struct nvme_dev *dev)
        } while (1);
        adminq->q_db = dev->dbs;
 
+ retry:
        /* Deregister the admin queue's interrupt */
        pci_free_irq(pdev, 0, adminq);
 
@@ -2185,25 +2205,34 @@ static int nvme_setup_io_queues(struct nvme_dev *dev)
        result = max(result - 1, 1);
        dev->max_qid = result + dev->io_queues[HCTX_TYPE_POLL];
 
-       dev_info(dev->ctrl.device, "%d/%d/%d default/read/poll queues\n",
-                                       dev->io_queues[HCTX_TYPE_DEFAULT],
-                                       dev->io_queues[HCTX_TYPE_READ],
-                                       dev->io_queues[HCTX_TYPE_POLL]);
-
        /*
         * Should investigate if there's a performance win from allocating
         * more queues than interrupt vectors; it might allow the submission
         * path to scale better, even if the receive path is limited by the
         * number of interrupts.
         */
-
        result = queue_request_irq(adminq);
        if (result) {
                adminq->cq_vector = -1;
                return result;
        }
        set_bit(NVMEQ_ENABLED, &adminq->flags);
-       return nvme_create_io_queues(dev);
+
+       result = nvme_create_io_queues(dev);
+       if (result || dev->online_queues < 2)
+               return result;
+
+       if (dev->online_queues - 1 < dev->max_qid) {
+               nr_io_queues = dev->online_queues - 1;
+               nvme_disable_io_queues(dev);
+               nvme_suspend_io_queues(dev);
+               goto retry;
+       }
+       dev_info(dev->ctrl.device, "%d/%d/%d default/read/poll queues\n",
+                                       dev->io_queues[HCTX_TYPE_DEFAULT],
+                                       dev->io_queues[HCTX_TYPE_READ],
+                                       dev->io_queues[HCTX_TYPE_POLL]);
+       return 0;
 }
 
 static void nvme_del_queue_end(struct request *req, blk_status_t error)
@@ -2248,7 +2277,7 @@ static int nvme_delete_queue(struct nvme_queue *nvmeq, u8 opcode)
        return 0;
 }
 
-static bool nvme_disable_io_queues(struct nvme_dev *dev, u8 opcode)
+static bool __nvme_disable_io_queues(struct nvme_dev *dev, u8 opcode)
 {
        int nr_queues = dev->online_queues - 1, sent = 0;
        unsigned long timeout;
@@ -2294,7 +2323,6 @@ static int nvme_dev_add(struct nvme_dev *dev)
                dev->tagset.nr_maps = 2; /* default + read */
                if (dev->io_queues[HCTX_TYPE_POLL])
                        dev->tagset.nr_maps++;
-               dev->tagset.nr_maps = HCTX_MAX_TYPES;
                dev->tagset.timeout = NVME_IO_TIMEOUT;
                dev->tagset.numa_node = dev_to_node(dev->dev);
                dev->tagset.queue_depth =
@@ -2410,7 +2438,6 @@ static void nvme_pci_disable(struct nvme_dev *dev)
 
 static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown)
 {
-       int i;
        bool dead = true;
        struct pci_dev *pdev = to_pci_dev(dev->dev);
 
@@ -2437,13 +2464,11 @@ static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown)
        nvme_stop_queues(&dev->ctrl);
 
        if (!dead && dev->ctrl.queue_count > 0) {
-               if (nvme_disable_io_queues(dev, nvme_admin_delete_sq))
-                       nvme_disable_io_queues(dev, nvme_admin_delete_cq);
+               nvme_disable_io_queues(dev);
                nvme_disable_admin_queue(dev, shutdown);
        }
-       for (i = dev->ctrl.queue_count - 1; i >= 0; i--)
-               nvme_suspend_queue(&dev->queues[i]);
-
+       nvme_suspend_io_queues(dev);
+       nvme_suspend_queue(&dev->queues[0]);
        nvme_pci_disable(dev);
 
        blk_mq_tagset_busy_iter(&dev->tagset, nvme_cancel_request, &dev->ctrl);
@@ -2946,6 +2971,8 @@ static const struct pci_device_id nvme_id_table[] = {
        { PCI_VDEVICE(INTEL, 0xf1a5),   /* Intel 600P/P3100 */
                .driver_data = NVME_QUIRK_NO_DEEPEST_PS |
                                NVME_QUIRK_MEDIUM_PRIO_SQ },
+       { PCI_VDEVICE(INTEL, 0xf1a6),   /* Intel 760p/Pro 7600p */
+               .driver_data = NVME_QUIRK_IGNORE_DEV_SUBNQN, },
        { PCI_VDEVICE(INTEL, 0x5845),   /* Qemu emulated controller */
                .driver_data = NVME_QUIRK_IDENTIFY_CNS, },
        { PCI_DEVICE(0x1bb1, 0x0100),   /* Seagate Nytro Flash Storage */