apple-nvme: add support for ->queue_rqs

Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c
index 001207aa7a81cc41a313006db89a9210513753bc..900cc18cda3b31f6b97421f8d0ba9db070246215 100644 (file)
--- a/drivers/nvme/host/apple.c
+++ b/drivers/nvme/host/apple.c
@@ -302,10 +302,9 @@ static void apple_nvmmu_inval(struct apple_nvme_queue *q, unsigned tag)
                dev_warn(anv->dev, "NVMMU TCB invalidation failed\n");
 }
 
-static void apple_nvme_submit_cmd(struct apple_nvme_queue *q,
-                                 struct nvme_command *cmd)
+static void apple_nvme_copy_cmd(struct apple_nvme_queue *q,
+                               struct nvme_command *cmd)
 {
-       struct apple_nvme *anv = queue_to_apple_nvme(q);
        u32 tag = nvme_tag_from_cid(cmd->common.command_id);
        struct apple_nvmmu_tcb *tcb = &q->tcbs[tag];
 
@@ -322,6 +321,16 @@ static void apple_nvme_submit_cmd(struct apple_nvme_queue *q,
                tcb->dma_flags |= APPLE_ANS_TCB_DMA_FROM_DEVICE;
 
        memcpy(&q->sqes[tag], cmd, sizeof(*cmd));
+}
+
+static void apple_nvme_submit_cmd(struct apple_nvme_queue *q,
+                                 struct nvme_command *cmd)
+{
+       u32 tag = nvme_tag_from_cid(cmd->common.command_id);
+       struct apple_nvme *anv = queue_to_apple_nvme(q);
+
+       apple_nvme_copy_cmd(q, cmd);
+
        spin_lock_irq(&anv->lock);
        writel(tag, q->sq_db);
        spin_unlock_irq(&anv->lock);
@@ -765,6 +774,35 @@ out:
        return false;
 }
 
+static blk_status_t apple_prep_rq(struct apple_nvme *anv, struct request *req)
+{
+       struct blk_mq_hw_ctx *hctx = req->mq_hctx;
+       struct nvme_ns *ns = hctx->queue->queuedata;
+       struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req);
+       struct nvme_command *cmnd = &iod->cmd;
+       blk_status_t ret;
+
+       iod->npages = -1;
+       iod->nents = 0;
+       iod->persistent = false;
+
+       ret = nvme_setup_cmd(ns, req);
+       if (ret)
+               return ret;
+
+       if (blk_rq_nr_phys_segments(req)) {
+               ret = apple_nvme_map_data(anv, req, cmnd);
+               if (ret)
+                       goto out_free_cmd;
+       }
+
+       blk_mq_start_request(req);
+       return BLK_STS_OK;
+out_free_cmd:
+       nvme_cleanup_cmd(req);
+       return ret;
+}
+
 static blk_status_t apple_nvme_queue_rq(struct blk_mq_hw_ctx *hctx,
                                        const struct blk_mq_queue_data *bd)
 {
@@ -776,10 +814,6 @@ static blk_status_t apple_nvme_queue_rq(struct blk_mq_hw_ctx *hctx,
        struct nvme_command *cmnd = &iod->cmd;
        blk_status_t ret;
 
-       iod->npages = -1;
-       iod->nents = 0;
-       iod->persistent = false;
-
        /*
         * We should not need to do this, but we're still using this to
         * ensure we can drain requests on a dying queue.
@@ -790,18 +824,10 @@ static blk_status_t apple_nvme_queue_rq(struct blk_mq_hw_ctx *hctx,
        if (!nvme_check_ready(&anv->ctrl, req, true))
                return nvme_fail_nonready_command(&anv->ctrl, req);
 
-       ret = nvme_setup_cmd(ns, req);
-       if (ret)
+       ret = apple_prep_rq(anv, req);
+       if (unlikely(ret))
                return ret;
 
-       if (blk_rq_nr_phys_segments(req)) {
-               ret = apple_nvme_map_data(anv, req, cmnd);
-               if (ret)
-                       goto out_free_cmd;
-       }
-
-       blk_mq_start_request(req);
-
        if (apple_nvme_delayed_flush(anv, ns, req)) {
                blk_mq_complete_request(req);
                return BLK_STS_OK;
@@ -809,10 +835,78 @@ static blk_status_t apple_nvme_queue_rq(struct blk_mq_hw_ctx *hctx,
 
        apple_nvme_submit_cmd(q, cmnd);
        return BLK_STS_OK;
+}
 
-out_free_cmd:
-       nvme_cleanup_cmd(req);
-       return ret;
+static bool apple_prep_rq_batch(struct apple_nvme_queue *q, struct request *req)
+{
+       struct apple_nvme *anv = queue_to_apple_nvme(q);
+
+       /*
+        * We should not need to do this, but we're still using this to
+        * ensure we can drain requests on a dying queue.
+        */
+       if (unlikely(!READ_ONCE(q->enabled)))
+               return BLK_STS_IOERR;
+
+       if (!nvme_check_ready(&anv->ctrl, req, true))
+               return nvme_fail_nonready_command(&anv->ctrl, req);
+
+       req->mq_hctx->tags->rqs[req->tag] = req;
+       return apple_prep_rq(anv, req) == BLK_STS_OK;
+}
+
+static void apple_nvme_submit_cmds(struct apple_nvme_queue *q,
+                                  struct request **rqlist)
+{
+       struct apple_nvme *anv = queue_to_apple_nvme(q);
+       struct request *req;
+
+       rq_list_for_each(rqlist, req) {
+               struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req);
+               struct nvme_command *cmd = &iod->cmd;
+
+               apple_nvme_copy_cmd(q, cmd);
+       }
+
+       spin_lock_irq(&anv->lock);
+       rq_list_for_each(rqlist, req) {
+               struct apple_nvme_iod *iod = blk_mq_rq_to_pdu(req);
+               struct nvme_command *cmd = &iod->cmd;
+               u32 tag = nvme_tag_from_cid(cmd->common.command_id);
+
+               writel(tag, q->sq_db);
+       }
+       spin_unlock_irq(&anv->lock);
+}
+
+static void apple_nvme_queue_rqs(struct request **rqlist)
+{
+       struct request *req, *next, *prev = NULL;
+       struct request *requeue_list = NULL;
+
+       rq_list_for_each_safe(rqlist, req, next) {
+               struct apple_nvme_queue *q = req->mq_hctx->driver_data;
+
+               if (!apple_prep_rq_batch(q, req)) {
+                       /* detach 'req' and add to remainder list */
+                       rq_list_move(rqlist, &requeue_list, req, prev);
+
+                       req = prev;
+                       if (!req)
+                               continue;
+               }
+
+               if (!next || req->mq_hctx != next->mq_hctx) {
+                       /* detach rest of list, and submit */
+                       req->rq_next = NULL;
+                       apple_nvme_submit_cmds(q, rqlist);
+                       *rqlist = next;
+                       prev = NULL;
+               } else
+                       prev = req;
+       }
+
+       *rqlist = requeue_list;
 }
 
 static int apple_nvme_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
@@ -1033,6 +1127,7 @@ static const struct blk_mq_ops apple_nvme_mq_admin_ops = {
 
 static const struct blk_mq_ops apple_nvme_mq_ops = {
        .queue_rq = apple_nvme_queue_rq,
+       .queue_rqs = apple_nvme_queue_rqs,
        .complete = apple_nvme_complete_rq,
        .init_hctx = apple_nvme_init_hctx,
        .init_request = apple_nvme_init_request,