block: locking fixesfor-2.6.38/multiqueue

Some of this code will want to be reordered, especially around the
plug handling.

Signed-off-by: Jens Axboe <jaxboe@fusionio.com>

4 files changed, 44 insertions, 8 deletions

diff --git a/block/blk-core.c b/block/blk-core.c
index 2127f5973f44..06e92db9d674 100644
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -396,6 +396,8 @@ EXPORT_SYMBOL(blk_sync_queue);
  */
 void __blk_run_queue(struct request_queue *q)
 {
+	assert_spin_locked(q->queue_lock);
+
 	blk_remove_plug(q);
 
 	if (unlikely(blk_queue_stopped(q)))
@@ -1295,9 +1297,11 @@ get_rq:
 	__elv_add_request(ctx, req, where, 0);
 out:
 	spin_unlock_irq(&ctx->lock);
-	if (unplug || !queue_should_plug(q))
+	if (unplug || !queue_should_plug(q)) {
+		spin_lock_irq(q->queue_lock);
 		__generic_unplug_device(q);
-	spin_unlock_irq(q->queue_lock);
+		spin_unlock_irq(q->queue_lock);
+	}
 	return 0;
 }
 
diff --git a/block/blk-exec.c b/block/blk-exec.c
index f16e6667da06..9445d915fb0e 100644
--- a/block/blk-exec.c
+++ b/block/blk-exec.c
@@ -54,8 +54,11 @@ void blk_execute_rq_nowait(struct request_queue *q, struct gendisk *bd_disk,
 	rq->rq_disk = bd_disk;
 	rq->end_io = done;
 	WARN_ON(irqs_disabled());
-	spin_lock_irq(q->queue_lock);
+	spin_lock_irq(&ctx->lock);
 	__elv_add_request(ctx, rq, where, 1);
+	spin_unlock(&ctx->lock);
+
+	spin_lock(q->queue_lock);
 	__generic_unplug_device(q);
 	/* the queue is stopped so it won't be plugged+unplugged */
 	if (rq->cmd_type == REQ_TYPE_PM_RESUME)
diff --git a/block/elevator.c b/block/elevator.c
index 22bc0b7c80f3..aef735dc1a4b 100644
--- a/block/elevator.c
+++ b/block/elevator.c
@@ -692,7 +692,13 @@ void elv_insert(struct blk_queue_ctx *ctx, struct request *rq, int where)
 
 	case ELEVATOR_INSERT_FRONT:
 		rq->cmd_flags |= REQ_SOFTBARRIER;
+		queue_ctx_lock_queue(q, ctx);
+
 		list_add(&rq->queuelist, &q->queue_head);
+		if (unplug_it && blk_queue_plugged(q) && !queue_in_flight(q))
+			__generic_unplug_device(q);
+
+		queue_ctx_unlock_queue(q, ctx);
 		break;
 
 	case ELEVATOR_INSERT_BACK:
@@ -709,7 +715,9 @@ void elv_insert(struct blk_queue_ctx *ctx, struct request *rq, int where)
 		 *   with anything.  There's no point in delaying queue
 		 *   processing.
 		 */
+		queue_ctx_lock_queue(q, ctx);
 		__blk_run_queue(q);
+		queue_ctx_unlock_queue(q, ctx);
 		break;
 
 	case ELEVATOR_INSERT_SORT:
@@ -729,6 +737,12 @@ void elv_insert(struct blk_queue_ctx *ctx, struct request *rq, int where)
 		 * elevator_add_req_fn.
 		 */
 		ctx->queue->elevator->ops->elevator_add_req_fn(ctx, rq);
+
+		if (unplug_it && blk_queue_plugged(q) && !queue_in_flight(q)) {
+			queue_ctx_lock_queue(q, ctx);
+			__generic_unplug_device(q);
+			queue_ctx_unlock_queue(q, ctx);
+		}
 		break;
 
 	default:
@@ -736,10 +750,6 @@ void elv_insert(struct blk_queue_ctx *ctx, struct request *rq, int where)
 		       __func__, where);
 		BUG();
 	}
-
-
-	if (unplug_it && blk_queue_plugged(q) && !queue_in_flight(q))
-		__generic_unplug_device(q);
 }
 
 void __elv_add_request(struct blk_queue_ctx *ctx, struct request *rq, int where,
@@ -747,6 +757,8 @@ void __elv_add_request(struct blk_queue_ctx *ctx, struct request *rq, int where,
 {
 	struct request_queue *q = ctx->queue;
 
+	assert_spin_locked(&ctx->lock);
+
 	if (rq->cmd_flags & REQ_SOFTBARRIER) {
 		/* barriers are scheduling boundary, update end_sector */
 		if (rq->cmd_type == REQ_TYPE_FS ||
@@ -758,8 +770,11 @@ void __elv_add_request(struct blk_queue_ctx *ctx, struct request *rq, int where,
 		    where == ELEVATOR_INSERT_SORT)
 		where = ELEVATOR_INSERT_BACK;
 
-	if (plug)
+	if (plug) {
+		queue_ctx_lock_queue(q, ctx);
 		blk_plug_device(q);
+		queue_ctx_unlock_queue(q, ctx);
+	}
 
 	elv_insert(ctx, rq, where);
 }
diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h
index 360f3173ea7f..3dfa06eb61ee 100644
--- a/include/linux/blk-mq.h
+++ b/include/linux/blk-mq.h
@@ -75,4 +75,18 @@ static inline int queue_elvpriv(struct request_queue *q)
 	return blk_ctx_sum(q, __ctx->rl.elvpriv);
 }
 
+static inline void queue_ctx_lock_queue(struct request_queue *q,
+					struct blk_queue_ctx *ctx)
+{
+	spin_unlock(&ctx->lock);
+	spin_lock(q->queue_lock);
+}
+
+static inline void queue_ctx_unlock_queue(struct request_queue *q,
+					  struct blk_queue_ctx *ctx)
+{
+	spin_unlock(q->queue_lock);
+	spin_lock(&ctx->lock);
+}
+
 #endif