[linux-2.6-block.git] / drivers / s390 / block / scm_blk.c

/*
 * Block driver for s390 storage class memory.
 *
 * Copyright IBM Corp. 2012
 * Author(s): Sebastian Ott <sebott@linux.vnet.ibm.com>
 */

#define KMSG_COMPONENT "scm_block"
#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt

#include <linux/interrupt.h>
#include <linux/spinlock.h>
#include <linux/module.h>
#include <linux/blkdev.h>
#include <linux/genhd.h>
#include <linux/slab.h>
#include <linux/list.h>
#include <asm/eadm.h>
#include "scm_blk.h"

debug_info_t *scm_debug;
static int scm_major;
static DEFINE_SPINLOCK(list_lock);
static LIST_HEAD(inactive_requests);
static unsigned int nr_requests = 64;
static atomic_t nr_devices = ATOMIC_INIT(0);
module_param(nr_requests, uint, S_IRUGO);
MODULE_PARM_DESC(nr_requests, "Number of parallel requests.");

MODULE_DESCRIPTION("Block driver for s390 storage class memory.");
MODULE_LICENSE("GPL");
MODULE_ALIAS("scm:scmdev*");

static void __scm_free_rq(struct scm_request *scmrq)
{
	struct aob_rq_header *aobrq = to_aobrq(scmrq);

	free_page((unsigned long) scmrq->aob);
	free_page((unsigned long) scmrq->aidaw);
	__scm_free_rq_cluster(scmrq);
	kfree(aobrq);
}

static void scm_free_rqs(void)
{
	struct list_head *iter, *safe;
	struct scm_request *scmrq;

	spin_lock_irq(&list_lock);
	list_for_each_safe(iter, safe, &inactive_requests) {
		scmrq = list_entry(iter, struct scm_request, list);
		list_del(&scmrq->list);
		__scm_free_rq(scmrq);
	}
	spin_unlock_irq(&list_lock);
}

static int __scm_alloc_rq(void)
{
	struct aob_rq_header *aobrq;
	struct scm_request *scmrq;

	aobrq = kzalloc(sizeof(*aobrq) + sizeof(*scmrq), GFP_KERNEL);
	if (!aobrq)
		return -ENOMEM;

	scmrq = (void *) aobrq->data;
	scmrq->aidaw = (void *) get_zeroed_page(GFP_DMA);
	scmrq->aob = (void *) get_zeroed_page(GFP_DMA);
	if (!scmrq->aob || !scmrq->aidaw) {
		__scm_free_rq(scmrq);
		return -ENOMEM;
	}

	if (__scm_alloc_rq_cluster(scmrq)) {
		__scm_free_rq(scmrq);
		return -ENOMEM;
	}

	INIT_LIST_HEAD(&scmrq->list);
	spin_lock_irq(&list_lock);
	list_add(&scmrq->list, &inactive_requests);
	spin_unlock_irq(&list_lock);

	return 0;
}

static int scm_alloc_rqs(unsigned int nrqs)
{
	int ret = 0;

	while (nrqs-- && !ret)
		ret = __scm_alloc_rq();

	return ret;
}

static struct scm_request *scm_request_fetch(void)
{
	struct scm_request *scmrq = NULL;

	spin_lock(&list_lock);
	if (list_empty(&inactive_requests))
		goto out;
	scmrq = list_first_entry(&inactive_requests, struct scm_request, list);
	list_del(&scmrq->list);
out:
	spin_unlock(&list_lock);
	return scmrq;
}

static void scm_request_done(struct scm_request *scmrq)
{
	unsigned long flags;

	spin_lock_irqsave(&list_lock, flags);
	list_add(&scmrq->list, &inactive_requests);
	spin_unlock_irqrestore(&list_lock, flags);
}

static int scm_open(struct block_device *blkdev, fmode_t mode)
{
	return scm_get_ref();
}

static void scm_release(struct gendisk *gendisk, fmode_t mode)
{
	scm_put_ref();
}

static const struct block_device_operations scm_blk_devops = {
	.owner = THIS_MODULE,
	.open = scm_open,
	.release = scm_release,
};

static bool scm_permit_request(struct scm_blk_dev *bdev, struct request *req)
{
	return rq_data_dir(req) != WRITE || bdev->state != SCM_WR_PROHIBIT;
}

static void scm_request_prepare(struct scm_request *scmrq)
{
	struct scm_blk_dev *bdev = scmrq->bdev;
	struct scm_device *scmdev = bdev->gendisk->private_data;
	struct aidaw *aidaw = scmrq->aidaw;
	struct msb *msb = &scmrq->aob->msb[0];
	struct req_iterator iter;
	struct bio_vec *bv;

	msb->bs = MSB_BS_4K;
	scmrq->aob->request.msb_count = 1;
	msb->scm_addr = scmdev->address +
		((u64) blk_rq_pos(scmrq->request) << 9);
	msb->oc = (rq_data_dir(scmrq->request) == READ) ?
		MSB_OC_READ : MSB_OC_WRITE;
	msb->flags |= MSB_FLAG_IDA;
	msb->data_addr = (u64) aidaw;

	rq_for_each_segment(bv, scmrq->request, iter) {
		WARN_ON(bv->bv_offset);
		msb->blk_count += bv->bv_len >> 12;
		aidaw->data_addr = (u64) page_address(bv->bv_page);
		aidaw++;
	}
}

static inline void scm_request_init(struct scm_blk_dev *bdev,
				    struct scm_request *scmrq,
				    struct request *req)
{
	struct aob_rq_header *aobrq = to_aobrq(scmrq);
	struct aob *aob = scmrq->aob;

	memset(aob, 0, sizeof(*aob));
	memset(scmrq->aidaw, 0, PAGE_SIZE);
	aobrq->scmdev = bdev->scmdev;
	aob->request.cmd_code = ARQB_CMD_MOVE;
	aob->request.data = (u64) aobrq;
	scmrq->request = req;
	scmrq->bdev = bdev;
	scmrq->retries = 4;
	scmrq->error = 0;
	scm_request_cluster_init(scmrq);
}

static void scm_ensure_queue_restart(struct scm_blk_dev *bdev)
{
	if (atomic_read(&bdev->queued_reqs)) {
		/* Queue restart is triggered by the next interrupt. */
		return;
	}
	blk_delay_queue(bdev->rq, SCM_QUEUE_DELAY);
}

void scm_request_requeue(struct scm_request *scmrq)
{
	struct scm_blk_dev *bdev = scmrq->bdev;

	scm_release_cluster(scmrq);
	blk_requeue_request(bdev->rq, scmrq->request);
	atomic_dec(&bdev->queued_reqs);
	scm_request_done(scmrq);
	scm_ensure_queue_restart(bdev);
}

void scm_request_finish(struct scm_request *scmrq)
{
	struct scm_blk_dev *bdev = scmrq->bdev;

	scm_release_cluster(scmrq);
	blk_end_request_all(scmrq->request, scmrq->error);
	atomic_dec(&bdev->queued_reqs);
	scm_request_done(scmrq);
}

static void scm_blk_request(struct request_queue *rq)
{
	struct scm_device *scmdev = rq->queuedata;
	struct scm_blk_dev *bdev = dev_get_drvdata(&scmdev->dev);
	struct scm_request *scmrq;
	struct request *req;
	int ret;

	while ((req = blk_peek_request(rq))) {
		if (req->cmd_type != REQ_TYPE_FS)
			continue;

		if (!scm_permit_request(bdev, req)) {
			scm_ensure_queue_restart(bdev);
			return;
		}
		scmrq = scm_request_fetch();
		if (!scmrq) {
			SCM_LOG(5, "no request");
			scm_ensure_queue_restart(bdev);
			return;
		}
		scm_request_init(bdev, scmrq, req);
		if (!scm_reserve_cluster(scmrq)) {
			SCM_LOG(5, "cluster busy");
			scm_request_done(scmrq);
			return;
		}
		if (scm_need_cluster_request(scmrq)) {
			atomic_inc(&bdev->queued_reqs);
			blk_start_request(req);
			scm_initiate_cluster_request(scmrq);
			return;
		}
		scm_request_prepare(scmrq);
		atomic_inc(&bdev->queued_reqs);
		blk_start_request(req);

		ret = scm_start_aob(scmrq->aob);
		if (ret) {
			SCM_LOG(5, "no subchannel");
			scm_request_requeue(scmrq);
			return;
		}
	}
}

static void __scmrq_log_error(struct scm_request *scmrq)
{
	struct aob *aob = scmrq->aob;

	if (scmrq->error == -ETIMEDOUT)
		SCM_LOG(1, "Request timeout");
	else {
		SCM_LOG(1, "Request error");
		SCM_LOG_HEX(1, &aob->response, sizeof(aob->response));
	}
	if (scmrq->retries)
		SCM_LOG(1, "Retry request");
	else
		pr_err("An I/O operation to SCM failed with rc=%d\n",
		       scmrq->error);
}

void scm_blk_irq(struct scm_device *scmdev, void *data, int error)
{
	struct scm_request *scmrq = data;
	struct scm_blk_dev *bdev = scmrq->bdev;

	scmrq->error = error;
	if (error)
		__scmrq_log_error(scmrq);

	spin_lock(&bdev->lock);
	list_add_tail(&scmrq->list, &bdev->finished_requests);
	spin_unlock(&bdev->lock);
	tasklet_hi_schedule(&bdev->tasklet);
}

static void scm_blk_handle_error(struct scm_request *scmrq)
{
	struct scm_blk_dev *bdev = scmrq->bdev;
	unsigned long flags;

	if (scmrq->error != -EIO)
		goto restart;

	/* For -EIO the response block is valid. */
	switch (scmrq->aob->response.eqc) {
	case EQC_WR_PROHIBIT:
		spin_lock_irqsave(&bdev->lock, flags);
		if (bdev->state != SCM_WR_PROHIBIT)
			pr_info("%lx: Write access to the SCM increment is suspended\n",
				(unsigned long) bdev->scmdev->address);
		bdev->state = SCM_WR_PROHIBIT;
		spin_unlock_irqrestore(&bdev->lock, flags);
		goto requeue;
	default:
		break;
	}

restart:
	if (!scm_start_aob(scmrq->aob))
		return;

requeue:
	spin_lock_irqsave(&bdev->rq_lock, flags);
	scm_request_requeue(scmrq);
	spin_unlock_irqrestore(&bdev->rq_lock, flags);
}

static void scm_blk_tasklet(struct scm_blk_dev *bdev)
{
	struct scm_request *scmrq;
	unsigned long flags;

	spin_lock_irqsave(&bdev->lock, flags);
	while (!list_empty(&bdev->finished_requests)) {
		scmrq = list_first_entry(&bdev->finished_requests,
					 struct scm_request, list);
		list_del(&scmrq->list);
		spin_unlock_irqrestore(&bdev->lock, flags);

		if (scmrq->error && scmrq->retries-- > 0) {
			scm_blk_handle_error(scmrq);

			/* Request restarted or requeued, handle next. */
			spin_lock_irqsave(&bdev->lock, flags);
			continue;
		}

		if (scm_test_cluster_request(scmrq)) {
			scm_cluster_request_irq(scmrq);
			spin_lock_irqsave(&bdev->lock, flags);
			continue;
		}

		scm_request_finish(scmrq);
		spin_lock_irqsave(&bdev->lock, flags);
	}
	spin_unlock_irqrestore(&bdev->lock, flags);
	/* Look out for more requests. */
	blk_run_queue(bdev->rq);
}

int scm_blk_dev_setup(struct scm_blk_dev *bdev, struct scm_device *scmdev)
{
	struct request_queue *rq;
	int len, ret = -ENOMEM;
	unsigned int devindex, nr_max_blk;

	devindex = atomic_inc_return(&nr_devices) - 1;
	/* scma..scmz + scmaa..scmzz */
	if (devindex > 701) {
		ret = -ENODEV;
		goto out;
	}

	bdev->scmdev = scmdev;
	bdev->state = SCM_OPER;
	spin_lock_init(&bdev->rq_lock);
	spin_lock_init(&bdev->lock);
	INIT_LIST_HEAD(&bdev->finished_requests);
	atomic_set(&bdev->queued_reqs, 0);
	tasklet_init(&bdev->tasklet,
		     (void (*)(unsigned long)) scm_blk_tasklet,
		     (unsigned long) bdev);

	rq = blk_init_queue(scm_blk_request, &bdev->rq_lock);
	if (!rq)
		goto out;

	bdev->rq = rq;
	nr_max_blk = min(scmdev->nr_max_block,
			 (unsigned int) (PAGE_SIZE / sizeof(struct aidaw)));

	blk_queue_logical_block_size(rq, 1 << 12);
	blk_queue_max_hw_sectors(rq, nr_max_blk << 3); /* 8 * 512 = blk_size */
	blk_queue_max_segments(rq, nr_max_blk);
	queue_flag_set_unlocked(QUEUE_FLAG_NONROT, rq);
	scm_blk_dev_cluster_setup(bdev);

	bdev->gendisk = alloc_disk(SCM_NR_PARTS);
	if (!bdev->gendisk)
		goto out_queue;

	rq->queuedata = scmdev;
	bdev->gendisk->driverfs_dev = &scmdev->dev;
	bdev->gendisk->private_data = scmdev;
	bdev->gendisk->fops = &scm_blk_devops;
	bdev->gendisk->queue = rq;
	bdev->gendisk->major = scm_major;
	bdev->gendisk->first_minor = devindex * SCM_NR_PARTS;

	len = snprintf(bdev->gendisk->disk_name, DISK_NAME_LEN, "scm");
	if (devindex > 25) {
		len += snprintf(bdev->gendisk->disk_name + len,
				DISK_NAME_LEN - len, "%c",
				'a' + (devindex / 26) - 1);
		devindex = devindex % 26;
	}
	snprintf(bdev->gendisk->disk_name + len, DISK_NAME_LEN - len, "%c",
		 'a' + devindex);

	/* 512 byte sectors */
	set_capacity(bdev->gendisk, scmdev->size >> 9);
	add_disk(bdev->gendisk);
	return 0;

out_queue:
	blk_cleanup_queue(rq);
out:
	atomic_dec(&nr_devices);
	return ret;
}

void scm_blk_dev_cleanup(struct scm_blk_dev *bdev)
{
	tasklet_kill(&bdev->tasklet);
	del_gendisk(bdev->gendisk);
	blk_cleanup_queue(bdev->gendisk->queue);
	put_disk(bdev->gendisk);
}

void scm_blk_set_available(struct scm_blk_dev *bdev)
{
	unsigned long flags;

	spin_lock_irqsave(&bdev->lock, flags);
	if (bdev->state == SCM_WR_PROHIBIT)
		pr_info("%lx: Write access to the SCM increment is restored\n",
			(unsigned long) bdev->scmdev->address);
	bdev->state = SCM_OPER;
	spin_unlock_irqrestore(&bdev->lock, flags);
}

static int __init scm_blk_init(void)
{
	int ret = -EINVAL;

	if (!scm_cluster_size_valid())
		goto out;

	ret = register_blkdev(0, "scm");
	if (ret < 0)
		goto out;

	scm_major = ret;
	ret = scm_alloc_rqs(nr_requests);
	if (ret)
		goto out_free;

	scm_debug = debug_register("scm_log", 16, 1, 16);
	if (!scm_debug) {
		ret = -ENOMEM;
		goto out_free;
	}

	debug_register_view(scm_debug, &debug_hex_ascii_view);
	debug_set_level(scm_debug, 2);

	ret = scm_drv_init();
	if (ret)
		goto out_dbf;

	return ret;

out_dbf:
	debug_unregister(scm_debug);
out_free:
	scm_free_rqs();
	unregister_blkdev(scm_major, "scm");
out:
	return ret;
}
module_init(scm_blk_init);

static void __exit scm_blk_cleanup(void)
{
	scm_drv_cleanup();
	debug_unregister(scm_debug);
	scm_free_rqs();
	unregister_blkdev(scm_major, "scm");
}
module_exit(scm_blk_cleanup);
Commit	Line	Data
f30664e2 SO	1	/*
	2	* Block driver for s390 storage class memory.
	3	*
	4	* Copyright IBM Corp. 2012
	5	* Author(s): Sebastian Ott <sebott@linux.vnet.ibm.com>
	6	*/
	7
	8	#define KMSG_COMPONENT "scm_block"
	9	#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
	10
	11	#include <linux/interrupt.h>
	12	#include <linux/spinlock.h>
	13	#include <linux/module.h>
	14	#include <linux/blkdev.h>
	15	#include <linux/genhd.h>
	16	#include <linux/slab.h>
	17	#include <linux/list.h>
	18	#include <asm/eadm.h>
	19	#include "scm_blk.h"
	20
	21	debug_info_t *scm_debug;
	22	static int scm_major;
	23	static DEFINE_SPINLOCK(list_lock);
	24	static LIST_HEAD(inactive_requests);
	25	static unsigned int nr_requests = 64;
	26	static atomic_t nr_devices = ATOMIC_INIT(0);
	27	module_param(nr_requests, uint, S_IRUGO);
	28	MODULE_PARM_DESC(nr_requests, "Number of parallel requests.");
	29
	30	MODULE_DESCRIPTION("Block driver for s390 storage class memory.");
	31	MODULE_LICENSE("GPL");
	32	MODULE_ALIAS("scm:scmdev*");
	33
	34	static void __scm_free_rq(struct scm_request *scmrq)
	35	{
	36	struct aob_rq_header *aobrq = to_aobrq(scmrq);
	37
	38	free_page((unsigned long) scmrq->aob);
	39	free_page((unsigned long) scmrq->aidaw);
0d804b20	40	__scm_free_rq_cluster(scmrq);
f30664e2 SO	41	kfree(aobrq);
	42	}
	43
	44	static void scm_free_rqs(void)
	45	{
	46	struct list_head iter, safe;
	47	struct scm_request *scmrq;
	48
	49	spin_lock_irq(&list_lock);
	50	list_for_each_safe(iter, safe, &inactive_requests) {
	51	scmrq = list_entry(iter, struct scm_request, list);
	52	list_del(&scmrq->list);
	53	__scm_free_rq(scmrq);
	54	}
	55	spin_unlock_irq(&list_lock);
	56	}
	57
	58	static int __scm_alloc_rq(void)
	59	{
	60	struct aob_rq_header *aobrq;
	61	struct scm_request *scmrq;
	62
	63	aobrq = kzalloc(sizeof(aobrq) + sizeof(scmrq), GFP_KERNEL);
	64	if (!aobrq)
	65	return -ENOMEM;
	66
	67	scmrq = (void *) aobrq->data;
	68	scmrq->aidaw = (void *) get_zeroed_page(GFP_DMA);
	69	scmrq->aob = (void *) get_zeroed_page(GFP_DMA);
	70	if (!scmrq->aob \|\| !scmrq->aidaw) {
	71	__scm_free_rq(scmrq);
	72	return -ENOMEM;
	73	}
0d804b20 SO	74
	75	if (__scm_alloc_rq_cluster(scmrq)) {
	76	__scm_free_rq(scmrq);
	77	return -ENOMEM;
	78	}
	79
f30664e2 SO	80	INIT_LIST_HEAD(&scmrq->list);
	81	spin_lock_irq(&list_lock);
	82	list_add(&scmrq->list, &inactive_requests);
	83	spin_unlock_irq(&list_lock);
	84
	85	return 0;
	86	}
	87
	88	static int scm_alloc_rqs(unsigned int nrqs)
	89	{
	90	int ret = 0;
	91
	92	while (nrqs-- && !ret)
	93	ret = __scm_alloc_rq();
	94
	95	return ret;
	96	}
	97
	98	static struct scm_request *scm_request_fetch(void)
	99	{
	100	struct scm_request *scmrq = NULL;
	101
	102	spin_lock(&list_lock);
	103	if (list_empty(&inactive_requests))
	104	goto out;
	105	scmrq = list_first_entry(&inactive_requests, struct scm_request, list);
	106	list_del(&scmrq->list);
	107	out:
	108	spin_unlock(&list_lock);
	109	return scmrq;
	110	}
	111
	112	static void scm_request_done(struct scm_request *scmrq)
	113	{
	114	unsigned long flags;
	115
	116	spin_lock_irqsave(&list_lock, flags);
	117	list_add(&scmrq->list, &inactive_requests);
	118	spin_unlock_irqrestore(&list_lock, flags);
	119	}
	120
	121	static int scm_open(struct block_device *blkdev, fmode_t mode)
	122	{
	123	return scm_get_ref();
	124	}
	125
db2a144b	126	static void scm_release(struct gendisk *gendisk, fmode_t mode)
f30664e2 SO	127	{
f30664e2 SO	128	scm_put_ref();
f30664e2 SO	129	}
	130
	131	static const struct block_device_operations scm_blk_devops = {
	132	.owner = THIS_MODULE,
	133	.open = scm_open,
	134	.release = scm_release,
	135	};
	136
4fa3c019 SO	137	static bool scm_permit_request(struct scm_blk_dev bdev, struct request req)
	138	{
	139	return rq_data_dir(req) != WRITE \|\| bdev->state != SCM_WR_PROHIBIT;
	140	}
	141
f30664e2 SO	142	static void scm_request_prepare(struct scm_request *scmrq)
	143	{
	144	struct scm_blk_dev *bdev = scmrq->bdev;
	145	struct scm_device *scmdev = bdev->gendisk->private_data;
	146	struct aidaw *aidaw = scmrq->aidaw;
	147	struct msb *msb = &scmrq->aob->msb[0];
	148	struct req_iterator iter;
	149	struct bio_vec *bv;
	150
	151	msb->bs = MSB_BS_4K;
	152	scmrq->aob->request.msb_count = 1;
	153	msb->scm_addr = scmdev->address +
	154	((u64) blk_rq_pos(scmrq->request) << 9);
	155	msb->oc = (rq_data_dir(scmrq->request) == READ) ?
	156	MSB_OC_READ : MSB_OC_WRITE;
	157	msb->flags \|= MSB_FLAG_IDA;
	158	msb->data_addr = (u64) aidaw;
	159
	160	rq_for_each_segment(bv, scmrq->request, iter) {
	161	WARN_ON(bv->bv_offset);
	162	msb->blk_count += bv->bv_len >> 12;
	163	aidaw->data_addr = (u64) page_address(bv->bv_page);
	164	aidaw++;
	165	}
	166	}
	167
	168	static inline void scm_request_init(struct scm_blk_dev *bdev,
	169	struct scm_request *scmrq,
	170	struct request *req)
	171	{
	172	struct aob_rq_header *aobrq = to_aobrq(scmrq);
	173	struct aob *aob = scmrq->aob;
	174
	175	memset(aob, 0, sizeof(*aob));
	176	memset(scmrq->aidaw, 0, PAGE_SIZE);
	177	aobrq->scmdev = bdev->scmdev;
	178	aob->request.cmd_code = ARQB_CMD_MOVE;
	179	aob->request.data = (u64) aobrq;
	180	scmrq->request = req;
	181	scmrq->bdev = bdev;
	182	scmrq->retries = 4;
	183	scmrq->error = 0;
0d804b20	184	scm_request_cluster_init(scmrq);
f30664e2 SO	185	}
	186
	187	static void scm_ensure_queue_restart(struct scm_blk_dev *bdev)
	188	{
	189	if (atomic_read(&bdev->queued_reqs)) {
	190	/* Queue restart is triggered by the next interrupt. */
	191	return;
	192	}
	193	blk_delay_queue(bdev->rq, SCM_QUEUE_DELAY);
	194	}
	195
0d804b20	196	void scm_request_requeue(struct scm_request *scmrq)
f30664e2 SO	197	{
	198	struct scm_blk_dev *bdev = scmrq->bdev;
	199
0d804b20	200	scm_release_cluster(scmrq);
f30664e2	201	blk_requeue_request(bdev->rq, scmrq->request);
8360cb5f	202	atomic_dec(&bdev->queued_reqs);
f30664e2 SO	203	scm_request_done(scmrq);
	204	scm_ensure_queue_restart(bdev);
	205	}
	206
0d804b20	207	void scm_request_finish(struct scm_request *scmrq)
f30664e2	208	{
8360cb5f SO	209	struct scm_blk_dev *bdev = scmrq->bdev;
8360cb5f SO	210
0d804b20	211	scm_release_cluster(scmrq);
f30664e2	212	blk_end_request_all(scmrq->request, scmrq->error);
8360cb5f	213	atomic_dec(&bdev->queued_reqs);
f30664e2 SO	214	scm_request_done(scmrq);
	215	}
	216
	217	static void scm_blk_request(struct request_queue *rq)
	218	{
	219	struct scm_device *scmdev = rq->queuedata;
	220	struct scm_blk_dev *bdev = dev_get_drvdata(&scmdev->dev);
	221	struct scm_request *scmrq;
	222	struct request *req;
	223	int ret;
	224
	225	while ((req = blk_peek_request(rq))) {
	226	if (req->cmd_type != REQ_TYPE_FS)
	227	continue;
	228
4fa3c019 SO	229	if (!scm_permit_request(bdev, req)) {
	230	scm_ensure_queue_restart(bdev);
	231	return;
	232	}
f30664e2 SO	233	scmrq = scm_request_fetch();
	234	if (!scmrq) {
	235	SCM_LOG(5, "no request");
	236	scm_ensure_queue_restart(bdev);
	237	return;
	238	}
	239	scm_request_init(bdev, scmrq, req);
0d804b20 SO	240	if (!scm_reserve_cluster(scmrq)) {
	241	SCM_LOG(5, "cluster busy");
	242	scm_request_done(scmrq);
	243	return;
	244	}
	245	if (scm_need_cluster_request(scmrq)) {
8360cb5f	246	atomic_inc(&bdev->queued_reqs);
0d804b20 SO	247	blk_start_request(req);
	248	scm_initiate_cluster_request(scmrq);
	249	return;
	250	}
f30664e2	251	scm_request_prepare(scmrq);
8360cb5f	252	atomic_inc(&bdev->queued_reqs);
f30664e2 SO	253	blk_start_request(req);
	254
	255	ret = scm_start_aob(scmrq->aob);
	256	if (ret) {
	257	SCM_LOG(5, "no subchannel");
	258	scm_request_requeue(scmrq);
	259	return;
	260	}
f30664e2 SO	261	}
	262	}
	263
	264	static void __scmrq_log_error(struct scm_request *scmrq)
	265	{
	266	struct aob *aob = scmrq->aob;
	267
	268	if (scmrq->error == -ETIMEDOUT)
	269	SCM_LOG(1, "Request timeout");
	270	else {
	271	SCM_LOG(1, "Request error");
	272	SCM_LOG_HEX(1, &aob->response, sizeof(aob->response));
	273	}
	274	if (scmrq->retries)
	275	SCM_LOG(1, "Retry request");
	276	else
	277	pr_err("An I/O operation to SCM failed with rc=%d\n",
	278	scmrq->error);
	279	}
	280
	281	void scm_blk_irq(struct scm_device scmdev, void data, int error)
	282	{
	283	struct scm_request *scmrq = data;
	284	struct scm_blk_dev *bdev = scmrq->bdev;
	285
	286	scmrq->error = error;
	287	if (error)
	288	__scmrq_log_error(scmrq);
	289
	290	spin_lock(&bdev->lock);
	291	list_add_tail(&scmrq->list, &bdev->finished_requests);
	292	spin_unlock(&bdev->lock);
	293	tasklet_hi_schedule(&bdev->tasklet);
	294	}
	295
4fa3c019 SO	296	static void scm_blk_handle_error(struct scm_request *scmrq)
	297	{
	298	struct scm_blk_dev *bdev = scmrq->bdev;
	299	unsigned long flags;
	300
	301	if (scmrq->error != -EIO)
	302	goto restart;
	303
	304	/* For -EIO the response block is valid. */
	305	switch (scmrq->aob->response.eqc) {
	306	case EQC_WR_PROHIBIT:
	307	spin_lock_irqsave(&bdev->lock, flags);
	308	if (bdev->state != SCM_WR_PROHIBIT)
3bff6038	309	pr_info("%lx: Write access to the SCM increment is suspended\n",
4fa3c019 SO	310	(unsigned long) bdev->scmdev->address);
	311	bdev->state = SCM_WR_PROHIBIT;
	312	spin_unlock_irqrestore(&bdev->lock, flags);
	313	goto requeue;
	314	default:
	315	break;
	316	}
	317
	318	restart:
	319	if (!scm_start_aob(scmrq->aob))
	320	return;
	321
	322	requeue:
	323	spin_lock_irqsave(&bdev->rq_lock, flags);
	324	scm_request_requeue(scmrq);
	325	spin_unlock_irqrestore(&bdev->rq_lock, flags);
	326	}
	327
f30664e2 SO	328	static void scm_blk_tasklet(struct scm_blk_dev *bdev)
	329	{
	330	struct scm_request *scmrq;
	331	unsigned long flags;
	332
	333	spin_lock_irqsave(&bdev->lock, flags);
	334	while (!list_empty(&bdev->finished_requests)) {
	335	scmrq = list_first_entry(&bdev->finished_requests,
	336	struct scm_request, list);
	337	list_del(&scmrq->list);
	338	spin_unlock_irqrestore(&bdev->lock, flags);
	339
	340	if (scmrq->error && scmrq->retries-- > 0) {
4fa3c019 SO	341	scm_blk_handle_error(scmrq);
4fa3c019 SO	342
f30664e2 SO	343	/* Request restarted or requeued, handle next. */
	344	spin_lock_irqsave(&bdev->lock, flags);
	345	continue;
	346	}
0d804b20 SO	347
	348	if (scm_test_cluster_request(scmrq)) {
	349	scm_cluster_request_irq(scmrq);
	350	spin_lock_irqsave(&bdev->lock, flags);
	351	continue;
	352	}
	353
f30664e2	354	scm_request_finish(scmrq);
f30664e2 SO	355	spin_lock_irqsave(&bdev->lock, flags);
	356	}
	357	spin_unlock_irqrestore(&bdev->lock, flags);
	358	/* Look out for more requests. */
	359	blk_run_queue(bdev->rq);
	360	}
	361
	362	int scm_blk_dev_setup(struct scm_blk_dev bdev, struct scm_device scmdev)
	363	{
	364	struct request_queue *rq;
	365	int len, ret = -ENOMEM;
	366	unsigned int devindex, nr_max_blk;
	367
	368	devindex = atomic_inc_return(&nr_devices) - 1;
	369	/* scma..scmz + scmaa..scmzz */
	370	if (devindex > 701) {
	371	ret = -ENODEV;
	372	goto out;
	373	}
	374
	375	bdev->scmdev = scmdev;
4fa3c019	376	bdev->state = SCM_OPER;
f30664e2 SO	377	spin_lock_init(&bdev->rq_lock);
	378	spin_lock_init(&bdev->lock);
	379	INIT_LIST_HEAD(&bdev->finished_requests);
	380	atomic_set(&bdev->queued_reqs, 0);
	381	tasklet_init(&bdev->tasklet,
	382	(void (*)(unsigned long)) scm_blk_tasklet,
	383	(unsigned long) bdev);
	384
	385	rq = blk_init_queue(scm_blk_request, &bdev->rq_lock);
	386	if (!rq)
	387	goto out;
	388
	389	bdev->rq = rq;
	390	nr_max_blk = min(scmdev->nr_max_block,
	391	(unsigned int) (PAGE_SIZE / sizeof(struct aidaw)));
	392
	393	blk_queue_logical_block_size(rq, 1 << 12);
	394	blk_queue_max_hw_sectors(rq, nr_max_blk << 3); /* 8 * 512 = blk_size */
	395	blk_queue_max_segments(rq, nr_max_blk);
	396	queue_flag_set_unlocked(QUEUE_FLAG_NONROT, rq);
0d804b20	397	scm_blk_dev_cluster_setup(bdev);
f30664e2 SO	398
	399	bdev->gendisk = alloc_disk(SCM_NR_PARTS);
	400	if (!bdev->gendisk)
	401	goto out_queue;
	402
	403	rq->queuedata = scmdev;
	404	bdev->gendisk->driverfs_dev = &scmdev->dev;
	405	bdev->gendisk->private_data = scmdev;
	406	bdev->gendisk->fops = &scm_blk_devops;
	407	bdev->gendisk->queue = rq;
	408	bdev->gendisk->major = scm_major;
	409	bdev->gendisk->first_minor = devindex * SCM_NR_PARTS;
	410
	411	len = snprintf(bdev->gendisk->disk_name, DISK_NAME_LEN, "scm");
	412	if (devindex > 25) {
	413	len += snprintf(bdev->gendisk->disk_name + len,
	414	DISK_NAME_LEN - len, "%c",
	415	'a' + (devindex / 26) - 1);
	416	devindex = devindex % 26;
	417	}
	418	snprintf(bdev->gendisk->disk_name + len, DISK_NAME_LEN - len, "%c",
	419	'a' + devindex);
	420
	421	/* 512 byte sectors */
	422	set_capacity(bdev->gendisk, scmdev->size >> 9);
	423	add_disk(bdev->gendisk);
	424	return 0;
	425
	426	out_queue:
	427	blk_cleanup_queue(rq);
	428	out:
	429	atomic_dec(&nr_devices);
	430	return ret;
	431	}
	432
	433	void scm_blk_dev_cleanup(struct scm_blk_dev *bdev)
	434	{
	435	tasklet_kill(&bdev->tasklet);
	436	del_gendisk(bdev->gendisk);
	437	blk_cleanup_queue(bdev->gendisk->queue);
	438	put_disk(bdev->gendisk);
	439	}
	440
4fa3c019 SO	441	void scm_blk_set_available(struct scm_blk_dev *bdev)
	442	{
	443	unsigned long flags;
	444
	445	spin_lock_irqsave(&bdev->lock, flags);
	446	if (bdev->state == SCM_WR_PROHIBIT)
3bff6038	447	pr_info("%lx: Write access to the SCM increment is restored\n",
4fa3c019 SO	448	(unsigned long) bdev->scmdev->address);
	449	bdev->state = SCM_OPER;
	450	spin_unlock_irqrestore(&bdev->lock, flags);
	451	}
	452
f30664e2 SO	453	static int __init scm_blk_init(void)
f30664e2 SO	454	{
0d804b20 SO	455	int ret = -EINVAL;
	456
	457	if (!scm_cluster_size_valid())
	458	goto out;
f30664e2 SO	459
	460	ret = register_blkdev(0, "scm");
	461	if (ret < 0)
	462	goto out;
	463
	464	scm_major = ret;
94f9852d WY	465	ret = scm_alloc_rqs(nr_requests);
94f9852d WY	466	if (ret)
fff60fab	467	goto out_free;
f30664e2 SO	468
f30664e2 SO	469	scm_debug = debug_register("scm_log", 16, 1, 16);
94f9852d WY	470	if (!scm_debug) {
94f9852d WY	471	ret = -ENOMEM;
f30664e2	472	goto out_free;
94f9852d	473	}
f30664e2 SO	474
	475	debug_register_view(scm_debug, &debug_hex_ascii_view);
	476	debug_set_level(scm_debug, 2);
	477
	478	ret = scm_drv_init();
	479	if (ret)
	480	goto out_dbf;
	481
	482	return ret;
	483
	484	out_dbf:
	485	debug_unregister(scm_debug);
	486	out_free:
	487	scm_free_rqs();
f30664e2 SO	488	unregister_blkdev(scm_major, "scm");
	489	out:
	490	return ret;
	491	}
	492	module_init(scm_blk_init);
	493
	494	static void __exit scm_blk_cleanup(void)
	495	{
	496	scm_drv_cleanup();
	497	debug_unregister(scm_debug);
	498	scm_free_rqs();
	499	unregister_blkdev(scm_major, "scm");
	500	}
	501	module_exit(scm_blk_cleanup);