[linux-2.6-block.git] / drivers / nvme / target / io-cmd-file.c

// SPDX-License-Identifier: GPL-2.0
/*
 * NVMe Over Fabrics Target File I/O commands implementation.
 * Copyright (c) 2017-2018 Western Digital Corporation or its
 * affiliates.
 */
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
#include <linux/uio.h>
#include <linux/falloc.h>
#include <linux/file.h>
#include "nvmet.h"

#define NVMET_MAX_MPOOL_BVEC		16
#define NVMET_MIN_MPOOL_OBJ		16

void nvmet_file_ns_disable(struct nvmet_ns *ns)
{
	if (ns->file) {
		if (ns->buffered_io)
			flush_workqueue(buffered_io_wq);
		mempool_destroy(ns->bvec_pool);
		ns->bvec_pool = NULL;
		kmem_cache_destroy(ns->bvec_cache);
		ns->bvec_cache = NULL;
		fput(ns->file);
		ns->file = NULL;
	}
}

int nvmet_file_ns_enable(struct nvmet_ns *ns)
{
	int flags = O_RDWR | O_LARGEFILE;
	struct kstat stat;
	int ret;

	if (!ns->buffered_io)
		flags |= O_DIRECT;

	ns->file = filp_open(ns->device_path, flags, 0);
	if (IS_ERR(ns->file)) {
		pr_err("failed to open file %s: (%ld)\n",
				ns->device_path, PTR_ERR(ns->file));
		return PTR_ERR(ns->file);
	}

	ret = vfs_getattr(&ns->file->f_path,
			&stat, STATX_SIZE, AT_STATX_FORCE_SYNC);
	if (ret)
		goto err;

	ns->size = stat.size;
	ns->blksize_shift = file_inode(ns->file)->i_blkbits;

	ns->bvec_cache = kmem_cache_create("nvmet-bvec",
			NVMET_MAX_MPOOL_BVEC * sizeof(struct bio_vec),
			0, SLAB_HWCACHE_ALIGN, NULL);
	if (!ns->bvec_cache) {
		ret = -ENOMEM;
		goto err;
	}

	ns->bvec_pool = mempool_create(NVMET_MIN_MPOOL_OBJ, mempool_alloc_slab,
			mempool_free_slab, ns->bvec_cache);

	if (!ns->bvec_pool) {
		ret = -ENOMEM;
		goto err;
	}

	return ret;
err:
	ns->size = 0;
	ns->blksize_shift = 0;
	nvmet_file_ns_disable(ns);
	return ret;
}

static void nvmet_file_init_bvec(struct bio_vec *bv, struct scatterlist *sg)
{
	bv->bv_page = sg_page(sg);
	bv->bv_offset = sg->offset;
	bv->bv_len = sg->length;
}

static ssize_t nvmet_file_submit_bvec(struct nvmet_req *req, loff_t pos,
		unsigned long nr_segs, size_t count, int ki_flags)
{
	struct kiocb *iocb = &req->f.iocb;
	ssize_t (*call_iter)(struct kiocb *iocb, struct iov_iter *iter);
	struct iov_iter iter;
	int rw;

	if (req->cmd->rw.opcode == nvme_cmd_write) {
		if (req->cmd->rw.control & cpu_to_le16(NVME_RW_FUA))
			ki_flags |= IOCB_DSYNC;
		call_iter = req->ns->file->f_op->write_iter;
		rw = WRITE;
	} else {
		call_iter = req->ns->file->f_op->read_iter;
		rw = READ;
	}

	iov_iter_bvec(&iter, rw, req->f.bvec, nr_segs, count);

	iocb->ki_pos = pos;
	iocb->ki_filp = req->ns->file;
	iocb->ki_flags = ki_flags | iocb_flags(req->ns->file);

	return call_iter(iocb, &iter);
}

static void nvmet_file_io_done(struct kiocb *iocb, long ret, long ret2)
{
	struct nvmet_req *req = container_of(iocb, struct nvmet_req, f.iocb);
	u16 status = NVME_SC_SUCCESS;

	if (req->f.bvec != req->inline_bvec) {
		if (likely(req->f.mpool_alloc == false))
			kfree(req->f.bvec);
		else
			mempool_free(req->f.bvec, req->ns->bvec_pool);
	}

	if (unlikely(ret != req->data_len))
		status = errno_to_nvme_status(req, ret);
	nvmet_req_complete(req, status);
}

static bool nvmet_file_execute_io(struct nvmet_req *req, int ki_flags)
{
	ssize_t nr_bvec = req->sg_cnt;
	unsigned long bv_cnt = 0;
	bool is_sync = false;
	size_t len = 0, total_len = 0;
	ssize_t ret = 0;
	loff_t pos;
	int i;
	struct scatterlist *sg;

	if (req->f.mpool_alloc && nr_bvec > NVMET_MAX_MPOOL_BVEC)
		is_sync = true;

	pos = le64_to_cpu(req->cmd->rw.slba) << req->ns->blksize_shift;
	if (unlikely(pos + req->data_len > req->ns->size)) {
		nvmet_req_complete(req, errno_to_nvme_status(req, -ENOSPC));
		return true;
	}

	memset(&req->f.iocb, 0, sizeof(struct kiocb));
	for_each_sg(req->sg, sg, req->sg_cnt, i) {
		nvmet_file_init_bvec(&req->f.bvec[bv_cnt], sg);
		len += req->f.bvec[bv_cnt].bv_len;
		total_len += req->f.bvec[bv_cnt].bv_len;
		bv_cnt++;

		WARN_ON_ONCE((nr_bvec - 1) < 0);

		if (unlikely(is_sync) &&
		    (nr_bvec - 1 == 0 || bv_cnt == NVMET_MAX_MPOOL_BVEC)) {
			ret = nvmet_file_submit_bvec(req, pos, bv_cnt, len, 0);
			if (ret < 0)
				goto complete;

			pos += len;
			bv_cnt = 0;
			len = 0;
		}
		nr_bvec--;
	}

	if (WARN_ON_ONCE(total_len != req->data_len)) {
		ret = -EIO;
		goto complete;
	}

	if (unlikely(is_sync)) {
		ret = total_len;
		goto complete;
	}

	/*
	 * A NULL ki_complete ask for synchronous execution, which we want
	 * for the IOCB_NOWAIT case.
	 */
	if (!(ki_flags & IOCB_NOWAIT))
		req->f.iocb.ki_complete = nvmet_file_io_done;

	ret = nvmet_file_submit_bvec(req, pos, bv_cnt, total_len, ki_flags);

	switch (ret) {
	case -EIOCBQUEUED:
		return true;
	case -EAGAIN:
		if (WARN_ON_ONCE(!(ki_flags & IOCB_NOWAIT)))
			goto complete;
		return false;
	case -EOPNOTSUPP:
		/*
		 * For file systems returning error -EOPNOTSUPP, handle
		 * IOCB_NOWAIT error case separately and retry without
		 * IOCB_NOWAIT.
		 */
		if ((ki_flags & IOCB_NOWAIT))
			return false;
		break;
	}

complete:
	nvmet_file_io_done(&req->f.iocb, ret, 0);
	return true;
}

static void nvmet_file_buffered_io_work(struct work_struct *w)
{
	struct nvmet_req *req = container_of(w, struct nvmet_req, f.work);

	nvmet_file_execute_io(req, 0);
}

static void nvmet_file_submit_buffered_io(struct nvmet_req *req)
{
	INIT_WORK(&req->f.work, nvmet_file_buffered_io_work);
	queue_work(buffered_io_wq, &req->f.work);
}

static void nvmet_file_execute_rw(struct nvmet_req *req)
{
	ssize_t nr_bvec = req->sg_cnt;

	if (!req->sg_cnt || !nr_bvec) {
		nvmet_req_complete(req, 0);
		return;
	}

	if (nr_bvec > NVMET_MAX_INLINE_BIOVEC)
		req->f.bvec = kmalloc_array(nr_bvec, sizeof(struct bio_vec),
				GFP_KERNEL);
	else
		req->f.bvec = req->inline_bvec;

	if (unlikely(!req->f.bvec)) {
		/* fallback under memory pressure */
		req->f.bvec = mempool_alloc(req->ns->bvec_pool, GFP_KERNEL);
		req->f.mpool_alloc = true;
	} else
		req->f.mpool_alloc = false;

	if (req->ns->buffered_io) {
		if (likely(!req->f.mpool_alloc) &&
				nvmet_file_execute_io(req, IOCB_NOWAIT))
			return;
		nvmet_file_submit_buffered_io(req);
	} else
		nvmet_file_execute_io(req, 0);
}

u16 nvmet_file_flush(struct nvmet_req *req)
{
	return errno_to_nvme_status(req, vfs_fsync(req->ns->file, 1));
}

static void nvmet_file_flush_work(struct work_struct *w)
{
	struct nvmet_req *req = container_of(w, struct nvmet_req, f.work);

	nvmet_req_complete(req, nvmet_file_flush(req));
}

static void nvmet_file_execute_flush(struct nvmet_req *req)
{
	INIT_WORK(&req->f.work, nvmet_file_flush_work);
	schedule_work(&req->f.work);
}

static void nvmet_file_execute_discard(struct nvmet_req *req)
{
	int mode = FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE;
	struct nvme_dsm_range range;
	loff_t offset, len;
	u16 status = 0;
	int ret;
	int i;

	for (i = 0; i <= le32_to_cpu(req->cmd->dsm.nr); i++) {
		status = nvmet_copy_from_sgl(req, i * sizeof(range), &range,
					sizeof(range));
		if (status)
			break;

		offset = le64_to_cpu(range.slba) << req->ns->blksize_shift;
		len = le32_to_cpu(range.nlb);
		len <<= req->ns->blksize_shift;
		if (offset + len > req->ns->size) {
			req->error_slba = le64_to_cpu(range.slba);
			status = errno_to_nvme_status(req, -ENOSPC);
			break;
		}

		ret = vfs_fallocate(req->ns->file, mode, offset, len);
		if (ret && ret != -EOPNOTSUPP) {
			req->error_slba = le64_to_cpu(range.slba);
			status = errno_to_nvme_status(req, ret);
			break;
		}
	}

	nvmet_req_complete(req, status);
}

static void nvmet_file_dsm_work(struct work_struct *w)
{
	struct nvmet_req *req = container_of(w, struct nvmet_req, f.work);

	switch (le32_to_cpu(req->cmd->dsm.attributes)) {
	case NVME_DSMGMT_AD:
		nvmet_file_execute_discard(req);
		return;
	case NVME_DSMGMT_IDR:
	case NVME_DSMGMT_IDW:
	default:
		/* Not supported yet */
		nvmet_req_complete(req, 0);
		return;
	}
}

static void nvmet_file_execute_dsm(struct nvmet_req *req)
{
	INIT_WORK(&req->f.work, nvmet_file_dsm_work);
	schedule_work(&req->f.work);
}

static void nvmet_file_write_zeroes_work(struct work_struct *w)
{
	struct nvmet_req *req = container_of(w, struct nvmet_req, f.work);
	struct nvme_write_zeroes_cmd *write_zeroes = &req->cmd->write_zeroes;
	int mode = FALLOC_FL_ZERO_RANGE | FALLOC_FL_KEEP_SIZE;
	loff_t offset;
	loff_t len;
	int ret;

	offset = le64_to_cpu(write_zeroes->slba) << req->ns->blksize_shift;
	len = (((sector_t)le16_to_cpu(write_zeroes->length) + 1) <<
			req->ns->blksize_shift);

	if (unlikely(offset + len > req->ns->size)) {
		nvmet_req_complete(req, errno_to_nvme_status(req, -ENOSPC));
		return;
	}

	ret = vfs_fallocate(req->ns->file, mode, offset, len);
	nvmet_req_complete(req, ret < 0 ? errno_to_nvme_status(req, ret) : 0);
}

static void nvmet_file_execute_write_zeroes(struct nvmet_req *req)
{
	INIT_WORK(&req->f.work, nvmet_file_write_zeroes_work);
	schedule_work(&req->f.work);
}

u16 nvmet_file_parse_io_cmd(struct nvmet_req *req)
{
	struct nvme_command *cmd = req->cmd;

	switch (cmd->common.opcode) {
	case nvme_cmd_read:
	case nvme_cmd_write:
		req->execute = nvmet_file_execute_rw;
		req->data_len = nvmet_rw_len(req);
		return 0;
	case nvme_cmd_flush:
		req->execute = nvmet_file_execute_flush;
		req->data_len = 0;
		return 0;
	case nvme_cmd_dsm:
		req->execute = nvmet_file_execute_dsm;
		req->data_len = (le32_to_cpu(cmd->dsm.nr) + 1) *
			sizeof(struct nvme_dsm_range);
		return 0;
	case nvme_cmd_write_zeroes:
		req->execute = nvmet_file_execute_write_zeroes;
		req->data_len = 0;
		return 0;
	default:
		pr_err("unhandled cmd for file ns %d on qid %d\n",
				cmd->common.opcode, req->sq->qid);
		req->error_loc = offsetof(struct nvme_common_command, opcode);
		return NVME_SC_INVALID_OPCODE | NVME_SC_DNR;
	}
}
Commit	Line	Data
d5eff33e CK	1	// SPDX-License-Identifier: GPL-2.0
	2	/*
	3	* NVMe Over Fabrics Target File I/O commands implementation.
	4	* Copyright (c) 2017-2018 Western Digital Corporation or its
	5	* affiliates.
	6	*/
	7	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
	8	#include <linux/uio.h>
	9	#include <linux/falloc.h>
	10	#include <linux/file.h>
	11	#include "nvmet.h"
	12
	13	#define NVMET_MAX_MPOOL_BVEC 16
	14	#define NVMET_MIN_MPOOL_OBJ 16
	15
	16	void nvmet_file_ns_disable(struct nvmet_ns *ns)
	17	{
	18	if (ns->file) {
55eb942e CK	19	if (ns->buffered_io)
55eb942e CK	20	flush_workqueue(buffered_io_wq);
d5eff33e CK	21	mempool_destroy(ns->bvec_pool);
	22	ns->bvec_pool = NULL;
	23	kmem_cache_destroy(ns->bvec_cache);
	24	ns->bvec_cache = NULL;
	25	fput(ns->file);
	26	ns->file = NULL;
	27	}
	28	}
	29
	30	int nvmet_file_ns_enable(struct nvmet_ns *ns)
	31	{
55eb942e	32	int flags = O_RDWR \| O_LARGEFILE;
d5eff33e	33	struct kstat stat;
55eb942e CK	34	int ret;
	35
	36	if (!ns->buffered_io)
	37	flags \|= O_DIRECT;
d5eff33e	38
55eb942e	39	ns->file = filp_open(ns->device_path, flags, 0);
d5eff33e CK	40	if (IS_ERR(ns->file)) {
d5eff33e CK	41	pr_err("failed to open file %s: (%ld)\n",
81cf54e0	42	ns->device_path, PTR_ERR(ns->file));
d5eff33e CK	43	return PTR_ERR(ns->file);
	44	}
	45
	46	ret = vfs_getattr(&ns->file->f_path,
	47	&stat, STATX_SIZE, AT_STATX_FORCE_SYNC);
	48	if (ret)
	49	goto err;
	50
	51	ns->size = stat.size;
	52	ns->blksize_shift = file_inode(ns->file)->i_blkbits;
	53
	54	ns->bvec_cache = kmem_cache_create("nvmet-bvec",
	55	NVMET_MAX_MPOOL_BVEC * sizeof(struct bio_vec),
	56	0, SLAB_HWCACHE_ALIGN, NULL);
1367bc82 WY	57	if (!ns->bvec_cache) {
1367bc82 WY	58	ret = -ENOMEM;
d5eff33e	59	goto err;
1367bc82	60	}
d5eff33e CK	61
	62	ns->bvec_pool = mempool_create(NVMET_MIN_MPOOL_OBJ, mempool_alloc_slab,
	63	mempool_free_slab, ns->bvec_cache);
	64
1367bc82 WY	65	if (!ns->bvec_pool) {
1367bc82 WY	66	ret = -ENOMEM;
d5eff33e	67	goto err;
1367bc82	68	}
d5eff33e CK	69
	70	return ret;
	71	err:
	72	ns->size = 0;
	73	ns->blksize_shift = 0;
	74	nvmet_file_ns_disable(ns);
	75	return ret;
	76	}
	77
02db9954	78	static void nvmet_file_init_bvec(struct bio_vec bv, struct scatterlist sg)
d5eff33e	79	{
02db9954 ML	80	bv->bv_page = sg_page(sg);
	81	bv->bv_offset = sg->offset;
	82	bv->bv_len = sg->length;
d5eff33e CK	83	}
	84
	85	static ssize_t nvmet_file_submit_bvec(struct nvmet_req *req, loff_t pos,
50a909db	86	unsigned long nr_segs, size_t count, int ki_flags)
d5eff33e CK	87	{
	88	struct kiocb *iocb = &req->f.iocb;
	89	ssize_t (call_iter)(struct kiocb iocb, struct iov_iter *iter);
	90	struct iov_iter iter;
50a909db	91	int rw;
d5eff33e CK	92
	93	if (req->cmd->rw.opcode == nvme_cmd_write) {
	94	if (req->cmd->rw.control & cpu_to_le16(NVME_RW_FUA))
50a909db	95	ki_flags \|= IOCB_DSYNC;
d5eff33e CK	96	call_iter = req->ns->file->f_op->write_iter;
	97	rw = WRITE;
	98	} else {
	99	call_iter = req->ns->file->f_op->read_iter;
	100	rw = READ;
	101	}
	102
aa563d7b	103	iov_iter_bvec(&iter, rw, req->f.bvec, nr_segs, count);
d5eff33e CK	104
	105	iocb->ki_pos = pos;
	106	iocb->ki_filp = req->ns->file;
55eb942e	107	iocb->ki_flags = ki_flags \| iocb_flags(req->ns->file);
d5eff33e	108
50a909db	109	return call_iter(iocb, &iter);
d5eff33e CK	110	}
	111
	112	static void nvmet_file_io_done(struct kiocb *iocb, long ret, long ret2)
	113	{
	114	struct nvmet_req *req = container_of(iocb, struct nvmet_req, f.iocb);
c6aa3542	115	u16 status = NVME_SC_SUCCESS;
d5eff33e CK	116
	117	if (req->f.bvec != req->inline_bvec) {
	118	if (likely(req->f.mpool_alloc == false))
	119	kfree(req->f.bvec);
	120	else
	121	mempool_free(req->f.bvec, req->ns->bvec_pool);
	122	}
	123
c6aa3542 CK	124	if (unlikely(ret != req->data_len))
	125	status = errno_to_nvme_status(req, ret);
	126	nvmet_req_complete(req, status);
d5eff33e CK	127	}
d5eff33e CK	128
50a909db	129	static bool nvmet_file_execute_io(struct nvmet_req *req, int ki_flags)
d5eff33e	130	{
02db9954	131	ssize_t nr_bvec = req->sg_cnt;
d5eff33e CK	132	unsigned long bv_cnt = 0;
	133	bool is_sync = false;
	134	size_t len = 0, total_len = 0;
	135	ssize_t ret = 0;
	136	loff_t pos;
02db9954 ML	137	int i;
02db9954 ML	138	struct scatterlist *sg;
50a909db CK	139
	140	if (req->f.mpool_alloc && nr_bvec > NVMET_MAX_MPOOL_BVEC)
	141	is_sync = true;
d5eff33e	142
9c891c13 SG	143	pos = le64_to_cpu(req->cmd->rw.slba) << req->ns->blksize_shift;
9c891c13 SG	144	if (unlikely(pos + req->data_len > req->ns->size)) {
c6aa3542	145	nvmet_req_complete(req, errno_to_nvme_status(req, -ENOSPC));
50a909db	146	return true;
d5eff33e CK	147	}
d5eff33e CK	148
d5eff33e	149	memset(&req->f.iocb, 0, sizeof(struct kiocb));
02db9954 ML	150	for_each_sg(req->sg, sg, req->sg_cnt, i) {
02db9954 ML	151	nvmet_file_init_bvec(&req->f.bvec[bv_cnt], sg);
d5eff33e CK	152	len += req->f.bvec[bv_cnt].bv_len;
	153	total_len += req->f.bvec[bv_cnt].bv_len;
	154	bv_cnt++;
	155
	156	WARN_ON_ONCE((nr_bvec - 1) < 0);
	157
	158	if (unlikely(is_sync) &&
	159	(nr_bvec - 1 == 0 \|\| bv_cnt == NVMET_MAX_MPOOL_BVEC)) {
50a909db	160	ret = nvmet_file_submit_bvec(req, pos, bv_cnt, len, 0);
d5eff33e	161	if (ret < 0)
50a909db CK	162	goto complete;
50a909db CK	163
d5eff33e CK	164	pos += len;
	165	bv_cnt = 0;
	166	len = 0;
	167	}
	168	nr_bvec--;
	169	}
	170
50a909db	171	if (WARN_ON_ONCE(total_len != req->data_len)) {
d5eff33e	172	ret = -EIO;
50a909db CK	173	goto complete;
	174	}
	175
	176	if (unlikely(is_sync)) {
	177	ret = total_len;
	178	goto complete;
d5eff33e	179	}
50a909db CK	180
	181	/*
	182	* A NULL ki_complete ask for synchronous execution, which we want
	183	* for the IOCB_NOWAIT case.
	184	*/
	185	if (!(ki_flags & IOCB_NOWAIT))
	186	req->f.iocb.ki_complete = nvmet_file_io_done;
	187
	188	ret = nvmet_file_submit_bvec(req, pos, bv_cnt, total_len, ki_flags);
	189
	190	switch (ret) {
	191	case -EIOCBQUEUED:
	192	return true;
	193	case -EAGAIN:
	194	if (WARN_ON_ONCE(!(ki_flags & IOCB_NOWAIT)))
	195	goto complete;
	196	return false;
	197	case -EOPNOTSUPP:
	198	/*
	199	* For file systems returning error -EOPNOTSUPP, handle
	200	* IOCB_NOWAIT error case separately and retry without
	201	* IOCB_NOWAIT.
	202	*/
	203	if ((ki_flags & IOCB_NOWAIT))
	204	return false;
	205	break;
	206	}
	207
	208	complete:
	209	nvmet_file_io_done(&req->f.iocb, ret, 0);
	210	return true;
d5eff33e CK	211	}
d5eff33e CK	212
55eb942e CK	213	static void nvmet_file_buffered_io_work(struct work_struct *w)
	214	{
	215	struct nvmet_req *req = container_of(w, struct nvmet_req, f.work);
	216
50a909db	217	nvmet_file_execute_io(req, 0);
55eb942e CK	218	}
55eb942e CK	219
50a909db	220	static void nvmet_file_submit_buffered_io(struct nvmet_req *req)
55eb942e CK	221	{
	222	INIT_WORK(&req->f.work, nvmet_file_buffered_io_work);
	223	queue_work(buffered_io_wq, &req->f.work);
	224	}
	225
50a909db CK	226	static void nvmet_file_execute_rw(struct nvmet_req *req)
50a909db CK	227	{
02db9954	228	ssize_t nr_bvec = req->sg_cnt;
50a909db CK	229
	230	if (!req->sg_cnt \|\| !nr_bvec) {
	231	nvmet_req_complete(req, 0);
	232	return;
	233	}
	234
	235	if (nr_bvec > NVMET_MAX_INLINE_BIOVEC)
	236	req->f.bvec = kmalloc_array(nr_bvec, sizeof(struct bio_vec),
	237	GFP_KERNEL);
	238	else
	239	req->f.bvec = req->inline_bvec;
	240
	241	if (unlikely(!req->f.bvec)) {
	242	/* fallback under memory pressure */
	243	req->f.bvec = mempool_alloc(req->ns->bvec_pool, GFP_KERNEL);
	244	req->f.mpool_alloc = true;
	245	} else
	246	req->f.mpool_alloc = false;
	247
	248	if (req->ns->buffered_io) {
	249	if (likely(!req->f.mpool_alloc) &&
	250	nvmet_file_execute_io(req, IOCB_NOWAIT))
	251	return;
	252	nvmet_file_submit_buffered_io(req);
	253	} else
	254	nvmet_file_execute_io(req, 0);
	255	}
	256
dedf0be5 CK	257	u16 nvmet_file_flush(struct nvmet_req *req)
dedf0be5 CK	258	{
c6aa3542	259	return errno_to_nvme_status(req, vfs_fsync(req->ns->file, 1));
dedf0be5 CK	260	}
dedf0be5 CK	261
d5eff33e CK	262	static void nvmet_file_flush_work(struct work_struct *w)
	263	{
	264	struct nvmet_req *req = container_of(w, struct nvmet_req, f.work);
d5eff33e	265
dedf0be5	266	nvmet_req_complete(req, nvmet_file_flush(req));
d5eff33e CK	267	}
	268
	269	static void nvmet_file_execute_flush(struct nvmet_req *req)
	270	{
	271	INIT_WORK(&req->f.work, nvmet_file_flush_work);
	272	schedule_work(&req->f.work);
	273	}
	274
	275	static void nvmet_file_execute_discard(struct nvmet_req *req)
	276	{
	277	int mode = FALLOC_FL_PUNCH_HOLE \| FALLOC_FL_KEEP_SIZE;
	278	struct nvme_dsm_range range;
1b72b71f	279	loff_t offset, len;
c6aa3542 CK	280	u16 status = 0;
c6aa3542 CK	281	int ret;
1b72b71f	282	int i;
d5eff33e CK	283
d5eff33e CK	284	for (i = 0; i <= le32_to_cpu(req->cmd->dsm.nr); i++) {
c6aa3542	285	status = nvmet_copy_from_sgl(req, i * sizeof(range), &range,
1b72b71f	286	sizeof(range));
c6aa3542	287	if (status)
d5eff33e	288	break;
9c891c13	289
d5eff33e	290	offset = le64_to_cpu(range.slba) << req->ns->blksize_shift;
8eacd1bd BVA	291	len = le32_to_cpu(range.nlb);
8eacd1bd BVA	292	len <<= req->ns->blksize_shift;
9c891c13	293	if (offset + len > req->ns->size) {
c6aa3542 CK	294	req->error_slba = le64_to_cpu(range.slba);
c6aa3542 CK	295	status = errno_to_nvme_status(req, -ENOSPC);
9c891c13 SG	296	break;
	297	}
	298
c6aa3542	299	ret = vfs_fallocate(req->ns->file, mode, offset, len);
005c674f	300	if (ret && ret != -EOPNOTSUPP) {
c6aa3542 CK	301	req->error_slba = le64_to_cpu(range.slba);
c6aa3542 CK	302	status = errno_to_nvme_status(req, ret);
d5eff33e	303	break;
1b72b71f	304	}
d5eff33e CK	305	}
d5eff33e CK	306
c6aa3542	307	nvmet_req_complete(req, status);
d5eff33e CK	308	}
	309
	310	static void nvmet_file_dsm_work(struct work_struct *w)
	311	{
	312	struct nvmet_req *req = container_of(w, struct nvmet_req, f.work);
	313
	314	switch (le32_to_cpu(req->cmd->dsm.attributes)) {
	315	case NVME_DSMGMT_AD:
	316	nvmet_file_execute_discard(req);
	317	return;
	318	case NVME_DSMGMT_IDR:
	319	case NVME_DSMGMT_IDW:
	320	default:
	321	/* Not supported yet */
	322	nvmet_req_complete(req, 0);
	323	return;
	324	}
	325	}
	326
	327	static void nvmet_file_execute_dsm(struct nvmet_req *req)
	328	{
	329	INIT_WORK(&req->f.work, nvmet_file_dsm_work);
	330	schedule_work(&req->f.work);
	331	}
	332
	333	static void nvmet_file_write_zeroes_work(struct work_struct *w)
	334	{
	335	struct nvmet_req *req = container_of(w, struct nvmet_req, f.work);
	336	struct nvme_write_zeroes_cmd *write_zeroes = &req->cmd->write_zeroes;
	337	int mode = FALLOC_FL_ZERO_RANGE \| FALLOC_FL_KEEP_SIZE;
	338	loff_t offset;
	339	loff_t len;
	340	int ret;
	341
	342	offset = le64_to_cpu(write_zeroes->slba) << req->ns->blksize_shift;
	343	len = (((sector_t)le16_to_cpu(write_zeroes->length) + 1) <<
	344	req->ns->blksize_shift);
	345
9c891c13	346	if (unlikely(offset + len > req->ns->size)) {
c6aa3542	347	nvmet_req_complete(req, errno_to_nvme_status(req, -ENOSPC));
9c891c13 SG	348	return;
	349	}
	350
d5eff33e	351	ret = vfs_fallocate(req->ns->file, mode, offset, len);
c6aa3542	352	nvmet_req_complete(req, ret < 0 ? errno_to_nvme_status(req, ret) : 0);
d5eff33e CK	353	}
	354
	355	static void nvmet_file_execute_write_zeroes(struct nvmet_req *req)
	356	{
	357	INIT_WORK(&req->f.work, nvmet_file_write_zeroes_work);
	358	schedule_work(&req->f.work);
	359	}
	360
	361	u16 nvmet_file_parse_io_cmd(struct nvmet_req *req)
	362	{
	363	struct nvme_command *cmd = req->cmd;
	364
	365	switch (cmd->common.opcode) {
	366	case nvme_cmd_read:
	367	case nvme_cmd_write:
50a909db	368	req->execute = nvmet_file_execute_rw;
d5eff33e CK	369	req->data_len = nvmet_rw_len(req);
	370	return 0;
	371	case nvme_cmd_flush:
	372	req->execute = nvmet_file_execute_flush;
	373	req->data_len = 0;
	374	return 0;
	375	case nvme_cmd_dsm:
	376	req->execute = nvmet_file_execute_dsm;
	377	req->data_len = (le32_to_cpu(cmd->dsm.nr) + 1) *
	378	sizeof(struct nvme_dsm_range);
	379	return 0;
	380	case nvme_cmd_write_zeroes:
	381	req->execute = nvmet_file_execute_write_zeroes;
	382	req->data_len = 0;
	383	return 0;
	384	default:
	385	pr_err("unhandled cmd for file ns %d on qid %d\n",
	386	cmd->common.opcode, req->sq->qid);
c6aa3542	387	req->error_loc = offsetof(struct nvme_common_command, opcode);
d5eff33e CK	388	return NVME_SC_INVALID_OPCODE \| NVME_SC_DNR;
	389	}
	390	}