[linux-2.6-block.git] / drivers / lightnvm / pblk-read.c

// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright (C) 2016 CNEX Labs
 * Initial release: Javier Gonzalez <javier@cnexlabs.com>
 *                  Matias Bjorling <matias@cnexlabs.com>
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License version
 * 2 as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * pblk-read.c - pblk's read path
 */

#include "pblk.h"

/*
 * There is no guarantee that the value read from cache has not been updated and
 * resides at another location in the cache. We guarantee though that if the
 * value is read from the cache, it belongs to the mapped lba. In order to
 * guarantee and order between writes and reads are ordered, a flush must be
 * issued.
 */
static int pblk_read_from_cache(struct pblk *pblk, struct bio *bio,
				sector_t lba, struct ppa_addr ppa)
{
#ifdef CONFIG_NVM_PBLK_DEBUG
	/* Callers must ensure that the ppa points to a cache address */
	BUG_ON(pblk_ppa_empty(ppa));
	BUG_ON(!pblk_addr_in_cache(ppa));
#endif

	return pblk_rb_copy_to_bio(&pblk->rwb, bio, lba, ppa);
}

static int pblk_read_ppalist_rq(struct pblk *pblk, struct nvm_rq *rqd,
				 struct bio *bio, sector_t blba,
				 bool *from_cache)
{
	void *meta_list = rqd->meta_list;
	int nr_secs, i;

retry:
	nr_secs = pblk_lookup_l2p_seq(pblk, rqd->ppa_list, blba, rqd->nr_ppas,
					from_cache);

	if (!*from_cache)
		goto end;

	for (i = 0; i < nr_secs; i++) {
		struct pblk_sec_meta *meta = pblk_get_meta(pblk, meta_list, i);
		sector_t lba = blba + i;

		if (pblk_ppa_empty(rqd->ppa_list[i])) {
			__le64 addr_empty = cpu_to_le64(ADDR_EMPTY);

			meta->lba = addr_empty;
		} else if (pblk_addr_in_cache(rqd->ppa_list[i])) {
			/*
			 * Try to read from write buffer. The address is later
			 * checked on the write buffer to prevent retrieving
			 * overwritten data.
			 */
			if (!pblk_read_from_cache(pblk, bio, lba,
							rqd->ppa_list[i])) {
				if (i == 0) {
					/*
					 * We didn't call with bio_advance()
					 * yet, so we can just retry.
					 */
					goto retry;
				} else {
					/*
					 * We already call bio_advance()
					 * so we cannot retry and we need
					 * to quit that function in order
					 * to allow caller to handle the bio
					 * splitting in the current sector
					 * position.
					 */
					nr_secs = i;
					goto end;
				}
			}
			meta->lba = cpu_to_le64(lba);
#ifdef CONFIG_NVM_PBLK_DEBUG
			atomic_long_inc(&pblk->cache_reads);
#endif
		}
		bio_advance(bio, PBLK_EXPOSED_PAGE_SIZE);
	}

end:
	if (pblk_io_aligned(pblk, nr_secs))
		rqd->is_seq = 1;

#ifdef CONFIG_NVM_PBLK_DEBUG
	atomic_long_add(nr_secs, &pblk->inflight_reads);
#endif

	return nr_secs;
}


static void pblk_read_check_seq(struct pblk *pblk, struct nvm_rq *rqd,
				sector_t blba)
{
	void *meta_list = rqd->meta_list;
	int nr_lbas = rqd->nr_ppas;
	int i;

	if (!pblk_is_oob_meta_supported(pblk))
		return;

	for (i = 0; i < nr_lbas; i++) {
		struct pblk_sec_meta *meta = pblk_get_meta(pblk, meta_list, i);
		u64 lba = le64_to_cpu(meta->lba);

		if (lba == ADDR_EMPTY)
			continue;

		if (lba != blba + i) {
#ifdef CONFIG_NVM_PBLK_DEBUG
			struct ppa_addr *ppa_list = nvm_rq_to_ppa_list(rqd);

			print_ppa(pblk, &ppa_list[i], "seq", i);
#endif
			pblk_err(pblk, "corrupted read LBA (%llu/%llu)\n",
							lba, (u64)blba + i);
			WARN_ON(1);
		}
	}
}

/*
 * There can be holes in the lba list.
 */
static void pblk_read_check_rand(struct pblk *pblk, struct nvm_rq *rqd,
				 u64 *lba_list, int nr_lbas)
{
	void *meta_lba_list = rqd->meta_list;
	int i, j;

	if (!pblk_is_oob_meta_supported(pblk))
		return;

	for (i = 0, j = 0; i < nr_lbas; i++) {
		struct pblk_sec_meta *meta = pblk_get_meta(pblk,
							   meta_lba_list, j);
		u64 lba = lba_list[i];
		u64 meta_lba;

		if (lba == ADDR_EMPTY)
			continue;

		meta_lba = le64_to_cpu(meta->lba);

		if (lba != meta_lba) {
#ifdef CONFIG_NVM_PBLK_DEBUG
			struct ppa_addr *ppa_list = nvm_rq_to_ppa_list(rqd);

			print_ppa(pblk, &ppa_list[j], "rnd", j);
#endif
			pblk_err(pblk, "corrupted read LBA (%llu/%llu)\n",
							meta_lba, lba);
			WARN_ON(1);
		}

		j++;
	}

	WARN_ONCE(j != rqd->nr_ppas, "pblk: corrupted random request\n");
}

static void pblk_end_user_read(struct bio *bio, int error)
{
	if (error && error != NVM_RSP_WARN_HIGHECC)
		bio_io_error(bio);
	else
		bio_endio(bio);
}

static void __pblk_end_io_read(struct pblk *pblk, struct nvm_rq *rqd,
			       bool put_line)
{
	struct pblk_g_ctx *r_ctx = nvm_rq_to_pdu(rqd);
	struct bio *int_bio = rqd->bio;
	unsigned long start_time = r_ctx->start_time;

	bio_end_io_acct(int_bio, start_time);

	if (rqd->error)
		pblk_log_read_err(pblk, rqd);

	pblk_read_check_seq(pblk, rqd, r_ctx->lba);
	bio_put(int_bio);

	if (put_line)
		pblk_rq_to_line_put(pblk, rqd);

#ifdef CONFIG_NVM_PBLK_DEBUG
	atomic_long_add(rqd->nr_ppas, &pblk->sync_reads);
	atomic_long_sub(rqd->nr_ppas, &pblk->inflight_reads);
#endif

	pblk_free_rqd(pblk, rqd, PBLK_READ);
	atomic_dec(&pblk->inflight_io);
}

static void pblk_end_io_read(struct nvm_rq *rqd)
{
	struct pblk *pblk = rqd->private;
	struct pblk_g_ctx *r_ctx = nvm_rq_to_pdu(rqd);
	struct bio *bio = (struct bio *)r_ctx->private;

	pblk_end_user_read(bio, rqd->error);
	__pblk_end_io_read(pblk, rqd, true);
}

static void pblk_read_rq(struct pblk *pblk, struct nvm_rq *rqd, struct bio *bio,
			 sector_t lba, bool *from_cache)
{
	struct pblk_sec_meta *meta = pblk_get_meta(pblk, rqd->meta_list, 0);
	struct ppa_addr ppa;

	pblk_lookup_l2p_seq(pblk, &ppa, lba, 1, from_cache);

#ifdef CONFIG_NVM_PBLK_DEBUG
	atomic_long_inc(&pblk->inflight_reads);
#endif

retry:
	if (pblk_ppa_empty(ppa)) {
		__le64 addr_empty = cpu_to_le64(ADDR_EMPTY);

		meta->lba = addr_empty;
		return;
	}

	/* Try to read from write buffer. The address is later checked on the
	 * write buffer to prevent retrieving overwritten data.
	 */
	if (pblk_addr_in_cache(ppa)) {
		if (!pblk_read_from_cache(pblk, bio, lba, ppa)) {
			pblk_lookup_l2p_seq(pblk, &ppa, lba, 1, from_cache);
			goto retry;
		}

		meta->lba = cpu_to_le64(lba);

#ifdef CONFIG_NVM_PBLK_DEBUG
		atomic_long_inc(&pblk->cache_reads);
#endif
	} else {
		rqd->ppa_addr = ppa;
	}
}

void pblk_submit_read(struct pblk *pblk, struct bio *bio)
{
	sector_t blba = pblk_get_lba(bio);
	unsigned int nr_secs = pblk_get_secs(bio);
	bool from_cache;
	struct pblk_g_ctx *r_ctx;
	struct nvm_rq *rqd;
	struct bio *int_bio, *split_bio;
	unsigned long start_time;

	start_time = bio_start_io_acct(bio);

	rqd = pblk_alloc_rqd(pblk, PBLK_READ);

	rqd->opcode = NVM_OP_PREAD;
	rqd->nr_ppas = nr_secs;
	rqd->private = pblk;
	rqd->end_io = pblk_end_io_read;

	r_ctx = nvm_rq_to_pdu(rqd);
	r_ctx->start_time = start_time;
	r_ctx->lba = blba;

	if (pblk_alloc_rqd_meta(pblk, rqd)) {
		bio_io_error(bio);
		pblk_free_rqd(pblk, rqd, PBLK_READ);
		return;
	}

	/* Clone read bio to deal internally with:
	 * -read errors when reading from drive
	 * -bio_advance() calls during cache reads
	 */
	int_bio = bio_clone_fast(bio, GFP_KERNEL, &pblk_bio_set);

	if (nr_secs > 1)
		nr_secs = pblk_read_ppalist_rq(pblk, rqd, int_bio, blba,
						&from_cache);
	else
		pblk_read_rq(pblk, rqd, int_bio, blba, &from_cache);

split_retry:
	r_ctx->private = bio; /* original bio */
	rqd->bio = int_bio; /* internal bio */

	if (from_cache && nr_secs == rqd->nr_ppas) {
		/* All data was read from cache, we can complete the IO. */
		pblk_end_user_read(bio, 0);
		atomic_inc(&pblk->inflight_io);
		__pblk_end_io_read(pblk, rqd, false);
	} else if (nr_secs != rqd->nr_ppas) {
		/* The read bio request could be partially filled by the write
		 * buffer, but there are some holes that need to be read from
		 * the drive. In order to handle this, we will use block layer
		 * mechanism to split this request in to smaller ones and make
		 * a chain of it.
		 */
		split_bio = bio_split(bio, nr_secs * NR_PHY_IN_LOG, GFP_KERNEL,
					&pblk_bio_set);
		bio_chain(split_bio, bio);
		submit_bio_noacct(bio);

		/* New bio contains first N sectors of the previous one, so
		 * we can continue to use existing rqd, but we need to shrink
		 * the number of PPAs in it. New bio is also guaranteed that
		 * it contains only either data from cache or from drive, newer
		 * mix of them.
		 */
		bio = split_bio;
		rqd->nr_ppas = nr_secs;
		if (rqd->nr_ppas == 1)
			rqd->ppa_addr = rqd->ppa_list[0];

		/* Recreate int_bio - existing might have some needed internal
		 * fields modified already.
		 */
		bio_put(int_bio);
		int_bio = bio_clone_fast(bio, GFP_KERNEL, &pblk_bio_set);
		goto split_retry;
	} else if (pblk_submit_io(pblk, rqd, NULL)) {
		/* Submitting IO to drive failed, let's report an error */
		rqd->error = -ENODEV;
		pblk_end_io_read(rqd);
	}
}

static int read_ppalist_rq_gc(struct pblk *pblk, struct nvm_rq *rqd,
			      struct pblk_line *line, u64 *lba_list,
			      u64 *paddr_list_gc, unsigned int nr_secs)
{
	struct ppa_addr ppa_list_l2p[NVM_MAX_VLBA];
	struct ppa_addr ppa_gc;
	int valid_secs = 0;
	int i;

	pblk_lookup_l2p_rand(pblk, ppa_list_l2p, lba_list, nr_secs);

	for (i = 0; i < nr_secs; i++) {
		if (lba_list[i] == ADDR_EMPTY)
			continue;

		ppa_gc = addr_to_gen_ppa(pblk, paddr_list_gc[i], line->id);
		if (!pblk_ppa_comp(ppa_list_l2p[i], ppa_gc)) {
			paddr_list_gc[i] = lba_list[i] = ADDR_EMPTY;
			continue;
		}

		rqd->ppa_list[valid_secs++] = ppa_list_l2p[i];
	}

#ifdef CONFIG_NVM_PBLK_DEBUG
	atomic_long_add(valid_secs, &pblk->inflight_reads);
#endif

	return valid_secs;
}

static int read_rq_gc(struct pblk *pblk, struct nvm_rq *rqd,
		      struct pblk_line *line, sector_t lba,
		      u64 paddr_gc)
{
	struct ppa_addr ppa_l2p, ppa_gc;
	int valid_secs = 0;

	if (lba == ADDR_EMPTY)
		goto out;

	/* logic error: lba out-of-bounds */
	if (lba >= pblk->capacity) {
		WARN(1, "pblk: read lba out of bounds\n");
		goto out;
	}

	spin_lock(&pblk->trans_lock);
	ppa_l2p = pblk_trans_map_get(pblk, lba);
	spin_unlock(&pblk->trans_lock);

	ppa_gc = addr_to_gen_ppa(pblk, paddr_gc, line->id);
	if (!pblk_ppa_comp(ppa_l2p, ppa_gc))
		goto out;

	rqd->ppa_addr = ppa_l2p;
	valid_secs = 1;

#ifdef CONFIG_NVM_PBLK_DEBUG
	atomic_long_inc(&pblk->inflight_reads);
#endif

out:
	return valid_secs;
}

int pblk_submit_read_gc(struct pblk *pblk, struct pblk_gc_rq *gc_rq)
{
	struct nvm_rq rqd;
	int ret = NVM_IO_OK;

	memset(&rqd, 0, sizeof(struct nvm_rq));

	ret = pblk_alloc_rqd_meta(pblk, &rqd);
	if (ret)
		return ret;

	if (gc_rq->nr_secs > 1) {
		gc_rq->secs_to_gc = read_ppalist_rq_gc(pblk, &rqd, gc_rq->line,
							gc_rq->lba_list,
							gc_rq->paddr_list,
							gc_rq->nr_secs);
		if (gc_rq->secs_to_gc == 1)
			rqd.ppa_addr = rqd.ppa_list[0];
	} else {
		gc_rq->secs_to_gc = read_rq_gc(pblk, &rqd, gc_rq->line,
							gc_rq->lba_list[0],
							gc_rq->paddr_list[0]);
	}

	if (!(gc_rq->secs_to_gc))
		goto out;

	rqd.opcode = NVM_OP_PREAD;
	rqd.nr_ppas = gc_rq->secs_to_gc;

	if (pblk_submit_io_sync(pblk, &rqd, gc_rq->data)) {
		ret = -EIO;
		goto err_free_dma;
	}

	pblk_read_check_rand(pblk, &rqd, gc_rq->lba_list, gc_rq->nr_secs);

	atomic_dec(&pblk->inflight_io);

	if (rqd.error) {
		atomic_long_inc(&pblk->read_failed_gc);
#ifdef CONFIG_NVM_PBLK_DEBUG
		pblk_print_failed_rqd(pblk, &rqd, rqd.error);
#endif
	}

#ifdef CONFIG_NVM_PBLK_DEBUG
	atomic_long_add(gc_rq->secs_to_gc, &pblk->sync_reads);
	atomic_long_add(gc_rq->secs_to_gc, &pblk->recov_gc_reads);
	atomic_long_sub(gc_rq->secs_to_gc, &pblk->inflight_reads);
#endif

out:
	pblk_free_rqd_meta(pblk, &rqd);
	return ret;

err_free_dma:
	pblk_free_rqd_meta(pblk, &rqd);
	return ret;
}
Commit	Line	Data
02a1520d	1	// SPDX-License-Identifier: GPL-2.0
a4bd217b JG	2	/*
	3	* Copyright (C) 2016 CNEX Labs
	4	* Initial release: Javier Gonzalez <javier@cnexlabs.com>
	5	* Matias Bjorling <matias@cnexlabs.com>
	6	*
	7	* This program is free software; you can redistribute it and/or
	8	* modify it under the terms of the GNU General Public License version
	9	* 2 as published by the Free Software Foundation.
	10	*
	11	* This program is distributed in the hope that it will be useful, but
	12	* WITHOUT ANY WARRANTY; without even the implied warranty of
	13	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	14	* General Public License for more details.
	15	*
	16	* pblk-read.c - pblk's read path
	17	*/
	18
	19	#include "pblk.h"
	20
	21	/*
	22	* There is no guarantee that the value read from cache has not been updated and
	23	* resides at another location in the cache. We guarantee though that if the
	24	* value is read from the cache, it belongs to the mapped lba. In order to
	25	* guarantee and order between writes and reads are ordered, a flush must be
	26	* issued.
	27	*/
	28	static int pblk_read_from_cache(struct pblk pblk, struct bio bio,
a96de64a	29	sector_t lba, struct ppa_addr ppa)
a4bd217b	30	{
880eda54	31	#ifdef CONFIG_NVM_PBLK_DEBUG
a4bd217b JG	32	/* Callers must ensure that the ppa points to a cache address */
	33	BUG_ON(pblk_ppa_empty(ppa));
	34	BUG_ON(!pblk_addr_in_cache(ppa));
	35	#endif
	36
a96de64a	37	return pblk_rb_copy_to_bio(&pblk->rwb, bio, lba, ppa);
a4bd217b JG	38	}
a4bd217b JG	39
a96de64a	40	static int pblk_read_ppalist_rq(struct pblk pblk, struct nvm_rq rqd,
87cc40bb	41	struct bio *bio, sector_t blba,
a96de64a	42	bool *from_cache)
a4bd217b	43	{
faa79f27	44	void *meta_list = rqd->meta_list;
a96de64a	45	int nr_secs, i;
a4bd217b	46
a96de64a IK	47	retry:
	48	nr_secs = pblk_lookup_l2p_seq(pblk, rqd->ppa_list, blba, rqd->nr_ppas,
	49	from_cache);
	50
	51	if (!*from_cache)
	52	goto end;
a4bd217b JG	53
a4bd217b JG	54	for (i = 0; i < nr_secs; i++) {
faa79f27	55	struct pblk_sec_meta *meta = pblk_get_meta(pblk, meta_list, i);
a4bd217b JG	56	sector_t lba = blba + i;
a4bd217b JG	57
a96de64a	58	if (pblk_ppa_empty(rqd->ppa_list[i])) {
faa79f27 IK	59	__le64 addr_empty = cpu_to_le64(ADDR_EMPTY);
faa79f27 IK	60
faa79f27	61	meta->lba = addr_empty;
a96de64a IK	62	} else if (pblk_addr_in_cache(rqd->ppa_list[i])) {
	63	/*
	64	* Try to read from write buffer. The address is later
	65	* checked on the write buffer to prevent retrieving
	66	* overwritten data.
	67	*/
	68	if (!pblk_read_from_cache(pblk, bio, lba,
	69	rqd->ppa_list[i])) {
	70	if (i == 0) {
	71	/*
	72	* We didn't call with bio_advance()
	73	* yet, so we can just retry.
	74	*/
	75	goto retry;
	76	} else {
	77	/*
	78	* We already call bio_advance()
	79	* so we cannot retry and we need
	80	* to quit that function in order
	81	* to allow caller to handle the bio
	82	* splitting in the current sector
	83	* position.
	84	*/
	85	nr_secs = i;
	86	goto end;
	87	}
75cb8e93	88	}
faa79f27	89	meta->lba = cpu_to_le64(lba);
880eda54	90	#ifdef CONFIG_NVM_PBLK_DEBUG
db7ada33 JG	91	atomic_long_inc(&pblk->cache_reads);
db7ada33 JG	92	#endif
a4bd217b	93	}
a96de64a	94	bio_advance(bio, PBLK_EXPOSED_PAGE_SIZE);
a4bd217b JG	95	}
a4bd217b JG	96
a96de64a	97	end:
f9c10152	98	if (pblk_io_aligned(pblk, nr_secs))
d7b68016	99	rqd->is_seq = 1;
f9c10152	100
880eda54	101	#ifdef CONFIG_NVM_PBLK_DEBUG
a4bd217b JG	102	atomic_long_add(nr_secs, &pblk->inflight_reads);
a4bd217b JG	103	#endif
a96de64a IK	104
a96de64a IK	105	return nr_secs;
a4bd217b JG	106	}
a4bd217b JG	107
a4bd217b	108
03a34b2d JG	109	static void pblk_read_check_seq(struct pblk pblk, struct nvm_rq rqd,
03a34b2d JG	110	sector_t blba)
a4809fee	111	{
faa79f27	112	void *meta_list = rqd->meta_list;
03a34b2d	113	int nr_lbas = rqd->nr_ppas;
a4809fee JG	114	int i;
a4809fee JG	115
55d8ec35 IK	116	if (!pblk_is_oob_meta_supported(pblk))
	117	return;
	118
a4809fee	119	for (i = 0; i < nr_lbas; i++) {
faa79f27 IK	120	struct pblk_sec_meta *meta = pblk_get_meta(pblk, meta_list, i);
faa79f27 IK	121	u64 lba = le64_to_cpu(meta->lba);
a4809fee JG	122
	123	if (lba == ADDR_EMPTY)
	124	continue;
	125
03a34b2d	126	if (lba != blba + i) {
880eda54	127	#ifdef CONFIG_NVM_PBLK_DEBUG
d68a9344	128	struct ppa_addr *ppa_list = nvm_rq_to_ppa_list(rqd);
03a34b2d	129
d68a9344	130	print_ppa(pblk, &ppa_list[i], "seq", i);
03a34b2d	131	#endif
4e495a46	132	pblk_err(pblk, "corrupted read LBA (%llu/%llu)\n",
03a34b2d JG	133	lba, (u64)blba + i);
	134	WARN_ON(1);
	135	}
a4809fee JG	136	}
	137	}
	138
310df582 JG	139	/*
	140	* There can be holes in the lba list.
	141	*/
03a34b2d JG	142	static void pblk_read_check_rand(struct pblk pblk, struct nvm_rq rqd,
03a34b2d JG	143	u64 *lba_list, int nr_lbas)
310df582	144	{
faa79f27	145	void *meta_lba_list = rqd->meta_list;
310df582 JG	146	int i, j;
310df582 JG	147
55d8ec35 IK	148	if (!pblk_is_oob_meta_supported(pblk))
	149	return;
	150
310df582	151	for (i = 0, j = 0; i < nr_lbas; i++) {
faa79f27 IK	152	struct pblk_sec_meta *meta = pblk_get_meta(pblk,
faa79f27 IK	153	meta_lba_list, j);
310df582 JG	154	u64 lba = lba_list[i];
	155	u64 meta_lba;
	156
	157	if (lba == ADDR_EMPTY)
	158	continue;
	159
faa79f27	160	meta_lba = le64_to_cpu(meta->lba);
310df582 JG	161
310df582 JG	162	if (lba != meta_lba) {
880eda54	163	#ifdef CONFIG_NVM_PBLK_DEBUG
d68a9344	164	struct ppa_addr *ppa_list = nvm_rq_to_ppa_list(rqd);
03a34b2d	165
ea1d24bc	166	print_ppa(pblk, &ppa_list[j], "rnd", j);
03a34b2d	167	#endif
4e495a46	168	pblk_err(pblk, "corrupted read LBA (%llu/%llu)\n",
ea1d24bc	169	meta_lba, lba);
310df582 JG	170	WARN_ON(1);
310df582 JG	171	}
03a34b2d JG	172
03a34b2d JG	173	j++;
310df582	174	}
03a34b2d JG	175
03a34b2d JG	176	WARN_ONCE(j != rqd->nr_ppas, "pblk: corrupted random request\n");
310df582 JG	177	}
310df582 JG	178
d38954ed	179	static void pblk_end_user_read(struct bio *bio, int error)
7bd4d370	180	{
d38954ed IK	181	if (error && error != NVM_RSP_WARN_HIGHECC)
d38954ed IK	182	bio_io_error(bio);
3e03f632 IK	183	else
3e03f632 IK	184	bio_endio(bio);
7bd4d370 JG	185	}
	186
	187	static void __pblk_end_io_read(struct pblk pblk, struct nvm_rq rqd,
	188	bool put_line)
a4bd217b	189	{
084ec9ba	190	struct pblk_g_ctx *r_ctx = nvm_rq_to_pdu(rqd);
87cc40bb	191	struct bio *int_bio = rqd->bio;
998ba629 JG	192	unsigned long start_time = r_ctx->start_time;
998ba629 JG	193
a8e45650	194	bio_end_io_acct(int_bio, start_time);
a4bd217b JG	195
	196	if (rqd->error)
	197	pblk_log_read_err(pblk, rqd);
a4bd217b	198
03a34b2d	199	pblk_read_check_seq(pblk, rqd, r_ctx->lba);
a96de64a	200	bio_put(int_bio);
084ec9ba	201
7bd4d370	202	if (put_line)
ae14cc04	203	pblk_rq_to_line_put(pblk, rqd);
a4bd217b	204
880eda54	205	#ifdef CONFIG_NVM_PBLK_DEBUG
a4bd217b JG	206	atomic_long_add(rqd->nr_ppas, &pblk->sync_reads);
	207	atomic_long_sub(rqd->nr_ppas, &pblk->inflight_reads);
	208	#endif
	209
e2cddf20	210	pblk_free_rqd(pblk, rqd, PBLK_READ);
588726d3	211	atomic_dec(&pblk->inflight_io);
a4bd217b JG	212	}
a4bd217b JG	213
7bd4d370 JG	214	static void pblk_end_io_read(struct nvm_rq *rqd)
	215	{
	216	struct pblk *pblk = rqd->private;
87cc40bb JG	217	struct pblk_g_ctx *r_ctx = nvm_rq_to_pdu(rqd);
87cc40bb JG	218	struct bio bio = (struct bio )r_ctx->private;
7bd4d370	219
d38954ed	220	pblk_end_user_read(bio, rqd->error);
7bd4d370 JG	221	__pblk_end_io_read(pblk, rqd, true);
	222	}
	223
87cc40bb	224	static void pblk_read_rq(struct pblk pblk, struct nvm_rq rqd, struct bio *bio,
a96de64a	225	sector_t lba, bool *from_cache)
a4bd217b	226	{
faa79f27	227	struct pblk_sec_meta *meta = pblk_get_meta(pblk, rqd->meta_list, 0);
a4bd217b	228	struct ppa_addr ppa;
a4bd217b	229
a96de64a	230	pblk_lookup_l2p_seq(pblk, &ppa, lba, 1, from_cache);
a4bd217b	231
880eda54	232	#ifdef CONFIG_NVM_PBLK_DEBUG
a4bd217b JG	233	atomic_long_inc(&pblk->inflight_reads);
	234	#endif
	235
	236	retry:
	237	if (pblk_ppa_empty(ppa)) {
faa79f27 IK	238	__le64 addr_empty = cpu_to_le64(ADDR_EMPTY);
faa79f27 IK	239
faa79f27	240	meta->lba = addr_empty;
a4bd217b JG	241	return;
	242	}
	243
	244	/* Try to read from write buffer. The address is later checked on the
	245	* write buffer to prevent retrieving overwritten data.
	246	*/
	247	if (pblk_addr_in_cache(ppa)) {
a96de64a IK	248	if (!pblk_read_from_cache(pblk, bio, lba, ppa)) {
a96de64a IK	249	pblk_lookup_l2p_seq(pblk, &ppa, lba, 1, from_cache);
a4bd217b JG	250	goto retry;
a4bd217b JG	251	}
a4809fee	252
faa79f27	253	meta->lba = cpu_to_le64(lba);
a4809fee	254
880eda54	255	#ifdef CONFIG_NVM_PBLK_DEBUG
7bd4d370	256	atomic_long_inc(&pblk->cache_reads);
db7ada33	257	#endif
a4bd217b JG	258	} else {
	259	rqd->ppa_addr = ppa;
	260	}
	261	}
	262
3e03f632	263	void pblk_submit_read(struct pblk pblk, struct bio bio)
a4bd217b	264	{
84454e6d	265	sector_t blba = pblk_get_lba(bio);
5bf1e1ee	266	unsigned int nr_secs = pblk_get_secs(bio);
a96de64a	267	bool from_cache;
a4809fee	268	struct pblk_g_ctx *r_ctx;
a4bd217b	269	struct nvm_rq *rqd;
a96de64a	270	struct bio int_bio, split_bio;
a8e45650	271	unsigned long start_time;
a4bd217b	272
a8e45650	273	start_time = bio_start_io_acct(bio);
998ba629	274
e2cddf20	275	rqd = pblk_alloc_rqd(pblk, PBLK_READ);
a4bd217b JG	276
a4bd217b JG	277	rqd->opcode = NVM_OP_PREAD;
a4bd217b JG	278	rqd->nr_ppas = nr_secs;
	279	rqd->private = pblk;
	280	rqd->end_io = pblk_end_io_read;
	281
a4809fee	282	r_ctx = nvm_rq_to_pdu(rqd);
a8e45650	283	r_ctx->start_time = start_time;
a4809fee JG	284	r_ctx->lba = blba;
a4809fee JG	285
3e03f632 IK	286	if (pblk_alloc_rqd_meta(pblk, rqd)) {
	287	bio_io_error(bio);
	288	pblk_free_rqd(pblk, rqd, PBLK_READ);
	289	return;
	290	}
	291
	292	/* Clone read bio to deal internally with:
	293	* -read errors when reading from drive
a96de64a	294	* -bio_advance() calls during cache reads
3e03f632 IK	295	*/
3e03f632 IK	296	int_bio = bio_clone_fast(bio, GFP_KERNEL, &pblk_bio_set);
a4bd217b	297
45dcf29b	298	if (nr_secs > 1)
a96de64a IK	299	nr_secs = pblk_read_ppalist_rq(pblk, rqd, int_bio, blba,
a96de64a IK	300	&from_cache);
45dcf29b	301	else
a96de64a	302	pblk_read_rq(pblk, rqd, int_bio, blba, &from_cache);
a4bd217b	303
a96de64a	304	split_retry:
3e03f632 IK	305	r_ctx->private = bio; /* original bio */
	306	rqd->bio = int_bio; /* internal bio */
	307
a96de64a IK	308	if (from_cache && nr_secs == rqd->nr_ppas) {
a96de64a IK	309	/* All data was read from cache, we can complete the IO. */
3e03f632	310	pblk_end_user_read(bio, 0);
588726d3	311	atomic_inc(&pblk->inflight_io);
7bd4d370	312	__pblk_end_io_read(pblk, rqd, false);
a96de64a	313	} else if (nr_secs != rqd->nr_ppas) {
3e03f632 IK	314	/* The read bio request could be partially filled by the write
3e03f632 IK	315	* buffer, but there are some holes that need to be read from
a96de64a IK	316	* the drive. In order to handle this, we will use block layer
	317	* mechanism to split this request in to smaller ones and make
	318	* a chain of it.
3e03f632	319	*/
a96de64a IK	320	split_bio = bio_split(bio, nr_secs * NR_PHY_IN_LOG, GFP_KERNEL,
	321	&pblk_bio_set);
	322	bio_chain(split_bio, bio);
ed00aabd	323	submit_bio_noacct(bio);
a96de64a IK	324
	325	/* New bio contains first N sectors of the previous one, so
	326	* we can continue to use existing rqd, but we need to shrink
	327	* the number of PPAs in it. New bio is also guaranteed that
	328	* it contains only either data from cache or from drive, newer
	329	* mix of them.
	330	*/
	331	bio = split_bio;
	332	rqd->nr_ppas = nr_secs;
	333	if (rqd->nr_ppas == 1)
	334	rqd->ppa_addr = rqd->ppa_list[0];
a4bd217b	335
a96de64a IK	336	/* Recreate int_bio - existing might have some needed internal
	337	* fields modified already.
	338	*/
	339	bio_put(int_bio);
	340	int_bio = bio_clone_fast(bio, GFP_KERNEL, &pblk_bio_set);
	341	goto split_retry;
48e5da72	342	} else if (pblk_submit_io(pblk, rqd, NULL)) {
a96de64a IK	343	/* Submitting IO to drive failed, let's report an error */
	344	rqd->error = -ENODEV;
	345	pblk_end_io_read(rqd);
3e03f632	346	}
a4bd217b JG	347	}
	348
	349	static int read_ppalist_rq_gc(struct pblk pblk, struct nvm_rq rqd,
	350	struct pblk_line line, u64 lba_list,
d340121e	351	u64 *paddr_list_gc, unsigned int nr_secs)
a4bd217b	352	{
afdc23c9	353	struct ppa_addr ppa_list_l2p[NVM_MAX_VLBA];
d340121e	354	struct ppa_addr ppa_gc;
a4bd217b JG	355	int valid_secs = 0;
	356	int i;
	357
d340121e	358	pblk_lookup_l2p_rand(pblk, ppa_list_l2p, lba_list, nr_secs);
a4bd217b JG	359
a4bd217b JG	360	for (i = 0; i < nr_secs; i++) {
d340121e JG	361	if (lba_list[i] == ADDR_EMPTY)
	362	continue;
	363
	364	ppa_gc = addr_to_gen_ppa(pblk, paddr_list_gc[i], line->id);
	365	if (!pblk_ppa_comp(ppa_list_l2p[i], ppa_gc)) {
	366	paddr_list_gc[i] = lba_list[i] = ADDR_EMPTY;
a4bd217b JG	367	continue;
	368	}
	369
d340121e	370	rqd->ppa_list[valid_secs++] = ppa_list_l2p[i];
a4bd217b JG	371	}
a4bd217b JG	372
880eda54	373	#ifdef CONFIG_NVM_PBLK_DEBUG
a4bd217b JG	374	atomic_long_add(valid_secs, &pblk->inflight_reads);
a4bd217b JG	375	#endif
d340121e	376
a4bd217b JG	377	return valid_secs;
	378	}
	379
	380	static int read_rq_gc(struct pblk pblk, struct nvm_rq rqd,
d340121e JG	381	struct pblk_line *line, sector_t lba,
d340121e JG	382	u64 paddr_gc)
a4bd217b	383	{
d340121e	384	struct ppa_addr ppa_l2p, ppa_gc;
a4bd217b JG	385	int valid_secs = 0;
a4bd217b JG	386
659226eb DC	387	if (lba == ADDR_EMPTY)
	388	goto out;
	389
a4bd217b	390	/* logic error: lba out-of-bounds */
847a3a27	391	if (lba >= pblk->capacity) {
2a79efd8	392	WARN(1, "pblk: read lba out of bounds\n");
a4bd217b JG	393	goto out;
	394	}
	395
a4bd217b	396	spin_lock(&pblk->trans_lock);
d340121e	397	ppa_l2p = pblk_trans_map_get(pblk, lba);
a4bd217b JG	398	spin_unlock(&pblk->trans_lock);
a4bd217b JG	399
d340121e JG	400	ppa_gc = addr_to_gen_ppa(pblk, paddr_gc, line->id);
d340121e JG	401	if (!pblk_ppa_comp(ppa_l2p, ppa_gc))
a4bd217b JG	402	goto out;
a4bd217b JG	403
d340121e	404	rqd->ppa_addr = ppa_l2p;
a4bd217b JG	405	valid_secs = 1;
a4bd217b JG	406
880eda54	407	#ifdef CONFIG_NVM_PBLK_DEBUG
a4bd217b JG	408	atomic_long_inc(&pblk->inflight_reads);
	409	#endif
	410
	411	out:
	412	return valid_secs;
	413	}
	414
d340121e	415	int pblk_submit_read_gc(struct pblk pblk, struct pblk_gc_rq gc_rq)
a4bd217b	416	{
a4bd217b	417	struct nvm_rq rqd;
d340121e	418	int ret = NVM_IO_OK;
a4bd217b JG	419
	420	memset(&rqd, 0, sizeof(struct nvm_rq));
	421
45dcf29b JG	422	ret = pblk_alloc_rqd_meta(pblk, &rqd);
	423	if (ret)
	424	return ret;
63e3809c	425
d340121e	426	if (gc_rq->nr_secs > 1) {
d340121e JG	427	gc_rq->secs_to_gc = read_ppalist_rq_gc(pblk, &rqd, gc_rq->line,
	428	gc_rq->lba_list,
	429	gc_rq->paddr_list,
	430	gc_rq->nr_secs);
	431	if (gc_rq->secs_to_gc == 1)
63e3809c	432	rqd.ppa_addr = rqd.ppa_list[0];
a4bd217b	433	} else {
d340121e JG	434	gc_rq->secs_to_gc = read_rq_gc(pblk, &rqd, gc_rq->line,
	435	gc_rq->lba_list[0],
	436	gc_rq->paddr_list[0]);
a4bd217b JG	437	}
a4bd217b JG	438
d340121e	439	if (!(gc_rq->secs_to_gc))
a4bd217b JG	440	goto out;
a4bd217b JG	441
a4bd217b	442	rqd.opcode = NVM_OP_PREAD;
d340121e	443	rqd.nr_ppas = gc_rq->secs_to_gc;
a4bd217b	444
48e5da72	445	if (pblk_submit_io_sync(pblk, &rqd, gc_rq->data)) {
d340121e	446	ret = -EIO;
48e5da72	447	goto err_free_dma;
a4bd217b JG	448	}
a4bd217b JG	449
03a34b2d	450	pblk_read_check_rand(pblk, &rqd, gc_rq->lba_list, gc_rq->nr_secs);
310df582	451
588726d3	452	atomic_dec(&pblk->inflight_io);
a4bd217b JG	453
	454	if (rqd.error) {
	455	atomic_long_inc(&pblk->read_failed_gc);
880eda54	456	#ifdef CONFIG_NVM_PBLK_DEBUG
a4bd217b JG	457	pblk_print_failed_rqd(pblk, &rqd, rqd.error);
	458	#endif
	459	}
	460
880eda54	461	#ifdef CONFIG_NVM_PBLK_DEBUG
d340121e JG	462	atomic_long_add(gc_rq->secs_to_gc, &pblk->sync_reads);
	463	atomic_long_add(gc_rq->secs_to_gc, &pblk->recov_gc_reads);
	464	atomic_long_sub(gc_rq->secs_to_gc, &pblk->inflight_reads);
a4bd217b JG	465	#endif
	466
	467	out:
45dcf29b	468	pblk_free_rqd_meta(pblk, &rqd);
d340121e	469	return ret;
a4bd217b JG	470
a4bd217b JG	471	err_free_dma:
45dcf29b	472	pblk_free_rqd_meta(pblk, &rqd);
d340121e	473	return ret;
a4bd217b	474	}