[linux-2.6-block.git] / drivers / lightnvm / pblk-gc.c

/*
 * Copyright (C) 2016 CNEX Labs
 * Initial release: Javier Gonzalez <javier@cnexlabs.com>
 *                  Matias Bjorling <matias@cnexlabs.com>
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License version
 * 2 as published by the Free Software Foundation.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * pblk-gc.c - pblk's garbage collector
 */

#include "pblk.h"
#include <linux/delay.h>

static void pblk_gc_free_gc_rq(struct pblk_gc_rq *gc_rq)
{
	if (gc_rq->data)
		vfree(gc_rq->data);
	kfree(gc_rq);
}

static int pblk_gc_write(struct pblk *pblk)
{
	struct pblk_gc *gc = &pblk->gc;
	struct pblk_gc_rq *gc_rq, *tgc_rq;
	LIST_HEAD(w_list);

	spin_lock(&gc->w_lock);
	if (list_empty(&gc->w_list)) {
		spin_unlock(&gc->w_lock);
		return 1;
	}

	list_cut_position(&w_list, &gc->w_list, gc->w_list.prev);
	gc->w_entries = 0;
	spin_unlock(&gc->w_lock);

	list_for_each_entry_safe(gc_rq, tgc_rq, &w_list, list) {
		pblk_write_gc_to_cache(pblk, gc_rq);
		list_del(&gc_rq->list);
		kref_put(&gc_rq->line->ref, pblk_line_put);
		pblk_gc_free_gc_rq(gc_rq);
	}

	return 0;
}

static void pblk_gc_writer_kick(struct pblk_gc *gc)
{
	wake_up_process(gc->gc_writer_ts);
}

static void pblk_put_line_back(struct pblk *pblk, struct pblk_line *line)
{
	struct pblk_line_mgmt *l_mg = &pblk->l_mg;
	struct list_head *move_list;

	spin_lock(&line->lock);
	WARN_ON(line->state != PBLK_LINESTATE_GC);
	line->state = PBLK_LINESTATE_CLOSED;
	move_list = pblk_line_gc_list(pblk, line);
	spin_unlock(&line->lock);

	if (move_list) {
		spin_lock(&l_mg->gc_lock);
		list_add_tail(&line->list, move_list);
		spin_unlock(&l_mg->gc_lock);
	}
}

static void pblk_gc_line_ws(struct work_struct *work)
{
	struct pblk_line_ws *gc_rq_ws = container_of(work,
						struct pblk_line_ws, ws);
	struct pblk *pblk = gc_rq_ws->pblk;
	struct nvm_tgt_dev *dev = pblk->dev;
	struct nvm_geo *geo = &dev->geo;
	struct pblk_gc *gc = &pblk->gc;
	struct pblk_line *line = gc_rq_ws->line;
	struct pblk_gc_rq *gc_rq = gc_rq_ws->priv;
	int ret;

	up(&gc->gc_sem);

	gc_rq->data = vmalloc(array_size(gc_rq->nr_secs, geo->csecs));
	if (!gc_rq->data) {
		pr_err("pblk: could not GC line:%d (%d/%d)\n",
					line->id, *line->vsc, gc_rq->nr_secs);
		goto out;
	}

	/* Read from GC victim block */
	ret = pblk_submit_read_gc(pblk, gc_rq);
	if (ret) {
		pr_err("pblk: failed GC read in line:%d (err:%d)\n",
								line->id, ret);
		goto out;
	}

	if (!gc_rq->secs_to_gc)
		goto out;

retry:
	spin_lock(&gc->w_lock);
	if (gc->w_entries >= PBLK_GC_RQ_QD) {
		spin_unlock(&gc->w_lock);
		pblk_gc_writer_kick(&pblk->gc);
		usleep_range(128, 256);
		goto retry;
	}
	gc->w_entries++;
	list_add_tail(&gc_rq->list, &gc->w_list);
	spin_unlock(&gc->w_lock);

	pblk_gc_writer_kick(&pblk->gc);

	kfree(gc_rq_ws);
	return;

out:
	pblk_gc_free_gc_rq(gc_rq);
	kref_put(&line->ref, pblk_line_put);
	kfree(gc_rq_ws);
}

static __le64 *get_lba_list_from_emeta(struct pblk *pblk,
				       struct pblk_line *line)
{
	struct line_emeta *emeta_buf;
	struct pblk_line_mgmt *l_mg = &pblk->l_mg;
	struct pblk_line_meta *lm = &pblk->lm;
	unsigned int lba_list_size = lm->emeta_len[2];
	__le64 *lba_list;
	int ret;

	emeta_buf = pblk_malloc(lm->emeta_len[0],
				l_mg->emeta_alloc_type, GFP_KERNEL);
	if (!emeta_buf)
		return NULL;

	ret = pblk_line_read_emeta(pblk, line, emeta_buf);
	if (ret) {
		pr_err("pblk: line %d read emeta failed (%d)\n",
				line->id, ret);
		pblk_mfree(emeta_buf, l_mg->emeta_alloc_type);
		return NULL;
	}

	/* If this read fails, it means that emeta is corrupted.
	 * For now, leave the line untouched.
	 * TODO: Implement a recovery routine that scans and moves
	 * all sectors on the line.
	 */

	ret = pblk_recov_check_emeta(pblk, emeta_buf);
	if (ret) {
		pr_err("pblk: inconsistent emeta (line %d)\n",
				line->id);
		pblk_mfree(emeta_buf, l_mg->emeta_alloc_type);
		return NULL;
	}

	lba_list = pblk_malloc(lba_list_size,
			       l_mg->emeta_alloc_type, GFP_KERNEL);
	if (lba_list)
		memcpy(lba_list, emeta_to_lbas(pblk, emeta_buf), lba_list_size);

	pblk_mfree(emeta_buf, l_mg->emeta_alloc_type);

	return lba_list;
}

static void pblk_gc_line_prepare_ws(struct work_struct *work)
{
	struct pblk_line_ws *line_ws = container_of(work, struct pblk_line_ws,
									ws);
	struct pblk *pblk = line_ws->pblk;
	struct pblk_line *line = line_ws->line;
	struct pblk_line_mgmt *l_mg = &pblk->l_mg;
	struct pblk_line_meta *lm = &pblk->lm;
	struct pblk_gc *gc = &pblk->gc;
	struct pblk_line_ws *gc_rq_ws;
	struct pblk_gc_rq *gc_rq;
	__le64 *lba_list;
	unsigned long *invalid_bitmap;
	int sec_left, nr_secs, bit;

	invalid_bitmap = kmalloc(lm->sec_bitmap_len, GFP_KERNEL);
	if (!invalid_bitmap)
		goto fail_free_ws;

	if (line->w_err_gc->has_write_err) {
		lba_list = line->w_err_gc->lba_list;
		line->w_err_gc->lba_list = NULL;
	} else {
		lba_list = get_lba_list_from_emeta(pblk, line);
		if (!lba_list) {
			pr_err("pblk: could not interpret emeta (line %d)\n",
					line->id);
			goto fail_free_invalid_bitmap;
		}
	}

	spin_lock(&line->lock);
	bitmap_copy(invalid_bitmap, line->invalid_bitmap, lm->sec_per_line);
	sec_left = pblk_line_vsc(line);
	spin_unlock(&line->lock);

	if (sec_left < 0) {
		pr_err("pblk: corrupted GC line (%d)\n", line->id);
		goto fail_free_lba_list;
	}

	bit = -1;
next_rq:
	gc_rq = kmalloc(sizeof(struct pblk_gc_rq), GFP_KERNEL);
	if (!gc_rq)
		goto fail_free_lba_list;

	nr_secs = 0;
	do {
		bit = find_next_zero_bit(invalid_bitmap, lm->sec_per_line,
								bit + 1);
		if (bit > line->emeta_ssec)
			break;

		gc_rq->paddr_list[nr_secs] = bit;
		gc_rq->lba_list[nr_secs++] = le64_to_cpu(lba_list[bit]);
	} while (nr_secs < pblk->max_write_pgs);

	if (unlikely(!nr_secs)) {
		kfree(gc_rq);
		goto out;
	}

	gc_rq->nr_secs = nr_secs;
	gc_rq->line = line;

	gc_rq_ws = kmalloc(sizeof(struct pblk_line_ws), GFP_KERNEL);
	if (!gc_rq_ws)
		goto fail_free_gc_rq;

	gc_rq_ws->pblk = pblk;
	gc_rq_ws->line = line;
	gc_rq_ws->priv = gc_rq;

	/* The write GC path can be much slower than the read GC one due to
	 * the budget imposed by the rate-limiter. Balance in case that we get
	 * back pressure from the write GC path.
	 */
	while (down_timeout(&gc->gc_sem, msecs_to_jiffies(30000)))
		io_schedule();

	kref_get(&line->ref);

	INIT_WORK(&gc_rq_ws->ws, pblk_gc_line_ws);
	queue_work(gc->gc_line_reader_wq, &gc_rq_ws->ws);

	sec_left -= nr_secs;
	if (sec_left > 0)
		goto next_rq;

out:
	pblk_mfree(lba_list, l_mg->emeta_alloc_type);
	kfree(line_ws);
	kfree(invalid_bitmap);

	kref_put(&line->ref, pblk_line_put);
	atomic_dec(&gc->read_inflight_gc);

	return;

fail_free_gc_rq:
	kfree(gc_rq);
fail_free_lba_list:
	pblk_mfree(lba_list, l_mg->emeta_alloc_type);
fail_free_invalid_bitmap:
	kfree(invalid_bitmap);
fail_free_ws:
	kfree(line_ws);

	pblk_put_line_back(pblk, line);
	kref_put(&line->ref, pblk_line_put);
	atomic_dec(&gc->read_inflight_gc);

	pr_err("pblk: Failed to GC line %d\n", line->id);
}

static int pblk_gc_line(struct pblk *pblk, struct pblk_line *line)
{
	struct pblk_gc *gc = &pblk->gc;
	struct pblk_line_ws *line_ws;

	pr_debug("pblk: line '%d' being reclaimed for GC\n", line->id);

	line_ws = kmalloc(sizeof(struct pblk_line_ws), GFP_KERNEL);
	if (!line_ws)
		return -ENOMEM;

	line_ws->pblk = pblk;
	line_ws->line = line;

	atomic_inc(&gc->pipeline_gc);
	INIT_WORK(&line_ws->ws, pblk_gc_line_prepare_ws);
	queue_work(gc->gc_reader_wq, &line_ws->ws);

	return 0;
}

static void pblk_gc_reader_kick(struct pblk_gc *gc)
{
	wake_up_process(gc->gc_reader_ts);
}

static void pblk_gc_kick(struct pblk *pblk)
{
	struct pblk_gc *gc = &pblk->gc;

	pblk_gc_writer_kick(gc);
	pblk_gc_reader_kick(gc);

	/* If we're shutting down GC, let's not start it up again */
	if (gc->gc_enabled) {
		wake_up_process(gc->gc_ts);
		mod_timer(&gc->gc_timer,
			  jiffies + msecs_to_jiffies(GC_TIME_MSECS));
	}
}

static int pblk_gc_read(struct pblk *pblk)
{
	struct pblk_gc *gc = &pblk->gc;
	struct pblk_line *line;

	spin_lock(&gc->r_lock);
	if (list_empty(&gc->r_list)) {
		spin_unlock(&gc->r_lock);
		return 1;
	}

	line = list_first_entry(&gc->r_list, struct pblk_line, list);
	list_del(&line->list);
	spin_unlock(&gc->r_lock);

	pblk_gc_kick(pblk);

	if (pblk_gc_line(pblk, line))
		pr_err("pblk: failed to GC line %d\n", line->id);

	return 0;
}

static struct pblk_line *pblk_gc_get_victim_line(struct pblk *pblk,
						 struct list_head *group_list)
{
	struct pblk_line *line, *victim;
	int line_vsc, victim_vsc;

	victim = list_first_entry(group_list, struct pblk_line, list);
	list_for_each_entry(line, group_list, list) {
		line_vsc = le32_to_cpu(*line->vsc);
		victim_vsc = le32_to_cpu(*victim->vsc);
		if (line_vsc < victim_vsc)
			victim = line;
	}

	return victim;
}

static bool pblk_gc_should_run(struct pblk_gc *gc, struct pblk_rl *rl)
{
	unsigned int nr_blocks_free, nr_blocks_need;
	unsigned int werr_lines = atomic_read(&rl->werr_lines);

	nr_blocks_need = pblk_rl_high_thrs(rl);
	nr_blocks_free = pblk_rl_nr_free_blks(rl);

	/* This is not critical, no need to take lock here */
	return ((werr_lines > 0) ||
		((gc->gc_active) && (nr_blocks_need > nr_blocks_free)));
}

void pblk_gc_free_full_lines(struct pblk *pblk)
{
	struct pblk_line_mgmt *l_mg = &pblk->l_mg;
	struct pblk_gc *gc = &pblk->gc;
	struct pblk_line *line;

	do {
		spin_lock(&l_mg->gc_lock);
		if (list_empty(&l_mg->gc_full_list)) {
			spin_unlock(&l_mg->gc_lock);
			return;
		}

		line = list_first_entry(&l_mg->gc_full_list,
							struct pblk_line, list);

		spin_lock(&line->lock);
		WARN_ON(line->state != PBLK_LINESTATE_CLOSED);
		line->state = PBLK_LINESTATE_GC;
		spin_unlock(&line->lock);

		list_del(&line->list);
		spin_unlock(&l_mg->gc_lock);

		atomic_inc(&gc->pipeline_gc);
		kref_put(&line->ref, pblk_line_put);
	} while (1);
}

/*
 * Lines with no valid sectors will be returned to the free list immediately. If
 * GC is activated - either because the free block count is under the determined
 * threshold, or because it is being forced from user space - only lines with a
 * high count of invalid sectors will be recycled.
 */
static void pblk_gc_run(struct pblk *pblk)
{
	struct pblk_line_mgmt *l_mg = &pblk->l_mg;
	struct pblk_gc *gc = &pblk->gc;
	struct pblk_line *line;
	struct list_head *group_list;
	bool run_gc;
	int read_inflight_gc, gc_group = 0, prev_group = 0;

	pblk_gc_free_full_lines(pblk);

	run_gc = pblk_gc_should_run(&pblk->gc, &pblk->rl);
	if (!run_gc || (atomic_read(&gc->read_inflight_gc) >= PBLK_GC_L_QD))
		return;

next_gc_group:
	group_list = l_mg->gc_lists[gc_group++];

	do {
		spin_lock(&l_mg->gc_lock);
		if (list_empty(group_list)) {
			spin_unlock(&l_mg->gc_lock);
			break;
		}

		line = pblk_gc_get_victim_line(pblk, group_list);

		spin_lock(&line->lock);
		WARN_ON(line->state != PBLK_LINESTATE_CLOSED);
		line->state = PBLK_LINESTATE_GC;
		spin_unlock(&line->lock);

		list_del(&line->list);
		spin_unlock(&l_mg->gc_lock);

		spin_lock(&gc->r_lock);
		list_add_tail(&line->list, &gc->r_list);
		spin_unlock(&gc->r_lock);

		read_inflight_gc = atomic_inc_return(&gc->read_inflight_gc);
		pblk_gc_reader_kick(gc);

		prev_group = 1;

		/* No need to queue up more GC lines than we can handle */
		run_gc = pblk_gc_should_run(&pblk->gc, &pblk->rl);
		if (!run_gc || read_inflight_gc >= PBLK_GC_L_QD)
			break;
	} while (1);

	if (!prev_group && pblk->rl.rb_state > gc_group &&
						gc_group < PBLK_GC_NR_LISTS)
		goto next_gc_group;
}

static void pblk_gc_timer(struct timer_list *t)
{
	struct pblk *pblk = from_timer(pblk, t, gc.gc_timer);

	pblk_gc_kick(pblk);
}

static int pblk_gc_ts(void *data)
{
	struct pblk *pblk = data;

	while (!kthread_should_stop()) {
		pblk_gc_run(pblk);
		set_current_state(TASK_INTERRUPTIBLE);
		io_schedule();
	}

	return 0;
}

static int pblk_gc_writer_ts(void *data)
{
	struct pblk *pblk = data;

	while (!kthread_should_stop()) {
		if (!pblk_gc_write(pblk))
			continue;
		set_current_state(TASK_INTERRUPTIBLE);
		io_schedule();
	}

	return 0;
}

static int pblk_gc_reader_ts(void *data)
{
	struct pblk *pblk = data;
	struct pblk_gc *gc = &pblk->gc;

	while (!kthread_should_stop()) {
		if (!pblk_gc_read(pblk))
			continue;
		set_current_state(TASK_INTERRUPTIBLE);
		io_schedule();
	}

#ifdef CONFIG_NVM_DEBUG
	pr_info("pblk: flushing gc pipeline, %d lines left\n",
		atomic_read(&gc->pipeline_gc));
#endif

	do {
		if (!atomic_read(&gc->pipeline_gc))
			break;

		schedule();
	} while (1);

	return 0;
}

static void pblk_gc_start(struct pblk *pblk)
{
	pblk->gc.gc_active = 1;
	pr_debug("pblk: gc start\n");
}

void pblk_gc_should_start(struct pblk *pblk)
{
	struct pblk_gc *gc = &pblk->gc;

	if (gc->gc_enabled && !gc->gc_active) {
		pblk_gc_start(pblk);
		pblk_gc_kick(pblk);
	}
}

void pblk_gc_should_stop(struct pblk *pblk)
{
	struct pblk_gc *gc = &pblk->gc;

	if (gc->gc_active && !gc->gc_forced)
		gc->gc_active = 0;
}

void pblk_gc_should_kick(struct pblk *pblk)
{
	pblk_rl_update_rates(&pblk->rl);
}

void pblk_gc_sysfs_state_show(struct pblk *pblk, int *gc_enabled,
			      int *gc_active)
{
	struct pblk_gc *gc = &pblk->gc;

	spin_lock(&gc->lock);
	*gc_enabled = gc->gc_enabled;
	*gc_active = gc->gc_active;
	spin_unlock(&gc->lock);
}

int pblk_gc_sysfs_force(struct pblk *pblk, int force)
{
	struct pblk_gc *gc = &pblk->gc;

	if (force < 0 || force > 1)
		return -EINVAL;

	spin_lock(&gc->lock);
	gc->gc_forced = force;

	if (force)
		gc->gc_enabled = 1;
	else
		gc->gc_enabled = 0;
	spin_unlock(&gc->lock);

	pblk_gc_should_start(pblk);

	return 0;
}

int pblk_gc_init(struct pblk *pblk)
{
	struct pblk_gc *gc = &pblk->gc;
	int ret;

	gc->gc_ts = kthread_create(pblk_gc_ts, pblk, "pblk-gc-ts");
	if (IS_ERR(gc->gc_ts)) {
		pr_err("pblk: could not allocate GC main kthread\n");
		return PTR_ERR(gc->gc_ts);
	}

	gc->gc_writer_ts = kthread_create(pblk_gc_writer_ts, pblk,
							"pblk-gc-writer-ts");
	if (IS_ERR(gc->gc_writer_ts)) {
		pr_err("pblk: could not allocate GC writer kthread\n");
		ret = PTR_ERR(gc->gc_writer_ts);
		goto fail_free_main_kthread;
	}

	gc->gc_reader_ts = kthread_create(pblk_gc_reader_ts, pblk,
							"pblk-gc-reader-ts");
	if (IS_ERR(gc->gc_reader_ts)) {
		pr_err("pblk: could not allocate GC reader kthread\n");
		ret = PTR_ERR(gc->gc_reader_ts);
		goto fail_free_writer_kthread;
	}

	timer_setup(&gc->gc_timer, pblk_gc_timer, 0);
	mod_timer(&gc->gc_timer, jiffies + msecs_to_jiffies(GC_TIME_MSECS));

	gc->gc_active = 0;
	gc->gc_forced = 0;
	gc->gc_enabled = 1;
	gc->w_entries = 0;
	atomic_set(&gc->read_inflight_gc, 0);
	atomic_set(&gc->pipeline_gc, 0);

	/* Workqueue that reads valid sectors from a line and submit them to the
	 * GC writer to be recycled.
	 */
	gc->gc_line_reader_wq = alloc_workqueue("pblk-gc-line-reader-wq",
			WQ_MEM_RECLAIM | WQ_UNBOUND, PBLK_GC_MAX_READERS);
	if (!gc->gc_line_reader_wq) {
		pr_err("pblk: could not allocate GC line reader workqueue\n");
		ret = -ENOMEM;
		goto fail_free_reader_kthread;
	}

	/* Workqueue that prepare lines for GC */
	gc->gc_reader_wq = alloc_workqueue("pblk-gc-line_wq",
					WQ_MEM_RECLAIM | WQ_UNBOUND, 1);
	if (!gc->gc_reader_wq) {
		pr_err("pblk: could not allocate GC reader workqueue\n");
		ret = -ENOMEM;
		goto fail_free_reader_line_wq;
	}

	spin_lock_init(&gc->lock);
	spin_lock_init(&gc->w_lock);
	spin_lock_init(&gc->r_lock);

	sema_init(&gc->gc_sem, PBLK_GC_RQ_QD);

	INIT_LIST_HEAD(&gc->w_list);
	INIT_LIST_HEAD(&gc->r_list);

	return 0;

fail_free_reader_line_wq:
	destroy_workqueue(gc->gc_line_reader_wq);
fail_free_reader_kthread:
	kthread_stop(gc->gc_reader_ts);
fail_free_writer_kthread:
	kthread_stop(gc->gc_writer_ts);
fail_free_main_kthread:
	kthread_stop(gc->gc_ts);

	return ret;
}

void pblk_gc_exit(struct pblk *pblk, bool graceful)
{
	struct pblk_gc *gc = &pblk->gc;

	gc->gc_enabled = 0;
	del_timer_sync(&gc->gc_timer);
	gc->gc_active = 0;

	if (gc->gc_ts)
		kthread_stop(gc->gc_ts);

	if (gc->gc_reader_ts)
		kthread_stop(gc->gc_reader_ts);

	if (graceful) {
		flush_workqueue(gc->gc_reader_wq);
		flush_workqueue(gc->gc_line_reader_wq);
	}

	destroy_workqueue(gc->gc_reader_wq);
	destroy_workqueue(gc->gc_line_reader_wq);

	if (gc->gc_writer_ts)
		kthread_stop(gc->gc_writer_ts);
}
Commit	Line	Data
a4bd217b JG	1	/*
	2	* Copyright (C) 2016 CNEX Labs
	3	* Initial release: Javier Gonzalez <javier@cnexlabs.com>
	4	* Matias Bjorling <matias@cnexlabs.com>
	5	*
	6	* This program is free software; you can redistribute it and/or
	7	* modify it under the terms of the GNU General Public License version
	8	* 2 as published by the Free Software Foundation.
	9	*
	10	* This program is distributed in the hope that it will be useful, but
	11	* WITHOUT ANY WARRANTY; without even the implied warranty of
	12	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	13	* General Public License for more details.
	14	*
	15	* pblk-gc.c - pblk's garbage collector
	16	*/
	17
	18	#include "pblk.h"
	19	#include <linux/delay.h>
	20
	21	static void pblk_gc_free_gc_rq(struct pblk_gc_rq *gc_rq)
	22	{
d340121e JG	23	if (gc_rq->data)
d340121e JG	24	vfree(gc_rq->data);
a4bd217b JG	25	kfree(gc_rq);
	26	}
	27
	28	static int pblk_gc_write(struct pblk *pblk)
	29	{
	30	struct pblk_gc *gc = &pblk->gc;
	31	struct pblk_gc_rq gc_rq, tgc_rq;
	32	LIST_HEAD(w_list);
	33
	34	spin_lock(&gc->w_lock);
	35	if (list_empty(&gc->w_list)) {
	36	spin_unlock(&gc->w_lock);
	37	return 1;
	38	}
	39
b20ba1bc JG	40	list_cut_position(&w_list, &gc->w_list, gc->w_list.prev);
b20ba1bc JG	41	gc->w_entries = 0;
a4bd217b JG	42	spin_unlock(&gc->w_lock);
	43
	44	list_for_each_entry_safe(gc_rq, tgc_rq, &w_list, list) {
d340121e	45	pblk_write_gc_to_cache(pblk, gc_rq);
a4bd217b	46	list_del(&gc_rq->list);
b20ba1bc	47	kref_put(&gc_rq->line->ref, pblk_line_put);
a4bd217b JG	48	pblk_gc_free_gc_rq(gc_rq);
	49	}
	50
	51	return 0;
	52	}
	53
	54	static void pblk_gc_writer_kick(struct pblk_gc *gc)
	55	{
	56	wake_up_process(gc->gc_writer_ts);
	57	}
	58
a4bd217b JG	59	static void pblk_put_line_back(struct pblk pblk, struct pblk_line line)
	60	{
	61	struct pblk_line_mgmt *l_mg = &pblk->l_mg;
	62	struct list_head *move_list;
	63
	64	spin_lock(&line->lock);
	65	WARN_ON(line->state != PBLK_LINESTATE_GC);
	66	line->state = PBLK_LINESTATE_CLOSED;
	67	move_list = pblk_line_gc_list(pblk, line);
	68	spin_unlock(&line->lock);
	69
	70	if (move_list) {
	71	spin_lock(&l_mg->gc_lock);
	72	list_add_tail(&line->list, move_list);
	73	spin_unlock(&l_mg->gc_lock);
	74	}
	75	}
	76
	77	static void pblk_gc_line_ws(struct work_struct *work)
b20ba1bc	78	{
b84ae4a8	79	struct pblk_line_ws *gc_rq_ws = container_of(work,
b20ba1bc	80	struct pblk_line_ws, ws);
b84ae4a8	81	struct pblk *pblk = gc_rq_ws->pblk;
2a19b10d JG	82	struct nvm_tgt_dev *dev = pblk->dev;
2a19b10d JG	83	struct nvm_geo *geo = &dev->geo;
b20ba1bc	84	struct pblk_gc *gc = &pblk->gc;
b84ae4a8 JG	85	struct pblk_line *line = gc_rq_ws->line;
b84ae4a8 JG	86	struct pblk_gc_rq *gc_rq = gc_rq_ws->priv;
2a19b10d	87	int ret;
b20ba1bc JG	88
	89	up(&gc->gc_sem);
	90
42bc47b3	91	gc_rq->data = vmalloc(array_size(gc_rq->nr_secs, geo->csecs));
2a19b10d JG	92	if (!gc_rq->data) {
	93	pr_err("pblk: could not GC line:%d (%d/%d)\n",
	94	line->id, *line->vsc, gc_rq->nr_secs);
	95	goto out;
	96	}
	97
	98	/* Read from GC victim block */
	99	ret = pblk_submit_read_gc(pblk, gc_rq);
	100	if (ret) {
	101	pr_err("pblk: failed GC read in line:%d (err:%d)\n",
	102	line->id, ret);
	103	goto out;
	104	}
	105
	106	if (!gc_rq->secs_to_gc)
	107	goto out;
	108
	109	retry:
	110	spin_lock(&gc->w_lock);
	111	if (gc->w_entries >= PBLK_GC_RQ_QD) {
	112	spin_unlock(&gc->w_lock);
	113	pblk_gc_writer_kick(&pblk->gc);
	114	usleep_range(128, 256);
	115	goto retry;
b20ba1bc	116	}
2a19b10d JG	117	gc->w_entries++;
	118	list_add_tail(&gc_rq->list, &gc->w_list);
	119	spin_unlock(&gc->w_lock);
	120
	121	pblk_gc_writer_kick(&pblk->gc);
b20ba1bc	122
b84ae4a8	123	kfree(gc_rq_ws);
2a19b10d JG	124	return;
	125
	126	out:
	127	pblk_gc_free_gc_rq(gc_rq);
	128	kref_put(&line->ref, pblk_line_put);
	129	kfree(gc_rq_ws);
b20ba1bc JG	130	}
b20ba1bc JG	131
48b8d208 HH	132	static __le64 get_lba_list_from_emeta(struct pblk pblk,
	133	struct pblk_line *line)
	134	{
	135	struct line_emeta *emeta_buf;
	136	struct pblk_line_mgmt *l_mg = &pblk->l_mg;
	137	struct pblk_line_meta *lm = &pblk->lm;
	138	unsigned int lba_list_size = lm->emeta_len[2];
	139	__le64 *lba_list;
	140	int ret;
	141
	142	emeta_buf = pblk_malloc(lm->emeta_len[0],
	143	l_mg->emeta_alloc_type, GFP_KERNEL);
	144	if (!emeta_buf)
	145	return NULL;
	146
	147	ret = pblk_line_read_emeta(pblk, line, emeta_buf);
	148	if (ret) {
	149	pr_err("pblk: line %d read emeta failed (%d)\n",
	150	line->id, ret);
	151	pblk_mfree(emeta_buf, l_mg->emeta_alloc_type);
	152	return NULL;
	153	}
	154
	155	/* If this read fails, it means that emeta is corrupted.
	156	* For now, leave the line untouched.
	157	* TODO: Implement a recovery routine that scans and moves
	158	* all sectors on the line.
	159	*/
	160
	161	ret = pblk_recov_check_emeta(pblk, emeta_buf);
	162	if (ret) {
	163	pr_err("pblk: inconsistent emeta (line %d)\n",
	164	line->id);
	165	pblk_mfree(emeta_buf, l_mg->emeta_alloc_type);
	166	return NULL;
	167	}
	168
	169	lba_list = pblk_malloc(lba_list_size,
	170	l_mg->emeta_alloc_type, GFP_KERNEL);
	171	if (lba_list)
	172	memcpy(lba_list, emeta_to_lbas(pblk, emeta_buf), lba_list_size);
	173
	174	pblk_mfree(emeta_buf, l_mg->emeta_alloc_type);
	175
	176	return lba_list;
	177	}
	178
b20ba1bc	179	static void pblk_gc_line_prepare_ws(struct work_struct *work)
a4bd217b JG	180	{
	181	struct pblk_line_ws *line_ws = container_of(work, struct pblk_line_ws,
	182	ws);
	183	struct pblk *pblk = line_ws->pblk;
a4bd217b	184	struct pblk_line *line = line_ws->line;
b20ba1bc	185	struct pblk_line_mgmt *l_mg = &pblk->l_mg;
a4bd217b	186	struct pblk_line_meta *lm = &pblk->lm;
b20ba1bc	187	struct pblk_gc *gc = &pblk->gc;
b84ae4a8	188	struct pblk_line_ws *gc_rq_ws;
b20ba1bc	189	struct pblk_gc_rq *gc_rq;
dd2a4343	190	__le64 *lba_list;
d340121e	191	unsigned long *invalid_bitmap;
b20ba1bc	192	int sec_left, nr_secs, bit;
a4bd217b	193
d340121e	194	invalid_bitmap = kmalloc(lm->sec_bitmap_len, GFP_KERNEL);
5da84cf6	195	if (!invalid_bitmap)
d340121e	196	goto fail_free_ws;
d340121e	197
48b8d208 HH	198	if (line->w_err_gc->has_write_err) {
	199	lba_list = line->w_err_gc->lba_list;
	200	line->w_err_gc->lba_list = NULL;
	201	} else {
	202	lba_list = get_lba_list_from_emeta(pblk, line);
	203	if (!lba_list) {
	204	pr_err("pblk: could not interpret emeta (line %d)\n",
	205	line->id);
0ec6937e	206	goto fail_free_invalid_bitmap;
48b8d208	207	}
a4bd217b	208	}
a4bd217b	209
d340121e JG	210	spin_lock(&line->lock);
d340121e JG	211	bitmap_copy(invalid_bitmap, line->invalid_bitmap, lm->sec_per_line);
b20ba1bc	212	sec_left = pblk_line_vsc(line);
d340121e JG	213	spin_unlock(&line->lock);
d340121e JG	214
a4bd217b JG	215	if (sec_left < 0) {
a4bd217b JG	216	pr_err("pblk: corrupted GC line (%d)\n", line->id);
48b8d208	217	goto fail_free_lba_list;
a4bd217b JG	218	}
	219
	220	bit = -1;
	221	next_rq:
b20ba1bc JG	222	gc_rq = kmalloc(sizeof(struct pblk_gc_rq), GFP_KERNEL);
b20ba1bc JG	223	if (!gc_rq)
48b8d208	224	goto fail_free_lba_list;
a4bd217b	225
b20ba1bc	226	nr_secs = 0;
a4bd217b	227	do {
d340121e	228	bit = find_next_zero_bit(invalid_bitmap, lm->sec_per_line,
a4bd217b JG	229	bit + 1);
	230	if (bit > line->emeta_ssec)
	231	break;
	232
d340121e	233	gc_rq->paddr_list[nr_secs] = bit;
b20ba1bc JG	234	gc_rq->lba_list[nr_secs++] = le64_to_cpu(lba_list[bit]);
b20ba1bc JG	235	} while (nr_secs < pblk->max_write_pgs);
a4bd217b	236
b20ba1bc JG	237	if (unlikely(!nr_secs)) {
b20ba1bc JG	238	kfree(gc_rq);
a4bd217b JG	239	goto out;
	240	}
	241
b20ba1bc JG	242	gc_rq->nr_secs = nr_secs;
	243	gc_rq->line = line;
	244
b84ae4a8 JG	245	gc_rq_ws = kmalloc(sizeof(struct pblk_line_ws), GFP_KERNEL);
b84ae4a8 JG	246	if (!gc_rq_ws)
b20ba1bc JG	247	goto fail_free_gc_rq;
b20ba1bc JG	248
b84ae4a8 JG	249	gc_rq_ws->pblk = pblk;
	250	gc_rq_ws->line = line;
	251	gc_rq_ws->priv = gc_rq;
b20ba1bc	252
8da10cce JG	253	/* The write GC path can be much slower than the read GC one due to
	254	* the budget imposed by the rate-limiter. Balance in case that we get
	255	* back pressure from the write GC path.
	256	*/
	257	while (down_timeout(&gc->gc_sem, msecs_to_jiffies(30000)))
	258	io_schedule();
	259
b20ba1bc JG	260	kref_get(&line->ref);
b20ba1bc JG	261
b84ae4a8 JG	262	INIT_WORK(&gc_rq_ws->ws, pblk_gc_line_ws);
b84ae4a8 JG	263	queue_work(gc->gc_line_reader_wq, &gc_rq_ws->ws);
a4bd217b	264
b20ba1bc	265	sec_left -= nr_secs;
a4bd217b JG	266	if (sec_left > 0)
	267	goto next_rq;
	268
	269	out:
48b8d208	270	pblk_mfree(lba_list, l_mg->emeta_alloc_type);
b84ae4a8	271	kfree(line_ws);
d340121e	272	kfree(invalid_bitmap);
b20ba1bc JG	273
b20ba1bc JG	274	kref_put(&line->ref, pblk_line_put);
d6b992f7	275	atomic_dec(&gc->read_inflight_gc);
b20ba1bc JG	276
	277	return;
	278
	279	fail_free_gc_rq:
	280	kfree(gc_rq);
48b8d208 HH	281	fail_free_lba_list:
48b8d208 HH	282	pblk_mfree(lba_list, l_mg->emeta_alloc_type);
0ec6937e	283	fail_free_invalid_bitmap:
d340121e JG	284	kfree(invalid_bitmap);
	285	fail_free_ws:
	286	kfree(line_ws);
	287
b20ba1bc JG	288	pblk_put_line_back(pblk, line);
b20ba1bc JG	289	kref_put(&line->ref, pblk_line_put);
d6b992f7	290	atomic_dec(&gc->read_inflight_gc);
b20ba1bc JG	291
b20ba1bc JG	292	pr_err("pblk: Failed to GC line %d\n", line->id);
a4bd217b JG	293	}
	294
	295	static int pblk_gc_line(struct pblk pblk, struct pblk_line line)
	296	{
b20ba1bc	297	struct pblk_gc *gc = &pblk->gc;
a4bd217b	298	struct pblk_line_ws *line_ws;
a4bd217b	299
b20ba1bc	300	pr_debug("pblk: line '%d' being reclaimed for GC\n", line->id);
a4bd217b	301
b84ae4a8	302	line_ws = kmalloc(sizeof(struct pblk_line_ws), GFP_KERNEL);
b20ba1bc JG	303	if (!line_ws)
b20ba1bc JG	304	return -ENOMEM;
a4bd217b	305
a4bd217b JG	306	line_ws->pblk = pblk;
a4bd217b JG	307	line_ws->line = line;
a4bd217b	308
d6b992f7	309	atomic_inc(&gc->pipeline_gc);
b20ba1bc JG	310	INIT_WORK(&line_ws->ws, pblk_gc_line_prepare_ws);
b20ba1bc JG	311	queue_work(gc->gc_reader_wq, &line_ws->ws);
a4bd217b JG	312
a4bd217b JG	313	return 0;
a4bd217b JG	314	}
a4bd217b JG	315
8bd40020 JG	316	static void pblk_gc_reader_kick(struct pblk_gc *gc)
	317	{
	318	wake_up_process(gc->gc_reader_ts);
	319	}
	320
	321	static void pblk_gc_kick(struct pblk *pblk)
	322	{
	323	struct pblk_gc *gc = &pblk->gc;
	324
	325	pblk_gc_writer_kick(gc);
	326	pblk_gc_reader_kick(gc);
	327
	328	/* If we're shutting down GC, let's not start it up again */
	329	if (gc->gc_enabled) {
	330	wake_up_process(gc->gc_ts);
	331	mod_timer(&gc->gc_timer,
	332	jiffies + msecs_to_jiffies(GC_TIME_MSECS));
	333	}
	334	}
	335
b20ba1bc	336	static int pblk_gc_read(struct pblk *pblk)
a4bd217b	337	{
b20ba1bc JG	338	struct pblk_gc *gc = &pblk->gc;
b20ba1bc JG	339	struct pblk_line *line;
a4bd217b	340
b20ba1bc JG	341	spin_lock(&gc->r_lock);
	342	if (list_empty(&gc->r_list)) {
	343	spin_unlock(&gc->r_lock);
	344	return 1;
a4bd217b	345	}
b20ba1bc JG	346
	347	line = list_first_entry(&gc->r_list, struct pblk_line, list);
	348	list_del(&line->list);
	349	spin_unlock(&gc->r_lock);
	350
	351	pblk_gc_kick(pblk);
	352
	353	if (pblk_gc_line(pblk, line))
	354	pr_err("pblk: failed to GC line %d\n", line->id);
	355
	356	return 0;
	357	}
	358
d45ebd47 JG	359	static struct pblk_line pblk_gc_get_victim_line(struct pblk pblk,
	360	struct list_head *group_list)
	361	{
	362	struct pblk_line line, victim;
f417aa0b	363	int line_vsc, victim_vsc;
d45ebd47 JG	364
	365	victim = list_first_entry(group_list, struct pblk_line, list);
	366	list_for_each_entry(line, group_list, list) {
f417aa0b JG	367	line_vsc = le32_to_cpu(*line->vsc);
	368	victim_vsc = le32_to_cpu(*victim->vsc);
	369	if (line_vsc < victim_vsc)
d45ebd47 JG	370	victim = line;
	371	}
	372
	373	return victim;
	374	}
	375
b20ba1bc JG	376	static bool pblk_gc_should_run(struct pblk_gc gc, struct pblk_rl rl)
	377	{
	378	unsigned int nr_blocks_free, nr_blocks_need;
48b8d208	379	unsigned int werr_lines = atomic_read(&rl->werr_lines);
b20ba1bc JG	380
	381	nr_blocks_need = pblk_rl_high_thrs(rl);
	382	nr_blocks_free = pblk_rl_nr_free_blks(rl);
	383
	384	/* This is not critical, no need to take lock here */
48b8d208 HH	385	return ((werr_lines > 0) \|\|
48b8d208 HH	386	((gc->gc_active) && (nr_blocks_need > nr_blocks_free)));
b20ba1bc JG	387	}
b20ba1bc JG	388
37ce33d5	389	void pblk_gc_free_full_lines(struct pblk *pblk)
a4bd217b JG	390	{
a4bd217b JG	391	struct pblk_line_mgmt *l_mg = &pblk->l_mg;
d6b992f7	392	struct pblk_gc *gc = &pblk->gc;
b20ba1bc	393	struct pblk_line *line;
b20ba1bc JG	394
	395	do {
	396	spin_lock(&l_mg->gc_lock);
	397	if (list_empty(&l_mg->gc_full_list)) {
	398	spin_unlock(&l_mg->gc_lock);
37ce33d5	399	return;
b20ba1bc JG	400	}
	401
	402	line = list_first_entry(&l_mg->gc_full_list,
	403	struct pblk_line, list);
a4bd217b	404
a4bd217b JG	405	spin_lock(&line->lock);
	406	WARN_ON(line->state != PBLK_LINESTATE_CLOSED);
	407	line->state = PBLK_LINESTATE_GC;
	408	spin_unlock(&line->lock);
	409
	410	list_del(&line->list);
b20ba1bc JG	411	spin_unlock(&l_mg->gc_lock);
b20ba1bc JG	412
d6b992f7	413	atomic_inc(&gc->pipeline_gc);
a4bd217b	414	kref_put(&line->ref, pblk_line_put);
b20ba1bc	415	} while (1);
37ce33d5 HH	416	}
	417
	418	/*
	419	* Lines with no valid sectors will be returned to the free list immediately. If
	420	* GC is activated - either because the free block count is under the determined
	421	* threshold, or because it is being forced from user space - only lines with a
	422	* high count of invalid sectors will be recycled.
	423	*/
	424	static void pblk_gc_run(struct pblk *pblk)
	425	{
	426	struct pblk_line_mgmt *l_mg = &pblk->l_mg;
	427	struct pblk_gc *gc = &pblk->gc;
	428	struct pblk_line *line;
	429	struct list_head *group_list;
	430	bool run_gc;
d6b992f7	431	int read_inflight_gc, gc_group = 0, prev_group = 0;
37ce33d5 HH	432
37ce33d5 HH	433	pblk_gc_free_full_lines(pblk);
a4bd217b	434
b20ba1bc	435	run_gc = pblk_gc_should_run(&pblk->gc, &pblk->rl);
d6b992f7	436	if (!run_gc \|\| (atomic_read(&gc->read_inflight_gc) >= PBLK_GC_L_QD))
b20ba1bc	437	return;
a4bd217b JG	438
	439	next_gc_group:
	440	group_list = l_mg->gc_lists[gc_group++];
b20ba1bc JG	441
	442	do {
	443	spin_lock(&l_mg->gc_lock);
	444	if (list_empty(group_list)) {
a4bd217b	445	spin_unlock(&l_mg->gc_lock);
b20ba1bc	446	break;
a4bd217b JG	447	}
a4bd217b JG	448
d45ebd47	449	line = pblk_gc_get_victim_line(pblk, group_list);
a4bd217b JG	450
	451	spin_lock(&line->lock);
	452	WARN_ON(line->state != PBLK_LINESTATE_CLOSED);
	453	line->state = PBLK_LINESTATE_GC;
a4bd217b JG	454	spin_unlock(&line->lock);
a4bd217b JG	455
b20ba1bc JG	456	list_del(&line->list);
	457	spin_unlock(&l_mg->gc_lock);
	458
	459	spin_lock(&gc->r_lock);
	460	list_add_tail(&line->list, &gc->r_list);
	461	spin_unlock(&gc->r_lock);
	462
d6b992f7	463	read_inflight_gc = atomic_inc_return(&gc->read_inflight_gc);
b20ba1bc	464	pblk_gc_reader_kick(gc);
a4bd217b	465
b20ba1bc	466	prev_group = 1;
a4bd217b	467
b20ba1bc JG	468	/* No need to queue up more GC lines than we can handle */
b20ba1bc JG	469	run_gc = pblk_gc_should_run(&pblk->gc, &pblk->rl);
d6b992f7	470	if (!run_gc \|\| read_inflight_gc >= PBLK_GC_L_QD)
b20ba1bc JG	471	break;
	472	} while (1);
	473
	474	if (!prev_group && pblk->rl.rb_state > gc_group &&
	475	gc_group < PBLK_GC_NR_LISTS)
a4bd217b JG	476	goto next_gc_group;
	477	}
	478
87c1d2d3	479	static void pblk_gc_timer(struct timer_list *t)
a4bd217b	480	{
87c1d2d3	481	struct pblk *pblk = from_timer(pblk, t, gc.gc_timer);
a4bd217b JG	482
	483	pblk_gc_kick(pblk);
	484	}
	485
	486	static int pblk_gc_ts(void *data)
	487	{
	488	struct pblk *pblk = data;
	489
	490	while (!kthread_should_stop()) {
	491	pblk_gc_run(pblk);
	492	set_current_state(TASK_INTERRUPTIBLE);
	493	io_schedule();
	494	}
	495
	496	return 0;
	497	}
	498
	499	static int pblk_gc_writer_ts(void *data)
	500	{
	501	struct pblk *pblk = data;
	502
	503	while (!kthread_should_stop()) {
	504	if (!pblk_gc_write(pblk))
	505	continue;
	506	set_current_state(TASK_INTERRUPTIBLE);
	507	io_schedule();
	508	}
	509
	510	return 0;
	511	}
	512
b20ba1bc	513	static int pblk_gc_reader_ts(void *data)
a4bd217b	514	{
b20ba1bc	515	struct pblk *pblk = data;
d6b992f7	516	struct pblk_gc *gc = &pblk->gc;
a4bd217b	517
b20ba1bc JG	518	while (!kthread_should_stop()) {
	519	if (!pblk_gc_read(pblk))
	520	continue;
	521	set_current_state(TASK_INTERRUPTIBLE);
	522	io_schedule();
	523	}
	524
d6b992f7 HH	525	#ifdef CONFIG_NVM_DEBUG
	526	pr_info("pblk: flushing gc pipeline, %d lines left\n",
	527	atomic_read(&gc->pipeline_gc));
	528	#endif
	529
	530	do {
	531	if (!atomic_read(&gc->pipeline_gc))
	532	break;
	533
	534	schedule();
	535	} while (1);
	536
b20ba1bc	537	return 0;
a4bd217b JG	538	}
a4bd217b JG	539
b20ba1bc	540	static void pblk_gc_start(struct pblk *pblk)
a4bd217b	541	{
b20ba1bc JG	542	pblk->gc.gc_active = 1;
b20ba1bc JG	543	pr_debug("pblk: gc start\n");
a4bd217b JG	544	}
a4bd217b JG	545
b20ba1bc	546	void pblk_gc_should_start(struct pblk *pblk)
a4bd217b JG	547	{
	548	struct pblk_gc *gc = &pblk->gc;
	549
3e3a5b8e	550	if (gc->gc_enabled && !gc->gc_active) {
a4bd217b	551	pblk_gc_start(pblk);
3e3a5b8e HH	552	pblk_gc_kick(pblk);
3e3a5b8e HH	553	}
a4bd217b JG	554	}
a4bd217b JG	555
a4bd217b JG	556	void pblk_gc_should_stop(struct pblk *pblk)
	557	{
	558	struct pblk_gc *gc = &pblk->gc;
	559
	560	if (gc->gc_active && !gc->gc_forced)
aed49e19	561	gc->gc_active = 0;
a4bd217b JG	562	}
a4bd217b JG	563
03661b5f HH	564	void pblk_gc_should_kick(struct pblk *pblk)
	565	{
	566	pblk_rl_update_rates(&pblk->rl);
	567	}
	568
a4bd217b JG	569	void pblk_gc_sysfs_state_show(struct pblk pblk, int gc_enabled,
	570	int *gc_active)
	571	{
	572	struct pblk_gc *gc = &pblk->gc;
	573
	574	spin_lock(&gc->lock);
	575	*gc_enabled = gc->gc_enabled;
	576	*gc_active = gc->gc_active;
	577	spin_unlock(&gc->lock);
	578	}
	579
b20ba1bc	580	int pblk_gc_sysfs_force(struct pblk *pblk, int force)
a4bd217b JG	581	{
a4bd217b JG	582	struct pblk_gc *gc = &pblk->gc;
b20ba1bc JG	583
	584	if (force < 0 \|\| force > 1)
	585	return -EINVAL;
a4bd217b JG	586
a4bd217b JG	587	spin_lock(&gc->lock);
a4bd217b	588	gc->gc_forced = force;
b20ba1bc JG	589
	590	if (force)
	591	gc->gc_enabled = 1;
	592	else
	593	gc->gc_enabled = 0;
a4bd217b	594	spin_unlock(&gc->lock);
b20ba1bc JG	595
	596	pblk_gc_should_start(pblk);
	597
	598	return 0;
a4bd217b JG	599	}
	600
	601	int pblk_gc_init(struct pblk *pblk)
	602	{
	603	struct pblk_gc *gc = &pblk->gc;
	604	int ret;
	605
	606	gc->gc_ts = kthread_create(pblk_gc_ts, pblk, "pblk-gc-ts");
	607	if (IS_ERR(gc->gc_ts)) {
	608	pr_err("pblk: could not allocate GC main kthread\n");
	609	return PTR_ERR(gc->gc_ts);
	610	}
	611
	612	gc->gc_writer_ts = kthread_create(pblk_gc_writer_ts, pblk,
	613	"pblk-gc-writer-ts");
	614	if (IS_ERR(gc->gc_writer_ts)) {
	615	pr_err("pblk: could not allocate GC writer kthread\n");
	616	ret = PTR_ERR(gc->gc_writer_ts);
	617	goto fail_free_main_kthread;
	618	}
	619
b20ba1bc JG	620	gc->gc_reader_ts = kthread_create(pblk_gc_reader_ts, pblk,
	621	"pblk-gc-reader-ts");
	622	if (IS_ERR(gc->gc_reader_ts)) {
	623	pr_err("pblk: could not allocate GC reader kthread\n");
	624	ret = PTR_ERR(gc->gc_reader_ts);
	625	goto fail_free_writer_kthread;
	626	}
	627
87c1d2d3	628	timer_setup(&gc->gc_timer, pblk_gc_timer, 0);
a4bd217b JG	629	mod_timer(&gc->gc_timer, jiffies + msecs_to_jiffies(GC_TIME_MSECS));
	630
	631	gc->gc_active = 0;
	632	gc->gc_forced = 0;
	633	gc->gc_enabled = 1;
a4bd217b	634	gc->w_entries = 0;
d6b992f7 HH	635	atomic_set(&gc->read_inflight_gc, 0);
d6b992f7 HH	636	atomic_set(&gc->pipeline_gc, 0);
a4bd217b	637
b20ba1bc JG	638	/* Workqueue that reads valid sectors from a line and submit them to the
	639	* GC writer to be recycled.
	640	*/
	641	gc->gc_line_reader_wq = alloc_workqueue("pblk-gc-line-reader-wq",
	642	WQ_MEM_RECLAIM \| WQ_UNBOUND, PBLK_GC_MAX_READERS);
	643	if (!gc->gc_line_reader_wq) {
	644	pr_err("pblk: could not allocate GC line reader workqueue\n");
	645	ret = -ENOMEM;
	646	goto fail_free_reader_kthread;
	647	}
	648
	649	/* Workqueue that prepare lines for GC */
	650	gc->gc_reader_wq = alloc_workqueue("pblk-gc-line_wq",
	651	WQ_MEM_RECLAIM \| WQ_UNBOUND, 1);
a4bd217b JG	652	if (!gc->gc_reader_wq) {
	653	pr_err("pblk: could not allocate GC reader workqueue\n");
	654	ret = -ENOMEM;
b20ba1bc	655	goto fail_free_reader_line_wq;
a4bd217b JG	656	}
	657
	658	spin_lock_init(&gc->lock);
	659	spin_lock_init(&gc->w_lock);
b20ba1bc JG	660	spin_lock_init(&gc->r_lock);
b20ba1bc JG	661
3627896a	662	sema_init(&gc->gc_sem, PBLK_GC_RQ_QD);
b20ba1bc	663
a4bd217b	664	INIT_LIST_HEAD(&gc->w_list);
b20ba1bc	665	INIT_LIST_HEAD(&gc->r_list);
a4bd217b JG	666
	667	return 0;
	668
b20ba1bc JG	669	fail_free_reader_line_wq:
	670	destroy_workqueue(gc->gc_line_reader_wq);
	671	fail_free_reader_kthread:
	672	kthread_stop(gc->gc_reader_ts);
a4bd217b JG	673	fail_free_writer_kthread:
a4bd217b JG	674	kthread_stop(gc->gc_writer_ts);
503ec94e DC	675	fail_free_main_kthread:
503ec94e DC	676	kthread_stop(gc->gc_ts);
a4bd217b JG	677
	678	return ret;
	679	}
	680
a7c9e910	681	void pblk_gc_exit(struct pblk *pblk, bool graceful)
a4bd217b JG	682	{
	683	struct pblk_gc *gc = &pblk->gc;
	684
3e3a5b8e HH	685	gc->gc_enabled = 0;
3e3a5b8e HH	686	del_timer_sync(&gc->gc_timer);
aed49e19	687	gc->gc_active = 0;
a4bd217b JG	688
	689	if (gc->gc_ts)
	690	kthread_stop(gc->gc_ts);
	691
1edebacf HH	692	if (gc->gc_reader_ts)
	693	kthread_stop(gc->gc_reader_ts);
	694
a7c9e910 JG	695	if (graceful) {
	696	flush_workqueue(gc->gc_reader_wq);
	697	flush_workqueue(gc->gc_line_reader_wq);
	698	}
b20ba1bc	699
a7c9e910	700	destroy_workqueue(gc->gc_reader_wq);
5565b0ca	701	destroy_workqueue(gc->gc_line_reader_wq);
a4bd217b JG	702
	703	if (gc->gc_writer_ts)
	704	kthread_stop(gc->gc_writer_ts);
	705	}