[linux-2.6-block.git] / drivers / block / zram / zcomp.c

/*
 * Copyright (C) 2014 Sergey Senozhatsky.
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License
 * as published by the Free Software Foundation; either version
 * 2 of the License, or (at your option) any later version.
 */

#include <linux/kernel.h>
#include <linux/string.h>
#include <linux/err.h>
#include <linux/slab.h>
#include <linux/wait.h>
#include <linux/sched.h>

#include "zcomp.h"
#include "zcomp_lzo.h"
#ifdef CONFIG_ZRAM_LZ4_COMPRESS
#include "zcomp_lz4.h"
#endif

/*
 * single zcomp_strm backend
 */
struct zcomp_strm_single {
	struct mutex strm_lock;
	struct zcomp_strm *zstrm;
};

/*
 * multi zcomp_strm backend
 */
struct zcomp_strm_multi {
	/* protect strm list */
	spinlock_t strm_lock;
	/* max possible number of zstrm streams */
	int max_strm;
	/* number of available zstrm streams */
	int avail_strm;
	/* list of available strms */
	struct list_head idle_strm;
	wait_queue_head_t strm_wait;
};

static struct zcomp_backend *backends[] = {
	&zcomp_lzo,
#ifdef CONFIG_ZRAM_LZ4_COMPRESS
	&zcomp_lz4,
#endif
	NULL
};

static struct zcomp_backend *find_backend(const char *compress)
{
	int i = 0;
	while (backends[i]) {
		if (sysfs_streq(compress, backends[i]->name))
			break;
		i++;
	}
	return backends[i];
}

static void zcomp_strm_free(struct zcomp *comp, struct zcomp_strm *zstrm)
{
	if (zstrm->private)
		comp->backend->destroy(zstrm->private);
	free_pages((unsigned long)zstrm->buffer, 1);
	kfree(zstrm);
}

/*
 * allocate new zcomp_strm structure with ->private initialized by
 * backend, return NULL on error
 */
static struct zcomp_strm *zcomp_strm_alloc(struct zcomp *comp)
{
	struct zcomp_strm *zstrm = kmalloc(sizeof(*zstrm), GFP_KERNEL);
	if (!zstrm)
		return NULL;

	zstrm->private = comp->backend->create();
	/*
	 * allocate 2 pages. 1 for compressed data, plus 1 extra for the
	 * case when compressed size is larger than the original one
	 */
	zstrm->buffer = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 1);
	if (!zstrm->private || !zstrm->buffer) {
		zcomp_strm_free(comp, zstrm);
		zstrm = NULL;
	}
	return zstrm;
}

/*
 * get idle zcomp_strm or wait until other process release
 * (zcomp_strm_release()) one for us
 */
static struct zcomp_strm *zcomp_strm_multi_find(struct zcomp *comp)
{
	struct zcomp_strm_multi *zs = comp->stream;
	struct zcomp_strm *zstrm;

	while (1) {
		spin_lock(&zs->strm_lock);
		if (!list_empty(&zs->idle_strm)) {
			zstrm = list_entry(zs->idle_strm.next,
					struct zcomp_strm, list);
			list_del(&zstrm->list);
			spin_unlock(&zs->strm_lock);
			return zstrm;
		}
		/* zstrm streams limit reached, wait for idle stream */
		if (zs->avail_strm >= zs->max_strm) {
			spin_unlock(&zs->strm_lock);
			wait_event(zs->strm_wait, !list_empty(&zs->idle_strm));
			continue;
		}
		/* allocate new zstrm stream */
		zs->avail_strm++;
		spin_unlock(&zs->strm_lock);

		zstrm = zcomp_strm_alloc(comp);
		if (!zstrm) {
			spin_lock(&zs->strm_lock);
			zs->avail_strm--;
			spin_unlock(&zs->strm_lock);
			wait_event(zs->strm_wait, !list_empty(&zs->idle_strm));
			continue;
		}
		break;
	}
	return zstrm;
}

/* add stream back to idle list and wake up waiter or free the stream */
static void zcomp_strm_multi_release(struct zcomp *comp, struct zcomp_strm *zstrm)
{
	struct zcomp_strm_multi *zs = comp->stream;

	spin_lock(&zs->strm_lock);
	if (zs->avail_strm <= zs->max_strm) {
		list_add(&zstrm->list, &zs->idle_strm);
		spin_unlock(&zs->strm_lock);
		wake_up(&zs->strm_wait);
		return;
	}

	zs->avail_strm--;
	spin_unlock(&zs->strm_lock);
	zcomp_strm_free(comp, zstrm);
}

/* change max_strm limit */
static bool zcomp_strm_multi_set_max_streams(struct zcomp *comp, int num_strm)
{
	struct zcomp_strm_multi *zs = comp->stream;
	struct zcomp_strm *zstrm;

	spin_lock(&zs->strm_lock);
	zs->max_strm = num_strm;
	/*
	 * if user has lowered the limit and there are idle streams,
	 * immediately free as much streams (and memory) as we can.
	 */
	while (zs->avail_strm > num_strm && !list_empty(&zs->idle_strm)) {
		zstrm = list_entry(zs->idle_strm.next,
				struct zcomp_strm, list);
		list_del(&zstrm->list);
		zcomp_strm_free(comp, zstrm);
		zs->avail_strm--;
	}
	spin_unlock(&zs->strm_lock);
	return true;
}

static void zcomp_strm_multi_destroy(struct zcomp *comp)
{
	struct zcomp_strm_multi *zs = comp->stream;
	struct zcomp_strm *zstrm;

	while (!list_empty(&zs->idle_strm)) {
		zstrm = list_entry(zs->idle_strm.next,
				struct zcomp_strm, list);
		list_del(&zstrm->list);
		zcomp_strm_free(comp, zstrm);
	}
	kfree(zs);
}

static int zcomp_strm_multi_create(struct zcomp *comp, int max_strm)
{
	struct zcomp_strm *zstrm;
	struct zcomp_strm_multi *zs;

	comp->destroy = zcomp_strm_multi_destroy;
	comp->strm_find = zcomp_strm_multi_find;
	comp->strm_release = zcomp_strm_multi_release;
	comp->set_max_streams = zcomp_strm_multi_set_max_streams;
	zs = kmalloc(sizeof(struct zcomp_strm_multi), GFP_KERNEL);
	if (!zs)
		return -ENOMEM;

	comp->stream = zs;
	spin_lock_init(&zs->strm_lock);
	INIT_LIST_HEAD(&zs->idle_strm);
	init_waitqueue_head(&zs->strm_wait);
	zs->max_strm = max_strm;
	zs->avail_strm = 1;

	zstrm = zcomp_strm_alloc(comp);
	if (!zstrm) {
		kfree(zs);
		return -ENOMEM;
	}
	list_add(&zstrm->list, &zs->idle_strm);
	return 0;
}

static struct zcomp_strm *zcomp_strm_single_find(struct zcomp *comp)
{
	struct zcomp_strm_single *zs = comp->stream;
	mutex_lock(&zs->strm_lock);
	return zs->zstrm;
}

static void zcomp_strm_single_release(struct zcomp *comp,
		struct zcomp_strm *zstrm)
{
	struct zcomp_strm_single *zs = comp->stream;
	mutex_unlock(&zs->strm_lock);
}

static bool zcomp_strm_single_set_max_streams(struct zcomp *comp, int num_strm)
{
	/* zcomp_strm_single support only max_comp_streams == 1 */
	return false;
}

static void zcomp_strm_single_destroy(struct zcomp *comp)
{
	struct zcomp_strm_single *zs = comp->stream;
	zcomp_strm_free(comp, zs->zstrm);
	kfree(zs);
}

static int zcomp_strm_single_create(struct zcomp *comp)
{
	struct zcomp_strm_single *zs;

	comp->destroy = zcomp_strm_single_destroy;
	comp->strm_find = zcomp_strm_single_find;
	comp->strm_release = zcomp_strm_single_release;
	comp->set_max_streams = zcomp_strm_single_set_max_streams;
	zs = kmalloc(sizeof(struct zcomp_strm_single), GFP_KERNEL);
	if (!zs)
		return -ENOMEM;

	comp->stream = zs;
	mutex_init(&zs->strm_lock);
	zs->zstrm = zcomp_strm_alloc(comp);
	if (!zs->zstrm) {
		kfree(zs);
		return -ENOMEM;
	}
	return 0;
}

/* show available compressors */
ssize_t zcomp_available_show(const char *comp, char *buf)
{
	ssize_t sz = 0;
	int i = 0;

	while (backends[i]) {
		if (sysfs_streq(comp, backends[i]->name))
			sz += sprintf(buf + sz, "[%s] ", backends[i]->name);
		else
			sz += sprintf(buf + sz, "%s ", backends[i]->name);
		i++;
	}
	sz += sprintf(buf + sz, "\n");
	return sz;
}

bool zcomp_set_max_streams(struct zcomp *comp, int num_strm)
{
	return comp->set_max_streams(comp, num_strm);
}

struct zcomp_strm *zcomp_strm_find(struct zcomp *comp)
{
	return comp->strm_find(comp);
}

void zcomp_strm_release(struct zcomp *comp, struct zcomp_strm *zstrm)
{
	comp->strm_release(comp, zstrm);
}

int zcomp_compress(struct zcomp *comp, struct zcomp_strm *zstrm,
		const unsigned char *src, size_t *dst_len)
{
	return comp->backend->compress(src, zstrm->buffer, dst_len,
			zstrm->private);
}

int zcomp_decompress(struct zcomp *comp, const unsigned char *src,
		size_t src_len, unsigned char *dst)
{
	return comp->backend->decompress(src, src_len, dst);
}

void zcomp_destroy(struct zcomp *comp)
{
	comp->destroy(comp);
	kfree(comp);
}

/*
 * search available compressors for requested algorithm.
 * allocate new zcomp and initialize it. return compressing
 * backend pointer or ERR_PTR if things went bad. ERR_PTR(-EINVAL)
 * if requested algorithm is not supported, ERR_PTR(-ENOMEM) in
 * case of allocation error.
 */
struct zcomp *zcomp_create(const char *compress, int max_strm)
{
	struct zcomp *comp;
	struct zcomp_backend *backend;

	backend = find_backend(compress);
	if (!backend)
		return ERR_PTR(-EINVAL);

	comp = kzalloc(sizeof(struct zcomp), GFP_KERNEL);
	if (!comp)
		return ERR_PTR(-ENOMEM);

	comp->backend = backend;
	if (max_strm > 1)
		zcomp_strm_multi_create(comp, max_strm);
	else
		zcomp_strm_single_create(comp);
	if (!comp->stream) {
		kfree(comp);
		return ERR_PTR(-ENOMEM);
	}
	return comp;
}
Commit	Line	Data
e7e1ef43 SS	1	/*
	2	* Copyright (C) 2014 Sergey Senozhatsky.
	3	*
	4	* This program is free software; you can redistribute it and/or
	5	* modify it under the terms of the GNU General Public License
	6	* as published by the Free Software Foundation; either version
	7	* 2 of the License, or (at your option) any later version.
	8	*/
	9
	10	#include <linux/kernel.h>
	11	#include <linux/string.h>
fcfa8d95	12	#include <linux/err.h>
e7e1ef43 SS	13	#include <linux/slab.h>
	14	#include <linux/wait.h>
	15	#include <linux/sched.h>
	16
	17	#include "zcomp.h"
	18	#include "zcomp_lzo.h"
6e76668e SS	19	#ifdef CONFIG_ZRAM_LZ4_COMPRESS
	20	#include "zcomp_lz4.h"
	21	#endif
e7e1ef43	22
9cc97529 SS	23	/*
	24	* single zcomp_strm backend
	25	*/
	26	struct zcomp_strm_single {
	27	struct mutex strm_lock;
	28	struct zcomp_strm *zstrm;
	29	};
	30
beca3ec7 SS	31	/*
	32	* multi zcomp_strm backend
	33	*/
	34	struct zcomp_strm_multi {
	35	/* protect strm list */
	36	spinlock_t strm_lock;
	37	/* max possible number of zstrm streams */
	38	int max_strm;
	39	/* number of available zstrm streams */
	40	int avail_strm;
	41	/* list of available strms */
	42	struct list_head idle_strm;
	43	wait_queue_head_t strm_wait;
	44	};
	45
e46b8a03 SS	46	static struct zcomp_backend *backends[] = {
e46b8a03 SS	47	&zcomp_lzo,
6e76668e SS	48	#ifdef CONFIG_ZRAM_LZ4_COMPRESS
	49	&zcomp_lz4,
	50	#endif
e46b8a03 SS	51	NULL
	52	};
	53
e7e1ef43 SS	54	static struct zcomp_backend find_backend(const char compress)
e7e1ef43 SS	55	{
e46b8a03 SS	56	int i = 0;
	57	while (backends[i]) {
	58	if (sysfs_streq(compress, backends[i]->name))
	59	break;
	60	i++;
	61	}
	62	return backends[i];
e7e1ef43 SS	63	}
	64
	65	static void zcomp_strm_free(struct zcomp comp, struct zcomp_strm zstrm)
	66	{
	67	if (zstrm->private)
	68	comp->backend->destroy(zstrm->private);
	69	free_pages((unsigned long)zstrm->buffer, 1);
	70	kfree(zstrm);
	71	}
	72
	73	/*
	74	* allocate new zcomp_strm structure with ->private initialized by
	75	* backend, return NULL on error
	76	*/
	77	static struct zcomp_strm zcomp_strm_alloc(struct zcomp comp)
	78	{
	79	struct zcomp_strm zstrm = kmalloc(sizeof(zstrm), GFP_KERNEL);
	80	if (!zstrm)
	81	return NULL;
	82
	83	zstrm->private = comp->backend->create();
	84	/*
	85	* allocate 2 pages. 1 for compressed data, plus 1 extra for the
	86	* case when compressed size is larger than the original one
	87	*/
	88	zstrm->buffer = (void *)__get_free_pages(GFP_KERNEL \| __GFP_ZERO, 1);
	89	if (!zstrm->private \|\| !zstrm->buffer) {
	90	zcomp_strm_free(comp, zstrm);
	91	zstrm = NULL;
	92	}
	93	return zstrm;
	94	}
	95
beca3ec7 SS	96	/*
	97	* get idle zcomp_strm or wait until other process release
	98	* (zcomp_strm_release()) one for us
	99	*/
	100	static struct zcomp_strm zcomp_strm_multi_find(struct zcomp comp)
	101	{
	102	struct zcomp_strm_multi *zs = comp->stream;
	103	struct zcomp_strm *zstrm;
	104
	105	while (1) {
	106	spin_lock(&zs->strm_lock);
	107	if (!list_empty(&zs->idle_strm)) {
	108	zstrm = list_entry(zs->idle_strm.next,
	109	struct zcomp_strm, list);
	110	list_del(&zstrm->list);
	111	spin_unlock(&zs->strm_lock);
	112	return zstrm;
	113	}
	114	/* zstrm streams limit reached, wait for idle stream */
	115	if (zs->avail_strm >= zs->max_strm) {
	116	spin_unlock(&zs->strm_lock);
	117	wait_event(zs->strm_wait, !list_empty(&zs->idle_strm));
	118	continue;
	119	}
	120	/* allocate new zstrm stream */
	121	zs->avail_strm++;
	122	spin_unlock(&zs->strm_lock);
	123
	124	zstrm = zcomp_strm_alloc(comp);
	125	if (!zstrm) {
	126	spin_lock(&zs->strm_lock);
	127	zs->avail_strm--;
	128	spin_unlock(&zs->strm_lock);
	129	wait_event(zs->strm_wait, !list_empty(&zs->idle_strm));
	130	continue;
	131	}
	132	break;
	133	}
	134	return zstrm;
	135	}
	136
	137	/* add stream back to idle list and wake up waiter or free the stream */
	138	static void zcomp_strm_multi_release(struct zcomp comp, struct zcomp_strm zstrm)
	139	{
	140	struct zcomp_strm_multi *zs = comp->stream;
	141
	142	spin_lock(&zs->strm_lock);
	143	if (zs->avail_strm <= zs->max_strm) {
	144	list_add(&zstrm->list, &zs->idle_strm);
	145	spin_unlock(&zs->strm_lock);
	146	wake_up(&zs->strm_wait);
	147	return;
	148	}
	149
	150	zs->avail_strm--;
	151	spin_unlock(&zs->strm_lock);
	152	zcomp_strm_free(comp, zstrm);
	153	}
	154
fe8eb122	155	/* change max_strm limit */
60a726e3	156	static bool zcomp_strm_multi_set_max_streams(struct zcomp *comp, int num_strm)
fe8eb122 SS	157	{
	158	struct zcomp_strm_multi *zs = comp->stream;
	159	struct zcomp_strm *zstrm;
	160
	161	spin_lock(&zs->strm_lock);
	162	zs->max_strm = num_strm;
	163	/*
	164	* if user has lowered the limit and there are idle streams,
	165	* immediately free as much streams (and memory) as we can.
	166	*/
	167	while (zs->avail_strm > num_strm && !list_empty(&zs->idle_strm)) {
	168	zstrm = list_entry(zs->idle_strm.next,
	169	struct zcomp_strm, list);
	170	list_del(&zstrm->list);
	171	zcomp_strm_free(comp, zstrm);
	172	zs->avail_strm--;
	173	}
	174	spin_unlock(&zs->strm_lock);
60a726e3	175	return true;
fe8eb122 SS	176	}
fe8eb122 SS	177
beca3ec7 SS	178	static void zcomp_strm_multi_destroy(struct zcomp *comp)
	179	{
	180	struct zcomp_strm_multi *zs = comp->stream;
	181	struct zcomp_strm *zstrm;
	182
	183	while (!list_empty(&zs->idle_strm)) {
	184	zstrm = list_entry(zs->idle_strm.next,
	185	struct zcomp_strm, list);
	186	list_del(&zstrm->list);
	187	zcomp_strm_free(comp, zstrm);
	188	}
	189	kfree(zs);
	190	}
	191
	192	static int zcomp_strm_multi_create(struct zcomp *comp, int max_strm)
	193	{
	194	struct zcomp_strm *zstrm;
	195	struct zcomp_strm_multi *zs;
	196
	197	comp->destroy = zcomp_strm_multi_destroy;
	198	comp->strm_find = zcomp_strm_multi_find;
	199	comp->strm_release = zcomp_strm_multi_release;
fe8eb122	200	comp->set_max_streams = zcomp_strm_multi_set_max_streams;
beca3ec7 SS	201	zs = kmalloc(sizeof(struct zcomp_strm_multi), GFP_KERNEL);
	202	if (!zs)
	203	return -ENOMEM;
	204
	205	comp->stream = zs;
	206	spin_lock_init(&zs->strm_lock);
	207	INIT_LIST_HEAD(&zs->idle_strm);
	208	init_waitqueue_head(&zs->strm_wait);
	209	zs->max_strm = max_strm;
	210	zs->avail_strm = 1;
	211
	212	zstrm = zcomp_strm_alloc(comp);
	213	if (!zstrm) {
	214	kfree(zs);
	215	return -ENOMEM;
	216	}
	217	list_add(&zstrm->list, &zs->idle_strm);
	218	return 0;
	219	}
	220
9cc97529 SS	221	static struct zcomp_strm zcomp_strm_single_find(struct zcomp comp)
	222	{
	223	struct zcomp_strm_single *zs = comp->stream;
	224	mutex_lock(&zs->strm_lock);
	225	return zs->zstrm;
	226	}
	227
	228	static void zcomp_strm_single_release(struct zcomp *comp,
	229	struct zcomp_strm *zstrm)
	230	{
	231	struct zcomp_strm_single *zs = comp->stream;
	232	mutex_unlock(&zs->strm_lock);
	233	}
	234
60a726e3	235	static bool zcomp_strm_single_set_max_streams(struct zcomp *comp, int num_strm)
fe8eb122 SS	236	{
fe8eb122 SS	237	/* zcomp_strm_single support only max_comp_streams == 1 */
60a726e3	238	return false;
fe8eb122 SS	239	}
fe8eb122 SS	240
9cc97529 SS	241	static void zcomp_strm_single_destroy(struct zcomp *comp)
	242	{
	243	struct zcomp_strm_single *zs = comp->stream;
	244	zcomp_strm_free(comp, zs->zstrm);
	245	kfree(zs);
	246	}
	247
	248	static int zcomp_strm_single_create(struct zcomp *comp)
	249	{
	250	struct zcomp_strm_single *zs;
	251
	252	comp->destroy = zcomp_strm_single_destroy;
	253	comp->strm_find = zcomp_strm_single_find;
	254	comp->strm_release = zcomp_strm_single_release;
fe8eb122	255	comp->set_max_streams = zcomp_strm_single_set_max_streams;
9cc97529 SS	256	zs = kmalloc(sizeof(struct zcomp_strm_single), GFP_KERNEL);
	257	if (!zs)
	258	return -ENOMEM;
	259
	260	comp->stream = zs;
	261	mutex_init(&zs->strm_lock);
	262	zs->zstrm = zcomp_strm_alloc(comp);
	263	if (!zs->zstrm) {
	264	kfree(zs);
	265	return -ENOMEM;
	266	}
	267	return 0;
	268	}
	269
e46b8a03 SS	270	/* show available compressors */
	271	ssize_t zcomp_available_show(const char comp, char buf)
	272	{
	273	ssize_t sz = 0;
	274	int i = 0;
	275
	276	while (backends[i]) {
	277	if (sysfs_streq(comp, backends[i]->name))
	278	sz += sprintf(buf + sz, "[%s] ", backends[i]->name);
	279	else
	280	sz += sprintf(buf + sz, "%s ", backends[i]->name);
	281	i++;
	282	}
	283	sz += sprintf(buf + sz, "\n");
	284	return sz;
	285	}
	286
60a726e3	287	bool zcomp_set_max_streams(struct zcomp *comp, int num_strm)
fe8eb122 SS	288	{
	289	return comp->set_max_streams(comp, num_strm);
	290	}
	291
e7e1ef43 SS	292	struct zcomp_strm zcomp_strm_find(struct zcomp comp)
e7e1ef43 SS	293	{
9cc97529	294	return comp->strm_find(comp);
e7e1ef43 SS	295	}
	296
	297	void zcomp_strm_release(struct zcomp comp, struct zcomp_strm zstrm)
	298	{
9cc97529	299	comp->strm_release(comp, zstrm);
e7e1ef43 SS	300	}
	301
	302	int zcomp_compress(struct zcomp comp, struct zcomp_strm zstrm,
	303	const unsigned char src, size_t dst_len)
	304	{
	305	return comp->backend->compress(src, zstrm->buffer, dst_len,
	306	zstrm->private);
	307	}
	308
	309	int zcomp_decompress(struct zcomp comp, const unsigned char src,
	310	size_t src_len, unsigned char *dst)
	311	{
	312	return comp->backend->decompress(src, src_len, dst);
	313	}
	314
	315	void zcomp_destroy(struct zcomp *comp)
	316	{
9cc97529	317	comp->destroy(comp);
e7e1ef43 SS	318	kfree(comp);
	319	}
	320
	321	/*
	322	* search available compressors for requested algorithm.
fcfa8d95 SS	323	* allocate new zcomp and initialize it. return compressing
	324	* backend pointer or ERR_PTR if things went bad. ERR_PTR(-EINVAL)
	325	* if requested algorithm is not supported, ERR_PTR(-ENOMEM) in
	326	* case of allocation error.
e7e1ef43	327	*/
beca3ec7	328	struct zcomp zcomp_create(const char compress, int max_strm)
e7e1ef43 SS	329	{
	330	struct zcomp *comp;
	331	struct zcomp_backend *backend;
	332
	333	backend = find_backend(compress);
	334	if (!backend)
fcfa8d95	335	return ERR_PTR(-EINVAL);
e7e1ef43 SS	336
	337	comp = kzalloc(sizeof(struct zcomp), GFP_KERNEL);
	338	if (!comp)
fcfa8d95	339	return ERR_PTR(-ENOMEM);
e7e1ef43 SS	340
e7e1ef43 SS	341	comp->backend = backend;
beca3ec7 SS	342	if (max_strm > 1)
	343	zcomp_strm_multi_create(comp, max_strm);
	344	else
	345	zcomp_strm_single_create(comp);
	346	if (!comp->stream) {
e7e1ef43	347	kfree(comp);
fcfa8d95	348	return ERR_PTR(-ENOMEM);
e7e1ef43 SS	349	}
	350	return comp;
	351	}