[linux-2.6-block.git] / drivers / xen / tmem.c

/*
 * Xen implementation for transcendent memory (tmem)
 *
 * Copyright (C) 2009-2011 Oracle Corp.  All rights reserved.
 * Author: Dan Magenheimer
 */

#define pr_fmt(fmt) "xen:" KBUILD_MODNAME ": " fmt

#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/types.h>
#include <linux/init.h>
#include <linux/pagemap.h>
#include <linux/cleancache.h>
#include <linux/frontswap.h>

#include <xen/xen.h>
#include <xen/interface/xen.h>
#include <xen/page.h>
#include <asm/xen/hypercall.h>
#include <asm/xen/hypervisor.h>
#include <xen/tmem.h>

#ifndef CONFIG_XEN_TMEM_MODULE
bool __read_mostly tmem_enabled = false;

static int __init enable_tmem(char *s)
{
	tmem_enabled = true;
	return 1;
}
__setup("tmem", enable_tmem);
#endif

#ifdef CONFIG_CLEANCACHE
static bool cleancache __read_mostly = true;
module_param(cleancache, bool, S_IRUGO);
static bool selfballooning __read_mostly = true;
module_param(selfballooning, bool, S_IRUGO);
#endif /* CONFIG_CLEANCACHE */

#ifdef CONFIG_FRONTSWAP
static bool frontswap __read_mostly = true;
module_param(frontswap, bool, S_IRUGO);
#else /* CONFIG_FRONTSWAP */
#define frontswap (0)
#endif /* CONFIG_FRONTSWAP */

#ifdef CONFIG_XEN_SELFBALLOONING
static bool selfshrinking __read_mostly = true;
module_param(selfshrinking, bool, S_IRUGO);
#endif /* CONFIG_XEN_SELFBALLOONING */

#define TMEM_CONTROL               0
#define TMEM_NEW_POOL              1
#define TMEM_DESTROY_POOL          2
#define TMEM_NEW_PAGE              3
#define TMEM_PUT_PAGE              4
#define TMEM_GET_PAGE              5
#define TMEM_FLUSH_PAGE            6
#define TMEM_FLUSH_OBJECT          7
#define TMEM_READ                  8
#define TMEM_WRITE                 9
#define TMEM_XCHG                 10

/* Bits for HYPERVISOR_tmem_op(TMEM_NEW_POOL) */
#define TMEM_POOL_PERSIST          1
#define TMEM_POOL_SHARED           2
#define TMEM_POOL_PAGESIZE_SHIFT   4
#define TMEM_VERSION_SHIFT        24


struct tmem_pool_uuid {
	u64 uuid_lo;
	u64 uuid_hi;
};

struct tmem_oid {
	u64 oid[3];
};

#define TMEM_POOL_PRIVATE_UUID	{ 0, 0 }

/* flags for tmem_ops.new_pool */
#define TMEM_POOL_PERSIST          1
#define TMEM_POOL_SHARED           2

/* xen tmem foundation ops/hypercalls */

static inline int xen_tmem_op(u32 tmem_cmd, u32 tmem_pool, struct tmem_oid oid,
	u32 index, unsigned long gmfn, u32 tmem_offset, u32 pfn_offset, u32 len)
{
	struct tmem_op op;
	int rc = 0;

	op.cmd = tmem_cmd;
	op.pool_id = tmem_pool;
	op.u.gen.oid[0] = oid.oid[0];
	op.u.gen.oid[1] = oid.oid[1];
	op.u.gen.oid[2] = oid.oid[2];
	op.u.gen.index = index;
	op.u.gen.tmem_offset = tmem_offset;
	op.u.gen.pfn_offset = pfn_offset;
	op.u.gen.len = len;
	set_xen_guest_handle(op.u.gen.gmfn, (void *)gmfn);
	rc = HYPERVISOR_tmem_op(&op);
	return rc;
}

static int xen_tmem_new_pool(struct tmem_pool_uuid uuid,
				u32 flags, unsigned long pagesize)
{
	struct tmem_op op;
	int rc = 0, pageshift;

	for (pageshift = 0; pagesize != 1; pageshift++)
		pagesize >>= 1;
	flags |= (pageshift - 12) << TMEM_POOL_PAGESIZE_SHIFT;
	flags |= TMEM_SPEC_VERSION << TMEM_VERSION_SHIFT;
	op.cmd = TMEM_NEW_POOL;
	op.u.new.uuid[0] = uuid.uuid_lo;
	op.u.new.uuid[1] = uuid.uuid_hi;
	op.u.new.flags = flags;
	rc = HYPERVISOR_tmem_op(&op);
	return rc;
}

/* xen generic tmem ops */

static int xen_tmem_put_page(u32 pool_id, struct tmem_oid oid,
			     u32 index, struct page *page)
{
	return xen_tmem_op(TMEM_PUT_PAGE, pool_id, oid, index,
			   xen_page_to_gfn(page), 0, 0, 0);
}

static int xen_tmem_get_page(u32 pool_id, struct tmem_oid oid,
			     u32 index, struct page *page)
{
	return xen_tmem_op(TMEM_GET_PAGE, pool_id, oid, index,
			   xen_page_to_gfn(page), 0, 0, 0);
}

static int xen_tmem_flush_page(u32 pool_id, struct tmem_oid oid, u32 index)
{
	return xen_tmem_op(TMEM_FLUSH_PAGE, pool_id, oid, index,
		0, 0, 0, 0);
}

static int xen_tmem_flush_object(u32 pool_id, struct tmem_oid oid)
{
	return xen_tmem_op(TMEM_FLUSH_OBJECT, pool_id, oid, 0, 0, 0, 0, 0);
}


#ifdef CONFIG_CLEANCACHE
static int xen_tmem_destroy_pool(u32 pool_id)
{
	struct tmem_oid oid = { { 0 } };

	return xen_tmem_op(TMEM_DESTROY_POOL, pool_id, oid, 0, 0, 0, 0, 0);
}

/* cleancache ops */

static void tmem_cleancache_put_page(int pool, struct cleancache_filekey key,
				     pgoff_t index, struct page *page)
{
	u32 ind = (u32) index;
	struct tmem_oid oid = *(struct tmem_oid *)&key;

	if (pool < 0)
		return;
	if (ind != index)
		return;
	mb(); /* ensure page is quiescent; tmem may address it with an alias */
	(void)xen_tmem_put_page((u32)pool, oid, ind, page);
}

static int tmem_cleancache_get_page(int pool, struct cleancache_filekey key,
				    pgoff_t index, struct page *page)
{
	u32 ind = (u32) index;
	struct tmem_oid oid = *(struct tmem_oid *)&key;
	int ret;

	/* translate return values to linux semantics */
	if (pool < 0)
		return -1;
	if (ind != index)
		return -1;
	ret = xen_tmem_get_page((u32)pool, oid, ind, page);
	if (ret == 1)
		return 0;
	else
		return -1;
}

static void tmem_cleancache_flush_page(int pool, struct cleancache_filekey key,
				       pgoff_t index)
{
	u32 ind = (u32) index;
	struct tmem_oid oid = *(struct tmem_oid *)&key;

	if (pool < 0)
		return;
	if (ind != index)
		return;
	(void)xen_tmem_flush_page((u32)pool, oid, ind);
}

static void tmem_cleancache_flush_inode(int pool, struct cleancache_filekey key)
{
	struct tmem_oid oid = *(struct tmem_oid *)&key;

	if (pool < 0)
		return;
	(void)xen_tmem_flush_object((u32)pool, oid);
}

static void tmem_cleancache_flush_fs(int pool)
{
	if (pool < 0)
		return;
	(void)xen_tmem_destroy_pool((u32)pool);
}

static int tmem_cleancache_init_fs(size_t pagesize)
{
	struct tmem_pool_uuid uuid_private = TMEM_POOL_PRIVATE_UUID;

	return xen_tmem_new_pool(uuid_private, 0, pagesize);
}

static int tmem_cleancache_init_shared_fs(char *uuid, size_t pagesize)
{
	struct tmem_pool_uuid shared_uuid;

	shared_uuid.uuid_lo = *(u64 *)uuid;
	shared_uuid.uuid_hi = *(u64 *)(&uuid[8]);
	return xen_tmem_new_pool(shared_uuid, TMEM_POOL_SHARED, pagesize);
}

static const struct cleancache_ops tmem_cleancache_ops = {
	.put_page = tmem_cleancache_put_page,
	.get_page = tmem_cleancache_get_page,
	.invalidate_page = tmem_cleancache_flush_page,
	.invalidate_inode = tmem_cleancache_flush_inode,
	.invalidate_fs = tmem_cleancache_flush_fs,
	.init_shared_fs = tmem_cleancache_init_shared_fs,
	.init_fs = tmem_cleancache_init_fs
};
#endif

#ifdef CONFIG_FRONTSWAP
/* frontswap tmem operations */

/* a single tmem poolid is used for all frontswap "types" (swapfiles) */
static int tmem_frontswap_poolid;

/*
 * Swizzling increases objects per swaptype, increasing tmem concurrency
 * for heavy swaploads.  Later, larger nr_cpus -> larger SWIZ_BITS
 */
#define SWIZ_BITS		4
#define SWIZ_MASK		((1 << SWIZ_BITS) - 1)
#define _oswiz(_type, _ind)	((_type << SWIZ_BITS) | (_ind & SWIZ_MASK))
#define iswiz(_ind)		(_ind >> SWIZ_BITS)

static inline struct tmem_oid oswiz(unsigned type, u32 ind)
{
	struct tmem_oid oid = { .oid = { 0 } };
	oid.oid[0] = _oswiz(type, ind);
	return oid;
}

/* returns 0 if the page was successfully put into frontswap, -1 if not */
static int tmem_frontswap_store(unsigned type, pgoff_t offset,
				   struct page *page)
{
	u64 ind64 = (u64)offset;
	u32 ind = (u32)offset;
	int pool = tmem_frontswap_poolid;
	int ret;

	if (pool < 0)
		return -1;
	if (ind64 != ind)
		return -1;
	mb(); /* ensure page is quiescent; tmem may address it with an alias */
	ret = xen_tmem_put_page(pool, oswiz(type, ind), iswiz(ind), page);
	/* translate Xen tmem return values to linux semantics */
	if (ret == 1)
		return 0;
	else
		return -1;
}

/*
 * returns 0 if the page was successfully gotten from frontswap, -1 if
 * was not present (should never happen!)
 */
static int tmem_frontswap_load(unsigned type, pgoff_t offset,
				   struct page *page)
{
	u64 ind64 = (u64)offset;
	u32 ind = (u32)offset;
	int pool = tmem_frontswap_poolid;
	int ret;

	if (pool < 0)
		return -1;
	if (ind64 != ind)
		return -1;
	ret = xen_tmem_get_page(pool, oswiz(type, ind), iswiz(ind), page);
	/* translate Xen tmem return values to linux semantics */
	if (ret == 1)
		return 0;
	else
		return -1;
}

/* flush a single page from frontswap */
static void tmem_frontswap_flush_page(unsigned type, pgoff_t offset)
{
	u64 ind64 = (u64)offset;
	u32 ind = (u32)offset;
	int pool = tmem_frontswap_poolid;

	if (pool < 0)
		return;
	if (ind64 != ind)
		return;
	(void) xen_tmem_flush_page(pool, oswiz(type, ind), iswiz(ind));
}

/* flush all pages from the passed swaptype */
static void tmem_frontswap_flush_area(unsigned type)
{
	int pool = tmem_frontswap_poolid;
	int ind;

	if (pool < 0)
		return;
	for (ind = SWIZ_MASK; ind >= 0; ind--)
		(void)xen_tmem_flush_object(pool, oswiz(type, ind));
}

static void tmem_frontswap_init(unsigned ignored)
{
	struct tmem_pool_uuid private = TMEM_POOL_PRIVATE_UUID;

	/* a single tmem poolid is used for all frontswap "types" (swapfiles) */
	if (tmem_frontswap_poolid < 0)
		tmem_frontswap_poolid =
		    xen_tmem_new_pool(private, TMEM_POOL_PERSIST, PAGE_SIZE);
}

static struct frontswap_ops tmem_frontswap_ops = {
	.store = tmem_frontswap_store,
	.load = tmem_frontswap_load,
	.invalidate_page = tmem_frontswap_flush_page,
	.invalidate_area = tmem_frontswap_flush_area,
	.init = tmem_frontswap_init
};
#endif

static int __init xen_tmem_init(void)
{
	if (!xen_domain())
		return 0;
#ifdef CONFIG_FRONTSWAP
	if (tmem_enabled && frontswap) {
		char *s = "";

		tmem_frontswap_poolid = -1;
		frontswap_register_ops(&tmem_frontswap_ops);
		pr_info("frontswap enabled, RAM provided by Xen Transcendent Memory%s\n",
			s);
	}
#endif
#ifdef CONFIG_CLEANCACHE
	BUILD_BUG_ON(sizeof(struct cleancache_filekey) != sizeof(struct tmem_oid));
	if (tmem_enabled && cleancache) {
		int err;

		err = cleancache_register_ops(&tmem_cleancache_ops);
		if (err)
			pr_warn("xen-tmem: failed to enable cleancache: %d\n",
				err);
		else
			pr_info("cleancache enabled, RAM provided by "
				"Xen Transcendent Memory\n");
	}
#endif
#ifdef CONFIG_XEN_SELFBALLOONING
	/*
	 * There is no point of driving pages to the swap system if they
	 * aren't going anywhere in tmem universe.
	 */
	if (!frontswap) {
		selfshrinking = false;
		selfballooning = false;
	}
	xen_selfballoon_init(selfballooning, selfshrinking);
#endif
	return 0;
}

module_init(xen_tmem_init)
MODULE_LICENSE("GPL");
MODULE_AUTHOR("Dan Magenheimer <dan.magenheimer@oracle.com>");
MODULE_DESCRIPTION("Shim to Xen transcendent memory");
Commit	Line	Data
5bc20fc5 DM	1	/*
	2	* Xen implementation for transcendent memory (tmem)
	3	*
afec6e04	4	* Copyright (C) 2009-2011 Oracle Corp. All rights reserved.
5bc20fc5 DM	5	* Author: Dan Magenheimer
	6	*/
	7
283c0972 JP	8	#define pr_fmt(fmt) "xen:" KBUILD_MODNAME ": " fmt
283c0972 JP	9
10a7a077	10	#include <linux/module.h>
5bc20fc5 DM	11	#include <linux/kernel.h>
	12	#include <linux/types.h>
	13	#include <linux/init.h>
	14	#include <linux/pagemap.h>
	15	#include <linux/cleancache.h>
afec6e04	16	#include <linux/frontswap.h>
afec6e04	17
5bc20fc5 DM	18	#include <xen/xen.h>
5bc20fc5 DM	19	#include <xen/interface/xen.h>
a9fd60e2	20	#include <xen/page.h>
5bc20fc5	21	#include <asm/xen/hypercall.h>
5bc20fc5	22	#include <asm/xen/hypervisor.h>
b8b0f559	23	#include <xen/tmem.h>
5bc20fc5	24
0cb401d4 KRW	25	#ifndef CONFIG_XEN_TMEM_MODULE
	26	bool __read_mostly tmem_enabled = false;
	27
	28	static int __init enable_tmem(char *s)
	29	{
	30	tmem_enabled = true;
	31	return 1;
	32	}
	33	__setup("tmem", enable_tmem);
	34	#endif
	35
	36	#ifdef CONFIG_CLEANCACHE
9fd19653	37	static bool cleancache __read_mostly = true;
9fd19653	38	module_param(cleancache, bool, S_IRUGO);
2ca62b04	39	static bool selfballooning __read_mostly = true;
9fd19653	40	module_param(selfballooning, bool, S_IRUGO);
0cb401d4 KRW	41	#endif /* CONFIG_CLEANCACHE */
	42
	43	#ifdef CONFIG_FRONTSWAP
9fd19653	44	static bool frontswap __read_mostly = true;
9fd19653	45	module_param(frontswap, bool, S_IRUGO);
1d7004f0 FC	46	#else /* CONFIG_FRONTSWAP */
1d7004f0 FC	47	#define frontswap (0)
e8f9cb03 KRW	48	#endif /* CONFIG_FRONTSWAP */
e8f9cb03 KRW	49
23972c63	50	#ifdef CONFIG_XEN_SELFBALLOONING
2ca62b04 KRW	51	static bool selfshrinking __read_mostly = true;
2ca62b04 KRW	52	module_param(selfshrinking, bool, S_IRUGO);
23972c63	53	#endif /* CONFIG_XEN_SELFBALLOONING */
0cb401d4	54
5bc20fc5 DM	55	#define TMEM_CONTROL 0
	56	#define TMEM_NEW_POOL 1
	57	#define TMEM_DESTROY_POOL 2
	58	#define TMEM_NEW_PAGE 3
	59	#define TMEM_PUT_PAGE 4
	60	#define TMEM_GET_PAGE 5
	61	#define TMEM_FLUSH_PAGE 6
	62	#define TMEM_FLUSH_OBJECT 7
	63	#define TMEM_READ 8
	64	#define TMEM_WRITE 9
	65	#define TMEM_XCHG 10
	66
	67	/* Bits for HYPERVISOR_tmem_op(TMEM_NEW_POOL) */
	68	#define TMEM_POOL_PERSIST 1
	69	#define TMEM_POOL_SHARED 2
	70	#define TMEM_POOL_PAGESIZE_SHIFT 4
	71	#define TMEM_VERSION_SHIFT 24
	72
	73
	74	struct tmem_pool_uuid {
	75	u64 uuid_lo;
	76	u64 uuid_hi;
	77	};
	78
	79	struct tmem_oid {
	80	u64 oid[3];
	81	};
	82
	83	#define TMEM_POOL_PRIVATE_UUID { 0, 0 }
	84
	85	/* flags for tmem_ops.new_pool */
	86	#define TMEM_POOL_PERSIST 1
	87	#define TMEM_POOL_SHARED 2
	88
	89	/* xen tmem foundation ops/hypercalls */
	90
	91	static inline int xen_tmem_op(u32 tmem_cmd, u32 tmem_pool, struct tmem_oid oid,
	92	u32 index, unsigned long gmfn, u32 tmem_offset, u32 pfn_offset, u32 len)
	93	{
	94	struct tmem_op op;
	95	int rc = 0;
	96
	97	op.cmd = tmem_cmd;
	98	op.pool_id = tmem_pool;
	99	op.u.gen.oid[0] = oid.oid[0];
	100	op.u.gen.oid[1] = oid.oid[1];
	101	op.u.gen.oid[2] = oid.oid[2];
	102	op.u.gen.index = index;
	103	op.u.gen.tmem_offset = tmem_offset;
	104	op.u.gen.pfn_offset = pfn_offset;
	105	op.u.gen.len = len;
	106	set_xen_guest_handle(op.u.gen.gmfn, (void *)gmfn);
	107	rc = HYPERVISOR_tmem_op(&op);
	108	return rc;
	109	}
	110
	111	static int xen_tmem_new_pool(struct tmem_pool_uuid uuid,
	112	u32 flags, unsigned long pagesize)
	113	{
	114	struct tmem_op op;
	115	int rc = 0, pageshift;
	116
	117	for (pageshift = 0; pagesize != 1; pageshift++)
	118	pagesize >>= 1;
119	flags \|= (pageshift - 12) << TMEM_POOL_PAGESIZE_SHIFT;
120	flags \|= TMEM_SPEC_VERSION << TMEM_VERSION_SHIFT;
121	op.cmd = TMEM_NEW_POOL;
122	op.u.new.uuid[0] = uuid.uuid_lo;
123	op.u.new.uuid[1] = uuid.uuid_hi;
124	op.u.new.flags = flags;
125	rc = HYPERVISOR_tmem_op(&op);
126	return rc;
127	}
128
129	/* xen generic tmem ops */
130
131	static int xen_tmem_put_page(u32 pool_id, struct tmem_oid oid,
a76e3cc3	132	u32 index, struct page *page)
5bc20fc5	133	{
5bc20fc5	134	return xen_tmem_op(TMEM_PUT_PAGE, pool_id, oid, index,
a76e3cc3	135	xen_page_to_gfn(page), 0, 0, 0);
5bc20fc5 DM	136	}
	137
	138	static int xen_tmem_get_page(u32 pool_id, struct tmem_oid oid,
a76e3cc3	139	u32 index, struct page *page)
5bc20fc5	140	{
5bc20fc5	141	return xen_tmem_op(TMEM_GET_PAGE, pool_id, oid, index,
a76e3cc3	142	xen_page_to_gfn(page), 0, 0, 0);
5bc20fc5 DM	143	}
	144
	145	static int xen_tmem_flush_page(u32 pool_id, struct tmem_oid oid, u32 index)
	146	{
	147	return xen_tmem_op(TMEM_FLUSH_PAGE, pool_id, oid, index,
	148	0, 0, 0, 0);
	149	}
	150
	151	static int xen_tmem_flush_object(u32 pool_id, struct tmem_oid oid)
	152	{
	153	return xen_tmem_op(TMEM_FLUSH_OBJECT, pool_id, oid, 0, 0, 0, 0, 0);
	154	}
	155
5bc20fc5	156
afec6e04 DM	157	#ifdef CONFIG_CLEANCACHE
	158	static int xen_tmem_destroy_pool(u32 pool_id)
	159	{
	160	struct tmem_oid oid = { { 0 } };
	161
	162	return xen_tmem_op(TMEM_DESTROY_POOL, pool_id, oid, 0, 0, 0, 0, 0);
	163	}
	164
5bc20fc5 DM	165	/* cleancache ops */
	166
	167	static void tmem_cleancache_put_page(int pool, struct cleancache_filekey key,
	168	pgoff_t index, struct page *page)
	169	{
	170	u32 ind = (u32) index;
	171	struct tmem_oid oid = (struct tmem_oid )&key;
5bc20fc5 DM	172
	173	if (pool < 0)
	174	return;
	175	if (ind != index)
	176	return;
	177	mb(); /* ensure page is quiescent; tmem may address it with an alias */
a76e3cc3	178	(void)xen_tmem_put_page((u32)pool, oid, ind, page);
5bc20fc5 DM	179	}
	180
	181	static int tmem_cleancache_get_page(int pool, struct cleancache_filekey key,
	182	pgoff_t index, struct page *page)
	183	{
	184	u32 ind = (u32) index;
	185	struct tmem_oid oid = (struct tmem_oid )&key;
5bc20fc5 DM	186	int ret;
	187
	188	/* translate return values to linux semantics */
	189	if (pool < 0)
	190	return -1;
	191	if (ind != index)
	192	return -1;
a76e3cc3	193	ret = xen_tmem_get_page((u32)pool, oid, ind, page);
5bc20fc5 DM	194	if (ret == 1)
	195	return 0;
	196	else
	197	return -1;
	198	}
	199
	200	static void tmem_cleancache_flush_page(int pool, struct cleancache_filekey key,
	201	pgoff_t index)
	202	{
	203	u32 ind = (u32) index;
	204	struct tmem_oid oid = (struct tmem_oid )&key;
	205
	206	if (pool < 0)
	207	return;
	208	if (ind != index)
	209	return;
	210	(void)xen_tmem_flush_page((u32)pool, oid, ind);
	211	}
	212
	213	static void tmem_cleancache_flush_inode(int pool, struct cleancache_filekey key)
	214	{
	215	struct tmem_oid oid = (struct tmem_oid )&key;
	216
	217	if (pool < 0)
	218	return;
	219	(void)xen_tmem_flush_object((u32)pool, oid);
	220	}
	221
	222	static void tmem_cleancache_flush_fs(int pool)
	223	{
	224	if (pool < 0)
	225	return;
	226	(void)xen_tmem_destroy_pool((u32)pool);
	227	}
	228
	229	static int tmem_cleancache_init_fs(size_t pagesize)
	230	{
	231	struct tmem_pool_uuid uuid_private = TMEM_POOL_PRIVATE_UUID;
	232
	233	return xen_tmem_new_pool(uuid_private, 0, pagesize);
	234	}
	235
	236	static int tmem_cleancache_init_shared_fs(char *uuid, size_t pagesize)
	237	{
	238	struct tmem_pool_uuid shared_uuid;
	239
	240	shared_uuid.uuid_lo = (u64 )uuid;
	241	shared_uuid.uuid_hi = (u64 )(&uuid[8]);
	242	return xen_tmem_new_pool(shared_uuid, TMEM_POOL_SHARED, pagesize);
	243	}
	244
b3c6de49	245	static const struct cleancache_ops tmem_cleancache_ops = {
5bc20fc5 DM	246	.put_page = tmem_cleancache_put_page,
5bc20fc5 DM	247	.get_page = tmem_cleancache_get_page,
91c6cc9b DM	248	.invalidate_page = tmem_cleancache_flush_page,
	249	.invalidate_inode = tmem_cleancache_flush_inode,
	250	.invalidate_fs = tmem_cleancache_flush_fs,
5bc20fc5 DM	251	.init_shared_fs = tmem_cleancache_init_shared_fs,
	252	.init_fs = tmem_cleancache_init_fs
	253	};
afec6e04	254	#endif
5bc20fc5	255
afec6e04 DM	256	#ifdef CONFIG_FRONTSWAP
	257	/* frontswap tmem operations */
	258
	259	/* a single tmem poolid is used for all frontswap "types" (swapfiles) */
	260	static int tmem_frontswap_poolid;
	261
	262	/*
	263	* Swizzling increases objects per swaptype, increasing tmem concurrency
	264	* for heavy swaploads. Later, larger nr_cpus -> larger SWIZ_BITS
	265	*/
	266	#define SWIZ_BITS 4
	267	#define SWIZ_MASK ((1 << SWIZ_BITS) - 1)
	268	#define _oswiz(_type, _ind) ((_type << SWIZ_BITS) \| (_ind & SWIZ_MASK))
	269	#define iswiz(_ind) (_ind >> SWIZ_BITS)
	270
	271	static inline struct tmem_oid oswiz(unsigned type, u32 ind)
5bc20fc5	272	{
afec6e04 DM	273	struct tmem_oid oid = { .oid = { 0 } };
	274	oid.oid[0] = _oswiz(type, ind);
	275	return oid;
	276	}
5bc20fc5	277
afec6e04	278	/* returns 0 if the page was successfully put into frontswap, -1 if not */
165c8aed	279	static int tmem_frontswap_store(unsigned type, pgoff_t offset,
afec6e04 DM	280	struct page *page)
	281	{
	282	u64 ind64 = (u64)offset;
	283	u32 ind = (u32)offset;
afec6e04 DM	284	int pool = tmem_frontswap_poolid;
	285	int ret;
	286
	287	if (pool < 0)
	288	return -1;
	289	if (ind64 != ind)
	290	return -1;
	291	mb(); /* ensure page is quiescent; tmem may address it with an alias */
a76e3cc3	292	ret = xen_tmem_put_page(pool, oswiz(type, ind), iswiz(ind), page);
afec6e04 DM	293	/* translate Xen tmem return values to linux semantics */
	294	if (ret == 1)
	295	return 0;
	296	else
	297	return -1;
	298	}
	299
	300	/*
	301	* returns 0 if the page was successfully gotten from frontswap, -1 if
	302	* was not present (should never happen!)
	303	*/
165c8aed	304	static int tmem_frontswap_load(unsigned type, pgoff_t offset,
afec6e04 DM	305	struct page *page)
	306	{
	307	u64 ind64 = (u64)offset;
	308	u32 ind = (u32)offset;
afec6e04 DM	309	int pool = tmem_frontswap_poolid;
	310	int ret;
	311
	312	if (pool < 0)
	313	return -1;
	314	if (ind64 != ind)
	315	return -1;
a76e3cc3	316	ret = xen_tmem_get_page(pool, oswiz(type, ind), iswiz(ind), page);
afec6e04 DM	317	/* translate Xen tmem return values to linux semantics */
	318	if (ret == 1)
	319	return 0;
	320	else
	321	return -1;
	322	}
	323
	324	/* flush a single page from frontswap */
	325	static void tmem_frontswap_flush_page(unsigned type, pgoff_t offset)
	326	{
	327	u64 ind64 = (u64)offset;
	328	u32 ind = (u32)offset;
	329	int pool = tmem_frontswap_poolid;
	330
	331	if (pool < 0)
	332	return;
	333	if (ind64 != ind)
	334	return;
	335	(void) xen_tmem_flush_page(pool, oswiz(type, ind), iswiz(ind));
	336	}
	337
	338	/* flush all pages from the passed swaptype */
	339	static void tmem_frontswap_flush_area(unsigned type)
	340	{
	341	int pool = tmem_frontswap_poolid;
	342	int ind;
	343
	344	if (pool < 0)
	345	return;
	346	for (ind = SWIZ_MASK; ind >= 0; ind--)
	347	(void)xen_tmem_flush_object(pool, oswiz(type, ind));
	348	}
	349
	350	static void tmem_frontswap_init(unsigned ignored)
	351	{
	352	struct tmem_pool_uuid private = TMEM_POOL_PRIVATE_UUID;
	353
	354	/* a single tmem poolid is used for all frontswap "types" (swapfiles) */
	355	if (tmem_frontswap_poolid < 0)
	356	tmem_frontswap_poolid =
	357	xen_tmem_new_pool(private, TMEM_POOL_PERSIST, PAGE_SIZE);
	358	}
	359
1e01c968	360	static struct frontswap_ops tmem_frontswap_ops = {
165c8aed KRW	361	.store = tmem_frontswap_store,
165c8aed KRW	362	.load = tmem_frontswap_load,
91c6cc9b DM	363	.invalidate_page = tmem_frontswap_flush_page,
91c6cc9b DM	364	.invalidate_area = tmem_frontswap_flush_area,
afec6e04 DM	365	.init = tmem_frontswap_init
	366	};
	367	#endif
	368
144c61ad	369	static int __init xen_tmem_init(void)
afec6e04	370	{
5bc20fc5 DM	371	if (!xen_domain())
5bc20fc5 DM	372	return 0;
afec6e04	373	#ifdef CONFIG_FRONTSWAP
9fd19653	374	if (tmem_enabled && frontswap) {
afec6e04	375	char *s = "";
afec6e04 DM	376
afec6e04 DM	377	tmem_frontswap_poolid = -1;
d1dc6f1b	378	frontswap_register_ops(&tmem_frontswap_ops);
283c0972 JP	379	pr_info("frontswap enabled, RAM provided by Xen Transcendent Memory%s\n",
283c0972 JP	380	s);
afec6e04 DM	381	}
afec6e04 DM	382	#endif
5bc20fc5	383	#ifdef CONFIG_CLEANCACHE
01b720f3	384	BUILD_BUG_ON(sizeof(struct cleancache_filekey) != sizeof(struct tmem_oid));
9fd19653	385	if (tmem_enabled && cleancache) {
53d85c98 VD	386	int err;
	387
	388	err = cleancache_register_ops(&tmem_cleancache_ops);
	389	if (err)
	390	pr_warn("xen-tmem: failed to enable cleancache: %d\n",
	391	err);
	392	else
	393	pr_info("cleancache enabled, RAM provided by "
	394	"Xen Transcendent Memory\n");
5bc20fc5	395	}
10a7a077 DM	396	#endif
10a7a077 DM	397	#ifdef CONFIG_XEN_SELFBALLOONING
37d46e15 KRW	398	/*
	399	* There is no point of driving pages to the swap system if they
	400	* aren't going anywhere in tmem universe.
	401	*/
	402	if (!frontswap) {
	403	selfshrinking = false;
	404	selfballooning = false;
	405	}
2ca62b04	406	xen_selfballoon_init(selfballooning, selfshrinking);
5bc20fc5 DM	407	#endif
	408	return 0;
	409	}
	410
	411	module_init(xen_tmem_init)
10a7a077 DM	412	MODULE_LICENSE("GPL");
	413	MODULE_AUTHOR("Dan Magenheimer <dan.magenheimer@oracle.com>");
	414	MODULE_DESCRIPTION("Shim to Xen transcendent memory");