[linux-2.6-block.git] / lib / test_xarray.c

// SPDX-License-Identifier: GPL-2.0+
/*
 * test_xarray.c: Test the XArray API
 * Copyright (c) 2017-2018 Microsoft Corporation
 * Author: Matthew Wilcox <willy@infradead.org>
 */

#include <linux/xarray.h>
#include <linux/module.h>

static unsigned int tests_run;
static unsigned int tests_passed;

#ifndef XA_DEBUG
# ifdef __KERNEL__
void xa_dump(const struct xarray *xa) { }
# endif
#undef XA_BUG_ON
#define XA_BUG_ON(xa, x) do {					\
	tests_run++;						\
	if (x) {						\
		printk("BUG at %s:%d\n", __func__, __LINE__);	\
		xa_dump(xa);					\
		dump_stack();					\
	} else {						\
		tests_passed++;					\
	}							\
} while (0)
#endif

static void *xa_store_index(struct xarray *xa, unsigned long index, gfp_t gfp)
{
	return xa_store(xa, index, xa_mk_value(index & LONG_MAX), gfp);
}

static void xa_erase_index(struct xarray *xa, unsigned long index)
{
	XA_BUG_ON(xa, xa_erase(xa, index) != xa_mk_value(index & LONG_MAX));
	XA_BUG_ON(xa, xa_load(xa, index) != NULL);
}

/*
 * If anyone needs this, please move it to xarray.c.  We have no current
 * users outside the test suite because all current multislot users want
 * to use the advanced API.
 */
static void *xa_store_order(struct xarray *xa, unsigned long index,
		unsigned order, void *entry, gfp_t gfp)
{
	XA_STATE_ORDER(xas, xa, index, order);
	void *curr;

	do {
		xas_lock(&xas);
		curr = xas_store(&xas, entry);
		xas_unlock(&xas);
	} while (xas_nomem(&xas, gfp));

	return curr;
}

static noinline void check_xa_err(struct xarray *xa)
{
	XA_BUG_ON(xa, xa_err(xa_store_index(xa, 0, GFP_NOWAIT)) != 0);
	XA_BUG_ON(xa, xa_err(xa_erase(xa, 0)) != 0);
#ifndef __KERNEL__
	/* The kernel does not fail GFP_NOWAIT allocations */
	XA_BUG_ON(xa, xa_err(xa_store_index(xa, 1, GFP_NOWAIT)) != -ENOMEM);
	XA_BUG_ON(xa, xa_err(xa_store_index(xa, 1, GFP_NOWAIT)) != -ENOMEM);
#endif
	XA_BUG_ON(xa, xa_err(xa_store_index(xa, 1, GFP_KERNEL)) != 0);
	XA_BUG_ON(xa, xa_err(xa_store(xa, 1, xa_mk_value(0), GFP_KERNEL)) != 0);
	XA_BUG_ON(xa, xa_err(xa_erase(xa, 1)) != 0);
// kills the test-suite :-(
//	XA_BUG_ON(xa, xa_err(xa_store(xa, 0, xa_mk_internal(0), 0)) != -EINVAL);
}

static noinline void check_xas_retry(struct xarray *xa)
{
	XA_STATE(xas, xa, 0);
	void *entry;

	xa_store_index(xa, 0, GFP_KERNEL);
	xa_store_index(xa, 1, GFP_KERNEL);

	rcu_read_lock();
	XA_BUG_ON(xa, xas_find(&xas, ULONG_MAX) != xa_mk_value(0));
	xa_erase_index(xa, 1);
	XA_BUG_ON(xa, !xa_is_retry(xas_reload(&xas)));
	XA_BUG_ON(xa, xas_retry(&xas, NULL));
	XA_BUG_ON(xa, xas_retry(&xas, xa_mk_value(0)));
	xas_reset(&xas);
	XA_BUG_ON(xa, xas.xa_node != XAS_RESTART);
	XA_BUG_ON(xa, xas_next_entry(&xas, ULONG_MAX) != xa_mk_value(0));
	XA_BUG_ON(xa, xas.xa_node != NULL);

	XA_BUG_ON(xa, xa_store_index(xa, 1, GFP_KERNEL) != NULL);
	XA_BUG_ON(xa, !xa_is_internal(xas_reload(&xas)));
	xas.xa_node = XAS_RESTART;
	XA_BUG_ON(xa, xas_next_entry(&xas, ULONG_MAX) != xa_mk_value(0));
	rcu_read_unlock();

	/* Make sure we can iterate through retry entries */
	xas_lock(&xas);
	xas_set(&xas, 0);
	xas_store(&xas, XA_RETRY_ENTRY);
	xas_set(&xas, 1);
	xas_store(&xas, XA_RETRY_ENTRY);

	xas_set(&xas, 0);
	xas_for_each(&xas, entry, ULONG_MAX) {
		xas_store(&xas, xa_mk_value(xas.xa_index));
	}
	xas_unlock(&xas);

	xa_erase_index(xa, 0);
	xa_erase_index(xa, 1);
}

static noinline void check_xa_load(struct xarray *xa)
{
	unsigned long i, j;

	for (i = 0; i < 1024; i++) {
		for (j = 0; j < 1024; j++) {
			void *entry = xa_load(xa, j);
			if (j < i)
				XA_BUG_ON(xa, xa_to_value(entry) != j);
			else
				XA_BUG_ON(xa, entry);
		}
		XA_BUG_ON(xa, xa_store_index(xa, i, GFP_KERNEL) != NULL);
	}

	for (i = 0; i < 1024; i++) {
		for (j = 0; j < 1024; j++) {
			void *entry = xa_load(xa, j);
			if (j >= i)
				XA_BUG_ON(xa, xa_to_value(entry) != j);
			else
				XA_BUG_ON(xa, entry);
		}
		xa_erase_index(xa, i);
	}
	XA_BUG_ON(xa, !xa_empty(xa));
}

static noinline void check_xa_mark_1(struct xarray *xa, unsigned long index)
{
	unsigned int order;
	unsigned int max_order = IS_ENABLED(CONFIG_XARRAY_MULTI) ? 8 : 1;

	/* NULL elements have no marks set */
	XA_BUG_ON(xa, xa_get_mark(xa, index, XA_MARK_0));
	xa_set_mark(xa, index, XA_MARK_0);
	XA_BUG_ON(xa, xa_get_mark(xa, index, XA_MARK_0));

	/* Storing a pointer will not make a mark appear */
	XA_BUG_ON(xa, xa_store_index(xa, index, GFP_KERNEL) != NULL);
	XA_BUG_ON(xa, xa_get_mark(xa, index, XA_MARK_0));
	xa_set_mark(xa, index, XA_MARK_0);
	XA_BUG_ON(xa, !xa_get_mark(xa, index, XA_MARK_0));

	/* Setting one mark will not set another mark */
	XA_BUG_ON(xa, xa_get_mark(xa, index + 1, XA_MARK_0));
	XA_BUG_ON(xa, xa_get_mark(xa, index, XA_MARK_1));

	/* Storing NULL clears marks, and they can't be set again */
	xa_erase_index(xa, index);
	XA_BUG_ON(xa, !xa_empty(xa));
	XA_BUG_ON(xa, xa_get_mark(xa, index, XA_MARK_0));
	xa_set_mark(xa, index, XA_MARK_0);
	XA_BUG_ON(xa, xa_get_mark(xa, index, XA_MARK_0));

	/*
	 * Storing a multi-index entry over entries with marks gives the
	 * entire entry the union of the marks
	 */
	BUG_ON((index % 4) != 0);
	for (order = 2; order < max_order; order++) {
		unsigned long base = round_down(index, 1UL << order);
		unsigned long next = base + (1UL << order);
		unsigned long i;

		XA_BUG_ON(xa, xa_store_index(xa, index + 1, GFP_KERNEL));
		xa_set_mark(xa, index + 1, XA_MARK_0);
		XA_BUG_ON(xa, xa_store_index(xa, index + 2, GFP_KERNEL));
		xa_set_mark(xa, index + 2, XA_MARK_1);
		XA_BUG_ON(xa, xa_store_index(xa, next, GFP_KERNEL));
		xa_store_order(xa, index, order, xa_mk_value(index),
				GFP_KERNEL);
		for (i = base; i < next; i++) {
			XA_BUG_ON(xa, !xa_get_mark(xa, i, XA_MARK_0));
			XA_BUG_ON(xa, !xa_get_mark(xa, i, XA_MARK_1));
			XA_BUG_ON(xa, xa_get_mark(xa, i, XA_MARK_2));
		}
		XA_BUG_ON(xa, xa_get_mark(xa, next, XA_MARK_0));
		XA_BUG_ON(xa, xa_get_mark(xa, next, XA_MARK_1));
		XA_BUG_ON(xa, xa_get_mark(xa, next, XA_MARK_2));
		xa_erase_index(xa, index);
		xa_erase_index(xa, next);
		XA_BUG_ON(xa, !xa_empty(xa));
	}
	XA_BUG_ON(xa, !xa_empty(xa));
}

static noinline void check_xa_mark(struct xarray *xa)
{
	unsigned long index;

	for (index = 0; index < 16384; index += 4)
		check_xa_mark_1(xa, index);
}

static noinline void check_xa_shrink(struct xarray *xa)
{
	XA_STATE(xas, xa, 1);
	struct xa_node *node;

	XA_BUG_ON(xa, !xa_empty(xa));
	XA_BUG_ON(xa, xa_store_index(xa, 0, GFP_KERNEL) != NULL);
	XA_BUG_ON(xa, xa_store_index(xa, 1, GFP_KERNEL) != NULL);

	/*
	 * Check that erasing the entry at 1 shrinks the tree and properly
	 * marks the node as being deleted.
	 */
	xas_lock(&xas);
	XA_BUG_ON(xa, xas_load(&xas) != xa_mk_value(1));
	node = xas.xa_node;
	XA_BUG_ON(xa, xa_entry_locked(xa, node, 0) != xa_mk_value(0));
	XA_BUG_ON(xa, xas_store(&xas, NULL) != xa_mk_value(1));
	XA_BUG_ON(xa, xa_load(xa, 1) != NULL);
	XA_BUG_ON(xa, xas.xa_node != XAS_BOUNDS);
	XA_BUG_ON(xa, xa_entry_locked(xa, node, 0) != XA_RETRY_ENTRY);
	XA_BUG_ON(xa, xas_load(&xas) != NULL);
	xas_unlock(&xas);
	XA_BUG_ON(xa, xa_load(xa, 0) != xa_mk_value(0));
	xa_erase_index(xa, 0);
	XA_BUG_ON(xa, !xa_empty(xa));
}

static noinline void check_cmpxchg(struct xarray *xa)
{
	void *FIVE = xa_mk_value(5);
	void *SIX = xa_mk_value(6);
	void *LOTS = xa_mk_value(12345678);

	XA_BUG_ON(xa, !xa_empty(xa));
	XA_BUG_ON(xa, xa_store_index(xa, 12345678, GFP_KERNEL) != NULL);
	XA_BUG_ON(xa, xa_insert(xa, 12345678, xa, GFP_KERNEL) != -EEXIST);
	XA_BUG_ON(xa, xa_cmpxchg(xa, 12345678, SIX, FIVE, GFP_KERNEL) != LOTS);
	XA_BUG_ON(xa, xa_cmpxchg(xa, 12345678, LOTS, FIVE, GFP_KERNEL) != LOTS);
	XA_BUG_ON(xa, xa_cmpxchg(xa, 12345678, FIVE, LOTS, GFP_KERNEL) != FIVE);
	XA_BUG_ON(xa, xa_cmpxchg(xa, 5, FIVE, NULL, GFP_KERNEL) != NULL);
	XA_BUG_ON(xa, xa_cmpxchg(xa, 5, NULL, FIVE, GFP_KERNEL) != NULL);
	xa_erase_index(xa, 12345678);
	xa_erase_index(xa, 5);
	XA_BUG_ON(xa, !xa_empty(xa));
}

static noinline void check_xas_erase(struct xarray *xa)
{
	XA_STATE(xas, xa, 0);
	void *entry;
	unsigned long i, j;

	for (i = 0; i < 200; i++) {
		for (j = i; j < 2 * i + 17; j++) {
			xas_set(&xas, j);
			do {
				xas_lock(&xas);
				xas_store(&xas, xa_mk_value(j));
				xas_unlock(&xas);
			} while (xas_nomem(&xas, GFP_KERNEL));
		}

		xas_set(&xas, ULONG_MAX);
		do {
			xas_lock(&xas);
			xas_store(&xas, xa_mk_value(0));
			xas_unlock(&xas);
		} while (xas_nomem(&xas, GFP_KERNEL));

		xas_lock(&xas);
		xas_store(&xas, NULL);

		xas_set(&xas, 0);
		j = i;
		xas_for_each(&xas, entry, ULONG_MAX) {
			XA_BUG_ON(xa, entry != xa_mk_value(j));
			xas_store(&xas, NULL);
			j++;
		}
		xas_unlock(&xas);
		XA_BUG_ON(xa, !xa_empty(xa));
	}
}

static noinline void check_multi_store(struct xarray *xa)
{
#ifdef CONFIG_XARRAY_MULTI
	unsigned long i, j, k;
	unsigned int max_order = (sizeof(long) == 4) ? 30 : 60;

	/* Loading from any position returns the same value */
	xa_store_order(xa, 0, 1, xa_mk_value(0), GFP_KERNEL);
	XA_BUG_ON(xa, xa_load(xa, 0) != xa_mk_value(0));
	XA_BUG_ON(xa, xa_load(xa, 1) != xa_mk_value(0));
	XA_BUG_ON(xa, xa_load(xa, 2) != NULL);
	rcu_read_lock();
	XA_BUG_ON(xa, xa_to_node(xa_head(xa))->count != 2);
	XA_BUG_ON(xa, xa_to_node(xa_head(xa))->nr_values != 2);
	rcu_read_unlock();

	/* Storing adjacent to the value does not alter the value */
	xa_store(xa, 3, xa, GFP_KERNEL);
	XA_BUG_ON(xa, xa_load(xa, 0) != xa_mk_value(0));
	XA_BUG_ON(xa, xa_load(xa, 1) != xa_mk_value(0));
	XA_BUG_ON(xa, xa_load(xa, 2) != NULL);
	rcu_read_lock();
	XA_BUG_ON(xa, xa_to_node(xa_head(xa))->count != 3);
	XA_BUG_ON(xa, xa_to_node(xa_head(xa))->nr_values != 2);
	rcu_read_unlock();

	/* Overwriting multiple indexes works */
	xa_store_order(xa, 0, 2, xa_mk_value(1), GFP_KERNEL);
	XA_BUG_ON(xa, xa_load(xa, 0) != xa_mk_value(1));
	XA_BUG_ON(xa, xa_load(xa, 1) != xa_mk_value(1));
	XA_BUG_ON(xa, xa_load(xa, 2) != xa_mk_value(1));
	XA_BUG_ON(xa, xa_load(xa, 3) != xa_mk_value(1));
	XA_BUG_ON(xa, xa_load(xa, 4) != NULL);
	rcu_read_lock();
	XA_BUG_ON(xa, xa_to_node(xa_head(xa))->count != 4);
	XA_BUG_ON(xa, xa_to_node(xa_head(xa))->nr_values != 4);
	rcu_read_unlock();

	/* We can erase multiple values with a single store */
	xa_store_order(xa, 0, 63, NULL, GFP_KERNEL);
	XA_BUG_ON(xa, !xa_empty(xa));

	/* Even when the first slot is empty but the others aren't */
	xa_store_index(xa, 1, GFP_KERNEL);
	xa_store_index(xa, 2, GFP_KERNEL);
	xa_store_order(xa, 0, 2, NULL, GFP_KERNEL);
	XA_BUG_ON(xa, !xa_empty(xa));

	for (i = 0; i < max_order; i++) {
		for (j = 0; j < max_order; j++) {
			xa_store_order(xa, 0, i, xa_mk_value(i), GFP_KERNEL);
			xa_store_order(xa, 0, j, xa_mk_value(j), GFP_KERNEL);

			for (k = 0; k < max_order; k++) {
				void *entry = xa_load(xa, (1UL << k) - 1);
				if ((i < k) && (j < k))
					XA_BUG_ON(xa, entry != NULL);
				else
					XA_BUG_ON(xa, entry != xa_mk_value(j));
			}

			xa_erase(xa, 0);
			XA_BUG_ON(xa, !xa_empty(xa));
		}
	}
#endif
}

static noinline void check_multi_find(struct xarray *xa)
{
#ifdef CONFIG_XARRAY_MULTI
	unsigned long index;

	xa_store_order(xa, 12, 2, xa_mk_value(12), GFP_KERNEL);
	XA_BUG_ON(xa, xa_store_index(xa, 16, GFP_KERNEL) != NULL);

	index = 0;
	XA_BUG_ON(xa, xa_find(xa, &index, ULONG_MAX, XA_PRESENT) !=
			xa_mk_value(12));
	XA_BUG_ON(xa, index != 12);
	index = 13;
	XA_BUG_ON(xa, xa_find(xa, &index, ULONG_MAX, XA_PRESENT) !=
			xa_mk_value(12));
	XA_BUG_ON(xa, (index < 12) || (index >= 16));
	XA_BUG_ON(xa, xa_find_after(xa, &index, ULONG_MAX, XA_PRESENT) !=
			xa_mk_value(16));
	XA_BUG_ON(xa, index != 16);

	xa_erase_index(xa, 12);
	xa_erase_index(xa, 16);
	XA_BUG_ON(xa, !xa_empty(xa));
#endif
}

static noinline void check_multi_find_2(struct xarray *xa)
{
	unsigned int max_order = IS_ENABLED(CONFIG_XARRAY_MULTI) ? 10 : 1;
	unsigned int i, j;
	void *entry;

	for (i = 0; i < max_order; i++) {
		unsigned long index = 1UL << i;
		for (j = 0; j < index; j++) {
			XA_STATE(xas, xa, j + index);
			xa_store_index(xa, index - 1, GFP_KERNEL);
			xa_store_order(xa, index, i, xa_mk_value(index),
					GFP_KERNEL);
			rcu_read_lock();
			xas_for_each(&xas, entry, ULONG_MAX) {
				xa_erase_index(xa, index);
			}
			rcu_read_unlock();
			xa_erase_index(xa, index - 1);
			XA_BUG_ON(xa, !xa_empty(xa));
		}
	}
}

static noinline void check_find(struct xarray *xa)
{
	unsigned long i, j, k;

	XA_BUG_ON(xa, !xa_empty(xa));

	/*
	 * Check xa_find with all pairs between 0 and 99 inclusive,
	 * starting at every index between 0 and 99
	 */
	for (i = 0; i < 100; i++) {
		XA_BUG_ON(xa, xa_store_index(xa, i, GFP_KERNEL) != NULL);
		xa_set_mark(xa, i, XA_MARK_0);
		for (j = 0; j < i; j++) {
			XA_BUG_ON(xa, xa_store_index(xa, j, GFP_KERNEL) !=
					NULL);
			xa_set_mark(xa, j, XA_MARK_0);
			for (k = 0; k < 100; k++) {
				unsigned long index = k;
				void *entry = xa_find(xa, &index, ULONG_MAX,
								XA_PRESENT);
				if (k <= j)
					XA_BUG_ON(xa, index != j);
				else if (k <= i)
					XA_BUG_ON(xa, index != i);
				else
					XA_BUG_ON(xa, entry != NULL);

				index = k;
				entry = xa_find(xa, &index, ULONG_MAX,
								XA_MARK_0);
				if (k <= j)
					XA_BUG_ON(xa, index != j);
				else if (k <= i)
					XA_BUG_ON(xa, index != i);
				else
					XA_BUG_ON(xa, entry != NULL);
			}
			xa_erase_index(xa, j);
			XA_BUG_ON(xa, xa_get_mark(xa, j, XA_MARK_0));
			XA_BUG_ON(xa, !xa_get_mark(xa, i, XA_MARK_0));
		}
		xa_erase_index(xa, i);
		XA_BUG_ON(xa, xa_get_mark(xa, i, XA_MARK_0));
	}
	XA_BUG_ON(xa, !xa_empty(xa));
	check_multi_find(xa);
	check_multi_find_2(xa);
}

static DEFINE_XARRAY(array);

static int xarray_checks(void)
{
	check_xa_err(&array);
	check_xas_retry(&array);
	check_xa_load(&array);
	check_xa_mark(&array);
	check_xa_shrink(&array);
	check_xas_erase(&array);
	check_cmpxchg(&array);
	check_multi_store(&array);
	check_find(&array);

	printk("XArray: %u of %u tests passed\n", tests_passed, tests_run);
	return (tests_run == tests_passed) ? 0 : -EINVAL;
}

static void xarray_exit(void)
{
}

module_init(xarray_checks);
module_exit(xarray_exit);
MODULE_AUTHOR("Matthew Wilcox <willy@infradead.org>");
MODULE_LICENSE("GPL");
Commit	Line	Data
ad3d6c72 MW	1	// SPDX-License-Identifier: GPL-2.0+
	2	/*
	3	* test_xarray.c: Test the XArray API
	4	* Copyright (c) 2017-2018 Microsoft Corporation
	5	* Author: Matthew Wilcox <willy@infradead.org>
	6	*/
	7
	8	#include <linux/xarray.h>
	9	#include <linux/module.h>
	10
	11	static unsigned int tests_run;
	12	static unsigned int tests_passed;
	13
	14	#ifndef XA_DEBUG
	15	# ifdef __KERNEL__
	16	void xa_dump(const struct xarray *xa) { }
	17	# endif
	18	#undef XA_BUG_ON
	19	#define XA_BUG_ON(xa, x) do { \
	20	tests_run++; \
	21	if (x) { \
	22	printk("BUG at %s:%d\n", __func__, __LINE__); \
	23	xa_dump(xa); \
	24	dump_stack(); \
	25	} else { \
	26	tests_passed++; \
	27	} \
	28	} while (0)
	29	#endif
	30
	31	static void xa_store_index(struct xarray xa, unsigned long index, gfp_t gfp)
	32	{
58d6ea30	33	return xa_store(xa, index, xa_mk_value(index & LONG_MAX), gfp);
ad3d6c72 MW	34	}
	35
	36	static void xa_erase_index(struct xarray *xa, unsigned long index)
	37	{
58d6ea30 MW	38	XA_BUG_ON(xa, xa_erase(xa, index) != xa_mk_value(index & LONG_MAX));
	39	XA_BUG_ON(xa, xa_load(xa, index) != NULL);
	40	}
	41
	42	/*
	43	* If anyone needs this, please move it to xarray.c. We have no current
	44	* users outside the test suite because all current multislot users want
	45	* to use the advanced API.
	46	*/
	47	static void xa_store_order(struct xarray xa, unsigned long index,
	48	unsigned order, void *entry, gfp_t gfp)
	49	{
	50	XA_STATE_ORDER(xas, xa, index, order);
	51	void *curr;
	52
	53	do {
	54	xas_lock(&xas);
	55	curr = xas_store(&xas, entry);
	56	xas_unlock(&xas);
	57	} while (xas_nomem(&xas, gfp));
	58
	59	return curr;
	60	}
	61
	62	static noinline void check_xa_err(struct xarray *xa)
	63	{
	64	XA_BUG_ON(xa, xa_err(xa_store_index(xa, 0, GFP_NOWAIT)) != 0);
	65	XA_BUG_ON(xa, xa_err(xa_erase(xa, 0)) != 0);
	66	#ifndef __KERNEL__
	67	/* The kernel does not fail GFP_NOWAIT allocations */
	68	XA_BUG_ON(xa, xa_err(xa_store_index(xa, 1, GFP_NOWAIT)) != -ENOMEM);
	69	XA_BUG_ON(xa, xa_err(xa_store_index(xa, 1, GFP_NOWAIT)) != -ENOMEM);
	70	#endif
	71	XA_BUG_ON(xa, xa_err(xa_store_index(xa, 1, GFP_KERNEL)) != 0);
	72	XA_BUG_ON(xa, xa_err(xa_store(xa, 1, xa_mk_value(0), GFP_KERNEL)) != 0);
	73	XA_BUG_ON(xa, xa_err(xa_erase(xa, 1)) != 0);
	74	// kills the test-suite :-(
	75	// XA_BUG_ON(xa, xa_err(xa_store(xa, 0, xa_mk_internal(0), 0)) != -EINVAL);
ad3d6c72 MW	76	}
ad3d6c72 MW	77
b803b428 MW	78	static noinline void check_xas_retry(struct xarray *xa)
	79	{
	80	XA_STATE(xas, xa, 0);
	81	void *entry;
	82
	83	xa_store_index(xa, 0, GFP_KERNEL);
	84	xa_store_index(xa, 1, GFP_KERNEL);
	85
	86	rcu_read_lock();
	87	XA_BUG_ON(xa, xas_find(&xas, ULONG_MAX) != xa_mk_value(0));
	88	xa_erase_index(xa, 1);
	89	XA_BUG_ON(xa, !xa_is_retry(xas_reload(&xas)));
	90	XA_BUG_ON(xa, xas_retry(&xas, NULL));
	91	XA_BUG_ON(xa, xas_retry(&xas, xa_mk_value(0)));
	92	xas_reset(&xas);
	93	XA_BUG_ON(xa, xas.xa_node != XAS_RESTART);
	94	XA_BUG_ON(xa, xas_next_entry(&xas, ULONG_MAX) != xa_mk_value(0));
	95	XA_BUG_ON(xa, xas.xa_node != NULL);
	96
	97	XA_BUG_ON(xa, xa_store_index(xa, 1, GFP_KERNEL) != NULL);
	98	XA_BUG_ON(xa, !xa_is_internal(xas_reload(&xas)));
	99	xas.xa_node = XAS_RESTART;
	100	XA_BUG_ON(xa, xas_next_entry(&xas, ULONG_MAX) != xa_mk_value(0));
	101	rcu_read_unlock();
	102
	103	/* Make sure we can iterate through retry entries */
	104	xas_lock(&xas);
	105	xas_set(&xas, 0);
	106	xas_store(&xas, XA_RETRY_ENTRY);
	107	xas_set(&xas, 1);
	108	xas_store(&xas, XA_RETRY_ENTRY);
	109
	110	xas_set(&xas, 0);
	111	xas_for_each(&xas, entry, ULONG_MAX) {
	112	xas_store(&xas, xa_mk_value(xas.xa_index));
	113	}
	114	xas_unlock(&xas);
	115
	116	xa_erase_index(xa, 0);
	117	xa_erase_index(xa, 1);
	118	}
	119
ad3d6c72 MW	120	static noinline void check_xa_load(struct xarray *xa)
	121	{
	122	unsigned long i, j;
	123
	124	for (i = 0; i < 1024; i++) {
	125	for (j = 0; j < 1024; j++) {
	126	void *entry = xa_load(xa, j);
	127	if (j < i)
	128	XA_BUG_ON(xa, xa_to_value(entry) != j);
	129	else
	130	XA_BUG_ON(xa, entry);
	131	}
	132	XA_BUG_ON(xa, xa_store_index(xa, i, GFP_KERNEL) != NULL);
	133	}
	134
	135	for (i = 0; i < 1024; i++) {
	136	for (j = 0; j < 1024; j++) {
	137	void *entry = xa_load(xa, j);
	138	if (j >= i)
	139	XA_BUG_ON(xa, xa_to_value(entry) != j);
	140	else
	141	XA_BUG_ON(xa, entry);
	142	}
	143	xa_erase_index(xa, i);
	144	}
	145	XA_BUG_ON(xa, !xa_empty(xa));
	146	}
	147
9b89a035 MW	148	static noinline void check_xa_mark_1(struct xarray *xa, unsigned long index)
9b89a035 MW	149	{
58d6ea30 MW	150	unsigned int order;
	151	unsigned int max_order = IS_ENABLED(CONFIG_XARRAY_MULTI) ? 8 : 1;
	152
9b89a035 MW	153	/* NULL elements have no marks set */
	154	XA_BUG_ON(xa, xa_get_mark(xa, index, XA_MARK_0));
	155	xa_set_mark(xa, index, XA_MARK_0);
	156	XA_BUG_ON(xa, xa_get_mark(xa, index, XA_MARK_0));
	157
	158	/* Storing a pointer will not make a mark appear */
	159	XA_BUG_ON(xa, xa_store_index(xa, index, GFP_KERNEL) != NULL);
	160	XA_BUG_ON(xa, xa_get_mark(xa, index, XA_MARK_0));
	161	xa_set_mark(xa, index, XA_MARK_0);
	162	XA_BUG_ON(xa, !xa_get_mark(xa, index, XA_MARK_0));
	163
	164	/* Setting one mark will not set another mark */
	165	XA_BUG_ON(xa, xa_get_mark(xa, index + 1, XA_MARK_0));
	166	XA_BUG_ON(xa, xa_get_mark(xa, index, XA_MARK_1));
	167
	168	/* Storing NULL clears marks, and they can't be set again */
	169	xa_erase_index(xa, index);
	170	XA_BUG_ON(xa, !xa_empty(xa));
	171	XA_BUG_ON(xa, xa_get_mark(xa, index, XA_MARK_0));
	172	xa_set_mark(xa, index, XA_MARK_0);
	173	XA_BUG_ON(xa, xa_get_mark(xa, index, XA_MARK_0));
58d6ea30 MW	174
	175	/*
	176	* Storing a multi-index entry over entries with marks gives the
	177	* entire entry the union of the marks
	178	*/
	179	BUG_ON((index % 4) != 0);
	180	for (order = 2; order < max_order; order++) {
	181	unsigned long base = round_down(index, 1UL << order);
	182	unsigned long next = base + (1UL << order);
	183	unsigned long i;
	184
	185	XA_BUG_ON(xa, xa_store_index(xa, index + 1, GFP_KERNEL));
	186	xa_set_mark(xa, index + 1, XA_MARK_0);
	187	XA_BUG_ON(xa, xa_store_index(xa, index + 2, GFP_KERNEL));
	188	xa_set_mark(xa, index + 2, XA_MARK_1);
	189	XA_BUG_ON(xa, xa_store_index(xa, next, GFP_KERNEL));
	190	xa_store_order(xa, index, order, xa_mk_value(index),
	191	GFP_KERNEL);
	192	for (i = base; i < next; i++) {
	193	XA_BUG_ON(xa, !xa_get_mark(xa, i, XA_MARK_0));
	194	XA_BUG_ON(xa, !xa_get_mark(xa, i, XA_MARK_1));
	195	XA_BUG_ON(xa, xa_get_mark(xa, i, XA_MARK_2));
	196	}
	197	XA_BUG_ON(xa, xa_get_mark(xa, next, XA_MARK_0));
	198	XA_BUG_ON(xa, xa_get_mark(xa, next, XA_MARK_1));
	199	XA_BUG_ON(xa, xa_get_mark(xa, next, XA_MARK_2));
	200	xa_erase_index(xa, index);
	201	xa_erase_index(xa, next);
	202	XA_BUG_ON(xa, !xa_empty(xa));
	203	}
	204	XA_BUG_ON(xa, !xa_empty(xa));
9b89a035 MW	205	}
	206
	207	static noinline void check_xa_mark(struct xarray *xa)
	208	{
	209	unsigned long index;
	210
	211	for (index = 0; index < 16384; index += 4)
	212	check_xa_mark_1(xa, index);
	213	}
	214
58d6ea30 MW	215	static noinline void check_xa_shrink(struct xarray *xa)
	216	{
	217	XA_STATE(xas, xa, 1);
	218	struct xa_node *node;
	219
	220	XA_BUG_ON(xa, !xa_empty(xa));
	221	XA_BUG_ON(xa, xa_store_index(xa, 0, GFP_KERNEL) != NULL);
	222	XA_BUG_ON(xa, xa_store_index(xa, 1, GFP_KERNEL) != NULL);
	223
	224	/*
	225	* Check that erasing the entry at 1 shrinks the tree and properly
	226	* marks the node as being deleted.
	227	*/
	228	xas_lock(&xas);
	229	XA_BUG_ON(xa, xas_load(&xas) != xa_mk_value(1));
	230	node = xas.xa_node;
	231	XA_BUG_ON(xa, xa_entry_locked(xa, node, 0) != xa_mk_value(0));
	232	XA_BUG_ON(xa, xas_store(&xas, NULL) != xa_mk_value(1));
	233	XA_BUG_ON(xa, xa_load(xa, 1) != NULL);
	234	XA_BUG_ON(xa, xas.xa_node != XAS_BOUNDS);
	235	XA_BUG_ON(xa, xa_entry_locked(xa, node, 0) != XA_RETRY_ENTRY);
	236	XA_BUG_ON(xa, xas_load(&xas) != NULL);
	237	xas_unlock(&xas);
	238	XA_BUG_ON(xa, xa_load(xa, 0) != xa_mk_value(0));
	239	xa_erase_index(xa, 0);
	240	XA_BUG_ON(xa, !xa_empty(xa));
	241	}
	242
41aec91f MW	243	static noinline void check_cmpxchg(struct xarray *xa)
	244	{
	245	void *FIVE = xa_mk_value(5);
	246	void *SIX = xa_mk_value(6);
	247	void *LOTS = xa_mk_value(12345678);
	248
	249	XA_BUG_ON(xa, !xa_empty(xa));
	250	XA_BUG_ON(xa, xa_store_index(xa, 12345678, GFP_KERNEL) != NULL);
	251	XA_BUG_ON(xa, xa_insert(xa, 12345678, xa, GFP_KERNEL) != -EEXIST);
	252	XA_BUG_ON(xa, xa_cmpxchg(xa, 12345678, SIX, FIVE, GFP_KERNEL) != LOTS);
	253	XA_BUG_ON(xa, xa_cmpxchg(xa, 12345678, LOTS, FIVE, GFP_KERNEL) != LOTS);
	254	XA_BUG_ON(xa, xa_cmpxchg(xa, 12345678, FIVE, LOTS, GFP_KERNEL) != FIVE);
	255	XA_BUG_ON(xa, xa_cmpxchg(xa, 5, FIVE, NULL, GFP_KERNEL) != NULL);
	256	XA_BUG_ON(xa, xa_cmpxchg(xa, 5, NULL, FIVE, GFP_KERNEL) != NULL);
	257	xa_erase_index(xa, 12345678);
	258	xa_erase_index(xa, 5);
	259	XA_BUG_ON(xa, !xa_empty(xa));
	260	}
	261
b803b428 MW	262	static noinline void check_xas_erase(struct xarray *xa)
	263	{
	264	XA_STATE(xas, xa, 0);
	265	void *entry;
	266	unsigned long i, j;
	267
	268	for (i = 0; i < 200; i++) {
	269	for (j = i; j < 2 * i + 17; j++) {
	270	xas_set(&xas, j);
	271	do {
	272	xas_lock(&xas);
	273	xas_store(&xas, xa_mk_value(j));
	274	xas_unlock(&xas);
	275	} while (xas_nomem(&xas, GFP_KERNEL));
	276	}
	277
	278	xas_set(&xas, ULONG_MAX);
	279	do {
	280	xas_lock(&xas);
	281	xas_store(&xas, xa_mk_value(0));
	282	xas_unlock(&xas);
	283	} while (xas_nomem(&xas, GFP_KERNEL));
	284
	285	xas_lock(&xas);
	286	xas_store(&xas, NULL);
	287
	288	xas_set(&xas, 0);
	289	j = i;
	290	xas_for_each(&xas, entry, ULONG_MAX) {
	291	XA_BUG_ON(xa, entry != xa_mk_value(j));
	292	xas_store(&xas, NULL);
	293	j++;
	294	}
	295	xas_unlock(&xas);
	296	XA_BUG_ON(xa, !xa_empty(xa));
	297	}
	298	}
	299
58d6ea30 MW	300	static noinline void check_multi_store(struct xarray *xa)
	301	{
	302	#ifdef CONFIG_XARRAY_MULTI
	303	unsigned long i, j, k;
	304	unsigned int max_order = (sizeof(long) == 4) ? 30 : 60;
	305
	306	/* Loading from any position returns the same value */
	307	xa_store_order(xa, 0, 1, xa_mk_value(0), GFP_KERNEL);
	308	XA_BUG_ON(xa, xa_load(xa, 0) != xa_mk_value(0));
	309	XA_BUG_ON(xa, xa_load(xa, 1) != xa_mk_value(0));
	310	XA_BUG_ON(xa, xa_load(xa, 2) != NULL);
	311	rcu_read_lock();
	312	XA_BUG_ON(xa, xa_to_node(xa_head(xa))->count != 2);
	313	XA_BUG_ON(xa, xa_to_node(xa_head(xa))->nr_values != 2);
	314	rcu_read_unlock();
	315
	316	/* Storing adjacent to the value does not alter the value */
	317	xa_store(xa, 3, xa, GFP_KERNEL);
	318	XA_BUG_ON(xa, xa_load(xa, 0) != xa_mk_value(0));
	319	XA_BUG_ON(xa, xa_load(xa, 1) != xa_mk_value(0));
	320	XA_BUG_ON(xa, xa_load(xa, 2) != NULL);
	321	rcu_read_lock();
	322	XA_BUG_ON(xa, xa_to_node(xa_head(xa))->count != 3);
	323	XA_BUG_ON(xa, xa_to_node(xa_head(xa))->nr_values != 2);
	324	rcu_read_unlock();
	325
	326	/* Overwriting multiple indexes works */
	327	xa_store_order(xa, 0, 2, xa_mk_value(1), GFP_KERNEL);
	328	XA_BUG_ON(xa, xa_load(xa, 0) != xa_mk_value(1));
	329	XA_BUG_ON(xa, xa_load(xa, 1) != xa_mk_value(1));
	330	XA_BUG_ON(xa, xa_load(xa, 2) != xa_mk_value(1));
	331	XA_BUG_ON(xa, xa_load(xa, 3) != xa_mk_value(1));
	332	XA_BUG_ON(xa, xa_load(xa, 4) != NULL);
	333	rcu_read_lock();
	334	XA_BUG_ON(xa, xa_to_node(xa_head(xa))->count != 4);
	335	XA_BUG_ON(xa, xa_to_node(xa_head(xa))->nr_values != 4);
	336	rcu_read_unlock();
	337
	338	/* We can erase multiple values with a single store */
	339	xa_store_order(xa, 0, 63, NULL, GFP_KERNEL);
	340	XA_BUG_ON(xa, !xa_empty(xa));
	341
	342	/* Even when the first slot is empty but the others aren't */
	343	xa_store_index(xa, 1, GFP_KERNEL);
	344	xa_store_index(xa, 2, GFP_KERNEL);
	345	xa_store_order(xa, 0, 2, NULL, GFP_KERNEL);
	346	XA_BUG_ON(xa, !xa_empty(xa));
	347
	348	for (i = 0; i < max_order; i++) {
	349	for (j = 0; j < max_order; j++) {
	350	xa_store_order(xa, 0, i, xa_mk_value(i), GFP_KERNEL);
	351	xa_store_order(xa, 0, j, xa_mk_value(j), GFP_KERNEL);
	352
	353	for (k = 0; k < max_order; k++) {
	354	void *entry = xa_load(xa, (1UL << k) - 1);
	355	if ((i < k) && (j < k))
	356	XA_BUG_ON(xa, entry != NULL);
	357	else
	358	XA_BUG_ON(xa, entry != xa_mk_value(j));
	359	}
	360
	361	xa_erase(xa, 0);
	362	XA_BUG_ON(xa, !xa_empty(xa));
	363	}
364	}
365	#endif
366	}
367
b803b428 MW	368	static noinline void check_multi_find(struct xarray *xa)
	369	{
	370	#ifdef CONFIG_XARRAY_MULTI
	371	unsigned long index;
	372
	373	xa_store_order(xa, 12, 2, xa_mk_value(12), GFP_KERNEL);
	374	XA_BUG_ON(xa, xa_store_index(xa, 16, GFP_KERNEL) != NULL);
	375
	376	index = 0;
	377	XA_BUG_ON(xa, xa_find(xa, &index, ULONG_MAX, XA_PRESENT) !=
	378	xa_mk_value(12));
	379	XA_BUG_ON(xa, index != 12);
	380	index = 13;
	381	XA_BUG_ON(xa, xa_find(xa, &index, ULONG_MAX, XA_PRESENT) !=
	382	xa_mk_value(12));
	383	XA_BUG_ON(xa, (index < 12) \|\| (index >= 16));
	384	XA_BUG_ON(xa, xa_find_after(xa, &index, ULONG_MAX, XA_PRESENT) !=
	385	xa_mk_value(16));
	386	XA_BUG_ON(xa, index != 16);
	387
	388	xa_erase_index(xa, 12);
	389	xa_erase_index(xa, 16);
	390	XA_BUG_ON(xa, !xa_empty(xa));
	391	#endif
	392	}
	393
	394	static noinline void check_multi_find_2(struct xarray *xa)
	395	{
	396	unsigned int max_order = IS_ENABLED(CONFIG_XARRAY_MULTI) ? 10 : 1;
	397	unsigned int i, j;
	398	void *entry;
	399
	400	for (i = 0; i < max_order; i++) {
	401	unsigned long index = 1UL << i;
	402	for (j = 0; j < index; j++) {
	403	XA_STATE(xas, xa, j + index);
	404	xa_store_index(xa, index - 1, GFP_KERNEL);
	405	xa_store_order(xa, index, i, xa_mk_value(index),
	406	GFP_KERNEL);
	407	rcu_read_lock();
	408	xas_for_each(&xas, entry, ULONG_MAX) {
	409	xa_erase_index(xa, index);
	410	}
	411	rcu_read_unlock();
	412	xa_erase_index(xa, index - 1);
	413	XA_BUG_ON(xa, !xa_empty(xa));
	414	}
	415	}
	416	}
	417
	418	static noinline void check_find(struct xarray *xa)
	419	{
	420	unsigned long i, j, k;
	421
	422	XA_BUG_ON(xa, !xa_empty(xa));
	423
	424	/*
	425	* Check xa_find with all pairs between 0 and 99 inclusive,
	426	* starting at every index between 0 and 99
	427	*/
	428	for (i = 0; i < 100; i++) {
	429	XA_BUG_ON(xa, xa_store_index(xa, i, GFP_KERNEL) != NULL);
	430	xa_set_mark(xa, i, XA_MARK_0);
	431	for (j = 0; j < i; j++) {
432	XA_BUG_ON(xa, xa_store_index(xa, j, GFP_KERNEL) !=
433	NULL);
434	xa_set_mark(xa, j, XA_MARK_0);
435	for (k = 0; k < 100; k++) {
436	unsigned long index = k;
437	void *entry = xa_find(xa, &index, ULONG_MAX,
438	XA_PRESENT);
439	if (k <= j)
440	XA_BUG_ON(xa, index != j);
441	else if (k <= i)
442	XA_BUG_ON(xa, index != i);
443	else
444	XA_BUG_ON(xa, entry != NULL);
445
446	index = k;
447	entry = xa_find(xa, &index, ULONG_MAX,
448	XA_MARK_0);
449	if (k <= j)
450	XA_BUG_ON(xa, index != j);
451	else if (k <= i)
452	XA_BUG_ON(xa, index != i);
453	else
454	XA_BUG_ON(xa, entry != NULL);
455	}
456	xa_erase_index(xa, j);
457	XA_BUG_ON(xa, xa_get_mark(xa, j, XA_MARK_0));
458	XA_BUG_ON(xa, !xa_get_mark(xa, i, XA_MARK_0));
459	}
460	xa_erase_index(xa, i);
461	XA_BUG_ON(xa, xa_get_mark(xa, i, XA_MARK_0));
462	}
463	XA_BUG_ON(xa, !xa_empty(xa));
464	check_multi_find(xa);
465	check_multi_find_2(xa);
466	}
467
58d6ea30	468	static DEFINE_XARRAY(array);
ad3d6c72 MW	469
	470	static int xarray_checks(void)
	471	{
58d6ea30	472	check_xa_err(&array);
b803b428	473	check_xas_retry(&array);
ad3d6c72	474	check_xa_load(&array);
9b89a035	475	check_xa_mark(&array);
58d6ea30	476	check_xa_shrink(&array);
b803b428	477	check_xas_erase(&array);
41aec91f	478	check_cmpxchg(&array);
58d6ea30	479	check_multi_store(&array);
b803b428	480	check_find(&array);
ad3d6c72 MW	481
	482	printk("XArray: %u of %u tests passed\n", tests_passed, tests_run);
	483	return (tests_run == tests_passed) ? 0 : -EINVAL;
	484	}
	485
	486	static void xarray_exit(void)
	487	{
	488	}
	489
	490	module_init(xarray_checks);
	491	module_exit(xarray_exit);
	492	MODULE_AUTHOR("Matthew Wilcox <willy@infradead.org>");
	493	MODULE_LICENSE("GPL");