[linux-2.6-block.git] / lib / memcpy_kunit.c

// SPDX-License-Identifier: GPL-2.0
/*
 * Test cases for memcpy(), memmove(), and memset().
 */
#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt

#include <kunit/test.h>
#include <linux/device.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/overflow.h>
#include <linux/slab.h>
#include <linux/types.h>
#include <linux/vmalloc.h>

struct some_bytes {
	union {
		u8 data[32];
		struct {
			u32 one;
			u16 two;
			u8  three;
			/* 1 byte hole */
			u32 four[4];
		};
	};
};

#define check(instance, v) do {	\
	BUILD_BUG_ON(sizeof(instance.data) != 32);	\
	for (size_t i = 0; i < sizeof(instance.data); i++) {	\
		KUNIT_ASSERT_EQ_MSG(test, instance.data[i], v, \
			"line %d: '%s' not initialized to 0x%02x @ %d (saw 0x%02x)\n", \
			__LINE__, #instance, v, i, instance.data[i]);	\
	}	\
} while (0)

#define compare(name, one, two) do { \
	BUILD_BUG_ON(sizeof(one) != sizeof(two)); \
	for (size_t i = 0; i < sizeof(one); i++) {	\
		KUNIT_EXPECT_EQ_MSG(test, one.data[i], two.data[i], \
			"line %d: %s.data[%d] (0x%02x) != %s.data[%d] (0x%02x)\n", \
			__LINE__, #one, i, one.data[i], #two, i, two.data[i]); \
	}	\
	kunit_info(test, "ok: " TEST_OP "() " name "\n");	\
} while (0)

static void memcpy_test(struct kunit *test)
{
#define TEST_OP "memcpy"
	struct some_bytes control = {
		.data = { 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
			  0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
			  0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
			  0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
			},
	};
	struct some_bytes zero = { };
	struct some_bytes middle = {
		.data = { 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
			  0x20, 0x20, 0x20, 0x20, 0x00, 0x00, 0x00, 0x00,
			  0x00, 0x00, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20,
			  0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
			},
	};
	struct some_bytes three = {
		.data = { 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
			  0x20, 0x00, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20,
			  0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
			  0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
			},
	};
	struct some_bytes dest = { };
	int count;
	u8 *ptr;

	/* Verify static initializers. */
	check(control, 0x20);
	check(zero, 0);
	compare("static initializers", dest, zero);

	/* Verify assignment. */
	dest = control;
	compare("direct assignment", dest, control);

	/* Verify complete overwrite. */
	memcpy(dest.data, zero.data, sizeof(dest.data));
	compare("complete overwrite", dest, zero);

	/* Verify middle overwrite. */
	dest = control;
	memcpy(dest.data + 12, zero.data, 7);
	compare("middle overwrite", dest, middle);

	/* Verify argument side-effects aren't repeated. */
	dest = control;
	ptr = dest.data;
	count = 1;
	memcpy(ptr++, zero.data, count++);
	ptr += 8;
	memcpy(ptr++, zero.data, count++);
	compare("argument side-effects", dest, three);
#undef TEST_OP
}

static unsigned char larger_array [2048];

static void memmove_test(struct kunit *test)
{
#define TEST_OP "memmove"
	struct some_bytes control = {
		.data = { 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99,
			  0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99,
			  0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99,
			  0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99,
			},
	};
	struct some_bytes zero = { };
	struct some_bytes middle = {
		.data = { 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99,
			  0x99, 0x99, 0x99, 0x99, 0x00, 0x00, 0x00, 0x00,
			  0x00, 0x00, 0x00, 0x99, 0x99, 0x99, 0x99, 0x99,
			  0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99,
			},
	};
	struct some_bytes five = {
		.data = { 0x00, 0x00, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99,
			  0x99, 0x99, 0x00, 0x00, 0x00, 0x99, 0x99, 0x99,
			  0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99,
			  0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99,
			},
	};
	struct some_bytes overlap = {
		.data = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
			  0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F,
			  0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99,
			  0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99,
			},
	};
	struct some_bytes overlap_expected = {
		.data = { 0x00, 0x01, 0x00, 0x01, 0x02, 0x03, 0x04, 0x07,
			  0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F,
			  0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99,
			  0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99,
			},
	};
	struct some_bytes dest = { };
	int count;
	u8 *ptr;

	/* Verify static initializers. */
	check(control, 0x99);
	check(zero, 0);
	compare("static initializers", zero, dest);

	/* Verify assignment. */
	dest = control;
	compare("direct assignment", dest, control);

	/* Verify complete overwrite. */
	memmove(dest.data, zero.data, sizeof(dest.data));
	compare("complete overwrite", dest, zero);

	/* Verify middle overwrite. */
	dest = control;
	memmove(dest.data + 12, zero.data, 7);
	compare("middle overwrite", dest, middle);

	/* Verify argument side-effects aren't repeated. */
	dest = control;
	ptr = dest.data;
	count = 2;
	memmove(ptr++, zero.data, count++);
	ptr += 9;
	memmove(ptr++, zero.data, count++);
	compare("argument side-effects", dest, five);

	/* Verify overlapping overwrite is correct. */
	ptr = &overlap.data[2];
	memmove(ptr, overlap.data, 5);
	compare("overlapping write", overlap, overlap_expected);

	/* Verify larger overlapping moves. */
	larger_array[256] = 0xAAu;
	/*
	 * Test a backwards overlapping memmove first. 256 and 1024 are
	 * important for i386 to use rep movsl.
	 */
	memmove(larger_array, larger_array + 256, 1024);
	KUNIT_ASSERT_EQ(test, larger_array[0], 0xAAu);
	KUNIT_ASSERT_EQ(test, larger_array[256], 0x00);
	KUNIT_ASSERT_NULL(test,
		memchr(larger_array + 1, 0xaa, ARRAY_SIZE(larger_array) - 1));
	/* Test a forwards overlapping memmove. */
	larger_array[0] = 0xBBu;
	memmove(larger_array + 256, larger_array, 1024);
	KUNIT_ASSERT_EQ(test, larger_array[0], 0xBBu);
	KUNIT_ASSERT_EQ(test, larger_array[256], 0xBBu);
	KUNIT_ASSERT_NULL(test, memchr(larger_array + 1, 0xBBu, 256 - 1));
	KUNIT_ASSERT_NULL(test,
		memchr(larger_array + 257, 0xBBu, ARRAY_SIZE(larger_array) - 257));
#undef TEST_OP
}

static void memset_test(struct kunit *test)
{
#define TEST_OP "memset"
	struct some_bytes control = {
		.data = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
			  0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
			  0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
			  0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
			},
	};
	struct some_bytes complete = {
		.data = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
			  0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
			  0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
			  0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
			},
	};
	struct some_bytes middle = {
		.data = { 0x30, 0x30, 0x30, 0x30, 0x31, 0x31, 0x31, 0x31,
			  0x31, 0x31, 0x31, 0x31, 0x31, 0x31, 0x31, 0x31,
			  0x31, 0x31, 0x31, 0x31, 0x30, 0x30, 0x30, 0x30,
			  0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
			},
	};
	struct some_bytes three = {
		.data = { 0x60, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
			  0x30, 0x61, 0x61, 0x30, 0x30, 0x30, 0x30, 0x30,
			  0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
			  0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
			},
	};
	struct some_bytes after = {
		.data = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x72,
			  0x72, 0x72, 0x72, 0x72, 0x72, 0x72, 0x72, 0x72,
			  0x72, 0x72, 0x72, 0x72, 0x72, 0x72, 0x72, 0x72,
			  0x72, 0x72, 0x72, 0x72, 0x72, 0x72, 0x72, 0x72,
			},
	};
	struct some_bytes startat = {
		.data = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
			  0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
			  0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
			  0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
			},
	};
	struct some_bytes dest = { };
	int count, value;
	u8 *ptr;

	/* Verify static initializers. */
	check(control, 0x30);
	check(dest, 0);

	/* Verify assignment. */
	dest = control;
	compare("direct assignment", dest, control);

	/* Verify complete overwrite. */
	memset(dest.data, 0xff, sizeof(dest.data));
	compare("complete overwrite", dest, complete);

	/* Verify middle overwrite. */
	dest = control;
	memset(dest.data + 4, 0x31, 16);
	compare("middle overwrite", dest, middle);

	/* Verify argument side-effects aren't repeated. */
	dest = control;
	ptr = dest.data;
	value = 0x60;
	count = 1;
	memset(ptr++, value++, count++);
	ptr += 8;
	memset(ptr++, value++, count++);
	compare("argument side-effects", dest, three);

	/* Verify memset_after() */
	dest = control;
	memset_after(&dest, 0x72, three);
	compare("memset_after()", dest, after);

	/* Verify memset_startat() */
	dest = control;
	memset_startat(&dest, 0x79, four);
	compare("memset_startat()", dest, startat);
#undef TEST_OP
}

static u8 large_src[1024];
static u8 large_dst[2048];
static const u8 large_zero[2048];

static void set_random_nonzero(struct kunit *test, u8 *byte)
{
	int failed_rng = 0;

	while (*byte == 0) {
		get_random_bytes(byte, 1);
		KUNIT_ASSERT_LT_MSG(test, failed_rng++, 100,
				    "Is the RNG broken?");
	}
}

static void init_large(struct kunit *test)
{

	/* Get many bit patterns. */
	get_random_bytes(large_src, ARRAY_SIZE(large_src));

	/* Make sure we have non-zero edges. */
	set_random_nonzero(test, &large_src[0]);
	set_random_nonzero(test, &large_src[ARRAY_SIZE(large_src) - 1]);

	/* Explicitly zero the entire destination. */
	memset(large_dst, 0, ARRAY_SIZE(large_dst));
}

/*
 * Instead of an indirect function call for "copy" or a giant macro,
 * use a bool to pick memcpy or memmove.
 */
static void copy_large_test(struct kunit *test, bool use_memmove)
{
	init_large(test);

	/* Copy a growing number of non-overlapping bytes ... */
	for (int bytes = 1; bytes <= ARRAY_SIZE(large_src); bytes++) {
		/* Over a shifting destination window ... */
		for (int offset = 0; offset < ARRAY_SIZE(large_src); offset++) {
			int right_zero_pos = offset + bytes;
			int right_zero_size = ARRAY_SIZE(large_dst) - right_zero_pos;

			/* Copy! */
			if (use_memmove)
				memmove(large_dst + offset, large_src, bytes);
			else
				memcpy(large_dst + offset, large_src, bytes);

			/* Did we touch anything before the copy area? */
			KUNIT_ASSERT_EQ_MSG(test,
				memcmp(large_dst, large_zero, offset), 0,
				"with size %d at offset %d", bytes, offset);
			/* Did we touch anything after the copy area? */
			KUNIT_ASSERT_EQ_MSG(test,
				memcmp(&large_dst[right_zero_pos], large_zero, right_zero_size), 0,
				"with size %d at offset %d", bytes, offset);

			/* Are we byte-for-byte exact across the copy? */
			KUNIT_ASSERT_EQ_MSG(test,
				memcmp(large_dst + offset, large_src, bytes), 0,
				"with size %d at offset %d", bytes, offset);

			/* Zero out what we copied for the next cycle. */
			memset(large_dst + offset, 0, bytes);
		}
		/* Avoid stall warnings if this loop gets slow. */
		cond_resched();
	}
}

static void memcpy_large_test(struct kunit *test)
{
	copy_large_test(test, false);
}

static void memmove_large_test(struct kunit *test)
{
	copy_large_test(test, true);
}

/*
 * On the assumption that boundary conditions are going to be the most
 * sensitive, instead of taking a full step (inc) each iteration,
 * take single index steps for at least the first "inc"-many indexes
 * from the "start" and at least the last "inc"-many indexes before
 * the "end". When in the middle, take full "inc"-wide steps. For
 * example, calling next_step(idx, 1, 15, 3) with idx starting at 0
 * would see the following pattern: 1 2 3 4 7 10 11 12 13 14 15.
 */
static int next_step(int idx, int start, int end, int inc)
{
	start += inc;
	end -= inc;

	if (idx < start || idx + inc > end)
		inc = 1;
	return idx + inc;
}

static void inner_loop(struct kunit *test, int bytes, int d_off, int s_off)
{
	int left_zero_pos, left_zero_size;
	int right_zero_pos, right_zero_size;
	int src_pos, src_orig_pos, src_size;
	int pos;

	/* Place the source in the destination buffer. */
	memcpy(&large_dst[s_off], large_src, bytes);

	/* Copy to destination offset. */
	memmove(&large_dst[d_off], &large_dst[s_off], bytes);

	/* Make sure destination entirely matches. */
	KUNIT_ASSERT_EQ_MSG(test, memcmp(&large_dst[d_off], large_src, bytes), 0,
		"with size %d at src offset %d and dest offset %d",
		bytes, s_off, d_off);

	/* Calculate the expected zero spans. */
	if (s_off < d_off) {
		left_zero_pos = 0;
		left_zero_size = s_off;

		right_zero_pos = d_off + bytes;
		right_zero_size = ARRAY_SIZE(large_dst) - right_zero_pos;

		src_pos = s_off;
		src_orig_pos = 0;
		src_size = d_off - s_off;
	} else {
		left_zero_pos = 0;
		left_zero_size = d_off;

		right_zero_pos = s_off + bytes;
		right_zero_size = ARRAY_SIZE(large_dst) - right_zero_pos;

		src_pos = d_off + bytes;
		src_orig_pos = src_pos - s_off;
		src_size = right_zero_pos - src_pos;
	}

	/* Check non-overlapping source is unchanged.*/
	KUNIT_ASSERT_EQ_MSG(test,
		memcmp(&large_dst[src_pos], &large_src[src_orig_pos], src_size), 0,
		"with size %d at src offset %d and dest offset %d",
		bytes, s_off, d_off);

	/* Check leading buffer contents are zero. */
	KUNIT_ASSERT_EQ_MSG(test,
		memcmp(&large_dst[left_zero_pos], large_zero, left_zero_size), 0,
		"with size %d at src offset %d and dest offset %d",
		bytes, s_off, d_off);
	/* Check trailing buffer contents are zero. */
	KUNIT_ASSERT_EQ_MSG(test,
		memcmp(&large_dst[right_zero_pos], large_zero, right_zero_size), 0,
		"with size %d at src offset %d and dest offset %d",
		bytes, s_off, d_off);

	/* Zero out everything not already zeroed.*/
	pos = left_zero_pos + left_zero_size;
	memset(&large_dst[pos], 0, right_zero_pos - pos);
}

static void memmove_overlap_test(struct kunit *test)
{
	/*
	 * Running all possible offset and overlap combinations takes a
	 * very long time. Instead, only check up to 128 bytes offset
	 * into the destination buffer (which should result in crossing
	 * cachelines), with a step size of 1 through 7 to try to skip some
	 * redundancy.
	 */
	static const int offset_max = 128; /* less than ARRAY_SIZE(large_src); */
	static const int bytes_step = 7;
	static const int window_step = 7;

	static const int bytes_start = 1;
	static const int bytes_end = ARRAY_SIZE(large_src) + 1;

	init_large(test);

	/* Copy a growing number of overlapping bytes ... */
	for (int bytes = bytes_start; bytes < bytes_end;
	     bytes = next_step(bytes, bytes_start, bytes_end, bytes_step)) {

		/* Over a shifting destination window ... */
		for (int d_off = 0; d_off < offset_max; d_off++) {
			int s_start = max(d_off - bytes, 0);
			int s_end = min_t(int, d_off + bytes, ARRAY_SIZE(large_src));

			/* Over a shifting source window ... */
			for (int s_off = s_start; s_off < s_end;
			     s_off = next_step(s_off, s_start, s_end, window_step))
				inner_loop(test, bytes, d_off, s_off);

			/* Avoid stall warnings. */
			cond_resched();
		}
	}
}

static void strtomem_test(struct kunit *test)
{
	static const char input[sizeof(unsigned long)] = "hi";
	static const char truncate[] = "this is too long";
	struct {
		unsigned long canary1;
		unsigned char output[sizeof(unsigned long)] __nonstring;
		unsigned long canary2;
	} wrap;

	memset(&wrap, 0xFF, sizeof(wrap));
	KUNIT_EXPECT_EQ_MSG(test, wrap.canary1, ULONG_MAX,
			    "bad initial canary value");
	KUNIT_EXPECT_EQ_MSG(test, wrap.canary2, ULONG_MAX,
			    "bad initial canary value");

	/* Check unpadded copy leaves surroundings untouched. */
	strtomem(wrap.output, input);
	KUNIT_EXPECT_EQ(test, wrap.canary1, ULONG_MAX);
	KUNIT_EXPECT_EQ(test, wrap.output[0], input[0]);
	KUNIT_EXPECT_EQ(test, wrap.output[1], input[1]);
	for (size_t i = 2; i < sizeof(wrap.output); i++)
		KUNIT_EXPECT_EQ(test, wrap.output[i], 0xFF);
	KUNIT_EXPECT_EQ(test, wrap.canary2, ULONG_MAX);

	/* Check truncated copy leaves surroundings untouched. */
	memset(&wrap, 0xFF, sizeof(wrap));
	strtomem(wrap.output, truncate);
	KUNIT_EXPECT_EQ(test, wrap.canary1, ULONG_MAX);
	for (size_t i = 0; i < sizeof(wrap.output); i++)
		KUNIT_EXPECT_EQ(test, wrap.output[i], truncate[i]);
	KUNIT_EXPECT_EQ(test, wrap.canary2, ULONG_MAX);

	/* Check padded copy leaves only string padded. */
	memset(&wrap, 0xFF, sizeof(wrap));
	strtomem_pad(wrap.output, input, 0xAA);
	KUNIT_EXPECT_EQ(test, wrap.canary1, ULONG_MAX);
	KUNIT_EXPECT_EQ(test, wrap.output[0], input[0]);
	KUNIT_EXPECT_EQ(test, wrap.output[1], input[1]);
	for (size_t i = 2; i < sizeof(wrap.output); i++)
		KUNIT_EXPECT_EQ(test, wrap.output[i], 0xAA);
	KUNIT_EXPECT_EQ(test, wrap.canary2, ULONG_MAX);

	/* Check truncated padded copy has no padding. */
	memset(&wrap, 0xFF, sizeof(wrap));
	strtomem(wrap.output, truncate);
	KUNIT_EXPECT_EQ(test, wrap.canary1, ULONG_MAX);
	for (size_t i = 0; i < sizeof(wrap.output); i++)
		KUNIT_EXPECT_EQ(test, wrap.output[i], truncate[i]);
	KUNIT_EXPECT_EQ(test, wrap.canary2, ULONG_MAX);
}

static struct kunit_case memcpy_test_cases[] = {
	KUNIT_CASE(memset_test),
	KUNIT_CASE(memcpy_test),
	KUNIT_CASE(memcpy_large_test),
	KUNIT_CASE(memmove_test),
	KUNIT_CASE(memmove_large_test),
	KUNIT_CASE(memmove_overlap_test),
	KUNIT_CASE(strtomem_test),
	{}
};

static struct kunit_suite memcpy_test_suite = {
	.name = "memcpy",
	.test_cases = memcpy_test_cases,
};

kunit_test_suite(memcpy_test_suite);

MODULE_LICENSE("GPL");
Commit	Line	Data
bb95ebbe KC	1	// SPDX-License-Identifier: GPL-2.0
	2	/*
	3	* Test cases for memcpy(), memmove(), and memset().
	4	*/
	5	#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
	6
	7	#include <kunit/test.h>
	8	#include <linux/device.h>
	9	#include <linux/init.h>
	10	#include <linux/kernel.h>
	11	#include <linux/mm.h>
	12	#include <linux/module.h>
	13	#include <linux/overflow.h>
	14	#include <linux/slab.h>
	15	#include <linux/types.h>
	16	#include <linux/vmalloc.h>
	17
	18	struct some_bytes {
	19	union {
	20	u8 data[32];
	21	struct {
	22	u32 one;
	23	u16 two;
	24	u8 three;
	25	/* 1 byte hole */
	26	u32 four[4];
	27	};
	28	};
	29	};
	30
	31	#define check(instance, v) do { \
bb95ebbe	32	BUILD_BUG_ON(sizeof(instance.data) != 32); \
dfbafa70	33	for (size_t i = 0; i < sizeof(instance.data); i++) { \
bb95ebbe KC	34	KUNIT_ASSERT_EQ_MSG(test, instance.data[i], v, \
	35	"line %d: '%s' not initialized to 0x%02x @ %d (saw 0x%02x)\n", \
	36	__LINE__, #instance, v, i, instance.data[i]); \
	37	} \
	38	} while (0)
	39
	40	#define compare(name, one, two) do { \
bb95ebbe	41	BUILD_BUG_ON(sizeof(one) != sizeof(two)); \
dfbafa70	42	for (size_t i = 0; i < sizeof(one); i++) { \
bb95ebbe KC	43	KUNIT_EXPECT_EQ_MSG(test, one.data[i], two.data[i], \
	44	"line %d: %s.data[%d] (0x%02x) != %s.data[%d] (0x%02x)\n", \
	45	__LINE__, #one, i, one.data[i], #two, i, two.data[i]); \
	46	} \
	47	kunit_info(test, "ok: " TEST_OP "() " name "\n"); \
	48	} while (0)
	49
	50	static void memcpy_test(struct kunit *test)
	51	{
	52	#define TEST_OP "memcpy"
	53	struct some_bytes control = {
	54	.data = { 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
	55	0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
	56	0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
	57	0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
	58	},
	59	};
	60	struct some_bytes zero = { };
	61	struct some_bytes middle = {
	62	.data = { 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
	63	0x20, 0x20, 0x20, 0x20, 0x00, 0x00, 0x00, 0x00,
	64	0x00, 0x00, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20,
	65	0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
	66	},
	67	};
	68	struct some_bytes three = {
	69	.data = { 0x00, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
	70	0x20, 0x00, 0x00, 0x20, 0x20, 0x20, 0x20, 0x20,
	71	0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
	72	0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20,
	73	},
	74	};
	75	struct some_bytes dest = { };
	76	int count;
	77	u8 *ptr;
	78
	79	/* Verify static initializers. */
	80	check(control, 0x20);
	81	check(zero, 0);
	82	compare("static initializers", dest, zero);
	83
	84	/* Verify assignment. */
	85	dest = control;
	86	compare("direct assignment", dest, control);
	87
	88	/* Verify complete overwrite. */
	89	memcpy(dest.data, zero.data, sizeof(dest.data));
	90	compare("complete overwrite", dest, zero);
	91
	92	/* Verify middle overwrite. */
	93	dest = control;
	94	memcpy(dest.data + 12, zero.data, 7);
	95	compare("middle overwrite", dest, middle);
	96
	97	/* Verify argument side-effects aren't repeated. */
	98	dest = control;
	99	ptr = dest.data;
	100	count = 1;
	101	memcpy(ptr++, zero.data, count++);
	102	ptr += 8;
	103	memcpy(ptr++, zero.data, count++);
	104	compare("argument side-effects", dest, three);
	105	#undef TEST_OP
	106	}
107
bce5a1e8 ND	108	static unsigned char larger_array [2048];
bce5a1e8 ND	109
bb95ebbe KC	110	static void memmove_test(struct kunit *test)
	111	{
	112	#define TEST_OP "memmove"
	113	struct some_bytes control = {
	114	.data = { 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99,
	115	0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99,
	116	0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99,
	117	0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99,
	118	},
	119	};
	120	struct some_bytes zero = { };
	121	struct some_bytes middle = {
	122	.data = { 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99,
	123	0x99, 0x99, 0x99, 0x99, 0x00, 0x00, 0x00, 0x00,
	124	0x00, 0x00, 0x00, 0x99, 0x99, 0x99, 0x99, 0x99,
	125	0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99,
	126	},
	127	};
	128	struct some_bytes five = {
	129	.data = { 0x00, 0x00, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99,
	130	0x99, 0x99, 0x00, 0x00, 0x00, 0x99, 0x99, 0x99,
	131	0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99,
	132	0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99,
	133	},
	134	};
	135	struct some_bytes overlap = {
	136	.data = { 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07,
	137	0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F,
	138	0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99,
	139	0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99,
	140	},
	141	};
	142	struct some_bytes overlap_expected = {
	143	.data = { 0x00, 0x01, 0x00, 0x01, 0x02, 0x03, 0x04, 0x07,
	144	0x08, 0x09, 0x0A, 0x0B, 0x0C, 0x0D, 0x0E, 0x0F,
	145	0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99,
	146	0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99,
	147	},
	148	};
	149	struct some_bytes dest = { };
	150	int count;
	151	u8 *ptr;
	152
	153	/* Verify static initializers. */
	154	check(control, 0x99);
	155	check(zero, 0);
	156	compare("static initializers", zero, dest);
	157
	158	/* Verify assignment. */
	159	dest = control;
	160	compare("direct assignment", dest, control);
	161
	162	/* Verify complete overwrite. */
	163	memmove(dest.data, zero.data, sizeof(dest.data));
	164	compare("complete overwrite", dest, zero);
	165
	166	/* Verify middle overwrite. */
	167	dest = control;
	168	memmove(dest.data + 12, zero.data, 7);
	169	compare("middle overwrite", dest, middle);
	170
	171	/* Verify argument side-effects aren't repeated. */
	172	dest = control;
	173	ptr = dest.data;
174	count = 2;
175	memmove(ptr++, zero.data, count++);
176	ptr += 9;
177	memmove(ptr++, zero.data, count++);
178	compare("argument side-effects", dest, five);
179
180	/* Verify overlapping overwrite is correct. */
181	ptr = &overlap.data[2];
182	memmove(ptr, overlap.data, 5);
183	compare("overlapping write", overlap, overlap_expected);
bce5a1e8 ND	184
	185	/* Verify larger overlapping moves. */
	186	larger_array[256] = 0xAAu;
	187	/*
	188	* Test a backwards overlapping memmove first. 256 and 1024 are
	189	* important for i386 to use rep movsl.
	190	*/
	191	memmove(larger_array, larger_array + 256, 1024);
	192	KUNIT_ASSERT_EQ(test, larger_array[0], 0xAAu);
	193	KUNIT_ASSERT_EQ(test, larger_array[256], 0x00);
	194	KUNIT_ASSERT_NULL(test,
	195	memchr(larger_array + 1, 0xaa, ARRAY_SIZE(larger_array) - 1));
	196	/* Test a forwards overlapping memmove. */
	197	larger_array[0] = 0xBBu;
	198	memmove(larger_array + 256, larger_array, 1024);
	199	KUNIT_ASSERT_EQ(test, larger_array[0], 0xBBu);
	200	KUNIT_ASSERT_EQ(test, larger_array[256], 0xBBu);
	201	KUNIT_ASSERT_NULL(test, memchr(larger_array + 1, 0xBBu, 256 - 1));
	202	KUNIT_ASSERT_NULL(test,
	203	memchr(larger_array + 257, 0xBBu, ARRAY_SIZE(larger_array) - 257));
bb95ebbe KC	204	#undef TEST_OP
	205	}
	206
	207	static void memset_test(struct kunit *test)
	208	{
	209	#define TEST_OP "memset"
	210	struct some_bytes control = {
	211	.data = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
	212	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
	213	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
	214	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
	215	},
	216	};
	217	struct some_bytes complete = {
	218	.data = { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
	219	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
	220	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
	221	0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff,
	222	},
	223	};
	224	struct some_bytes middle = {
	225	.data = { 0x30, 0x30, 0x30, 0x30, 0x31, 0x31, 0x31, 0x31,
	226	0x31, 0x31, 0x31, 0x31, 0x31, 0x31, 0x31, 0x31,
	227	0x31, 0x31, 0x31, 0x31, 0x30, 0x30, 0x30, 0x30,
	228	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
	229	},
	230	};
	231	struct some_bytes three = {
	232	.data = { 0x60, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
	233	0x30, 0x61, 0x61, 0x30, 0x30, 0x30, 0x30, 0x30,
	234	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
	235	0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
	236	},
	237	};
4797632f KC	238	struct some_bytes after = {
	239	.data = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x72,
	240	0x72, 0x72, 0x72, 0x72, 0x72, 0x72, 0x72, 0x72,
	241	0x72, 0x72, 0x72, 0x72, 0x72, 0x72, 0x72, 0x72,
	242	0x72, 0x72, 0x72, 0x72, 0x72, 0x72, 0x72, 0x72,
	243	},
	244	};
6dbefad4 KC	245	struct some_bytes startat = {
	246	.data = { 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30, 0x30,
	247	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
	248	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
	249	0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79, 0x79,
	250	},
	251	};
bb95ebbe KC	252	struct some_bytes dest = { };
	253	int count, value;
	254	u8 *ptr;
	255
	256	/* Verify static initializers. */
	257	check(control, 0x30);
	258	check(dest, 0);
	259
	260	/* Verify assignment. */
	261	dest = control;
	262	compare("direct assignment", dest, control);
	263
	264	/* Verify complete overwrite. */
	265	memset(dest.data, 0xff, sizeof(dest.data));
	266	compare("complete overwrite", dest, complete);
	267
	268	/* Verify middle overwrite. */
	269	dest = control;
	270	memset(dest.data + 4, 0x31, 16);
	271	compare("middle overwrite", dest, middle);
	272
	273	/* Verify argument side-effects aren't repeated. */
	274	dest = control;
	275	ptr = dest.data;
	276	value = 0x60;
	277	count = 1;
	278	memset(ptr++, value++, count++);
	279	ptr += 8;
	280	memset(ptr++, value++, count++);
	281	compare("argument side-effects", dest, three);
4797632f KC	282
	283	/* Verify memset_after() */
	284	dest = control;
	285	memset_after(&dest, 0x72, three);
	286	compare("memset_after()", dest, after);
	287
6dbefad4 KC	288	/* Verify memset_startat() */
	289	dest = control;
	290	memset_startat(&dest, 0x79, four);
	291	compare("memset_startat()", dest, startat);
bb95ebbe KC	292	#undef TEST_OP
	293	}
	294
96fce387 KC	295	static u8 large_src[1024];
	296	static u8 large_dst[2048];
	297	static const u8 large_zero[2048];
	298
	299	static void set_random_nonzero(struct kunit test, u8 byte)
	300	{
	301	int failed_rng = 0;
	302
	303	while (*byte == 0) {
	304	get_random_bytes(byte, 1);
	305	KUNIT_ASSERT_LT_MSG(test, failed_rng++, 100,
	306	"Is the RNG broken?");
	307	}
	308	}
	309
	310	static void init_large(struct kunit *test)
	311	{
	312
	313	/* Get many bit patterns. */
	314	get_random_bytes(large_src, ARRAY_SIZE(large_src));
	315
	316	/* Make sure we have non-zero edges. */
	317	set_random_nonzero(test, &large_src[0]);
	318	set_random_nonzero(test, &large_src[ARRAY_SIZE(large_src) - 1]);
	319
	320	/* Explicitly zero the entire destination. */
	321	memset(large_dst, 0, ARRAY_SIZE(large_dst));
	322	}
	323
	324	/*
	325	* Instead of an indirect function call for "copy" or a giant macro,
	326	* use a bool to pick memcpy or memmove.
	327	*/
	328	static void copy_large_test(struct kunit *test, bool use_memmove)
	329	{
	330	init_large(test);
	331
	332	/* Copy a growing number of non-overlapping bytes ... */
	333	for (int bytes = 1; bytes <= ARRAY_SIZE(large_src); bytes++) {
	334	/* Over a shifting destination window ... */
	335	for (int offset = 0; offset < ARRAY_SIZE(large_src); offset++) {
	336	int right_zero_pos = offset + bytes;
	337	int right_zero_size = ARRAY_SIZE(large_dst) - right_zero_pos;
	338
	339	/* Copy! */
	340	if (use_memmove)
	341	memmove(large_dst + offset, large_src, bytes);
	342	else
	343	memcpy(large_dst + offset, large_src, bytes);
	344
	345	/* Did we touch anything before the copy area? */
	346	KUNIT_ASSERT_EQ_MSG(test,
	347	memcmp(large_dst, large_zero, offset), 0,
	348	"with size %d at offset %d", bytes, offset);
	349	/* Did we touch anything after the copy area? */
	350	KUNIT_ASSERT_EQ_MSG(test,
	351	memcmp(&large_dst[right_zero_pos], large_zero, right_zero_size), 0,
	352	"with size %d at offset %d", bytes, offset);
	353
	354	/* Are we byte-for-byte exact across the copy? */
	355	KUNIT_ASSERT_EQ_MSG(test,
	356	memcmp(large_dst + offset, large_src, bytes), 0,
	357	"with size %d at offset %d", bytes, offset);
	358
359	/* Zero out what we copied for the next cycle. */
360	memset(large_dst + offset, 0, bytes);
361	}
362	/* Avoid stall warnings if this loop gets slow. */
363	cond_resched();
364	}
365	}
366
367	static void memcpy_large_test(struct kunit *test)
368	{
369	copy_large_test(test, false);
370	}
371
372	static void memmove_large_test(struct kunit *test)
373	{
374	copy_large_test(test, true);
375	}
376
377	/*
378	* On the assumption that boundary conditions are going to be the most
379	* sensitive, instead of taking a full step (inc) each iteration,
380	* take single index steps for at least the first "inc"-many indexes
381	* from the "start" and at least the last "inc"-many indexes before
382	* the "end". When in the middle, take full "inc"-wide steps. For
383	* example, calling next_step(idx, 1, 15, 3) with idx starting at 0
384	* would see the following pattern: 1 2 3 4 7 10 11 12 13 14 15.
385	*/
386	static int next_step(int idx, int start, int end, int inc)
387	{
388	start += inc;
389	end -= inc;
390
391	if (idx < start \|\| idx + inc > end)
392	inc = 1;
393	return idx + inc;
394	}
395
396	static void inner_loop(struct kunit *test, int bytes, int d_off, int s_off)
397	{
398	int left_zero_pos, left_zero_size;
399	int right_zero_pos, right_zero_size;
400	int src_pos, src_orig_pos, src_size;
401	int pos;
402
403	/* Place the source in the destination buffer. */
404	memcpy(&large_dst[s_off], large_src, bytes);
405
406	/* Copy to destination offset. */
407	memmove(&large_dst[d_off], &large_dst[s_off], bytes);
408
409	/* Make sure destination entirely matches. */
410	KUNIT_ASSERT_EQ_MSG(test, memcmp(&large_dst[d_off], large_src, bytes), 0,
411	"with size %d at src offset %d and dest offset %d",
412	bytes, s_off, d_off);
413
414	/* Calculate the expected zero spans. */
415	if (s_off < d_off) {
416	left_zero_pos = 0;
417	left_zero_size = s_off;
418
419	right_zero_pos = d_off + bytes;
420	right_zero_size = ARRAY_SIZE(large_dst) - right_zero_pos;
421
422	src_pos = s_off;
423	src_orig_pos = 0;
424	src_size = d_off - s_off;
425	} else {
426	left_zero_pos = 0;
427	left_zero_size = d_off;
428
429	right_zero_pos = s_off + bytes;
430	right_zero_size = ARRAY_SIZE(large_dst) - right_zero_pos;
431
432	src_pos = d_off + bytes;
433	src_orig_pos = src_pos - s_off;
434	src_size = right_zero_pos - src_pos;
435	}
436
437	/* Check non-overlapping source is unchanged.*/
438	KUNIT_ASSERT_EQ_MSG(test,
439	memcmp(&large_dst[src_pos], &large_src[src_orig_pos], src_size), 0,
440	"with size %d at src offset %d and dest offset %d",
441	bytes, s_off, d_off);
442
443	/* Check leading buffer contents are zero. */
444	KUNIT_ASSERT_EQ_MSG(test,
445	memcmp(&large_dst[left_zero_pos], large_zero, left_zero_size), 0,
446	"with size %d at src offset %d and dest offset %d",
447	bytes, s_off, d_off);
448	/* Check trailing buffer contents are zero. */
449	KUNIT_ASSERT_EQ_MSG(test,
450	memcmp(&large_dst[right_zero_pos], large_zero, right_zero_size), 0,
451	"with size %d at src offset %d and dest offset %d",
452	bytes, s_off, d_off);
453
454	/* Zero out everything not already zeroed.*/
455	pos = left_zero_pos + left_zero_size;
456	memset(&large_dst[pos], 0, right_zero_pos - pos);
457	}
458
459	static void memmove_overlap_test(struct kunit *test)
460	{
461	/*
462	* Running all possible offset and overlap combinations takes a
463	* very long time. Instead, only check up to 128 bytes offset
464	* into the destination buffer (which should result in crossing
465	* cachelines), with a step size of 1 through 7 to try to skip some
466	* redundancy.
467	*/
468	static const int offset_max = 128; /* less than ARRAY_SIZE(large_src); */
469	static const int bytes_step = 7;
470	static const int window_step = 7;
471
472	static const int bytes_start = 1;
473	static const int bytes_end = ARRAY_SIZE(large_src) + 1;
474
475	init_large(test);
476
477	/* Copy a growing number of overlapping bytes ... */
478	for (int bytes = bytes_start; bytes < bytes_end;
479	bytes = next_step(bytes, bytes_start, bytes_end, bytes_step)) {
480
481	/* Over a shifting destination window ... */
482	for (int d_off = 0; d_off < offset_max; d_off++) {
483	int s_start = max(d_off - bytes, 0);
484	int s_end = min_t(int, d_off + bytes, ARRAY_SIZE(large_src));
485
486	/* Over a shifting source window ... */
487	for (int s_off = s_start; s_off < s_end;
488	s_off = next_step(s_off, s_start, s_end, window_step))
489	inner_loop(test, bytes, d_off, s_off);
490
491	/* Avoid stall warnings. */
492	cond_resched();
493	}
494	}
495	}
496
dfbafa70 KC	497	static void strtomem_test(struct kunit *test)
dfbafa70 KC	498	{
66cb2a36	499	static const char input[sizeof(unsigned long)] = "hi";
dfbafa70 KC	500	static const char truncate[] = "this is too long";
	501	struct {
	502	unsigned long canary1;
	503	unsigned char output[sizeof(unsigned long)] __nonstring;
	504	unsigned long canary2;
	505	} wrap;
	506
	507	memset(&wrap, 0xFF, sizeof(wrap));
	508	KUNIT_EXPECT_EQ_MSG(test, wrap.canary1, ULONG_MAX,
	509	"bad initial canary value");
	510	KUNIT_EXPECT_EQ_MSG(test, wrap.canary2, ULONG_MAX,
	511	"bad initial canary value");
	512
	513	/* Check unpadded copy leaves surroundings untouched. */
	514	strtomem(wrap.output, input);
	515	KUNIT_EXPECT_EQ(test, wrap.canary1, ULONG_MAX);
	516	KUNIT_EXPECT_EQ(test, wrap.output[0], input[0]);
	517	KUNIT_EXPECT_EQ(test, wrap.output[1], input[1]);
	518	for (size_t i = 2; i < sizeof(wrap.output); i++)
	519	KUNIT_EXPECT_EQ(test, wrap.output[i], 0xFF);
	520	KUNIT_EXPECT_EQ(test, wrap.canary2, ULONG_MAX);
	521
	522	/* Check truncated copy leaves surroundings untouched. */
	523	memset(&wrap, 0xFF, sizeof(wrap));
	524	strtomem(wrap.output, truncate);
	525	KUNIT_EXPECT_EQ(test, wrap.canary1, ULONG_MAX);
	526	for (size_t i = 0; i < sizeof(wrap.output); i++)
	527	KUNIT_EXPECT_EQ(test, wrap.output[i], truncate[i]);
	528	KUNIT_EXPECT_EQ(test, wrap.canary2, ULONG_MAX);
	529
	530	/* Check padded copy leaves only string padded. */
	531	memset(&wrap, 0xFF, sizeof(wrap));
	532	strtomem_pad(wrap.output, input, 0xAA);
	533	KUNIT_EXPECT_EQ(test, wrap.canary1, ULONG_MAX);
	534	KUNIT_EXPECT_EQ(test, wrap.output[0], input[0]);
	535	KUNIT_EXPECT_EQ(test, wrap.output[1], input[1]);
	536	for (size_t i = 2; i < sizeof(wrap.output); i++)
	537	KUNIT_EXPECT_EQ(test, wrap.output[i], 0xAA);
	538	KUNIT_EXPECT_EQ(test, wrap.canary2, ULONG_MAX);
	539
	540	/* Check truncated padded copy has no padding. */
	541	memset(&wrap, 0xFF, sizeof(wrap));
	542	strtomem(wrap.output, truncate);
	543	KUNIT_EXPECT_EQ(test, wrap.canary1, ULONG_MAX);
	544	for (size_t i = 0; i < sizeof(wrap.output); i++)
	545	KUNIT_EXPECT_EQ(test, wrap.output[i], truncate[i]);
	546	KUNIT_EXPECT_EQ(test, wrap.canary2, ULONG_MAX);
	547	}
	548
bb95ebbe KC	549	static struct kunit_case memcpy_test_cases[] = {
	550	KUNIT_CASE(memset_test),
	551	KUNIT_CASE(memcpy_test),
96fce387	552	KUNIT_CASE(memcpy_large_test),
bb95ebbe	553	KUNIT_CASE(memmove_test),
96fce387 KC	554	KUNIT_CASE(memmove_large_test),
96fce387 KC	555	KUNIT_CASE(memmove_overlap_test),
dfbafa70	556	KUNIT_CASE(strtomem_test),
bb95ebbe KC	557	{}
	558	};
	559
	560	static struct kunit_suite memcpy_test_suite = {
	561	.name = "memcpy",
	562	.test_cases = memcpy_test_cases,
	563	};
	564
	565	kunit_test_suite(memcpy_test_suite);
	566
	567	MODULE_LICENSE("GPL");