[linux-block.git] / tools / testing / selftests / vm / mremap_test.c

// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright 2020 Google LLC
 */
#define _GNU_SOURCE

#include <errno.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <sys/mman.h>
#include <time.h>
#include <stdbool.h>

#include "../kselftest.h"

#define EXPECT_SUCCESS 0
#define EXPECT_FAILURE 1
#define NON_OVERLAPPING 0
#define OVERLAPPING 1
#define NS_PER_SEC 1000000000ULL
#define VALIDATION_DEFAULT_THRESHOLD 4	/* 4MB */
#define VALIDATION_NO_THRESHOLD 0	/* Verify the entire region */

#define MIN(X, Y) ((X) < (Y) ? (X) : (Y))

struct config {
	unsigned long long src_alignment;
	unsigned long long dest_alignment;
	unsigned long long region_size;
	int overlapping;
};

struct test {
	const char *name;
	struct config config;
	int expect_failure;
};

enum {
	_1KB = 1ULL << 10,	/* 1KB -> not page aligned */
	_4KB = 4ULL << 10,
	_8KB = 8ULL << 10,
	_1MB = 1ULL << 20,
	_2MB = 2ULL << 20,
	_4MB = 4ULL << 20,
	_1GB = 1ULL << 30,
	_2GB = 2ULL << 30,
	PMD = _2MB,
	PUD = _1GB,
};

#define PTE page_size

#define MAKE_TEST(source_align, destination_align, size,	\
		  overlaps, should_fail, test_name)		\
(struct test){							\
	.name = test_name,					\
	.config = {						\
		.src_alignment = source_align,			\
		.dest_alignment = destination_align,		\
		.region_size = size,				\
		.overlapping = overlaps,			\
	},							\
	.expect_failure = should_fail				\
}

/*
 * Returns false if the requested remap region overlaps with an
 * existing mapping (e.g text, stack) else returns true.
 */
static bool is_remap_region_valid(void *addr, unsigned long long size)
{
	void *remap_addr = NULL;
	bool ret = true;

	/* Use MAP_FIXED_NOREPLACE flag to ensure region is not mapped */
	remap_addr = mmap(addr, size, PROT_READ | PROT_WRITE,
					 MAP_FIXED_NOREPLACE | MAP_ANONYMOUS | MAP_SHARED,
					 -1, 0);

	if (remap_addr == MAP_FAILED) {
		if (errno == EEXIST)
			ret = false;
	} else {
		munmap(remap_addr, size);
	}

	return ret;
}

/* Returns mmap_min_addr sysctl tunable from procfs */
static unsigned long long get_mmap_min_addr(void)
{
	FILE *fp;
	int n_matched;
	static unsigned long long addr;

	if (addr)
		return addr;

	fp = fopen("/proc/sys/vm/mmap_min_addr", "r");
	if (fp == NULL) {
		ksft_print_msg("Failed to open /proc/sys/vm/mmap_min_addr: %s\n",
			strerror(errno));
		exit(KSFT_SKIP);
	}

	n_matched = fscanf(fp, "%llu", &addr);
	if (n_matched != 1) {
		ksft_print_msg("Failed to read /proc/sys/vm/mmap_min_addr: %s\n",
			strerror(errno));
		fclose(fp);
		exit(KSFT_SKIP);
	}

	fclose(fp);
	return addr;
}

/*
 * This test validates that merge is called when expanding a mapping.
 * Mapping containing three pages is created, middle page is unmapped
 * and then the mapping containing the first page is expanded so that
 * it fills the created hole. The two parts should merge creating
 * single mapping with three pages.
 */
static void mremap_expand_merge(unsigned long page_size)
{
	char *test_name = "mremap expand merge";
	FILE *fp;
	char *line = NULL;
	size_t len = 0;
	bool success = false;
	char *start = mmap(NULL, 3 * page_size, PROT_READ | PROT_WRITE,
			   MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);

	munmap(start + page_size, page_size);
	mremap(start, page_size, 2 * page_size, 0);

	fp = fopen("/proc/self/maps", "r");
	if (fp == NULL) {
		ksft_test_result_fail("%s\n", test_name);
		return;
	}

	while (getline(&line, &len, fp) != -1) {
		char *first = strtok(line, "- ");
		void *first_val = (void *)strtol(first, NULL, 16);
		char *second = strtok(NULL, "- ");
		void *second_val = (void *) strtol(second, NULL, 16);

		if (first_val == start && second_val == start + 3 * page_size) {
			success = true;
			break;
		}
	}
	if (success)
		ksft_test_result_pass("%s\n", test_name);
	else
		ksft_test_result_fail("%s\n", test_name);
	fclose(fp);
}

/*
 * Returns the start address of the mapping on success, else returns
 * NULL on failure.
 */
static void *get_source_mapping(struct config c)
{
	unsigned long long addr = 0ULL;
	void *src_addr = NULL;
	unsigned long long mmap_min_addr;

	mmap_min_addr = get_mmap_min_addr();

retry:
	addr += c.src_alignment;
	if (addr < mmap_min_addr)
		goto retry;

	src_addr = mmap((void *) addr, c.region_size, PROT_READ | PROT_WRITE,
					MAP_FIXED_NOREPLACE | MAP_ANONYMOUS | MAP_SHARED,
					-1, 0);
	if (src_addr == MAP_FAILED) {
		if (errno == EPERM || errno == EEXIST)
			goto retry;
		goto error;
	}
	/*
	 * Check that the address is aligned to the specified alignment.
	 * Addresses which have alignments that are multiples of that
	 * specified are not considered valid. For instance, 1GB address is
	 * 2MB-aligned, however it will not be considered valid for a
	 * requested alignment of 2MB. This is done to reduce coincidental
	 * alignment in the tests.
	 */
	if (((unsigned long long) src_addr & (c.src_alignment - 1)) ||
			!((unsigned long long) src_addr & c.src_alignment)) {
		munmap(src_addr, c.region_size);
		goto retry;
	}

	if (!src_addr)
		goto error;

	return src_addr;
error:
	ksft_print_msg("Failed to map source region: %s\n",
			strerror(errno));
	return NULL;
}

/* Returns the time taken for the remap on success else returns -1. */
static long long remap_region(struct config c, unsigned int threshold_mb,
			      char pattern_seed)
{
	void *addr, *src_addr, *dest_addr;
	unsigned long long i;
	struct timespec t_start = {0, 0}, t_end = {0, 0};
	long long  start_ns, end_ns, align_mask, ret, offset;
	unsigned long long threshold;

	if (threshold_mb == VALIDATION_NO_THRESHOLD)
		threshold = c.region_size;
	else
		threshold = MIN(threshold_mb * _1MB, c.region_size);

	src_addr = get_source_mapping(c);
	if (!src_addr) {
		ret = -1;
		goto out;
	}

	/* Set byte pattern */
	srand(pattern_seed);
	for (i = 0; i < threshold; i++)
		memset((char *) src_addr + i, (char) rand(), 1);

	/* Mask to zero out lower bits of address for alignment */
	align_mask = ~(c.dest_alignment - 1);
	/* Offset of destination address from the end of the source region */
	offset = (c.overlapping) ? -c.dest_alignment : c.dest_alignment;
	addr = (void *) (((unsigned long long) src_addr + c.region_size
			  + offset) & align_mask);

	/* See comment in get_source_mapping() */
	if (!((unsigned long long) addr & c.dest_alignment))
		addr = (void *) ((unsigned long long) addr | c.dest_alignment);

	/* Don't destroy existing mappings unless expected to overlap */
	while (!is_remap_region_valid(addr, c.region_size) && !c.overlapping) {
		/* Check for unsigned overflow */
		if (addr + c.dest_alignment < addr) {
			ksft_print_msg("Couldn't find a valid region to remap to\n");
			ret = -1;
			goto out;
		}
		addr += c.dest_alignment;
	}

	clock_gettime(CLOCK_MONOTONIC, &t_start);
	dest_addr = mremap(src_addr, c.region_size, c.region_size,
					  MREMAP_MAYMOVE|MREMAP_FIXED, (char *) addr);
	clock_gettime(CLOCK_MONOTONIC, &t_end);

	if (dest_addr == MAP_FAILED) {
		ksft_print_msg("mremap failed: %s\n", strerror(errno));
		ret = -1;
		goto clean_up_src;
	}

	/* Verify byte pattern after remapping */
	srand(pattern_seed);
	for (i = 0; i < threshold; i++) {
		char c = (char) rand();

		if (((char *) dest_addr)[i] != c) {
			ksft_print_msg("Data after remap doesn't match at offset %d\n",
				       i);
			ksft_print_msg("Expected: %#x\t Got: %#x\n", c & 0xff,
					((char *) dest_addr)[i] & 0xff);
			ret = -1;
			goto clean_up_dest;
		}
	}

	start_ns = t_start.tv_sec * NS_PER_SEC + t_start.tv_nsec;
	end_ns = t_end.tv_sec * NS_PER_SEC + t_end.tv_nsec;
	ret = end_ns - start_ns;

/*
 * Since the destination address is specified using MREMAP_FIXED, subsequent
 * mremap will unmap any previous mapping at the address range specified by
 * dest_addr and region_size. This significantly affects the remap time of
 * subsequent tests. So we clean up mappings after each test.
 */
clean_up_dest:
	munmap(dest_addr, c.region_size);
clean_up_src:
	munmap(src_addr, c.region_size);
out:
	return ret;
}

static void run_mremap_test_case(struct test test_case, int *failures,
				 unsigned int threshold_mb,
				 unsigned int pattern_seed)
{
	long long remap_time = remap_region(test_case.config, threshold_mb,
					    pattern_seed);

	if (remap_time < 0) {
		if (test_case.expect_failure)
			ksft_test_result_xfail("%s\n\tExpected mremap failure\n",
					      test_case.name);
		else {
			ksft_test_result_fail("%s\n", test_case.name);
			*failures += 1;
		}
	} else {
		/*
		 * Comparing mremap time is only applicable if entire region
		 * was faulted in.
		 */
		if (threshold_mb == VALIDATION_NO_THRESHOLD ||
		    test_case.config.region_size <= threshold_mb * _1MB)
			ksft_test_result_pass("%s\n\tmremap time: %12lldns\n",
					      test_case.name, remap_time);
		else
			ksft_test_result_pass("%s\n", test_case.name);
	}
}

static void usage(const char *cmd)
{
	fprintf(stderr,
		"Usage: %s [[-t <threshold_mb>] [-p <pattern_seed>]]\n"
		"-t\t only validate threshold_mb of the remapped region\n"
		"  \t if 0 is supplied no threshold is used; all tests\n"
		"  \t are run and remapped regions validated fully.\n"
		"  \t The default threshold used is 4MB.\n"
		"-p\t provide a seed to generate the random pattern for\n"
		"  \t validating the remapped region.\n", cmd);
}

static int parse_args(int argc, char **argv, unsigned int *threshold_mb,
		      unsigned int *pattern_seed)
{
	const char *optstr = "t:p:";
	int opt;

	while ((opt = getopt(argc, argv, optstr)) != -1) {
		switch (opt) {
		case 't':
			*threshold_mb = atoi(optarg);
			break;
		case 'p':
			*pattern_seed = atoi(optarg);
			break;
		default:
			usage(argv[0]);
			return -1;
		}
	}

	if (optind < argc) {
		usage(argv[0]);
		return -1;
	}

	return 0;
}

#define MAX_TEST 13
#define MAX_PERF_TEST 3
int main(int argc, char **argv)
{
	int failures = 0;
	int i, run_perf_tests;
	unsigned int threshold_mb = VALIDATION_DEFAULT_THRESHOLD;
	unsigned int pattern_seed;
	int num_expand_tests = 1;
	struct test test_cases[MAX_TEST];
	struct test perf_test_cases[MAX_PERF_TEST];
	int page_size;
	time_t t;

	pattern_seed = (unsigned int) time(&t);

	if (parse_args(argc, argv, &threshold_mb, &pattern_seed) < 0)
		exit(EXIT_FAILURE);

	ksft_print_msg("Test configs:\n\tthreshold_mb=%u\n\tpattern_seed=%u\n\n",
		       threshold_mb, pattern_seed);

	page_size = sysconf(_SC_PAGESIZE);

	/* Expected mremap failures */
	test_cases[0] =	MAKE_TEST(page_size, page_size, page_size,
				  OVERLAPPING, EXPECT_FAILURE,
				  "mremap - Source and Destination Regions Overlapping");

	test_cases[1] = MAKE_TEST(page_size, page_size/4, page_size,
				  NON_OVERLAPPING, EXPECT_FAILURE,
				  "mremap - Destination Address Misaligned (1KB-aligned)");
	test_cases[2] = MAKE_TEST(page_size/4, page_size, page_size,
				  NON_OVERLAPPING, EXPECT_FAILURE,
				  "mremap - Source Address Misaligned (1KB-aligned)");

	/* Src addr PTE aligned */
	test_cases[3] = MAKE_TEST(PTE, PTE, PTE * 2,
				  NON_OVERLAPPING, EXPECT_SUCCESS,
				  "8KB mremap - Source PTE-aligned, Destination PTE-aligned");

	/* Src addr 1MB aligned */
	test_cases[4] = MAKE_TEST(_1MB, PTE, _2MB, NON_OVERLAPPING, EXPECT_SUCCESS,
				  "2MB mremap - Source 1MB-aligned, Destination PTE-aligned");
	test_cases[5] = MAKE_TEST(_1MB, _1MB, _2MB, NON_OVERLAPPING, EXPECT_SUCCESS,
				  "2MB mremap - Source 1MB-aligned, Destination 1MB-aligned");

	/* Src addr PMD aligned */
	test_cases[6] = MAKE_TEST(PMD, PTE, _4MB, NON_OVERLAPPING, EXPECT_SUCCESS,
				  "4MB mremap - Source PMD-aligned, Destination PTE-aligned");
	test_cases[7] =	MAKE_TEST(PMD, _1MB, _4MB, NON_OVERLAPPING, EXPECT_SUCCESS,
				  "4MB mremap - Source PMD-aligned, Destination 1MB-aligned");
	test_cases[8] = MAKE_TEST(PMD, PMD, _4MB, NON_OVERLAPPING, EXPECT_SUCCESS,
				  "4MB mremap - Source PMD-aligned, Destination PMD-aligned");

	/* Src addr PUD aligned */
	test_cases[9] = MAKE_TEST(PUD, PTE, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS,
				  "2GB mremap - Source PUD-aligned, Destination PTE-aligned");
	test_cases[10] = MAKE_TEST(PUD, _1MB, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS,
				   "2GB mremap - Source PUD-aligned, Destination 1MB-aligned");
	test_cases[11] = MAKE_TEST(PUD, PMD, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS,
				   "2GB mremap - Source PUD-aligned, Destination PMD-aligned");
	test_cases[12] = MAKE_TEST(PUD, PUD, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS,
				   "2GB mremap - Source PUD-aligned, Destination PUD-aligned");

	perf_test_cases[0] =  MAKE_TEST(page_size, page_size, _1GB, NON_OVERLAPPING, EXPECT_SUCCESS,
					"1GB mremap - Source PTE-aligned, Destination PTE-aligned");
	/*
	 * mremap 1GB region - Page table level aligned time
	 * comparison.
	 */
	perf_test_cases[1] = MAKE_TEST(PMD, PMD, _1GB, NON_OVERLAPPING, EXPECT_SUCCESS,
				       "1GB mremap - Source PMD-aligned, Destination PMD-aligned");
	perf_test_cases[2] = MAKE_TEST(PUD, PUD, _1GB, NON_OVERLAPPING, EXPECT_SUCCESS,
				       "1GB mremap - Source PUD-aligned, Destination PUD-aligned");

	run_perf_tests =  (threshold_mb == VALIDATION_NO_THRESHOLD) ||
				(threshold_mb * _1MB >= _1GB);

	ksft_set_plan(ARRAY_SIZE(test_cases) + (run_perf_tests ?
		      ARRAY_SIZE(perf_test_cases) : 0) + num_expand_tests);

	for (i = 0; i < ARRAY_SIZE(test_cases); i++)
		run_mremap_test_case(test_cases[i], &failures, threshold_mb,
				     pattern_seed);

	mremap_expand_merge(page_size);

	if (run_perf_tests) {
		ksft_print_msg("\n%s\n",
		 "mremap HAVE_MOVE_PMD/PUD optimization time comparison for 1GB region:");
		for (i = 0; i < ARRAY_SIZE(perf_test_cases); i++)
			run_mremap_test_case(perf_test_cases[i], &failures,
					     threshold_mb, pattern_seed);
	}

	if (failures > 0)
		ksft_exit_fail();
	else
		ksft_exit_pass();
}
Commit	Line	Data
7df66625 KS	1	// SPDX-License-Identifier: GPL-2.0
	2	/*
	3	* Copyright 2020 Google LLC
	4	*/
	5	#define _GNU_SOURCE
	6
	7	#include <errno.h>
	8	#include <stdlib.h>
9c85a9ba	9	#include <stdio.h>
7df66625 KS	10	#include <string.h>
	11	#include <sys/mman.h>
	12	#include <time.h>
18d609da	13	#include <stdbool.h>
7df66625 KS	14
	15	#include "../kselftest.h"
	16
	17	#define EXPECT_SUCCESS 0
	18	#define EXPECT_FAILURE 1
	19	#define NON_OVERLAPPING 0
	20	#define OVERLAPPING 1
	21	#define NS_PER_SEC 1000000000ULL
	22	#define VALIDATION_DEFAULT_THRESHOLD 4 /* 4MB */
	23	#define VALIDATION_NO_THRESHOLD 0 /* Verify the entire region */
	24
7df66625 KS	25	#define MIN(X, Y) ((X) < (Y) ? (X) : (Y))
	26
	27	struct config {
	28	unsigned long long src_alignment;
	29	unsigned long long dest_alignment;
	30	unsigned long long region_size;
	31	int overlapping;
	32	};
	33
	34	struct test {
	35	const char *name;
	36	struct config config;
	37	int expect_failure;
	38	};
	39
	40	enum {
	41	_1KB = 1ULL << 10, /* 1KB -> not page aligned */
	42	_4KB = 4ULL << 10,
	43	_8KB = 8ULL << 10,
	44	_1MB = 1ULL << 20,
	45	_2MB = 2ULL << 20,
	46	_4MB = 4ULL << 20,
	47	_1GB = 1ULL << 30,
	48	_2GB = 2ULL << 30,
7df66625 KS	49	PMD = _2MB,
	50	PUD = _1GB,
	51	};
	52
f27a5c93 AK	53	#define PTE page_size
f27a5c93 AK	54
7df66625 KS	55	#define MAKE_TEST(source_align, destination_align, size, \
7df66625 KS	56	overlaps, should_fail, test_name) \
f27a5c93	57	(struct test){ \
7df66625 KS	58	.name = test_name, \
	59	.config = { \
	60	.src_alignment = source_align, \
	61	.dest_alignment = destination_align, \
	62	.region_size = size, \
	63	.overlapping = overlaps, \
	64	}, \
	65	.expect_failure = should_fail \
	66	}
	67
18d609da SK	68	/*
	69	* Returns false if the requested remap region overlaps with an
	70	* existing mapping (e.g text, stack) else returns true.
	71	*/
	72	static bool is_remap_region_valid(void *addr, unsigned long long size)
	73	{
	74	void *remap_addr = NULL;
	75	bool ret = true;
	76
	77	/* Use MAP_FIXED_NOREPLACE flag to ensure region is not mapped */
	78	remap_addr = mmap(addr, size, PROT_READ \| PROT_WRITE,
	79	MAP_FIXED_NOREPLACE \| MAP_ANONYMOUS \| MAP_SHARED,
	80	-1, 0);
	81
	82	if (remap_addr == MAP_FAILED) {
	83	if (errno == EEXIST)
	84	ret = false;
	85	} else {
	86	munmap(remap_addr, size);
	87	}
	88
	89	return ret;
	90	}
	91
9c85a9ba SK	92	/* Returns mmap_min_addr sysctl tunable from procfs */
	93	static unsigned long long get_mmap_min_addr(void)
	94	{
	95	FILE *fp;
	96	int n_matched;
	97	static unsigned long long addr;
	98
	99	if (addr)
	100	return addr;
	101
	102	fp = fopen("/proc/sys/vm/mmap_min_addr", "r");
	103	if (fp == NULL) {
	104	ksft_print_msg("Failed to open /proc/sys/vm/mmap_min_addr: %s\n",
	105	strerror(errno));
	106	exit(KSFT_SKIP);
	107	}
	108
	109	n_matched = fscanf(fp, "%llu", &addr);
	110	if (n_matched != 1) {
	111	ksft_print_msg("Failed to read /proc/sys/vm/mmap_min_addr: %s\n",
	112	strerror(errno));
	113	fclose(fp);
	114	exit(KSFT_SKIP);
	115	}
	116
	117	fclose(fp);
	118	return addr;
	119	}
	120
ca3d76b0 JM	121	/*
	122	* This test validates that merge is called when expanding a mapping.
	123	* Mapping containing three pages is created, middle page is unmapped
	124	* and then the mapping containing the first page is expanded so that
	125	* it fills the created hole. The two parts should merge creating
	126	* single mapping with three pages.
	127	*/
	128	static void mremap_expand_merge(unsigned long page_size)
	129	{
	130	char *test_name = "mremap expand merge";
	131	FILE *fp;
	132	char *line = NULL;
	133	size_t len = 0;
	134	bool success = false;
	135	char start = mmap(NULL, 3 page_size, PROT_READ \| PROT_WRITE,
	136	MAP_PRIVATE \| MAP_ANONYMOUS, -1, 0);
	137
	138	munmap(start + page_size, page_size);
	139	mremap(start, page_size, 2 * page_size, 0);
	140
	141	fp = fopen("/proc/self/maps", "r");
	142	if (fp == NULL) {
	143	ksft_test_result_fail("%s\n", test_name);
	144	return;
	145	}
	146
	147	while (getline(&line, &len, fp) != -1) {
	148	char *first = strtok(line, "- ");
	149	void first_val = (void )strtol(first, NULL, 16);
	150	char *second = strtok(NULL, "- ");
	151	void second_val = (void ) strtol(second, NULL, 16);
	152
	153	if (first_val == start && second_val == start + 3 * page_size) {
	154	success = true;
	155	break;
	156	}
	157	}
	158	if (success)
	159	ksft_test_result_pass("%s\n", test_name);
	160	else
	161	ksft_test_result_fail("%s\n", test_name);
	162	fclose(fp);
	163	}
	164
7df66625 KS	165	/*
	166	* Returns the start address of the mapping on success, else returns
	167	* NULL on failure.
	168	*/
	169	static void *get_source_mapping(struct config c)
	170	{
	171	unsigned long long addr = 0ULL;
	172	void *src_addr = NULL;
9c85a9ba SK	173	unsigned long long mmap_min_addr;
	174
	175	mmap_min_addr = get_mmap_min_addr();
	176
7df66625 KS	177	retry:
7df66625 KS	178	addr += c.src_alignment;
9c85a9ba SK	179	if (addr < mmap_min_addr)
	180	goto retry;
	181
7df66625	182	src_addr = mmap((void *) addr, c.region_size, PROT_READ \| PROT_WRITE,
18d609da SK	183	MAP_FIXED_NOREPLACE \| MAP_ANONYMOUS \| MAP_SHARED,
18d609da SK	184	-1, 0);
7df66625	185	if (src_addr == MAP_FAILED) {
a9cc9c34	186	if (errno == EPERM \|\| errno == EEXIST)
7df66625 KS	187	goto retry;
	188	goto error;
	189	}
	190	/*
	191	* Check that the address is aligned to the specified alignment.
	192	* Addresses which have alignments that are multiples of that
	193	* specified are not considered valid. For instance, 1GB address is
	194	* 2MB-aligned, however it will not be considered valid for a
	195	* requested alignment of 2MB. This is done to reduce coincidental
	196	* alignment in the tests.
	197	*/
	198	if (((unsigned long long) src_addr & (c.src_alignment - 1)) \|\|
9c85a9ba SK	199	!((unsigned long long) src_addr & c.src_alignment)) {
9c85a9ba SK	200	munmap(src_addr, c.region_size);
7df66625	201	goto retry;
9c85a9ba	202	}
7df66625 KS	203
	204	if (!src_addr)
	205	goto error;
	206
	207	return src_addr;
	208	error:
	209	ksft_print_msg("Failed to map source region: %s\n",
	210	strerror(errno));
	211	return NULL;
	212	}
	213
	214	/* Returns the time taken for the remap on success else returns -1. */
	215	static long long remap_region(struct config c, unsigned int threshold_mb,
	216	char pattern_seed)
	217	{
	218	void addr, src_addr, *dest_addr;
	219	unsigned long long i;
	220	struct timespec t_start = {0, 0}, t_end = {0, 0};
	221	long long start_ns, end_ns, align_mask, ret, offset;
	222	unsigned long long threshold;
	223
	224	if (threshold_mb == VALIDATION_NO_THRESHOLD)
	225	threshold = c.region_size;
	226	else
	227	threshold = MIN(threshold_mb * _1MB, c.region_size);
	228
	229	src_addr = get_source_mapping(c);
	230	if (!src_addr) {
	231	ret = -1;
	232	goto out;
	233	}
	234
	235	/* Set byte pattern */
	236	srand(pattern_seed);
	237	for (i = 0; i < threshold; i++)
	238	memset((char *) src_addr + i, (char) rand(), 1);
	239
	240	/* Mask to zero out lower bits of address for alignment */
	241	align_mask = ~(c.dest_alignment - 1);
	242	/* Offset of destination address from the end of the source region */
	243	offset = (c.overlapping) ? -c.dest_alignment : c.dest_alignment;
	244	addr = (void *) (((unsigned long long) src_addr + c.region_size
	245	+ offset) & align_mask);
	246
	247	/* See comment in get_source_mapping() */
	248	if (!((unsigned long long) addr & c.dest_alignment))
	249	addr = (void *) ((unsigned long long) addr \| c.dest_alignment);
	250
18d609da SK	251	/* Don't destroy existing mappings unless expected to overlap */
	252	while (!is_remap_region_valid(addr, c.region_size) && !c.overlapping) {
	253	/* Check for unsigned overflow */
	254	if (addr + c.dest_alignment < addr) {
	255	ksft_print_msg("Couldn't find a valid region to remap to\n");
	256	ret = -1;
	257	goto out;
	258	}
	259	addr += c.dest_alignment;
	260	}
	261
7df66625 KS	262	clock_gettime(CLOCK_MONOTONIC, &t_start);
7df66625 KS	263	dest_addr = mremap(src_addr, c.region_size, c.region_size,
18d609da	264	MREMAP_MAYMOVE\|MREMAP_FIXED, (char *) addr);
7df66625 KS	265	clock_gettime(CLOCK_MONOTONIC, &t_end);
	266
	267	if (dest_addr == MAP_FAILED) {
	268	ksft_print_msg("mremap failed: %s\n", strerror(errno));
	269	ret = -1;
	270	goto clean_up_src;
	271	}
	272
	273	/* Verify byte pattern after remapping */
	274	srand(pattern_seed);
	275	for (i = 0; i < threshold; i++) {
	276	char c = (char) rand();
	277
	278	if (((char *) dest_addr)[i] != c) {
	279	ksft_print_msg("Data after remap doesn't match at offset %d\n",
	280	i);
	281	ksft_print_msg("Expected: %#x\t Got: %#x\n", c & 0xff,
	282	((char *) dest_addr)[i] & 0xff);
	283	ret = -1;
	284	goto clean_up_dest;
	285	}
	286	}
	287
	288	start_ns = t_start.tv_sec * NS_PER_SEC + t_start.tv_nsec;
	289	end_ns = t_end.tv_sec * NS_PER_SEC + t_end.tv_nsec;
	290	ret = end_ns - start_ns;
	291
	292	/*
	293	* Since the destination address is specified using MREMAP_FIXED, subsequent
	294	* mremap will unmap any previous mapping at the address range specified by
	295	* dest_addr and region_size. This significantly affects the remap time of
	296	* subsequent tests. So we clean up mappings after each test.
	297	*/
	298	clean_up_dest:
	299	munmap(dest_addr, c.region_size);
	300	clean_up_src:
	301	munmap(src_addr, c.region_size);
	302	out:
	303	return ret;
	304	}
	305
	306	static void run_mremap_test_case(struct test test_case, int *failures,
	307	unsigned int threshold_mb,
	308	unsigned int pattern_seed)
	309	{
	310	long long remap_time = remap_region(test_case.config, threshold_mb,
	311	pattern_seed);
	312
	313	if (remap_time < 0) {
	314	if (test_case.expect_failure)
e5508fc5	315	ksft_test_result_xfail("%s\n\tExpected mremap failure\n",
7df66625 KS	316	test_case.name);
	317	else {
	318	ksft_test_result_fail("%s\n", test_case.name);
	319	*failures += 1;
	320	}
	321	} else {
	322	/*
	323	* Comparing mremap time is only applicable if entire region
	324	* was faulted in.
	325	*/
	326	if (threshold_mb == VALIDATION_NO_THRESHOLD \|\|
	327	test_case.config.region_size <= threshold_mb * _1MB)
	328	ksft_test_result_pass("%s\n\tmremap time: %12lldns\n",
	329	test_case.name, remap_time);
	330	else
	331	ksft_test_result_pass("%s\n", test_case.name);
	332	}
	333	}
	334
	335	static void usage(const char *cmd)
	336	{
	337	fprintf(stderr,
	338	"Usage: %s [[-t <threshold_mb>] [-p <pattern_seed>]]\n"
	339	"-t\t only validate threshold_mb of the remapped region\n"
	340	" \t if 0 is supplied no threshold is used; all tests\n"
	341	" \t are run and remapped regions validated fully.\n"
	342	" \t The default threshold used is 4MB.\n"
	343	"-p\t provide a seed to generate the random pattern for\n"
	344	" \t validating the remapped region.\n", cmd);
	345	}
	346
	347	static int parse_args(int argc, char *argv, unsigned int threshold_mb,
	348	unsigned int *pattern_seed)
	349	{
	350	const char *optstr = "t:p:";
	351	int opt;
	352
	353	while ((opt = getopt(argc, argv, optstr)) != -1) {
	354	switch (opt) {
	355	case 't':
	356	*threshold_mb = atoi(optarg);
	357	break;
	358	case 'p':
	359	*pattern_seed = atoi(optarg);
	360	break;
	361	default:
	362	usage(argv[0]);
	363	return -1;
	364	}
	365	}
	366
	367	if (optind < argc) {
	368	usage(argv[0]);
	369	return -1;
	370	}
	371
	372	return 0;
	373	}
	374
f27a5c93 AK	375	#define MAX_TEST 13
f27a5c93 AK	376	#define MAX_PERF_TEST 3
7df66625 KS	377	int main(int argc, char **argv)
	378	{
	379	int failures = 0;
	380	int i, run_perf_tests;
	381	unsigned int threshold_mb = VALIDATION_DEFAULT_THRESHOLD;
	382	unsigned int pattern_seed;
ca3d76b0	383	int num_expand_tests = 1;
f27a5c93 AK	384	struct test test_cases[MAX_TEST];
	385	struct test perf_test_cases[MAX_PERF_TEST];
	386	int page_size;
7df66625 KS	387	time_t t;
	388
	389	pattern_seed = (unsigned int) time(&t);
	390
	391	if (parse_args(argc, argv, &threshold_mb, &pattern_seed) < 0)
	392	exit(EXIT_FAILURE);
	393
	394	ksft_print_msg("Test configs:\n\tthreshold_mb=%u\n\tpattern_seed=%u\n\n",
	395	threshold_mb, pattern_seed);
	396
f27a5c93 AK	397	page_size = sysconf(_SC_PAGESIZE);
	398
	399	/* Expected mremap failures */
	400	test_cases[0] = MAKE_TEST(page_size, page_size, page_size,
	401	OVERLAPPING, EXPECT_FAILURE,
	402	"mremap - Source and Destination Regions Overlapping");
	403
	404	test_cases[1] = MAKE_TEST(page_size, page_size/4, page_size,
	405	NON_OVERLAPPING, EXPECT_FAILURE,
	406	"mremap - Destination Address Misaligned (1KB-aligned)");
	407	test_cases[2] = MAKE_TEST(page_size/4, page_size, page_size,
	408	NON_OVERLAPPING, EXPECT_FAILURE,
	409	"mremap - Source Address Misaligned (1KB-aligned)");
	410
	411	/* Src addr PTE aligned */
	412	test_cases[3] = MAKE_TEST(PTE, PTE, PTE * 2,
	413	NON_OVERLAPPING, EXPECT_SUCCESS,
	414	"8KB mremap - Source PTE-aligned, Destination PTE-aligned");
	415
	416	/* Src addr 1MB aligned */
	417	test_cases[4] = MAKE_TEST(_1MB, PTE, _2MB, NON_OVERLAPPING, EXPECT_SUCCESS,
	418	"2MB mremap - Source 1MB-aligned, Destination PTE-aligned");
	419	test_cases[5] = MAKE_TEST(_1MB, _1MB, _2MB, NON_OVERLAPPING, EXPECT_SUCCESS,
	420	"2MB mremap - Source 1MB-aligned, Destination 1MB-aligned");
	421
	422	/* Src addr PMD aligned */
	423	test_cases[6] = MAKE_TEST(PMD, PTE, _4MB, NON_OVERLAPPING, EXPECT_SUCCESS,
	424	"4MB mremap - Source PMD-aligned, Destination PTE-aligned");
	425	test_cases[7] = MAKE_TEST(PMD, _1MB, _4MB, NON_OVERLAPPING, EXPECT_SUCCESS,
	426	"4MB mremap - Source PMD-aligned, Destination 1MB-aligned");
	427	test_cases[8] = MAKE_TEST(PMD, PMD, _4MB, NON_OVERLAPPING, EXPECT_SUCCESS,
	428	"4MB mremap - Source PMD-aligned, Destination PMD-aligned");
	429
	430	/* Src addr PUD aligned */
	431	test_cases[9] = MAKE_TEST(PUD, PTE, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS,
	432	"2GB mremap - Source PUD-aligned, Destination PTE-aligned");
	433	test_cases[10] = MAKE_TEST(PUD, _1MB, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS,
	434	"2GB mremap - Source PUD-aligned, Destination 1MB-aligned");
	435	test_cases[11] = MAKE_TEST(PUD, PMD, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS,
	436	"2GB mremap - Source PUD-aligned, Destination PMD-aligned");
	437	test_cases[12] = MAKE_TEST(PUD, PUD, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS,
	438	"2GB mremap - Source PUD-aligned, Destination PUD-aligned");
	439
	440	perf_test_cases[0] = MAKE_TEST(page_size, page_size, _1GB, NON_OVERLAPPING, EXPECT_SUCCESS,
	441	"1GB mremap - Source PTE-aligned, Destination PTE-aligned");
	442	/*
	443	* mremap 1GB region - Page table level aligned time
	444	* comparison.
	445	*/
	446	perf_test_cases[1] = MAKE_TEST(PMD, PMD, _1GB, NON_OVERLAPPING, EXPECT_SUCCESS,
	447	"1GB mremap - Source PMD-aligned, Destination PMD-aligned");
	448	perf_test_cases[2] = MAKE_TEST(PUD, PUD, _1GB, NON_OVERLAPPING, EXPECT_SUCCESS,
	449	"1GB mremap - Source PUD-aligned, Destination PUD-aligned");
7df66625 KS	450
	451	run_perf_tests = (threshold_mb == VALIDATION_NO_THRESHOLD) \|\|
	452	(threshold_mb * _1MB >= _1GB);
	453
	454	ksft_set_plan(ARRAY_SIZE(test_cases) + (run_perf_tests ?
ca3d76b0	455	ARRAY_SIZE(perf_test_cases) : 0) + num_expand_tests);
7df66625 KS	456
	457	for (i = 0; i < ARRAY_SIZE(test_cases); i++)
	458	run_mremap_test_case(test_cases[i], &failures, threshold_mb,
	459	pattern_seed);
	460
ca3d76b0 JM	461	mremap_expand_merge(page_size);
ca3d76b0 JM	462
7df66625 KS	463	if (run_perf_tests) {
	464	ksft_print_msg("\n%s\n",
	465	"mremap HAVE_MOVE_PMD/PUD optimization time comparison for 1GB region:");
	466	for (i = 0; i < ARRAY_SIZE(perf_test_cases); i++)
	467	run_mremap_test_case(perf_test_cases[i], &failures,
	468	threshold_mb, pattern_seed);
	469	}
	470
	471	if (failures > 0)
	472	ksft_exit_fail();
	473	else
	474	ksft_exit_pass();
	475	}