Commit | Line | Data |
---|---|---|
7df66625 KS |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* | |
3 | * Copyright 2020 Google LLC | |
4 | */ | |
5 | #define _GNU_SOURCE | |
6 | ||
7 | #include <errno.h> | |
8 | #include <stdlib.h> | |
9c85a9ba | 9 | #include <stdio.h> |
7df66625 KS |
10 | #include <string.h> |
11 | #include <sys/mman.h> | |
12 | #include <time.h> | |
18d609da | 13 | #include <stdbool.h> |
7df66625 KS |
14 | |
15 | #include "../kselftest.h" | |
16 | ||
17 | #define EXPECT_SUCCESS 0 | |
18 | #define EXPECT_FAILURE 1 | |
19 | #define NON_OVERLAPPING 0 | |
20 | #define OVERLAPPING 1 | |
21 | #define NS_PER_SEC 1000000000ULL | |
22 | #define VALIDATION_DEFAULT_THRESHOLD 4 /* 4MB */ | |
23 | #define VALIDATION_NO_THRESHOLD 0 /* Verify the entire region */ | |
24 | ||
1a251f52 | 25 | #ifndef MIN |
7df66625 | 26 | #define MIN(X, Y) ((X) < (Y) ? (X) : (Y)) |
c7876a0c | 27 | #define MAX(X, Y) ((X) > (Y) ? (X) : (Y)) |
1a251f52 | 28 | #endif |
85a22845 | 29 | #define SIZE_MB(m) ((size_t)m * (1024 * 1024)) |
7b709f38 | 30 | #define SIZE_KB(k) ((size_t)k * 1024) |
7df66625 KS |
31 | |
32 | struct config { | |
33 | unsigned long long src_alignment; | |
34 | unsigned long long dest_alignment; | |
35 | unsigned long long region_size; | |
36 | int overlapping; | |
a4cb3b24 | 37 | int dest_preamble_size; |
7df66625 KS |
38 | }; |
39 | ||
40 | struct test { | |
41 | const char *name; | |
42 | struct config config; | |
43 | int expect_failure; | |
44 | }; | |
45 | ||
46 | enum { | |
47 | _1KB = 1ULL << 10, /* 1KB -> not page aligned */ | |
48 | _4KB = 4ULL << 10, | |
49 | _8KB = 8ULL << 10, | |
50 | _1MB = 1ULL << 20, | |
51 | _2MB = 2ULL << 20, | |
52 | _4MB = 4ULL << 20, | |
8ed873d8 | 53 | _5MB = 5ULL << 20, |
7df66625 KS |
54 | _1GB = 1ULL << 30, |
55 | _2GB = 2ULL << 30, | |
7df66625 KS |
56 | PMD = _2MB, |
57 | PUD = _1GB, | |
58 | }; | |
59 | ||
f27a5c93 AK |
60 | #define PTE page_size |
61 | ||
7df66625 KS |
62 | #define MAKE_TEST(source_align, destination_align, size, \ |
63 | overlaps, should_fail, test_name) \ | |
f27a5c93 | 64 | (struct test){ \ |
7df66625 KS |
65 | .name = test_name, \ |
66 | .config = { \ | |
67 | .src_alignment = source_align, \ | |
68 | .dest_alignment = destination_align, \ | |
69 | .region_size = size, \ | |
70 | .overlapping = overlaps, \ | |
71 | }, \ | |
72 | .expect_failure = should_fail \ | |
73 | } | |
74 | ||
7033c6cc DJ |
75 | /* compute square root using binary search */ |
76 | static unsigned long get_sqrt(unsigned long val) | |
77 | { | |
78 | unsigned long low = 1; | |
79 | ||
80 | /* assuming rand_size is less than 1TB */ | |
81 | unsigned long high = (1UL << 20); | |
82 | ||
83 | while (low <= high) { | |
84 | unsigned long mid = low + (high - low) / 2; | |
85 | unsigned long temp = mid * mid; | |
86 | ||
87 | if (temp == val) | |
88 | return mid; | |
89 | if (temp < val) | |
90 | low = mid + 1; | |
91 | high = mid - 1; | |
92 | } | |
93 | return low; | |
94 | } | |
95 | ||
18d609da SK |
96 | /* |
97 | * Returns false if the requested remap region overlaps with an | |
98 | * existing mapping (e.g text, stack) else returns true. | |
99 | */ | |
100 | static bool is_remap_region_valid(void *addr, unsigned long long size) | |
101 | { | |
102 | void *remap_addr = NULL; | |
103 | bool ret = true; | |
104 | ||
105 | /* Use MAP_FIXED_NOREPLACE flag to ensure region is not mapped */ | |
106 | remap_addr = mmap(addr, size, PROT_READ | PROT_WRITE, | |
107 | MAP_FIXED_NOREPLACE | MAP_ANONYMOUS | MAP_SHARED, | |
108 | -1, 0); | |
109 | ||
110 | if (remap_addr == MAP_FAILED) { | |
111 | if (errno == EEXIST) | |
112 | ret = false; | |
113 | } else { | |
114 | munmap(remap_addr, size); | |
115 | } | |
116 | ||
117 | return ret; | |
118 | } | |
119 | ||
9c85a9ba SK |
120 | /* Returns mmap_min_addr sysctl tunable from procfs */ |
121 | static unsigned long long get_mmap_min_addr(void) | |
122 | { | |
123 | FILE *fp; | |
124 | int n_matched; | |
125 | static unsigned long long addr; | |
126 | ||
127 | if (addr) | |
128 | return addr; | |
129 | ||
130 | fp = fopen("/proc/sys/vm/mmap_min_addr", "r"); | |
131 | if (fp == NULL) { | |
132 | ksft_print_msg("Failed to open /proc/sys/vm/mmap_min_addr: %s\n", | |
133 | strerror(errno)); | |
134 | exit(KSFT_SKIP); | |
135 | } | |
136 | ||
137 | n_matched = fscanf(fp, "%llu", &addr); | |
138 | if (n_matched != 1) { | |
139 | ksft_print_msg("Failed to read /proc/sys/vm/mmap_min_addr: %s\n", | |
140 | strerror(errno)); | |
141 | fclose(fp); | |
142 | exit(KSFT_SKIP); | |
143 | } | |
144 | ||
145 | fclose(fp); | |
146 | return addr; | |
147 | } | |
148 | ||
ca3d76b0 | 149 | /* |
da0618c1 LS |
150 | * Using /proc/self/maps, assert that the specified address range is contained |
151 | * within a single mapping. | |
ca3d76b0 | 152 | */ |
e1e13262 DJ |
153 | static bool is_range_mapped(FILE *maps_fp, unsigned long start, |
154 | unsigned long end) | |
ca3d76b0 | 155 | { |
ca3d76b0 JM |
156 | char *line = NULL; |
157 | size_t len = 0; | |
158 | bool success = false; | |
e1e13262 | 159 | unsigned long first_val, second_val; |
ca3d76b0 | 160 | |
da0618c1 | 161 | rewind(maps_fp); |
ca3d76b0 | 162 | |
da0618c1 | 163 | while (getline(&line, &len, maps_fp) != -1) { |
e1e13262 DJ |
164 | if (sscanf(line, "%lx-%lx", &first_val, &second_val) != 2) { |
165 | ksft_exit_fail_msg("cannot parse /proc/self/maps\n"); | |
166 | break; | |
167 | } | |
ca3d76b0 | 168 | |
da0618c1 | 169 | if (first_val <= start && second_val >= end) { |
ca3d76b0 JM |
170 | success = true; |
171 | break; | |
172 | } | |
173 | } | |
da0618c1 LS |
174 | |
175 | return success; | |
176 | } | |
177 | ||
7b709f38 JF |
178 | /* |
179 | * Returns the start address of the mapping on success, else returns | |
180 | * NULL on failure. | |
181 | */ | |
182 | static void *get_source_mapping(struct config c) | |
183 | { | |
184 | unsigned long long addr = 0ULL; | |
185 | void *src_addr = NULL; | |
186 | unsigned long long mmap_min_addr; | |
187 | ||
188 | mmap_min_addr = get_mmap_min_addr(); | |
189 | /* | |
190 | * For some tests, we need to not have any mappings below the | |
191 | * source mapping. Add some headroom to mmap_min_addr for this. | |
192 | */ | |
193 | mmap_min_addr += 10 * _4MB; | |
194 | ||
195 | retry: | |
196 | addr += c.src_alignment; | |
197 | if (addr < mmap_min_addr) | |
198 | goto retry; | |
199 | ||
200 | src_addr = mmap((void *) addr, c.region_size, PROT_READ | PROT_WRITE, | |
201 | MAP_FIXED_NOREPLACE | MAP_ANONYMOUS | MAP_SHARED, | |
202 | -1, 0); | |
203 | if (src_addr == MAP_FAILED) { | |
204 | if (errno == EPERM || errno == EEXIST) | |
205 | goto retry; | |
206 | goto error; | |
207 | } | |
208 | /* | |
209 | * Check that the address is aligned to the specified alignment. | |
210 | * Addresses which have alignments that are multiples of that | |
211 | * specified are not considered valid. For instance, 1GB address is | |
212 | * 2MB-aligned, however it will not be considered valid for a | |
213 | * requested alignment of 2MB. This is done to reduce coincidental | |
214 | * alignment in the tests. | |
215 | */ | |
216 | if (((unsigned long long) src_addr & (c.src_alignment - 1)) || | |
217 | !((unsigned long long) src_addr & c.src_alignment)) { | |
218 | munmap(src_addr, c.region_size); | |
219 | goto retry; | |
220 | } | |
221 | ||
222 | if (!src_addr) | |
223 | goto error; | |
224 | ||
225 | return src_addr; | |
226 | error: | |
227 | ksft_print_msg("Failed to map source region: %s\n", | |
228 | strerror(errno)); | |
229 | return NULL; | |
230 | } | |
231 | ||
da0618c1 LS |
232 | /* |
233 | * This test validates that merge is called when expanding a mapping. | |
234 | * Mapping containing three pages is created, middle page is unmapped | |
235 | * and then the mapping containing the first page is expanded so that | |
236 | * it fills the created hole. The two parts should merge creating | |
237 | * single mapping with three pages. | |
238 | */ | |
239 | static void mremap_expand_merge(FILE *maps_fp, unsigned long page_size) | |
240 | { | |
241 | char *test_name = "mremap expand merge"; | |
242 | bool success = false; | |
243 | char *remap, *start; | |
244 | ||
245 | start = mmap(NULL, 3 * page_size, PROT_READ | PROT_WRITE, | |
246 | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); | |
247 | ||
248 | if (start == MAP_FAILED) { | |
249 | ksft_print_msg("mmap failed: %s\n", strerror(errno)); | |
250 | goto out; | |
251 | } | |
252 | ||
253 | munmap(start + page_size, page_size); | |
254 | remap = mremap(start, page_size, 2 * page_size, 0); | |
255 | if (remap == MAP_FAILED) { | |
256 | ksft_print_msg("mremap failed: %s\n", strerror(errno)); | |
257 | munmap(start, page_size); | |
258 | munmap(start + 2 * page_size, page_size); | |
259 | goto out; | |
260 | } | |
261 | ||
e1e13262 DJ |
262 | success = is_range_mapped(maps_fp, (unsigned long)start, |
263 | (unsigned long)(start + 3 * page_size)); | |
da0618c1 LS |
264 | munmap(start, 3 * page_size); |
265 | ||
266 | out: | |
267 | if (success) | |
268 | ksft_test_result_pass("%s\n", test_name); | |
269 | else | |
270 | ksft_test_result_fail("%s\n", test_name); | |
271 | } | |
272 | ||
273 | /* | |
274 | * Similar to mremap_expand_merge() except instead of removing the middle page, | |
275 | * we remove the last then attempt to remap offset from the second page. This | |
276 | * should result in the mapping being restored to its former state. | |
277 | */ | |
278 | static void mremap_expand_merge_offset(FILE *maps_fp, unsigned long page_size) | |
279 | { | |
280 | ||
281 | char *test_name = "mremap expand merge offset"; | |
282 | bool success = false; | |
283 | char *remap, *start; | |
284 | ||
285 | start = mmap(NULL, 3 * page_size, PROT_READ | PROT_WRITE, | |
286 | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); | |
287 | ||
288 | if (start == MAP_FAILED) { | |
289 | ksft_print_msg("mmap failed: %s\n", strerror(errno)); | |
290 | goto out; | |
291 | } | |
292 | ||
293 | /* Unmap final page to ensure we have space to expand. */ | |
294 | munmap(start + 2 * page_size, page_size); | |
295 | remap = mremap(start + page_size, page_size, 2 * page_size, 0); | |
296 | if (remap == MAP_FAILED) { | |
297 | ksft_print_msg("mremap failed: %s\n", strerror(errno)); | |
298 | munmap(start, 2 * page_size); | |
299 | goto out; | |
300 | } | |
301 | ||
e1e13262 DJ |
302 | success = is_range_mapped(maps_fp, (unsigned long)start, |
303 | (unsigned long)(start + 3 * page_size)); | |
da0618c1 LS |
304 | munmap(start, 3 * page_size); |
305 | ||
306 | out: | |
ca3d76b0 JM |
307 | if (success) |
308 | ksft_test_result_pass("%s\n", test_name); | |
309 | else | |
310 | ksft_test_result_fail("%s\n", test_name); | |
ca3d76b0 JM |
311 | } |
312 | ||
7df66625 | 313 | /* |
85a22845 JFG |
314 | * Verify that an mremap within a range does not cause corruption |
315 | * of unrelated part of range. | |
316 | * | |
317 | * Consider the following range which is 2MB aligned and is | |
318 | * a part of a larger 20MB range which is not shown. Each | |
319 | * character is 256KB below making the source and destination | |
320 | * 2MB each. The lower case letters are moved (s to d) and the | |
321 | * upper case letters are not moved. The below test verifies | |
322 | * that the upper case S letters are not corrupted by the | |
323 | * adjacent mremap. | |
324 | * | |
325 | * |DDDDddddSSSSssss| | |
7df66625 | 326 | */ |
c7876a0c | 327 | static void mremap_move_within_range(unsigned int pattern_seed, char *rand_addr) |
7df66625 | 328 | { |
85a22845 JFG |
329 | char *test_name = "mremap mremap move within range"; |
330 | void *src, *dest; | |
331 | int i, success = 1; | |
332 | ||
333 | size_t size = SIZE_MB(20); | |
334 | void *ptr = mmap(NULL, size, PROT_READ | PROT_WRITE, | |
335 | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); | |
336 | if (ptr == MAP_FAILED) { | |
337 | perror("mmap"); | |
338 | success = 0; | |
339 | goto out; | |
340 | } | |
341 | memset(ptr, 0, size); | |
9c85a9ba | 342 | |
85a22845 JFG |
343 | src = ptr + SIZE_MB(6); |
344 | src = (void *)((unsigned long)src & ~(SIZE_MB(2) - 1)); | |
9c85a9ba | 345 | |
85a22845 | 346 | /* Set byte pattern for source block. */ |
c7876a0c | 347 | memcpy(src, rand_addr, SIZE_MB(2)); |
9c85a9ba | 348 | |
85a22845 JFG |
349 | dest = src - SIZE_MB(2); |
350 | ||
351 | void *new_ptr = mremap(src + SIZE_MB(1), SIZE_MB(1), SIZE_MB(1), | |
352 | MREMAP_MAYMOVE | MREMAP_FIXED, dest + SIZE_MB(1)); | |
353 | if (new_ptr == MAP_FAILED) { | |
354 | perror("mremap"); | |
355 | success = 0; | |
356 | goto out; | |
7df66625 | 357 | } |
85a22845 JFG |
358 | |
359 | /* Verify byte pattern after remapping */ | |
360 | srand(pattern_seed); | |
361 | for (i = 0; i < SIZE_MB(1); i++) { | |
362 | char c = (char) rand(); | |
363 | ||
364 | if (((char *)src)[i] != c) { | |
365 | ksft_print_msg("Data at src at %d got corrupted due to unrelated mremap\n", | |
366 | i); | |
367 | ksft_print_msg("Expected: %#x\t Got: %#x\n", c & 0xff, | |
368 | ((char *) src)[i] & 0xff); | |
369 | success = 0; | |
370 | } | |
9c85a9ba | 371 | } |
7df66625 | 372 | |
85a22845 JFG |
373 | out: |
374 | if (munmap(ptr, size) == -1) | |
375 | perror("munmap"); | |
7df66625 | 376 | |
85a22845 JFG |
377 | if (success) |
378 | ksft_test_result_pass("%s\n", test_name); | |
379 | else | |
380 | ksft_test_result_fail("%s\n", test_name); | |
7df66625 KS |
381 | } |
382 | ||
383 | /* Returns the time taken for the remap on success else returns -1. */ | |
384 | static long long remap_region(struct config c, unsigned int threshold_mb, | |
7033c6cc | 385 | char *rand_addr) |
7df66625 | 386 | { |
a4cb3b24 | 387 | void *addr, *src_addr, *dest_addr, *dest_preamble_addr; |
7033c6cc | 388 | unsigned long long t, d; |
7df66625 KS |
389 | struct timespec t_start = {0, 0}, t_end = {0, 0}; |
390 | long long start_ns, end_ns, align_mask, ret, offset; | |
391 | unsigned long long threshold; | |
7033c6cc | 392 | unsigned long num_chunks; |
7df66625 KS |
393 | |
394 | if (threshold_mb == VALIDATION_NO_THRESHOLD) | |
395 | threshold = c.region_size; | |
396 | else | |
397 | threshold = MIN(threshold_mb * _1MB, c.region_size); | |
398 | ||
399 | src_addr = get_source_mapping(c); | |
400 | if (!src_addr) { | |
401 | ret = -1; | |
402 | goto out; | |
403 | } | |
404 | ||
a4cb3b24 | 405 | /* Set byte pattern for source block. */ |
c7876a0c | 406 | memcpy(src_addr, rand_addr, threshold); |
7df66625 KS |
407 | |
408 | /* Mask to zero out lower bits of address for alignment */ | |
409 | align_mask = ~(c.dest_alignment - 1); | |
410 | /* Offset of destination address from the end of the source region */ | |
411 | offset = (c.overlapping) ? -c.dest_alignment : c.dest_alignment; | |
412 | addr = (void *) (((unsigned long long) src_addr + c.region_size | |
413 | + offset) & align_mask); | |
414 | ||
a4cb3b24 JFG |
415 | /* Remap after the destination block preamble. */ |
416 | addr += c.dest_preamble_size; | |
417 | ||
7df66625 KS |
418 | /* See comment in get_source_mapping() */ |
419 | if (!((unsigned long long) addr & c.dest_alignment)) | |
420 | addr = (void *) ((unsigned long long) addr | c.dest_alignment); | |
421 | ||
18d609da SK |
422 | /* Don't destroy existing mappings unless expected to overlap */ |
423 | while (!is_remap_region_valid(addr, c.region_size) && !c.overlapping) { | |
424 | /* Check for unsigned overflow */ | |
425 | if (addr + c.dest_alignment < addr) { | |
426 | ksft_print_msg("Couldn't find a valid region to remap to\n"); | |
427 | ret = -1; | |
99eb26d5 | 428 | goto clean_up_src; |
18d609da SK |
429 | } |
430 | addr += c.dest_alignment; | |
431 | } | |
432 | ||
a4cb3b24 JFG |
433 | if (c.dest_preamble_size) { |
434 | dest_preamble_addr = mmap((void *) addr - c.dest_preamble_size, c.dest_preamble_size, | |
435 | PROT_READ | PROT_WRITE, | |
436 | MAP_FIXED_NOREPLACE | MAP_ANONYMOUS | MAP_SHARED, | |
437 | -1, 0); | |
438 | if (dest_preamble_addr == MAP_FAILED) { | |
439 | ksft_print_msg("Failed to map dest preamble region: %s\n", | |
440 | strerror(errno)); | |
441 | ret = -1; | |
442 | goto clean_up_src; | |
443 | } | |
444 | ||
445 | /* Set byte pattern for the dest preamble block. */ | |
c7876a0c | 446 | memcpy(dest_preamble_addr, rand_addr, c.dest_preamble_size); |
a4cb3b24 JFG |
447 | } |
448 | ||
7df66625 KS |
449 | clock_gettime(CLOCK_MONOTONIC, &t_start); |
450 | dest_addr = mremap(src_addr, c.region_size, c.region_size, | |
18d609da | 451 | MREMAP_MAYMOVE|MREMAP_FIXED, (char *) addr); |
7df66625 KS |
452 | clock_gettime(CLOCK_MONOTONIC, &t_end); |
453 | ||
454 | if (dest_addr == MAP_FAILED) { | |
455 | ksft_print_msg("mremap failed: %s\n", strerror(errno)); | |
456 | ret = -1; | |
a4cb3b24 | 457 | goto clean_up_dest_preamble; |
7df66625 KS |
458 | } |
459 | ||
7033c6cc DJ |
460 | /* |
461 | * Verify byte pattern after remapping. Employ an algorithm with a | |
462 | * square root time complexity in threshold: divide the range into | |
463 | * chunks, if memcmp() returns non-zero, only then perform an | |
464 | * iteration in that chunk to find the mismatch index. | |
465 | */ | |
466 | num_chunks = get_sqrt(threshold); | |
467 | for (unsigned long i = 0; i < num_chunks; ++i) { | |
468 | size_t chunk_size = threshold / num_chunks; | |
469 | unsigned long shift = i * chunk_size; | |
470 | ||
471 | if (!memcmp(dest_addr + shift, rand_addr + shift, chunk_size)) | |
472 | continue; | |
473 | ||
474 | /* brute force iteration only over mismatch segment */ | |
475 | for (t = shift; t < shift + chunk_size; ++t) { | |
476 | if (((char *) dest_addr)[t] != rand_addr[t]) { | |
477 | ksft_print_msg("Data after remap doesn't match at offset %llu\n", | |
478 | t); | |
479 | ksft_print_msg("Expected: %#x\t Got: %#x\n", rand_addr[t] & 0xff, | |
480 | ((char *) dest_addr)[t] & 0xff); | |
481 | ret = -1; | |
482 | goto clean_up_dest; | |
483 | } | |
484 | } | |
485 | } | |
7df66625 | 486 | |
7033c6cc DJ |
487 | /* |
488 | * if threshold is not divisible by num_chunks, then check the | |
489 | * last chunk | |
490 | */ | |
491 | for (t = num_chunks * (threshold / num_chunks); t < threshold; ++t) { | |
492 | if (((char *) dest_addr)[t] != rand_addr[t]) { | |
d3772e7b | 493 | ksft_print_msg("Data after remap doesn't match at offset %llu\n", |
7033c6cc DJ |
494 | t); |
495 | ksft_print_msg("Expected: %#x\t Got: %#x\n", rand_addr[t] & 0xff, | |
f8ee4361 | 496 | ((char *) dest_addr)[t] & 0xff); |
7df66625 KS |
497 | ret = -1; |
498 | goto clean_up_dest; | |
499 | } | |
500 | } | |
501 | ||
a4cb3b24 | 502 | /* Verify the dest preamble byte pattern after remapping */ |
7033c6cc DJ |
503 | if (!c.dest_preamble_size) |
504 | goto no_preamble; | |
505 | ||
506 | num_chunks = get_sqrt(c.dest_preamble_size); | |
507 | ||
508 | for (unsigned long i = 0; i < num_chunks; ++i) { | |
509 | size_t chunk_size = c.dest_preamble_size / num_chunks; | |
510 | unsigned long shift = i * chunk_size; | |
511 | ||
512 | if (!memcmp(dest_preamble_addr + shift, rand_addr + shift, | |
513 | chunk_size)) | |
514 | continue; | |
515 | ||
516 | /* brute force iteration only over mismatched segment */ | |
517 | for (d = shift; d < shift + chunk_size; ++d) { | |
518 | if (((char *) dest_preamble_addr)[d] != rand_addr[d]) { | |
519 | ksft_print_msg("Preamble data after remap doesn't match at offset %llu\n", | |
520 | d); | |
521 | ksft_print_msg("Expected: %#x\t Got: %#x\n", rand_addr[d] & 0xff, | |
522 | ((char *) dest_preamble_addr)[d] & 0xff); | |
a4cb3b24 JFG |
523 | ret = -1; |
524 | goto clean_up_dest; | |
525 | } | |
526 | } | |
527 | } | |
528 | ||
7033c6cc DJ |
529 | for (d = num_chunks * (c.dest_preamble_size / num_chunks); d < c.dest_preamble_size; ++d) { |
530 | if (((char *) dest_preamble_addr)[d] != rand_addr[d]) { | |
531 | ksft_print_msg("Preamble data after remap doesn't match at offset %llu\n", | |
532 | d); | |
533 | ksft_print_msg("Expected: %#x\t Got: %#x\n", rand_addr[d] & 0xff, | |
534 | ((char *) dest_preamble_addr)[d] & 0xff); | |
535 | ret = -1; | |
536 | goto clean_up_dest; | |
537 | } | |
538 | } | |
539 | ||
540 | no_preamble: | |
7df66625 KS |
541 | start_ns = t_start.tv_sec * NS_PER_SEC + t_start.tv_nsec; |
542 | end_ns = t_end.tv_sec * NS_PER_SEC + t_end.tv_nsec; | |
543 | ret = end_ns - start_ns; | |
544 | ||
545 | /* | |
546 | * Since the destination address is specified using MREMAP_FIXED, subsequent | |
547 | * mremap will unmap any previous mapping at the address range specified by | |
548 | * dest_addr and region_size. This significantly affects the remap time of | |
549 | * subsequent tests. So we clean up mappings after each test. | |
550 | */ | |
551 | clean_up_dest: | |
552 | munmap(dest_addr, c.region_size); | |
a4cb3b24 JFG |
553 | clean_up_dest_preamble: |
554 | if (c.dest_preamble_size && dest_preamble_addr) | |
555 | munmap(dest_preamble_addr, c.dest_preamble_size); | |
7df66625 KS |
556 | clean_up_src: |
557 | munmap(src_addr, c.region_size); | |
558 | out: | |
559 | return ret; | |
560 | } | |
561 | ||
7b709f38 JF |
562 | /* |
563 | * Verify that an mremap aligning down does not destroy | |
564 | * the beginning of the mapping just because the aligned | |
565 | * down address landed on a mapping that maybe does not exist. | |
566 | */ | |
c7876a0c DJ |
567 | static void mremap_move_1mb_from_start(unsigned int pattern_seed, |
568 | char *rand_addr) | |
7b709f38 JF |
569 | { |
570 | char *test_name = "mremap move 1mb from start at 1MB+256KB aligned src"; | |
571 | void *src = NULL, *dest = NULL; | |
572 | int i, success = 1; | |
573 | ||
574 | /* Config to reuse get_source_mapping() to do an aligned mmap. */ | |
575 | struct config c = { | |
576 | .src_alignment = SIZE_MB(1) + SIZE_KB(256), | |
577 | .region_size = SIZE_MB(6) | |
578 | }; | |
579 | ||
580 | src = get_source_mapping(c); | |
581 | if (!src) { | |
582 | success = 0; | |
583 | goto out; | |
584 | } | |
585 | ||
586 | c.src_alignment = SIZE_MB(1) + SIZE_KB(256); | |
587 | dest = get_source_mapping(c); | |
588 | if (!dest) { | |
589 | success = 0; | |
590 | goto out; | |
591 | } | |
592 | ||
593 | /* Set byte pattern for source block. */ | |
c7876a0c | 594 | memcpy(src, rand_addr, SIZE_MB(2)); |
7b709f38 JF |
595 | |
596 | /* | |
597 | * Unmap the beginning of dest so that the aligned address | |
598 | * falls on no mapping. | |
599 | */ | |
600 | munmap(dest, SIZE_MB(1)); | |
601 | ||
602 | void *new_ptr = mremap(src + SIZE_MB(1), SIZE_MB(1), SIZE_MB(1), | |
603 | MREMAP_MAYMOVE | MREMAP_FIXED, dest + SIZE_MB(1)); | |
604 | if (new_ptr == MAP_FAILED) { | |
605 | perror("mremap"); | |
606 | success = 0; | |
607 | goto out; | |
608 | } | |
609 | ||
610 | /* Verify byte pattern after remapping */ | |
611 | srand(pattern_seed); | |
612 | for (i = 0; i < SIZE_MB(1); i++) { | |
613 | char c = (char) rand(); | |
614 | ||
615 | if (((char *)src)[i] != c) { | |
616 | ksft_print_msg("Data at src at %d got corrupted due to unrelated mremap\n", | |
617 | i); | |
618 | ksft_print_msg("Expected: %#x\t Got: %#x\n", c & 0xff, | |
619 | ((char *) src)[i] & 0xff); | |
620 | success = 0; | |
621 | } | |
622 | } | |
623 | ||
624 | out: | |
625 | if (src && munmap(src, c.region_size) == -1) | |
626 | perror("munmap src"); | |
627 | ||
628 | if (dest && munmap(dest, c.region_size) == -1) | |
629 | perror("munmap dest"); | |
630 | ||
631 | if (success) | |
632 | ksft_test_result_pass("%s\n", test_name); | |
633 | else | |
634 | ksft_test_result_fail("%s\n", test_name); | |
635 | } | |
636 | ||
7df66625 KS |
637 | static void run_mremap_test_case(struct test test_case, int *failures, |
638 | unsigned int threshold_mb, | |
c7876a0c | 639 | unsigned int pattern_seed, char *rand_addr) |
7df66625 KS |
640 | { |
641 | long long remap_time = remap_region(test_case.config, threshold_mb, | |
7033c6cc | 642 | rand_addr); |
7df66625 KS |
643 | |
644 | if (remap_time < 0) { | |
645 | if (test_case.expect_failure) | |
e5508fc5 | 646 | ksft_test_result_xfail("%s\n\tExpected mremap failure\n", |
7df66625 KS |
647 | test_case.name); |
648 | else { | |
649 | ksft_test_result_fail("%s\n", test_case.name); | |
650 | *failures += 1; | |
651 | } | |
652 | } else { | |
653 | /* | |
654 | * Comparing mremap time is only applicable if entire region | |
655 | * was faulted in. | |
656 | */ | |
657 | if (threshold_mb == VALIDATION_NO_THRESHOLD || | |
658 | test_case.config.region_size <= threshold_mb * _1MB) | |
659 | ksft_test_result_pass("%s\n\tmremap time: %12lldns\n", | |
660 | test_case.name, remap_time); | |
661 | else | |
662 | ksft_test_result_pass("%s\n", test_case.name); | |
663 | } | |
664 | } | |
665 | ||
666 | static void usage(const char *cmd) | |
667 | { | |
668 | fprintf(stderr, | |
669 | "Usage: %s [[-t <threshold_mb>] [-p <pattern_seed>]]\n" | |
670 | "-t\t only validate threshold_mb of the remapped region\n" | |
671 | " \t if 0 is supplied no threshold is used; all tests\n" | |
672 | " \t are run and remapped regions validated fully.\n" | |
673 | " \t The default threshold used is 4MB.\n" | |
674 | "-p\t provide a seed to generate the random pattern for\n" | |
675 | " \t validating the remapped region.\n", cmd); | |
676 | } | |
677 | ||
678 | static int parse_args(int argc, char **argv, unsigned int *threshold_mb, | |
679 | unsigned int *pattern_seed) | |
680 | { | |
681 | const char *optstr = "t:p:"; | |
682 | int opt; | |
683 | ||
684 | while ((opt = getopt(argc, argv, optstr)) != -1) { | |
685 | switch (opt) { | |
686 | case 't': | |
687 | *threshold_mb = atoi(optarg); | |
688 | break; | |
689 | case 'p': | |
690 | *pattern_seed = atoi(optarg); | |
691 | break; | |
692 | default: | |
693 | usage(argv[0]); | |
694 | return -1; | |
695 | } | |
696 | } | |
697 | ||
698 | if (optind < argc) { | |
699 | usage(argv[0]); | |
700 | return -1; | |
701 | } | |
702 | ||
703 | return 0; | |
704 | } | |
705 | ||
a4cb3b24 | 706 | #define MAX_TEST 15 |
f27a5c93 | 707 | #define MAX_PERF_TEST 3 |
7df66625 KS |
708 | int main(int argc, char **argv) |
709 | { | |
710 | int failures = 0; | |
711 | int i, run_perf_tests; | |
712 | unsigned int threshold_mb = VALIDATION_DEFAULT_THRESHOLD; | |
c7876a0c DJ |
713 | |
714 | /* hard-coded test configs */ | |
715 | size_t max_test_variable_region_size = _2GB; | |
716 | size_t max_test_constant_region_size = _2MB; | |
717 | size_t dest_preamble_size = 10 * _4MB; | |
718 | ||
7df66625 | 719 | unsigned int pattern_seed; |
c7876a0c DJ |
720 | char *rand_addr; |
721 | size_t rand_size; | |
da0618c1 | 722 | int num_expand_tests = 2; |
7b709f38 | 723 | int num_misc_tests = 2; |
a4cb3b24 | 724 | struct test test_cases[MAX_TEST] = {}; |
f27a5c93 AK |
725 | struct test perf_test_cases[MAX_PERF_TEST]; |
726 | int page_size; | |
7df66625 | 727 | time_t t; |
da0618c1 | 728 | FILE *maps_fp; |
7df66625 KS |
729 | |
730 | pattern_seed = (unsigned int) time(&t); | |
731 | ||
732 | if (parse_args(argc, argv, &threshold_mb, &pattern_seed) < 0) | |
733 | exit(EXIT_FAILURE); | |
734 | ||
735 | ksft_print_msg("Test configs:\n\tthreshold_mb=%u\n\tpattern_seed=%u\n\n", | |
736 | threshold_mb, pattern_seed); | |
737 | ||
c7876a0c DJ |
738 | /* |
739 | * set preallocated random array according to test configs; see the | |
740 | * functions for the logic of setting the size | |
741 | */ | |
742 | if (!threshold_mb) | |
743 | rand_size = MAX(max_test_variable_region_size, | |
744 | max_test_constant_region_size); | |
745 | else | |
746 | rand_size = MAX(MIN(threshold_mb * _1MB, | |
747 | max_test_variable_region_size), | |
748 | max_test_constant_region_size); | |
749 | rand_size = MAX(dest_preamble_size, rand_size); | |
750 | ||
751 | rand_addr = (char *)mmap(NULL, rand_size, PROT_READ | PROT_WRITE, | |
752 | MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); | |
753 | if (rand_addr == MAP_FAILED) { | |
754 | perror("mmap"); | |
755 | ksft_exit_fail_msg("cannot mmap rand_addr\n"); | |
756 | } | |
757 | ||
758 | /* fill stream of random bytes */ | |
759 | srand(pattern_seed); | |
760 | for (unsigned long i = 0; i < rand_size; ++i) | |
761 | rand_addr[i] = (char) rand(); | |
762 | ||
f27a5c93 AK |
763 | page_size = sysconf(_SC_PAGESIZE); |
764 | ||
765 | /* Expected mremap failures */ | |
766 | test_cases[0] = MAKE_TEST(page_size, page_size, page_size, | |
767 | OVERLAPPING, EXPECT_FAILURE, | |
768 | "mremap - Source and Destination Regions Overlapping"); | |
769 | ||
770 | test_cases[1] = MAKE_TEST(page_size, page_size/4, page_size, | |
771 | NON_OVERLAPPING, EXPECT_FAILURE, | |
772 | "mremap - Destination Address Misaligned (1KB-aligned)"); | |
773 | test_cases[2] = MAKE_TEST(page_size/4, page_size, page_size, | |
774 | NON_OVERLAPPING, EXPECT_FAILURE, | |
775 | "mremap - Source Address Misaligned (1KB-aligned)"); | |
776 | ||
777 | /* Src addr PTE aligned */ | |
778 | test_cases[3] = MAKE_TEST(PTE, PTE, PTE * 2, | |
779 | NON_OVERLAPPING, EXPECT_SUCCESS, | |
780 | "8KB mremap - Source PTE-aligned, Destination PTE-aligned"); | |
781 | ||
782 | /* Src addr 1MB aligned */ | |
783 | test_cases[4] = MAKE_TEST(_1MB, PTE, _2MB, NON_OVERLAPPING, EXPECT_SUCCESS, | |
784 | "2MB mremap - Source 1MB-aligned, Destination PTE-aligned"); | |
785 | test_cases[5] = MAKE_TEST(_1MB, _1MB, _2MB, NON_OVERLAPPING, EXPECT_SUCCESS, | |
786 | "2MB mremap - Source 1MB-aligned, Destination 1MB-aligned"); | |
787 | ||
788 | /* Src addr PMD aligned */ | |
789 | test_cases[6] = MAKE_TEST(PMD, PTE, _4MB, NON_OVERLAPPING, EXPECT_SUCCESS, | |
790 | "4MB mremap - Source PMD-aligned, Destination PTE-aligned"); | |
791 | test_cases[7] = MAKE_TEST(PMD, _1MB, _4MB, NON_OVERLAPPING, EXPECT_SUCCESS, | |
792 | "4MB mremap - Source PMD-aligned, Destination 1MB-aligned"); | |
793 | test_cases[8] = MAKE_TEST(PMD, PMD, _4MB, NON_OVERLAPPING, EXPECT_SUCCESS, | |
794 | "4MB mremap - Source PMD-aligned, Destination PMD-aligned"); | |
795 | ||
796 | /* Src addr PUD aligned */ | |
797 | test_cases[9] = MAKE_TEST(PUD, PTE, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS, | |
798 | "2GB mremap - Source PUD-aligned, Destination PTE-aligned"); | |
799 | test_cases[10] = MAKE_TEST(PUD, _1MB, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS, | |
800 | "2GB mremap - Source PUD-aligned, Destination 1MB-aligned"); | |
801 | test_cases[11] = MAKE_TEST(PUD, PMD, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS, | |
802 | "2GB mremap - Source PUD-aligned, Destination PMD-aligned"); | |
803 | test_cases[12] = MAKE_TEST(PUD, PUD, _2GB, NON_OVERLAPPING, EXPECT_SUCCESS, | |
804 | "2GB mremap - Source PUD-aligned, Destination PUD-aligned"); | |
805 | ||
8ed873d8 JFG |
806 | /* Src and Dest addr 1MB aligned. 5MB mremap. */ |
807 | test_cases[13] = MAKE_TEST(_1MB, _1MB, _5MB, NON_OVERLAPPING, EXPECT_SUCCESS, | |
808 | "5MB mremap - Source 1MB-aligned, Destination 1MB-aligned"); | |
809 | ||
a4cb3b24 JFG |
810 | /* Src and Dest addr 1MB aligned. 5MB mremap. */ |
811 | test_cases[14] = MAKE_TEST(_1MB, _1MB, _5MB, NON_OVERLAPPING, EXPECT_SUCCESS, | |
812 | "5MB mremap - Source 1MB-aligned, Dest 1MB-aligned with 40MB Preamble"); | |
813 | test_cases[14].config.dest_preamble_size = 10 * _4MB; | |
814 | ||
f27a5c93 AK |
815 | perf_test_cases[0] = MAKE_TEST(page_size, page_size, _1GB, NON_OVERLAPPING, EXPECT_SUCCESS, |
816 | "1GB mremap - Source PTE-aligned, Destination PTE-aligned"); | |
817 | /* | |
818 | * mremap 1GB region - Page table level aligned time | |
819 | * comparison. | |
820 | */ | |
821 | perf_test_cases[1] = MAKE_TEST(PMD, PMD, _1GB, NON_OVERLAPPING, EXPECT_SUCCESS, | |
822 | "1GB mremap - Source PMD-aligned, Destination PMD-aligned"); | |
823 | perf_test_cases[2] = MAKE_TEST(PUD, PUD, _1GB, NON_OVERLAPPING, EXPECT_SUCCESS, | |
824 | "1GB mremap - Source PUD-aligned, Destination PUD-aligned"); | |
7df66625 KS |
825 | |
826 | run_perf_tests = (threshold_mb == VALIDATION_NO_THRESHOLD) || | |
827 | (threshold_mb * _1MB >= _1GB); | |
828 | ||
829 | ksft_set_plan(ARRAY_SIZE(test_cases) + (run_perf_tests ? | |
85a22845 | 830 | ARRAY_SIZE(perf_test_cases) : 0) + num_expand_tests + num_misc_tests); |
7df66625 KS |
831 | |
832 | for (i = 0; i < ARRAY_SIZE(test_cases); i++) | |
833 | run_mremap_test_case(test_cases[i], &failures, threshold_mb, | |
c7876a0c | 834 | pattern_seed, rand_addr); |
7df66625 | 835 | |
da0618c1 LS |
836 | maps_fp = fopen("/proc/self/maps", "r"); |
837 | ||
838 | if (maps_fp == NULL) { | |
c7876a0c DJ |
839 | munmap(rand_addr, rand_size); |
840 | ksft_exit_fail_msg("Failed to read /proc/self/maps: %s\n", strerror(errno)); | |
da0618c1 LS |
841 | } |
842 | ||
843 | mremap_expand_merge(maps_fp, page_size); | |
844 | mremap_expand_merge_offset(maps_fp, page_size); | |
845 | ||
846 | fclose(maps_fp); | |
ca3d76b0 | 847 | |
c7876a0c DJ |
848 | mremap_move_within_range(pattern_seed, rand_addr); |
849 | mremap_move_1mb_from_start(pattern_seed, rand_addr); | |
85a22845 | 850 | |
7df66625 KS |
851 | if (run_perf_tests) { |
852 | ksft_print_msg("\n%s\n", | |
853 | "mremap HAVE_MOVE_PMD/PUD optimization time comparison for 1GB region:"); | |
854 | for (i = 0; i < ARRAY_SIZE(perf_test_cases); i++) | |
855 | run_mremap_test_case(perf_test_cases[i], &failures, | |
c7876a0c DJ |
856 | threshold_mb, pattern_seed, |
857 | rand_addr); | |
7df66625 KS |
858 | } |
859 | ||
c7876a0c DJ |
860 | munmap(rand_addr, rand_size); |
861 | ||
7df66625 KS |
862 | if (failures > 0) |
863 | ksft_exit_fail(); | |
864 | else | |
865 | ksft_exit_pass(); | |
866 | } |