Commit | Line | Data |
---|---|---|
fc44ef5a | 1 | // SPDX-License-Identifier: GPL-2.0-only |
192dc405 ED |
2 | /* |
3 | * Copyright 2018 Google Inc. | |
4 | * Author: Eric Dumazet (edumazet@google.com) | |
5 | * | |
6 | * Reference program demonstrating tcp mmap() usage, | |
7 | * and SO_RCVLOWAT hints for receiver. | |
8 | * | |
9 | * Note : NIC with header split is needed to use mmap() on TCP : | |
10 | * Each incoming frame must be a multiple of PAGE_SIZE bytes of TCP payload. | |
11 | * | |
12 | * How to use on loopback interface : | |
13 | * | |
14 | * ifconfig lo mtu 61512 # 15*4096 + 40 (ipv6 header) + 32 (TCP with TS option header) | |
15 | * tcp_mmap -s -z & | |
16 | * tcp_mmap -H ::1 -z | |
17 | * | |
18 | * Or leave default lo mtu, but use -M option to set TCP_MAXSEG option to (4096 + 12) | |
19 | * (4096 : page size on x86, 12: TCP TS option length) | |
20 | * tcp_mmap -s -z -M $((4096+12)) & | |
21 | * tcp_mmap -H ::1 -z -M $((4096+12)) | |
22 | * | |
23 | * Note: -z option on sender uses MSG_ZEROCOPY, which forces a copy when packets go through loopback interface. | |
24 | * We might use sendfile() instead, but really this test program is about mmap(), for receivers ;) | |
25 | * | |
26 | * $ ./tcp_mmap -s & # Without mmap() | |
27 | * $ for i in {1..4}; do ./tcp_mmap -H ::1 -z ; done | |
28 | * received 32768 MB (0 % mmap'ed) in 14.1157 s, 19.4732 Gbit | |
29 | * cpu usage user:0.057 sys:7.815, 240.234 usec per MB, 65531 c-switches | |
30 | * received 32768 MB (0 % mmap'ed) in 14.6833 s, 18.7204 Gbit | |
31 | * cpu usage user:0.043 sys:8.103, 248.596 usec per MB, 65524 c-switches | |
32 | * received 32768 MB (0 % mmap'ed) in 11.143 s, 24.6682 Gbit | |
33 | * cpu usage user:0.044 sys:6.576, 202.026 usec per MB, 65519 c-switches | |
34 | * received 32768 MB (0 % mmap'ed) in 14.9056 s, 18.4413 Gbit | |
35 | * cpu usage user:0.036 sys:8.193, 251.129 usec per MB, 65530 c-switches | |
36 | * $ kill %1 # kill tcp_mmap server | |
37 | * | |
38 | * $ ./tcp_mmap -s -z & # With mmap() | |
39 | * $ for i in {1..4}; do ./tcp_mmap -H ::1 -z ; done | |
40 | * received 32768 MB (99.9939 % mmap'ed) in 6.73792 s, 40.7956 Gbit | |
41 | * cpu usage user:0.045 sys:2.827, 87.6465 usec per MB, 65532 c-switches | |
42 | * received 32768 MB (99.9939 % mmap'ed) in 7.26732 s, 37.8238 Gbit | |
43 | * cpu usage user:0.037 sys:3.087, 95.3369 usec per MB, 65532 c-switches | |
44 | * received 32768 MB (99.9939 % mmap'ed) in 7.61661 s, 36.0893 Gbit | |
45 | * cpu usage user:0.046 sys:3.559, 110.016 usec per MB, 65529 c-switches | |
46 | * received 32768 MB (99.9939 % mmap'ed) in 7.43764 s, 36.9577 Gbit | |
47 | * cpu usage user:0.035 sys:3.467, 106.873 usec per MB, 65530 c-switches | |
192dc405 ED |
48 | */ |
49 | #define _GNU_SOURCE | |
50 | #include <pthread.h> | |
51 | #include <sys/types.h> | |
52 | #include <fcntl.h> | |
53 | #include <error.h> | |
54 | #include <sys/socket.h> | |
55 | #include <sys/mman.h> | |
56 | #include <sys/resource.h> | |
57 | #include <unistd.h> | |
58 | #include <string.h> | |
59 | #include <stdlib.h> | |
60 | #include <stdio.h> | |
61 | #include <errno.h> | |
62 | #include <time.h> | |
63 | #include <sys/time.h> | |
64 | #include <netinet/in.h> | |
192dc405 ED |
65 | #include <arpa/inet.h> |
66 | #include <poll.h> | |
aacb0c2e ED |
67 | #include <linux/tcp.h> |
68 | #include <assert.h> | |
5c5945dc | 69 | #include <openssl/pem.h> |
192dc405 ED |
70 | |
71 | #ifndef MSG_ZEROCOPY | |
72 | #define MSG_ZEROCOPY 0x4000000 | |
73 | #endif | |
74 | ||
5c5945dc XL |
75 | #ifndef min |
76 | #define min(a, b) ((a) < (b) ? (a) : (b)) | |
77 | #endif | |
78 | ||
e698a237 | 79 | #define FILE_SZ (1ULL << 35) |
192dc405 ED |
80 | static int cfg_family = AF_INET6; |
81 | static socklen_t cfg_alen = sizeof(struct sockaddr_in6); | |
82 | static int cfg_port = 8787; | |
83 | ||
84 | static int rcvbuf; /* Default: autotuning. Can be set with -r <integer> option */ | |
85 | static int sndbuf; /* Default: autotuning. Can be set with -w <integer> option */ | |
86 | static int zflg; /* zero copy option. (MSG_ZEROCOPY for sender, mmap() for receiver */ | |
87 | static int xflg; /* hash received data (simple xor) (-h option) */ | |
88 | static int keepflag; /* -k option: receiver shall keep all received file in memory (no munmap() calls) */ | |
5c5945dc | 89 | static int integrity; /* -i option: sender and receiver compute sha256 over the data.*/ |
192dc405 | 90 | |
597b01ed ED |
91 | static size_t chunk_size = 512*1024; |
92 | ||
93 | static size_t map_align; | |
192dc405 ED |
94 | |
95 | unsigned long htotal; | |
5c5945dc | 96 | unsigned int digest_len; |
192dc405 ED |
97 | |
98 | static inline void prefetch(const void *x) | |
99 | { | |
100 | #if defined(__x86_64__) | |
101 | asm volatile("prefetcht0 %P0" : : "m" (*(const char *)x)); | |
102 | #endif | |
103 | } | |
104 | ||
105 | void hash_zone(void *zone, unsigned int length) | |
106 | { | |
107 | unsigned long temp = htotal; | |
108 | ||
109 | while (length >= 8*sizeof(long)) { | |
110 | prefetch(zone + 384); | |
111 | temp ^= *(unsigned long *)zone; | |
112 | temp ^= *(unsigned long *)(zone + sizeof(long)); | |
113 | temp ^= *(unsigned long *)(zone + 2*sizeof(long)); | |
114 | temp ^= *(unsigned long *)(zone + 3*sizeof(long)); | |
115 | temp ^= *(unsigned long *)(zone + 4*sizeof(long)); | |
116 | temp ^= *(unsigned long *)(zone + 5*sizeof(long)); | |
117 | temp ^= *(unsigned long *)(zone + 6*sizeof(long)); | |
118 | temp ^= *(unsigned long *)(zone + 7*sizeof(long)); | |
119 | zone += 8*sizeof(long); | |
120 | length -= 8*sizeof(long); | |
121 | } | |
122 | while (length >= 1) { | |
123 | temp ^= *(unsigned char *)zone; | |
124 | zone += 1; | |
125 | length--; | |
126 | } | |
127 | htotal = temp; | |
128 | } | |
129 | ||
597b01ed ED |
130 | #define ALIGN_UP(x, align_to) (((x) + ((align_to)-1)) & ~((align_to)-1)) |
131 | #define ALIGN_PTR_UP(p, ptr_align_to) ((typeof(p))ALIGN_UP((unsigned long)(p), ptr_align_to)) | |
132 | ||
72653ae5 ED |
133 | |
134 | static void *mmap_large_buffer(size_t need, size_t *allocated) | |
135 | { | |
136 | void *buffer; | |
137 | size_t sz; | |
138 | ||
139 | /* Attempt to use huge pages if possible. */ | |
140 | sz = ALIGN_UP(need, map_align); | |
141 | buffer = mmap(NULL, sz, PROT_READ | PROT_WRITE, | |
142 | MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, 0); | |
143 | ||
144 | if (buffer == (void *)-1) { | |
145 | sz = need; | |
146 | buffer = mmap(NULL, sz, PROT_READ | PROT_WRITE, | |
057fb031 ED |
147 | MAP_PRIVATE | MAP_ANONYMOUS | MAP_POPULATE, |
148 | -1, 0); | |
72653ae5 ED |
149 | if (buffer != (void *)-1) |
150 | fprintf(stderr, "MAP_HUGETLB attempt failed, look at /sys/kernel/mm/hugepages for optimal performance\n"); | |
151 | } | |
152 | *allocated = sz; | |
153 | return buffer; | |
154 | } | |
155 | ||
bd60438e WB |
156 | static uint32_t tcp_info_get_rcv_mss(int fd) |
157 | { | |
158 | socklen_t sz = sizeof(struct tcp_info); | |
159 | struct tcp_info info; | |
160 | ||
161 | if (getsockopt(fd, IPPROTO_TCP, TCP_INFO, &info, &sz)) { | |
162 | fprintf(stderr, "Error fetching TCP_INFO\n"); | |
163 | return 0; | |
164 | } | |
165 | ||
166 | return info.tcpi_rcv_mss; | |
167 | } | |
168 | ||
192dc405 ED |
169 | void *child_thread(void *arg) |
170 | { | |
5c5945dc | 171 | unsigned char digest[SHA256_DIGEST_LENGTH]; |
192dc405 | 172 | unsigned long total_mmap = 0, total = 0; |
aacb0c2e | 173 | struct tcp_zerocopy_receive zc; |
5c5945dc | 174 | unsigned char *buffer = NULL; |
192dc405 | 175 | unsigned long delta_usec; |
5c5945dc | 176 | EVP_MD_CTX *ctx = NULL; |
192dc405 ED |
177 | int flags = MAP_SHARED; |
178 | struct timeval t0, t1; | |
597b01ed | 179 | void *raddr = NULL; |
aacb0c2e | 180 | void *addr = NULL; |
192dc405 ED |
181 | double throughput; |
182 | struct rusage ru; | |
59c0d319 | 183 | size_t buffer_sz; |
192dc405 ED |
184 | int lu, fd; |
185 | ||
186 | fd = (int)(unsigned long)arg; | |
187 | ||
188 | gettimeofday(&t0, NULL); | |
189 | ||
190 | fcntl(fd, F_SETFL, O_NDELAY); | |
59c0d319 ED |
191 | buffer = mmap_large_buffer(chunk_size, &buffer_sz); |
192 | if (buffer == (void *)-1) { | |
193 | perror("mmap"); | |
192dc405 ED |
194 | goto error; |
195 | } | |
aacb0c2e | 196 | if (zflg) { |
597b01ed ED |
197 | raddr = mmap(NULL, chunk_size + map_align, PROT_READ, flags, fd, 0); |
198 | if (raddr == (void *)-1) { | |
199 | perror("mmap"); | |
aacb0c2e | 200 | zflg = 0; |
597b01ed ED |
201 | } else { |
202 | addr = ALIGN_PTR_UP(raddr, map_align); | |
203 | } | |
aacb0c2e | 204 | } |
5c5945dc XL |
205 | if (integrity) { |
206 | ctx = EVP_MD_CTX_new(); | |
207 | if (!ctx) { | |
208 | perror("cannot enable SHA computing"); | |
209 | goto error; | |
210 | } | |
211 | EVP_DigestInit_ex(ctx, EVP_sha256(), NULL); | |
212 | } | |
192dc405 ED |
213 | while (1) { |
214 | struct pollfd pfd = { .fd = fd, .events = POLLIN, }; | |
215 | int sub; | |
216 | ||
217 | poll(&pfd, 1, 10000); | |
218 | if (zflg) { | |
aacb0c2e ED |
219 | socklen_t zc_len = sizeof(zc); |
220 | int res; | |
221 | ||
bf5525f3 | 222 | memset(&zc, 0, sizeof(zc)); |
e698a237 | 223 | zc.address = (__u64)((unsigned long)addr); |
905a9eb5 | 224 | zc.length = min(chunk_size, FILE_SZ - total); |
bf5525f3 | 225 | |
aacb0c2e ED |
226 | res = getsockopt(fd, IPPROTO_TCP, TCP_ZEROCOPY_RECEIVE, |
227 | &zc, &zc_len); | |
228 | if (res == -1) | |
192dc405 | 229 | break; |
aacb0c2e ED |
230 | |
231 | if (zc.length) { | |
232 | assert(zc.length <= chunk_size); | |
5c5945dc XL |
233 | if (integrity) |
234 | EVP_DigestUpdate(ctx, addr, zc.length); | |
aacb0c2e ED |
235 | total_mmap += zc.length; |
236 | if (xflg) | |
237 | hash_zone(addr, zc.length); | |
0d894193 ED |
238 | /* It is more efficient to unmap the pages right now, |
239 | * instead of doing this in next TCP_ZEROCOPY_RECEIVE. | |
240 | */ | |
241 | madvise(addr, zc.length, MADV_DONTNEED); | |
aacb0c2e | 242 | total += zc.length; |
192dc405 | 243 | } |
aacb0c2e ED |
244 | if (zc.recv_skip_hint) { |
245 | assert(zc.recv_skip_hint <= chunk_size); | |
5c5945dc XL |
246 | lu = read(fd, buffer, min(zc.recv_skip_hint, |
247 | FILE_SZ - total)); | |
aacb0c2e | 248 | if (lu > 0) { |
5c5945dc XL |
249 | if (integrity) |
250 | EVP_DigestUpdate(ctx, buffer, lu); | |
aacb0c2e ED |
251 | if (xflg) |
252 | hash_zone(buffer, lu); | |
253 | total += lu; | |
254 | } | |
5c5945dc XL |
255 | if (lu == 0) |
256 | goto end; | |
192dc405 ED |
257 | } |
258 | continue; | |
259 | } | |
192dc405 ED |
260 | sub = 0; |
261 | while (sub < chunk_size) { | |
5c5945dc XL |
262 | lu = read(fd, buffer + sub, min(chunk_size - sub, |
263 | FILE_SZ - total)); | |
192dc405 ED |
264 | if (lu == 0) |
265 | goto end; | |
266 | if (lu < 0) | |
267 | break; | |
5c5945dc XL |
268 | if (integrity) |
269 | EVP_DigestUpdate(ctx, buffer + sub, lu); | |
192dc405 ED |
270 | if (xflg) |
271 | hash_zone(buffer + sub, lu); | |
272 | total += lu; | |
273 | sub += lu; | |
274 | } | |
275 | } | |
276 | end: | |
277 | gettimeofday(&t1, NULL); | |
278 | delta_usec = (t1.tv_sec - t0.tv_sec) * 1000000 + t1.tv_usec - t0.tv_usec; | |
279 | ||
5c5945dc XL |
280 | if (integrity) { |
281 | fcntl(fd, F_SETFL, 0); | |
282 | EVP_DigestFinal_ex(ctx, digest, &digest_len); | |
283 | lu = read(fd, buffer, SHA256_DIGEST_LENGTH); | |
284 | if (lu != SHA256_DIGEST_LENGTH) | |
285 | perror("Error: Cannot read SHA256\n"); | |
286 | ||
287 | if (memcmp(digest, buffer, | |
288 | SHA256_DIGEST_LENGTH)) | |
289 | fprintf(stderr, "Error: SHA256 of the data is not right\n"); | |
290 | else | |
291 | printf("\nSHA256 is correct\n"); | |
292 | } | |
293 | ||
192dc405 ED |
294 | throughput = 0; |
295 | if (delta_usec) | |
296 | throughput = total * 8.0 / (double)delta_usec / 1000.0; | |
297 | getrusage(RUSAGE_THREAD, &ru); | |
298 | if (total > 1024*1024) { | |
299 | unsigned long total_usec; | |
300 | unsigned long mb = total >> 20; | |
301 | total_usec = 1000000*ru.ru_utime.tv_sec + ru.ru_utime.tv_usec + | |
302 | 1000000*ru.ru_stime.tv_sec + ru.ru_stime.tv_usec; | |
303 | printf("received %lg MB (%lg %% mmap'ed) in %lg s, %lg Gbit\n" | |
bd60438e | 304 | " cpu usage user:%lg sys:%lg, %lg usec per MB, %lu c-switches, rcv_mss %u\n", |
192dc405 ED |
305 | total / (1024.0 * 1024.0), |
306 | 100.0*total_mmap/total, | |
307 | (double)delta_usec / 1000000.0, | |
308 | throughput, | |
309 | (double)ru.ru_utime.tv_sec + (double)ru.ru_utime.tv_usec / 1000000.0, | |
310 | (double)ru.ru_stime.tv_sec + (double)ru.ru_stime.tv_usec / 1000000.0, | |
311 | (double)total_usec/mb, | |
bd60438e WB |
312 | ru.ru_nvcsw, |
313 | tcp_info_get_rcv_mss(fd)); | |
192dc405 ED |
314 | } |
315 | error: | |
59c0d319 | 316 | munmap(buffer, buffer_sz); |
192dc405 | 317 | close(fd); |
aacb0c2e | 318 | if (zflg) |
597b01ed | 319 | munmap(raddr, chunk_size + map_align); |
192dc405 ED |
320 | pthread_exit(0); |
321 | } | |
322 | ||
323 | static void apply_rcvsnd_buf(int fd) | |
324 | { | |
325 | if (rcvbuf && setsockopt(fd, SOL_SOCKET, | |
326 | SO_RCVBUF, &rcvbuf, sizeof(rcvbuf)) == -1) { | |
327 | perror("setsockopt SO_RCVBUF"); | |
328 | } | |
329 | ||
330 | if (sndbuf && setsockopt(fd, SOL_SOCKET, | |
331 | SO_SNDBUF, &sndbuf, sizeof(sndbuf)) == -1) { | |
332 | perror("setsockopt SO_SNDBUF"); | |
333 | } | |
334 | } | |
335 | ||
336 | ||
337 | static void setup_sockaddr(int domain, const char *str_addr, | |
338 | struct sockaddr_storage *sockaddr) | |
339 | { | |
340 | struct sockaddr_in6 *addr6 = (void *) sockaddr; | |
341 | struct sockaddr_in *addr4 = (void *) sockaddr; | |
342 | ||
343 | switch (domain) { | |
344 | case PF_INET: | |
345 | memset(addr4, 0, sizeof(*addr4)); | |
346 | addr4->sin_family = AF_INET; | |
347 | addr4->sin_port = htons(cfg_port); | |
348 | if (str_addr && | |
349 | inet_pton(AF_INET, str_addr, &(addr4->sin_addr)) != 1) | |
350 | error(1, 0, "ipv4 parse error: %s", str_addr); | |
351 | break; | |
352 | case PF_INET6: | |
353 | memset(addr6, 0, sizeof(*addr6)); | |
354 | addr6->sin6_family = AF_INET6; | |
355 | addr6->sin6_port = htons(cfg_port); | |
356 | if (str_addr && | |
357 | inet_pton(AF_INET6, str_addr, &(addr6->sin6_addr)) != 1) | |
358 | error(1, 0, "ipv6 parse error: %s", str_addr); | |
359 | break; | |
360 | default: | |
361 | error(1, 0, "illegal domain"); | |
362 | } | |
363 | } | |
364 | ||
365 | static void do_accept(int fdlisten) | |
366 | { | |
20021578 | 367 | pthread_attr_t attr; |
a8472417 | 368 | int rcvlowat; |
20021578 ED |
369 | |
370 | pthread_attr_init(&attr); | |
371 | pthread_attr_setdetachstate(&attr, PTHREAD_CREATE_DETACHED); | |
372 | ||
a8472417 | 373 | rcvlowat = chunk_size; |
192dc405 | 374 | if (setsockopt(fdlisten, SOL_SOCKET, SO_RCVLOWAT, |
a8472417 | 375 | &rcvlowat, sizeof(rcvlowat)) == -1) { |
192dc405 ED |
376 | perror("setsockopt SO_RCVLOWAT"); |
377 | } | |
378 | ||
379 | apply_rcvsnd_buf(fdlisten); | |
380 | ||
381 | while (1) { | |
382 | struct sockaddr_in addr; | |
383 | socklen_t addrlen = sizeof(addr); | |
384 | pthread_t th; | |
385 | int fd, res; | |
386 | ||
387 | fd = accept(fdlisten, (struct sockaddr *)&addr, &addrlen); | |
388 | if (fd == -1) { | |
389 | perror("accept"); | |
390 | continue; | |
391 | } | |
20021578 | 392 | res = pthread_create(&th, &attr, child_thread, |
192dc405 ED |
393 | (void *)(unsigned long)fd); |
394 | if (res) { | |
395 | errno = res; | |
396 | perror("pthread_create"); | |
397 | close(fd); | |
398 | } | |
399 | } | |
400 | } | |
401 | ||
597b01ed ED |
402 | /* Each thread should reserve a big enough vma to avoid |
403 | * spinlock collisions in ptl locks. | |
404 | * This size is 2MB on x86_64, and is exported in /proc/meminfo. | |
405 | */ | |
406 | static unsigned long default_huge_page_size(void) | |
407 | { | |
408 | FILE *f = fopen("/proc/meminfo", "r"); | |
409 | unsigned long hps = 0; | |
410 | size_t linelen = 0; | |
411 | char *line = NULL; | |
412 | ||
413 | if (!f) | |
414 | return 0; | |
415 | while (getline(&line, &linelen, f) > 0) { | |
416 | if (sscanf(line, "Hugepagesize: %lu kB", &hps) == 1) { | |
417 | hps <<= 10; | |
418 | break; | |
419 | } | |
420 | } | |
421 | free(line); | |
422 | fclose(f); | |
423 | return hps; | |
424 | } | |
425 | ||
5c5945dc XL |
426 | static void randomize(void *target, size_t count) |
427 | { | |
428 | static int urandom = -1; | |
429 | ssize_t got; | |
430 | ||
431 | urandom = open("/dev/urandom", O_RDONLY); | |
432 | if (urandom < 0) { | |
433 | perror("open /dev/urandom"); | |
434 | exit(1); | |
435 | } | |
436 | got = read(urandom, target, count); | |
437 | if (got != count) { | |
438 | perror("read /dev/urandom"); | |
439 | exit(1); | |
440 | } | |
441 | } | |
442 | ||
192dc405 ED |
443 | int main(int argc, char *argv[]) |
444 | { | |
5c5945dc | 445 | unsigned char digest[SHA256_DIGEST_LENGTH]; |
192dc405 ED |
446 | struct sockaddr_storage listenaddr, addr; |
447 | unsigned int max_pacing_rate = 0; | |
5c5945dc XL |
448 | EVP_MD_CTX *ctx = NULL; |
449 | unsigned char *buffer; | |
94b6c13b | 450 | uint64_t total = 0; |
192dc405 ED |
451 | char *host = NULL; |
452 | int fd, c, on = 1; | |
72653ae5 | 453 | size_t buffer_sz; |
192dc405 ED |
454 | int sflg = 0; |
455 | int mss = 0; | |
456 | ||
5c5945dc | 457 | while ((c = getopt(argc, argv, "46p:svr:w:H:zxkP:M:C:a:i")) != -1) { |
192dc405 ED |
458 | switch (c) { |
459 | case '4': | |
460 | cfg_family = PF_INET; | |
461 | cfg_alen = sizeof(struct sockaddr_in); | |
462 | break; | |
463 | case '6': | |
464 | cfg_family = PF_INET6; | |
465 | cfg_alen = sizeof(struct sockaddr_in6); | |
466 | break; | |
467 | case 'p': | |
468 | cfg_port = atoi(optarg); | |
469 | break; | |
470 | case 'H': | |
471 | host = optarg; | |
472 | break; | |
473 | case 's': /* server : listen for incoming connections */ | |
474 | sflg++; | |
475 | break; | |
476 | case 'r': | |
477 | rcvbuf = atoi(optarg); | |
478 | break; | |
479 | case 'w': | |
480 | sndbuf = atoi(optarg); | |
481 | break; | |
482 | case 'z': | |
483 | zflg = 1; | |
484 | break; | |
485 | case 'M': | |
486 | mss = atoi(optarg); | |
487 | break; | |
488 | case 'x': | |
489 | xflg = 1; | |
490 | break; | |
491 | case 'k': | |
492 | keepflag = 1; | |
493 | break; | |
494 | case 'P': | |
495 | max_pacing_rate = atoi(optarg) ; | |
496 | break; | |
597b01ed ED |
497 | case 'C': |
498 | chunk_size = atol(optarg); | |
499 | break; | |
500 | case 'a': | |
501 | map_align = atol(optarg); | |
502 | break; | |
5c5945dc XL |
503 | case 'i': |
504 | integrity = 1; | |
505 | break; | |
192dc405 ED |
506 | default: |
507 | exit(1); | |
508 | } | |
509 | } | |
597b01ed ED |
510 | if (!map_align) { |
511 | map_align = default_huge_page_size(); | |
512 | /* if really /proc/meminfo is not helping, | |
513 | * we use the default x86_64 hugepagesize. | |
514 | */ | |
515 | if (!map_align) | |
516 | map_align = 2*1024*1024; | |
517 | } | |
192dc405 ED |
518 | if (sflg) { |
519 | int fdlisten = socket(cfg_family, SOCK_STREAM, 0); | |
520 | ||
521 | if (fdlisten == -1) { | |
522 | perror("socket"); | |
523 | exit(1); | |
524 | } | |
525 | apply_rcvsnd_buf(fdlisten); | |
526 | setsockopt(fdlisten, SOL_SOCKET, SO_REUSEADDR, &on, sizeof(on)); | |
527 | ||
528 | setup_sockaddr(cfg_family, host, &listenaddr); | |
529 | ||
530 | if (mss && | |
aacb0c2e ED |
531 | setsockopt(fdlisten, IPPROTO_TCP, TCP_MAXSEG, |
532 | &mss, sizeof(mss)) == -1) { | |
192dc405 ED |
533 | perror("setsockopt TCP_MAXSEG"); |
534 | exit(1); | |
535 | } | |
536 | if (bind(fdlisten, (const struct sockaddr *)&listenaddr, cfg_alen) == -1) { | |
537 | perror("bind"); | |
538 | exit(1); | |
539 | } | |
540 | if (listen(fdlisten, 128) == -1) { | |
541 | perror("listen"); | |
542 | exit(1); | |
543 | } | |
544 | do_accept(fdlisten); | |
545 | } | |
72653ae5 ED |
546 | |
547 | buffer = mmap_large_buffer(chunk_size, &buffer_sz); | |
5c5945dc | 548 | if (buffer == (unsigned char *)-1) { |
192dc405 ED |
549 | perror("mmap"); |
550 | exit(1); | |
551 | } | |
552 | ||
258fe208 | 553 | fd = socket(cfg_family, SOCK_STREAM, 0); |
192dc405 ED |
554 | if (fd == -1) { |
555 | perror("socket"); | |
556 | exit(1); | |
557 | } | |
558 | apply_rcvsnd_buf(fd); | |
559 | ||
560 | setup_sockaddr(cfg_family, host, &addr); | |
561 | ||
562 | if (mss && | |
aacb0c2e | 563 | setsockopt(fd, IPPROTO_TCP, TCP_MAXSEG, &mss, sizeof(mss)) == -1) { |
192dc405 ED |
564 | perror("setsockopt TCP_MAXSEG"); |
565 | exit(1); | |
566 | } | |
567 | if (connect(fd, (const struct sockaddr *)&addr, cfg_alen) == -1) { | |
568 | perror("connect"); | |
569 | exit(1); | |
570 | } | |
571 | if (max_pacing_rate && | |
572 | setsockopt(fd, SOL_SOCKET, SO_MAX_PACING_RATE, | |
573 | &max_pacing_rate, sizeof(max_pacing_rate)) == -1) | |
574 | perror("setsockopt SO_MAX_PACING_RATE"); | |
575 | ||
576 | if (zflg && setsockopt(fd, SOL_SOCKET, SO_ZEROCOPY, | |
577 | &on, sizeof(on)) == -1) { | |
578 | perror("setsockopt SO_ZEROCOPY, (-z option disabled)"); | |
579 | zflg = 0; | |
580 | } | |
5c5945dc XL |
581 | if (integrity) { |
582 | randomize(buffer, buffer_sz); | |
583 | ctx = EVP_MD_CTX_new(); | |
584 | if (!ctx) { | |
585 | perror("cannot enable SHA computing"); | |
586 | exit(1); | |
587 | } | |
588 | EVP_DigestInit_ex(ctx, EVP_sha256(), NULL); | |
589 | } | |
192dc405 | 590 | while (total < FILE_SZ) { |
5c5945dc | 591 | size_t offset = total % chunk_size; |
94b6c13b | 592 | int64_t wr = FILE_SZ - total; |
192dc405 | 593 | |
5c5945dc XL |
594 | if (wr > chunk_size - offset) |
595 | wr = chunk_size - offset; | |
596 | /* Note : we just want to fill the pipe with random bytes */ | |
597 | wr = send(fd, buffer + offset, | |
598 | (size_t)wr, zflg ? MSG_ZEROCOPY : 0); | |
192dc405 ED |
599 | if (wr <= 0) |
600 | break; | |
5c5945dc XL |
601 | if (integrity) |
602 | EVP_DigestUpdate(ctx, buffer + offset, wr); | |
192dc405 ED |
603 | total += wr; |
604 | } | |
5c5945dc XL |
605 | if (integrity && total == FILE_SZ) { |
606 | EVP_DigestFinal_ex(ctx, digest, &digest_len); | |
607 | send(fd, digest, (size_t)SHA256_DIGEST_LENGTH, 0); | |
608 | } | |
192dc405 | 609 | close(fd); |
72653ae5 | 610 | munmap(buffer, buffer_sz); |
192dc405 ED |
611 | return 0; |
612 | } |