projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
selftests/bpf: test_sockmap, fix data verification
[linux-2.6-block.git] / tools / testing / selftests / bpf / test_sockmap.c
CommitLineData
16962b24
JF		 1// SPDX-License-Identifier: GPL-2.0
2// Copyright (c) 2017-2018 Covalent IO, Inc. http://covalent.io
3#include <stdio.h>
4#include <stdlib.h>
5#include <sys/socket.h>
6#include <sys/ioctl.h>
7#include <sys/select.h>
8#include <netinet/in.h>
9#include <arpa/inet.h>
10#include <unistd.h>
11#include <string.h>
12#include <errno.h>
13#include <sys/ioctl.h>
14#include <stdbool.h>
15#include <signal.h>
16#include <fcntl.h>
17#include <sys/wait.h>
18#include <time.h>
19#include <sched.h>
20
21#include <sys/time.h>
22#include <sys/resource.h>
23#include <sys/types.h>
24#include <sys/sendfile.h>
25
26#include <linux/netlink.h>
27#include <linux/socket.h>
28#include <linux/sock_diag.h>
29#include <linux/bpf.h>
30#include <linux/if_link.h>
31#include <assert.h>
32#include <libgen.h>
33
34#include <getopt.h>
35
36#include <bpf/bpf.h>
37#include <bpf/libbpf.h>
38
39#include "bpf_util.h"
40#include "bpf_rlimit.h"
41#include "cgroup_helpers.h"
42
43int running;
44static void running_handler(int a);
45
46/* randomly selected ports for testing on lo */
47#define S1_PORT 10000
48#define S2_PORT 10001
49
b8b394fa
JF		 50#define BPF_SOCKMAP_FILENAME "test_sockmap_kern.o"
51#define BPF_SOCKHASH_FILENAME "test_sockhash_kern.o"
16962b24
JF		 52#define CG_PATH "/sockmap"
53
54/* global sockets */
55int s1, s2, c1, c2, p1, p2;
56int test_cnt;
57int passed;
58int failed;
59int map_fd[8];
60struct bpf_map *maps[8];
61int prog_fd[11];
62
63int txmsg_pass;
64int txmsg_noisy;
65int txmsg_redir;
66int txmsg_redir_noisy;
67int txmsg_drop;
68int txmsg_apply;
69int txmsg_cork;
70int txmsg_start;
71int txmsg_end;
72int txmsg_ingress;
73int txmsg_skb;
74
75static const struct option long_options[] = {
76 {"help", no_argument, NULL, 'h' },
77 {"cgroup", required_argument, NULL, 'c' },
78 {"rate", required_argument, NULL, 'r' },
79 {"verbose", no_argument, NULL, 'v' },
80 {"iov_count", required_argument, NULL, 'i' },
81 {"length", required_argument, NULL, 'l' },
82 {"test", required_argument, NULL, 't' },
83 {"data_test", no_argument, NULL, 'd' },
84 {"txmsg", no_argument, &txmsg_pass, 1 },
85 {"txmsg_noisy", no_argument, &txmsg_noisy, 1 },
86 {"txmsg_redir", no_argument, &txmsg_redir, 1 },
87 {"txmsg_redir_noisy", no_argument, &txmsg_redir_noisy, 1},
88 {"txmsg_drop", no_argument, &txmsg_drop, 1 },
89 {"txmsg_apply", required_argument, NULL, 'a'},
90 {"txmsg_cork", required_argument, NULL, 'k'},
91 {"txmsg_start", required_argument, NULL, 's'},
92 {"txmsg_end", required_argument, NULL, 'e'},
93 {"txmsg_ingress", no_argument, &txmsg_ingress, 1 },
94 {"txmsg_skb", no_argument, &txmsg_skb, 1 },
95 {0, 0, NULL, 0 }
96};
97
98static void usage(char *argv[])
99{
100 int i;
101
102 printf(" Usage: %s --cgroup <cgroup_path>\n", argv[0]);
103 printf(" options:\n");
104 for (i = 0; long_options[i].name != 0; i++) {
105 printf(" --%-12s", long_options[i].name);
106 if (long_options[i].flag != NULL)
107 printf(" flag (internal value:%d)\n",
108 *long_options[i].flag);
109 else
110 printf(" -%c\n", long_options[i].val);
111 }
112 printf("\n");
113}
114
115static int sockmap_init_sockets(int verbose)
116{
117 int i, err, one = 1;
118 struct sockaddr_in addr;
119 int *fds[4] = {&s1, &s2, &c1, &c2};
120
121 s1 = s2 = p1 = p2 = c1 = c2 = 0;
122
123 /* Init sockets */
124 for (i = 0; i < 4; i++) {
125 *fds[i] = socket(AF_INET, SOCK_STREAM, 0);
126 if (*fds[i] < 0) {
127 perror("socket s1 failed()");
128 return errno;
129 }
130 }
131
132 /* Allow reuse */
133 for (i = 0; i < 2; i++) {
134 err = setsockopt(*fds[i], SOL_SOCKET, SO_REUSEADDR,
135 (char *)&one, sizeof(one));
136 if (err) {
137 perror("setsockopt failed()");
138 return errno;
139 }
140 }
141
142 /* Non-blocking sockets */
143 for (i = 0; i < 2; i++) {
144 err = ioctl(*fds[i], FIONBIO, (char *)&one);
145 if (err < 0) {
146 perror("ioctl s1 failed()");
147 return errno;
148 }
149 }
150
151 /* Bind server sockets */
152 memset(&addr, 0, sizeof(struct sockaddr_in));
153 addr.sin_family = AF_INET;
154 addr.sin_addr.s_addr = inet_addr("127.0.0.1");
155
156 addr.sin_port = htons(S1_PORT);
157 err = bind(s1, (struct sockaddr *)&addr, sizeof(addr));
158 if (err < 0) {
159 perror("bind s1 failed()\n");
160 return errno;
161 }
162
163 addr.sin_port = htons(S2_PORT);
164 err = bind(s2, (struct sockaddr *)&addr, sizeof(addr));
165 if (err < 0) {
166 perror("bind s2 failed()\n");
167 return errno;
168 }
169
170 /* Listen server sockets */
171 addr.sin_port = htons(S1_PORT);
172 err = listen(s1, 32);
173 if (err < 0) {
174 perror("listen s1 failed()\n");
175 return errno;
176 }
177
178 addr.sin_port = htons(S2_PORT);
179 err = listen(s2, 32);
180 if (err < 0) {
181 perror("listen s1 failed()\n");
182 return errno;
183 }
184
185 /* Initiate Connect */
186 addr.sin_port = htons(S1_PORT);
187 err = connect(c1, (struct sockaddr *)&addr, sizeof(addr));
188 if (err < 0 && errno != EINPROGRESS) {
189 perror("connect c1 failed()\n");
190 return errno;
191 }
192
193 addr.sin_port = htons(S2_PORT);
194 err = connect(c2, (struct sockaddr *)&addr, sizeof(addr));
195 if (err < 0 && errno != EINPROGRESS) {
196 perror("connect c2 failed()\n");
197 return errno;
198 } else if (err < 0) {
199 err = 0;
200 }
201
202 /* Accept Connecrtions */
203 p1 = accept(s1, NULL, NULL);
204 if (p1 < 0) {
205 perror("accept s1 failed()\n");
206 return errno;
207 }
208
209 p2 = accept(s2, NULL, NULL);
210 if (p2 < 0) {
211 perror("accept s1 failed()\n");
212 return errno;
213 }
214
215 if (verbose) {
216 printf("connected sockets: c1 <-> p1, c2 <-> p2\n");
217 printf("cgroups binding: c1(%i) <-> s1(%i) - - - c2(%i) <-> s2(%i)\n",
218 c1, s1, c2, s2);
219 }
220 return 0;
221}
222
223struct msg_stats {
224 size_t bytes_sent;
225 size_t bytes_recvd;
226 struct timespec start;
227 struct timespec end;
228};
229
230struct sockmap_options {
231 int verbose;
232 bool base;
233 bool sendpage;
234 bool data_test;
235 bool drop_expected;
236 int iov_count;
237 int iov_length;
238 int rate;
239};
240
241static int msg_loop_sendpage(int fd, int iov_length, int cnt,
242 struct msg_stats *s,
243 struct sockmap_options *opt)
244{
245 bool drop = opt->drop_expected;
246 unsigned char k = 0;
247 FILE *file;
248 int i, fp;
249
250 file = fopen(".sendpage_tst.tmp", "w+");
251 for (i = 0; i < iov_length * cnt; i++, k++)
252 fwrite(&k, sizeof(char), 1, file);
253 fflush(file);
254 fseek(file, 0, SEEK_SET);
255 fclose(file);
256
257 fp = open(".sendpage_tst.tmp", O_RDONLY);
258 clock_gettime(CLOCK_MONOTONIC, &s->start);
259 for (i = 0; i < cnt; i++) {
260 int sent = sendfile(fd, fp, NULL, iov_length);
261
262 if (!drop && sent < 0) {
263 perror("send loop error:");
264 close(fp);
265 return sent;
266 } else if (drop && sent >= 0) {
267 printf("sendpage loop error expected: %i\n", sent);
268 close(fp);
269 return -EIO;
270 }
271
272 if (sent > 0)
273 s->bytes_sent += sent;
274 }
275 clock_gettime(CLOCK_MONOTONIC, &s->end);
276 close(fp);
277 return 0;
278}
279
280static int msg_loop(int fd, int iov_count, int iov_length, int cnt,
281 struct msg_stats *s, bool tx,
282 struct sockmap_options *opt)
283{
284 struct msghdr msg = {0};
285 int err, i, flags = MSG_NOSIGNAL;
286 struct iovec *iov;
287 unsigned char k;
288 bool data_test = opt->data_test;
289 bool drop = opt->drop_expected;
290
291 iov = calloc(iov_count, sizeof(struct iovec));
292 if (!iov)
293 return errno;
294
295 k = 0;
296 for (i = 0; i < iov_count; i++) {
297 unsigned char *d = calloc(iov_length, sizeof(char));
298
299 if (!d) {
300 fprintf(stderr, "iov_count %i/%i OOM\n", i, iov_count);
301 goto out_errno;
302 }
303 iov[i].iov_base = d;
304 iov[i].iov_len = iov_length;
305
306 if (data_test && tx) {
307 int j;
308
309 for (j = 0; j < iov_length; j++)
310 d[j] = k++;
311 }
312 }
313
314 msg.msg_iov = iov;
315 msg.msg_iovlen = iov_count;
316 k = 0;
317
318 if (tx) {
319 clock_gettime(CLOCK_MONOTONIC, &s->start);
320 for (i = 0; i < cnt; i++) {
321 int sent = sendmsg(fd, &msg, flags);
322
323 if (!drop && sent < 0) {
324 perror("send loop error:");
325 goto out_errno;
326 } else if (drop && sent >= 0) {
327 printf("send loop error expected: %i\n", sent);
328 errno = -EIO;
329 goto out_errno;
330 }
331 if (sent > 0)
332 s->bytes_sent += sent;
333 }
334 clock_gettime(CLOCK_MONOTONIC, &s->end);
335 } else {
336 int slct, recv, max_fd = fd;
337 int fd_flags = O_NONBLOCK;
338 struct timeval timeout;
339 float total_bytes;
d825e12f
PB		 340 int bytes_cnt = 0;
341 int chunk_sz;
16962b24
JF		 342 fd_set w;
343
d825e12f
PB		 344 if (opt->sendpage)
345 chunk_sz = iov_length * cnt;
346 else
347 chunk_sz = iov_length * iov_count;
348
16962b24
JF		 349 fcntl(fd, fd_flags);
350 total_bytes = (float)iov_count * (float)iov_length * (float)cnt;
351 err = clock_gettime(CLOCK_MONOTONIC, &s->start);
352 if (err < 0)
353 perror("recv start time: ");
354 while (s->bytes_recvd < total_bytes) {
a009f1f3
PB		 355 if (txmsg_cork) {
356 timeout.tv_sec = 0;
357 timeout.tv_usec = 1000;
358 } else {
359 timeout.tv_sec = 1;
360 timeout.tv_usec = 0;
361 }
16962b24
JF		 362
363 /* FD sets */
364 FD_ZERO(&w);
365 FD_SET(fd, &w);
366
367 slct = select(max_fd + 1, &w, NULL, NULL, &timeout);
368 if (slct == -1) {
369 perror("select()");
370 clock_gettime(CLOCK_MONOTONIC, &s->end);
371 goto out_errno;
372 } else if (!slct) {
373 if (opt->verbose)
374 fprintf(stderr, "unexpected timeout\n");
375 errno = -EIO;
376 clock_gettime(CLOCK_MONOTONIC, &s->end);
377 goto out_errno;
378 }
379
380 recv = recvmsg(fd, &msg, flags);
381 if (recv < 0) {
382 if (errno != EWOULDBLOCK) {
383 clock_gettime(CLOCK_MONOTONIC, &s->end);
384 perror("recv failed()\n");
385 goto out_errno;
386 }
387 }
388
389 s->bytes_recvd += recv;
390
391 if (data_test) {
392 int j;
393
394 for (i = 0; i < msg.msg_iovlen; i++) {
395 unsigned char *d = iov[i].iov_base;
396
397 for (j = 0;
398 j < iov[i].iov_len && recv; j++) {
399 if (d[j] != k++) {
400 errno = -EIO;
401 fprintf(stderr,
402 "detected data corruption @iov[%i]:%i %02x != %02x, %02x ?= %02x\n",
d825e12f 403 i, j, d[j], k - 1, d[j+1], k);
16962b24
JF		 404 goto out_errno;
405 }
d825e12f
PB		 406 bytes_cnt++;
407 if (bytes_cnt == chunk_sz) {
408 k = 0;
409 bytes_cnt = 0;
410 }
16962b24
JF		 411 recv--;
412 }
413 }
414 }
415 }
416 clock_gettime(CLOCK_MONOTONIC, &s->end);
417 }
418
419 for (i = 0; i < iov_count; i++)
420 free(iov[i].iov_base);
421 free(iov);
422 return 0;
423out_errno:
424 for (i = 0; i < iov_count; i++)
425 free(iov[i].iov_base);
426 free(iov);
427 return errno;
428}
429
430static float giga = 1000000000;
431
432static inline float sentBps(struct msg_stats s)
433{
434 return s.bytes_sent / (s.end.tv_sec - s.start.tv_sec);
435}
436
437static inline float recvdBps(struct msg_stats s)
438{
439 return s.bytes_recvd / (s.end.tv_sec - s.start.tv_sec);
440}
441
442static int sendmsg_test(struct sockmap_options *opt)
443{
444 float sent_Bps = 0, recvd_Bps = 0;
445 int rx_fd, txpid, rxpid, err = 0;
446 struct msg_stats s = {0};
447 int iov_count = opt->iov_count;
448 int iov_buf = opt->iov_length;
16edddfe 449 int rx_status, tx_status;
16962b24 450 int cnt = opt->rate;
16962b24
JF		 451
452 errno = 0;
453
454 if (opt->base)
455 rx_fd = p1;
456 else
457 rx_fd = p2;
458
459 rxpid = fork();
460 if (rxpid == 0) {
461 if (opt->drop_expected)
16edddfe 462 exit(0);
16962b24
JF		 463
464 if (opt->sendpage)
465 iov_count = 1;
466 err = msg_loop(rx_fd, iov_count, iov_buf,
467 cnt, &s, false, opt);
468 if (err && opt->verbose)
469 fprintf(stderr,
470 "msg_loop_rx: iov_count %i iov_buf %i cnt %i err %i\n",
471 iov_count, iov_buf, cnt, err);
472 shutdown(p2, SHUT_RDWR);
473 shutdown(p1, SHUT_RDWR);
474 if (s.end.tv_sec - s.start.tv_sec) {
475 sent_Bps = sentBps(s);
476 recvd_Bps = recvdBps(s);
477 }
478 if (opt->verbose)
479 fprintf(stdout,
480 "rx_sendmsg: TX: %zuB %fB/s %fGB/s RX: %zuB %fB/s %fGB/s\n",
481 s.bytes_sent, sent_Bps, sent_Bps/giga,
482 s.bytes_recvd, recvd_Bps, recvd_Bps/giga);
16edddfe
PB		 483 if (err && txmsg_cork)
484 err = 0;
485 exit(err ? 1 : 0);
16962b24
JF		 486 } else if (rxpid == -1) {
487 perror("msg_loop_rx: ");
488 return errno;
489 }
490
491 txpid = fork();
492 if (txpid == 0) {
493 if (opt->sendpage)
494 err = msg_loop_sendpage(c1, iov_buf, cnt, &s, opt);
495 else
496 err = msg_loop(c1, iov_count, iov_buf,
497 cnt, &s, true, opt);
498
499 if (err)
500 fprintf(stderr,
501 "msg_loop_tx: iov_count %i iov_buf %i cnt %i err %i\n",
502 iov_count, iov_buf, cnt, err);
503 shutdown(c1, SHUT_RDWR);
504 if (s.end.tv_sec - s.start.tv_sec) {
505 sent_Bps = sentBps(s);
506 recvd_Bps = recvdBps(s);
507 }
508 if (opt->verbose)
509 fprintf(stdout,
510 "tx_sendmsg: TX: %zuB %fB/s %f GB/s RX: %zuB %fB/s %fGB/s\n",
511 s.bytes_sent, sent_Bps, sent_Bps/giga,
512 s.bytes_recvd, recvd_Bps, recvd_Bps/giga);
16edddfe 513 exit(err ? 1 : 0);
16962b24
JF		 514 } else if (txpid == -1) {
515 perror("msg_loop_tx: ");
516 return errno;
517 }
518
16edddfe
PB		 519 assert(waitpid(rxpid, &rx_status, 0) == rxpid);
520 assert(waitpid(txpid, &tx_status, 0) == txpid);
521 if (WIFEXITED(rx_status)) {
522 err = WEXITSTATUS(rx_status);
523 if (err) {
524 fprintf(stderr, "rx thread exited with err %d. ", err);
525 goto out;
526 }
527 }
528 if (WIFEXITED(tx_status)) {
529 err = WEXITSTATUS(tx_status);
530 if (err)
531 fprintf(stderr, "tx thread exited with err %d. ", err);
532 }
533out:
16962b24
JF		 534 return err;
535}
536
537static int forever_ping_pong(int rate, struct sockmap_options *opt)
538{
539 struct timeval timeout;
540 char buf[1024] = {0};
541 int sc;
542
543 timeout.tv_sec = 10;
544 timeout.tv_usec = 0;
545
546 /* Ping/Pong data from client to server */
547 sc = send(c1, buf, sizeof(buf), 0);
548 if (sc < 0) {
549 perror("send failed()\n");
550 return sc;
551 }
552
553 do {
554 int s, rc, i, max_fd = p2;
555 fd_set w;
556
557 /* FD sets */
558 FD_ZERO(&w);
559 FD_SET(c1, &w);
560 FD_SET(c2, &w);
561 FD_SET(p1, &w);
562 FD_SET(p2, &w);
563
564 s = select(max_fd + 1, &w, NULL, NULL, &timeout);
565 if (s == -1) {
566 perror("select()");
567 break;
568 } else if (!s) {
569 fprintf(stderr, "unexpected timeout\n");
570 break;
571 }
572
573 for (i = 0; i <= max_fd && s > 0; ++i) {
574 if (!FD_ISSET(i, &w))
575 continue;
576
577 s--;
578
579 rc = recv(i, buf, sizeof(buf), 0);
580 if (rc < 0) {
581 if (errno != EWOULDBLOCK) {
582 perror("recv failed()\n");
583 return rc;
584 }
585 }
586
587 if (rc == 0) {
588 close(i);
589 break;
590 }
591
592 sc = send(i, buf, rc, 0);
593 if (sc < 0) {
594 perror("send failed()\n");
595 return sc;
596 }
597 }
598
599 if (rate)
600 sleep(rate);
601
602 if (opt->verbose) {
603 printf(".");
604 fflush(stdout);
605
606 }
607 } while (running);
608
609 return 0;
610}
611
612enum {
613 PING_PONG,
614 SENDMSG,
615 BASE,
616 BASE_SENDPAGE,
617 SENDPAGE,
618};
619
620static int run_options(struct sockmap_options *options, int cg_fd, int test)
621{
622 int i, key, next_key, err, tx_prog_fd = -1, zero = 0;
623
624 /* If base test skip BPF setup */
625 if (test == BASE || test == BASE_SENDPAGE)
626 goto run;
627
628 /* Attach programs to sockmap */
629 err = bpf_prog_attach(prog_fd[0], map_fd[0],
630 BPF_SK_SKB_STREAM_PARSER, 0);
631 if (err) {
632 fprintf(stderr,
633 "ERROR: bpf_prog_attach (sockmap %i->%i): %d (%s)\n",
634 prog_fd[0], map_fd[0], err, strerror(errno));
635 return err;
636 }
637
638 err = bpf_prog_attach(prog_fd[1], map_fd[0],
639 BPF_SK_SKB_STREAM_VERDICT, 0);
640 if (err) {
641 fprintf(stderr, "ERROR: bpf_prog_attach (sockmap): %d (%s)\n",
642 err, strerror(errno));
643 return err;
644 }
645
646 /* Attach to cgroups */
647 err = bpf_prog_attach(prog_fd[2], cg_fd, BPF_CGROUP_SOCK_OPS, 0);
648 if (err) {
649 fprintf(stderr, "ERROR: bpf_prog_attach (groups): %d (%s)\n",
650 err, strerror(errno));
651 return err;
652 }
653
654run:
655 err = sockmap_init_sockets(options->verbose);
656 if (err) {
657 fprintf(stderr, "ERROR: test socket failed: %d\n", err);
658 goto out;
659 }
660
661 /* Attach txmsg program to sockmap */
662 if (txmsg_pass)
663 tx_prog_fd = prog_fd[3];
664 else if (txmsg_noisy)
665 tx_prog_fd = prog_fd[4];
666 else if (txmsg_redir)
667 tx_prog_fd = prog_fd[5];
668 else if (txmsg_redir_noisy)
669 tx_prog_fd = prog_fd[6];
670 else if (txmsg_drop)
671 tx_prog_fd = prog_fd[9];
672 /* apply and cork must be last */
673 else if (txmsg_apply)
674 tx_prog_fd = prog_fd[7];
675 else if (txmsg_cork)
676 tx_prog_fd = prog_fd[8];
677 else
678 tx_prog_fd = 0;
679
680 if (tx_prog_fd) {
681 int redir_fd, i = 0;
682
683 err = bpf_prog_attach(tx_prog_fd,
684 map_fd[1], BPF_SK_MSG_VERDICT, 0);
685 if (err) {
686 fprintf(stderr,
687 "ERROR: bpf_prog_attach (txmsg): %d (%s)\n",
688 err, strerror(errno));
689 goto out;
690 }
691
692 err = bpf_map_update_elem(map_fd[1], &i, &c1, BPF_ANY);
693 if (err) {
694 fprintf(stderr,
695 "ERROR: bpf_map_update_elem (txmsg): %d (%s\n",
696 err, strerror(errno));
697 goto out;
698 }
699
700 if (txmsg_redir || txmsg_redir_noisy)
701 redir_fd = c2;
702 else
703 redir_fd = c1;
704
705 err = bpf_map_update_elem(map_fd[2], &i, &redir_fd, BPF_ANY);
706 if (err) {
707 fprintf(stderr,
708 "ERROR: bpf_map_update_elem (txmsg): %d (%s\n",
709 err, strerror(errno));
710 goto out;
711 }
712
713 if (txmsg_apply) {
714 err = bpf_map_update_elem(map_fd[3],
715 &i, &txmsg_apply, BPF_ANY);
716 if (err) {
717 fprintf(stderr,
718 "ERROR: bpf_map_update_elem (apply_bytes): %d (%s\n",
719 err, strerror(errno));
720 goto out;
721 }
722 }
723
724 if (txmsg_cork) {
725 err = bpf_map_update_elem(map_fd[4],
726 &i, &txmsg_cork, BPF_ANY);
727 if (err) {
728 fprintf(stderr,
729 "ERROR: bpf_map_update_elem (cork_bytes): %d (%s\n",
730 err, strerror(errno));
731 goto out;
732 }
733 }
734
735 if (txmsg_start) {
736 err = bpf_map_update_elem(map_fd[5],
737 &i, &txmsg_start, BPF_ANY);
738 if (err) {
739 fprintf(stderr,
740 "ERROR: bpf_map_update_elem (txmsg_start): %d (%s)\n",
741 err, strerror(errno));
742 goto out;
743 }
744 }
745
746 if (txmsg_end) {
747 i = 1;
748 err = bpf_map_update_elem(map_fd[5],
749 &i, &txmsg_end, BPF_ANY);
750 if (err) {
751 fprintf(stderr,
752 "ERROR: bpf_map_update_elem (txmsg_end): %d (%s)\n",
753 err, strerror(errno));
754 goto out;
755 }
756 }
757
758 if (txmsg_ingress) {
759 int in = BPF_F_INGRESS;
760
761 i = 0;
762 err = bpf_map_update_elem(map_fd[6], &i, &in, BPF_ANY);
763 if (err) {
764 fprintf(stderr,
765 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
766 err, strerror(errno));
767 }
768 i = 1;
769 err = bpf_map_update_elem(map_fd[1], &i, &p1, BPF_ANY);
770 if (err) {
771 fprintf(stderr,
772 "ERROR: bpf_map_update_elem (p1 txmsg): %d (%s)\n",
773 err, strerror(errno));
774 }
775 err = bpf_map_update_elem(map_fd[2], &i, &p1, BPF_ANY);
776 if (err) {
777 fprintf(stderr,
778 "ERROR: bpf_map_update_elem (p1 redir): %d (%s)\n",
779 err, strerror(errno));
780 }
781
782 i = 2;
783 err = bpf_map_update_elem(map_fd[2], &i, &p2, BPF_ANY);
784 if (err) {
785 fprintf(stderr,
786 "ERROR: bpf_map_update_elem (p2 txmsg): %d (%s)\n",
787 err, strerror(errno));
788 }
789 }
790
791 if (txmsg_skb) {
792 int skb_fd = (test == SENDMSG || test == SENDPAGE) ?
793 p2 : p1;
794 int ingress = BPF_F_INGRESS;
795
796 i = 0;
797 err = bpf_map_update_elem(map_fd[7],
798 &i, &ingress, BPF_ANY);
799 if (err) {
800 fprintf(stderr,
801 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
802 err, strerror(errno));
803 }
804
805 i = 3;
806 err = bpf_map_update_elem(map_fd[0],
807 &i, &skb_fd, BPF_ANY);
808 if (err) {
809 fprintf(stderr,
810 "ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n",
811 err, strerror(errno));
812 }
813 }
814 }
815
816 if (txmsg_drop)
817 options->drop_expected = true;
818
819 if (test == PING_PONG)
820 err = forever_ping_pong(options->rate, options);
821 else if (test == SENDMSG) {
822 options->base = false;
823 options->sendpage = false;
824 err = sendmsg_test(options);
825 } else if (test == SENDPAGE) {
826 options->base = false;
827 options->sendpage = true;
828 err = sendmsg_test(options);
829 } else if (test == BASE) {
830 options->base = true;
831 options->sendpage = false;
832 err = sendmsg_test(options);
833 } else if (test == BASE_SENDPAGE) {
834 options->base = true;
835 options->sendpage = true;
836 err = sendmsg_test(options);
837 } else
838 fprintf(stderr, "unknown test\n");
839out:
840 /* Detatch and zero all the maps */
841 bpf_prog_detach2(prog_fd[2], cg_fd, BPF_CGROUP_SOCK_OPS);
842 bpf_prog_detach2(prog_fd[0], map_fd[0], BPF_SK_SKB_STREAM_PARSER);
843 bpf_prog_detach2(prog_fd[1], map_fd[0], BPF_SK_SKB_STREAM_VERDICT);
844 if (tx_prog_fd >= 0)
845 bpf_prog_detach2(tx_prog_fd, map_fd[1], BPF_SK_MSG_VERDICT);
846
847 for (i = 0; i < 8; i++) {
848 key = next_key = 0;
849 bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY);
850 while (bpf_map_get_next_key(map_fd[i], &key, &next_key) == 0) {
851 bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY);
852 key = next_key;
853 }
854 }
855
856 close(s1);
857 close(s2);
858 close(p1);
859 close(p2);
860 close(c1);
861 close(c2);
862 return err;
863}
864
865static char *test_to_str(int test)
866{
867 switch (test) {
868 case SENDMSG:
869 return "sendmsg";
870 case SENDPAGE:
871 return "sendpage";
872 }
873 return "unknown";
874}
875
876#define OPTSTRING 60
877static void test_options(char *options)
878{
879 memset(options, 0, OPTSTRING);
880
881 if (txmsg_pass)
882 strncat(options, "pass,", OPTSTRING);
883 if (txmsg_noisy)
884 strncat(options, "pass_noisy,", OPTSTRING);
885 if (txmsg_redir)
886 strncat(options, "redir,", OPTSTRING);
887 if (txmsg_redir_noisy)
888 strncat(options, "redir_noisy,", OPTSTRING);
889 if (txmsg_drop)
890 strncat(options, "drop,", OPTSTRING);
891 if (txmsg_apply)
892 strncat(options, "apply,", OPTSTRING);
893 if (txmsg_cork)
894 strncat(options, "cork,", OPTSTRING);
895 if (txmsg_start)
896 strncat(options, "start,", OPTSTRING);
897 if (txmsg_end)
898 strncat(options, "end,", OPTSTRING);
899 if (txmsg_ingress)
900 strncat(options, "ingress,", OPTSTRING);
901 if (txmsg_skb)
902 strncat(options, "skb,", OPTSTRING);
903}
904
905static int __test_exec(int cgrp, int test, struct sockmap_options *opt)
906{
907 char *options = calloc(60, sizeof(char));
908 int err;
909
910 if (test == SENDPAGE)
911 opt->sendpage = true;
912 else
913 opt->sendpage = false;
914
915 if (txmsg_drop)
916 opt->drop_expected = true;
917 else
918 opt->drop_expected = false;
919
920 test_options(options);
921
922 fprintf(stdout,
923 "[TEST %i]: (%i, %i, %i, %s, %s): ",
924 test_cnt, opt->rate, opt->iov_count, opt->iov_length,
925 test_to_str(test), options);
926 fflush(stdout);
927 err = run_options(opt, cgrp, test);
928 fprintf(stdout, "%s\n", !err ? "PASS" : "FAILED");
929 test_cnt++;
930 !err ? passed++ : failed++;
931 free(options);
932 return err;
933}
934
935static int test_exec(int cgrp, struct sockmap_options *opt)
936{
937 int err = __test_exec(cgrp, SENDMSG, opt);
938
16962b24
JF		 939 if (err)
940 goto out;
941
942 err = __test_exec(cgrp, SENDPAGE, opt);
16962b24
JF		 943out:
944 return err;
945}
946
947static int test_loop(int cgrp)
948{
949 struct sockmap_options opt;
950
951 int err, i, l, r;
952
953 opt.verbose = 0;
954 opt.base = false;
955 opt.sendpage = false;
956 opt.data_test = false;
957 opt.drop_expected = false;
958 opt.iov_count = 0;
959 opt.iov_length = 0;
960 opt.rate = 0;
961
a18fda1a
JF		 962 r = 1;
963 for (i = 1; i < 100; i += 33) {
964 for (l = 1; l < 100; l += 33) {
965 opt.rate = r;
966 opt.iov_count = i;
967 opt.iov_length = l;
968 err = test_exec(cgrp, &opt);
969 if (err)
970 goto out;
16962b24
JF		 971 }
972 }
a18fda1a 973 sched_yield();
16962b24
JF		 974out:
975 return err;
976}
977
978static int test_txmsg(int cgrp)
979{
980 int err;
981
982 txmsg_pass = txmsg_noisy = txmsg_redir_noisy = txmsg_drop = 0;
983 txmsg_apply = txmsg_cork = 0;
984 txmsg_ingress = txmsg_skb = 0;
985
986 txmsg_pass = 1;
987 err = test_loop(cgrp);
988 txmsg_pass = 0;
989 if (err)
990 goto out;
991
992 txmsg_redir = 1;
993 err = test_loop(cgrp);
994 txmsg_redir = 0;
995 if (err)
996 goto out;
997
998 txmsg_drop = 1;
999 err = test_loop(cgrp);
1000 txmsg_drop = 0;
1001 if (err)
1002 goto out;
1003
1004 txmsg_redir = 1;
1005 txmsg_ingress = 1;
1006 err = test_loop(cgrp);
1007 txmsg_redir = 0;
1008 txmsg_ingress = 0;
1009 if (err)
1010 goto out;
1011out:
1012 txmsg_pass = 0;
1013 txmsg_redir = 0;
1014 txmsg_drop = 0;
1015 return err;
1016}
1017
1018static int test_send(struct sockmap_options *opt, int cgrp)
1019{
1020 int err;
1021
1022 opt->iov_length = 1;
1023 opt->iov_count = 1;
1024 opt->rate = 1;
1025 err = test_exec(cgrp, opt);
1026 if (err)
1027 goto out;
1028
1029 opt->iov_length = 1;
1030 opt->iov_count = 1024;
1031 opt->rate = 1;
1032 err = test_exec(cgrp, opt);
1033 if (err)
1034 goto out;
1035
1036 opt->iov_length = 1024;
1037 opt->iov_count = 1;
1038 opt->rate = 1;
1039 err = test_exec(cgrp, opt);
1040 if (err)
1041 goto out;
1042
1043 opt->iov_length = 1;
1044 opt->iov_count = 1;
a009f1f3 1045 opt->rate = 512;
16962b24
JF		 1046 err = test_exec(cgrp, opt);
1047 if (err)
1048 goto out;
1049
1050 opt->iov_length = 256;
1051 opt->iov_count = 1024;
a009f1f3 1052 opt->rate = 2;
16962b24
JF		 1053 err = test_exec(cgrp, opt);
1054 if (err)
1055 goto out;
1056
1057 opt->rate = 100;
1058 opt->iov_count = 1;
1059 opt->iov_length = 5;
1060 err = test_exec(cgrp, opt);
1061 if (err)
1062 goto out;
1063out:
a18fda1a 1064 sched_yield();
16962b24
JF		 1065 return err;
1066}
1067
1068static int test_mixed(int cgrp)
1069{
1070 struct sockmap_options opt = {0};
1071 int err;
1072
1073 txmsg_pass = txmsg_noisy = txmsg_redir_noisy = txmsg_drop = 0;
1074 txmsg_apply = txmsg_cork = 0;
1075 txmsg_start = txmsg_end = 0;
1076 /* Test small and large iov_count values with pass/redir/apply/cork */
1077 txmsg_pass = 1;
1078 txmsg_redir = 0;
1079 txmsg_apply = 1;
1080 txmsg_cork = 0;
1081 err = test_send(&opt, cgrp);
1082 if (err)
1083 goto out;
1084
1085 txmsg_pass = 1;
1086 txmsg_redir = 0;
1087 txmsg_apply = 0;
1088 txmsg_cork = 1;
1089 err = test_send(&opt, cgrp);
1090 if (err)
1091 goto out;
1092
1093 txmsg_pass = 1;
1094 txmsg_redir = 0;
1095 txmsg_apply = 1;
1096 txmsg_cork = 1;
1097 err = test_send(&opt, cgrp);
1098 if (err)
1099 goto out;
1100
1101 txmsg_pass = 1;
1102 txmsg_redir = 0;
1103 txmsg_apply = 1024;
1104 txmsg_cork = 0;
1105 err = test_send(&opt, cgrp);
1106 if (err)
1107 goto out;
1108
1109 txmsg_pass = 1;
1110 txmsg_redir = 0;
1111 txmsg_apply = 0;
1112 txmsg_cork = 1024;
1113 err = test_send(&opt, cgrp);
1114 if (err)
1115 goto out;
1116
1117 txmsg_pass = 1;
1118 txmsg_redir = 0;
1119 txmsg_apply = 1024;
1120 txmsg_cork = 1024;
1121 err = test_send(&opt, cgrp);
1122 if (err)
1123 goto out;
1124
1125 txmsg_pass = 1;
1126 txmsg_redir = 0;
1127 txmsg_cork = 4096;
1128 txmsg_apply = 4096;
1129 err = test_send(&opt, cgrp);
1130 if (err)
1131 goto out;
1132
1133 txmsg_pass = 0;
1134 txmsg_redir = 1;
1135 txmsg_apply = 1;
1136 txmsg_cork = 0;
1137 err = test_send(&opt, cgrp);
1138 if (err)
1139 goto out;
1140
1141 txmsg_pass = 0;
1142 txmsg_redir = 1;
1143 txmsg_apply = 0;
1144 txmsg_cork = 1;
1145 err = test_send(&opt, cgrp);
1146 if (err)
1147 goto out;
1148
1149 txmsg_pass = 0;
1150 txmsg_redir = 1;
1151 txmsg_apply = 1024;
1152 txmsg_cork = 0;
1153 err = test_send(&opt, cgrp);
1154 if (err)
1155 goto out;
1156
1157 txmsg_pass = 0;
1158 txmsg_redir = 1;
1159 txmsg_apply = 0;
1160 txmsg_cork = 1024;
1161 err = test_send(&opt, cgrp);
1162 if (err)
1163 goto out;
1164
1165 txmsg_pass = 0;
1166 txmsg_redir = 1;
1167 txmsg_apply = 1024;
1168 txmsg_cork = 1024;
1169 err = test_send(&opt, cgrp);
1170 if (err)
1171 goto out;
1172
1173 txmsg_pass = 0;
1174 txmsg_redir = 1;
1175 txmsg_cork = 4096;
1176 txmsg_apply = 4096;
1177 err = test_send(&opt, cgrp);
1178 if (err)
1179 goto out;
1180out:
1181 return err;
1182}
1183
1184static int test_start_end(int cgrp)
1185{
1186 struct sockmap_options opt = {0};
1187 int err, i;
1188
1189 /* Test basic start/end with lots of iov_count and iov_lengths */
1190 txmsg_start = 1;
1191 txmsg_end = 2;
1192 err = test_txmsg(cgrp);
1193 if (err)
1194 goto out;
1195
1196 /* Test start/end with cork */
1197 opt.rate = 16;
1198 opt.iov_count = 1;
1199 opt.iov_length = 100;
1200 txmsg_cork = 1600;
1201
a18fda1a 1202 for (i = 99; i <= 1600; i += 500) {
16962b24
JF		 1203 txmsg_start = 0;
1204 txmsg_end = i;
1205 err = test_exec(cgrp, &opt);
1206 if (err)
1207 goto out;
1208 }
1209
1210 /* Test start/end with cork but pull data in middle */
a18fda1a 1211 for (i = 199; i <= 1600; i += 500) {
16962b24
JF		 1212 txmsg_start = 100;
1213 txmsg_end = i;
1214 err = test_exec(cgrp, &opt);
1215 if (err)
1216 goto out;
1217 }
1218
1219 /* Test start/end with cork pulling last sg entry */
1220 txmsg_start = 1500;
1221 txmsg_end = 1600;
1222 err = test_exec(cgrp, &opt);
1223 if (err)
1224 goto out;
1225
1226 /* Test start/end pull of single byte in last page */
1227 txmsg_start = 1111;
1228 txmsg_end = 1112;
1229 err = test_exec(cgrp, &opt);
1230 if (err)
1231 goto out;
1232
1233 /* Test start/end with end < start */
1234 txmsg_start = 1111;
1235 txmsg_end = 0;
1236 err = test_exec(cgrp, &opt);
1237 if (err)
1238 goto out;
1239
1240 /* Test start/end with end > data */
1241 txmsg_start = 0;
1242 txmsg_end = 1601;
1243 err = test_exec(cgrp, &opt);
1244 if (err)
1245 goto out;
1246
1247 /* Test start/end with start > data */
1248 txmsg_start = 1601;
1249 txmsg_end = 1600;
1250 err = test_exec(cgrp, &opt);
1251
1252out:
1253 txmsg_start = 0;
1254 txmsg_end = 0;
a18fda1a 1255 sched_yield();
16962b24
JF		 1256 return err;
1257}
1258
1259char *map_names[] = {
1260 "sock_map",
1261 "sock_map_txmsg",
1262 "sock_map_redir",
1263 "sock_apply_bytes",
1264 "sock_cork_bytes",
1265 "sock_pull_bytes",
1266 "sock_redir_flags",
1267 "sock_skb_opts",
1268};
1269
1270int prog_attach_type[] = {
1271 BPF_SK_SKB_STREAM_PARSER,
1272 BPF_SK_SKB_STREAM_VERDICT,
1273 BPF_CGROUP_SOCK_OPS,
1274 BPF_SK_MSG_VERDICT,
1275 BPF_SK_MSG_VERDICT,
1276 BPF_SK_MSG_VERDICT,
1277 BPF_SK_MSG_VERDICT,
1278 BPF_SK_MSG_VERDICT,
1279 BPF_SK_MSG_VERDICT,
1280 BPF_SK_MSG_VERDICT,
1281};
1282
1283int prog_type[] = {
1284 BPF_PROG_TYPE_SK_SKB,
1285 BPF_PROG_TYPE_SK_SKB,
1286 BPF_PROG_TYPE_SOCK_OPS,
1287 BPF_PROG_TYPE_SK_MSG,
1288 BPF_PROG_TYPE_SK_MSG,
1289 BPF_PROG_TYPE_SK_MSG,
1290 BPF_PROG_TYPE_SK_MSG,
1291 BPF_PROG_TYPE_SK_MSG,
1292 BPF_PROG_TYPE_SK_MSG,
1293 BPF_PROG_TYPE_SK_MSG,
1294};
1295
b8b394fa 1296static int populate_progs(char *bpf_file)
16962b24 1297{
16962b24
JF		 1298 struct bpf_program *prog;
1299 struct bpf_object *obj;
1300 int i = 0;
1301 long err;
1302
1303 obj = bpf_object__open(bpf_file);
1304 err = libbpf_get_error(obj);
1305 if (err) {
1306 char err_buf[256];
1307
1308 libbpf_strerror(err, err_buf, sizeof(err_buf));
1309 printf("Unable to load eBPF objects in file '%s' : %s\n",
1310 bpf_file, err_buf);
1311 return -1;
1312 }
1313
1314 bpf_object__for_each_program(prog, obj) {
1315 bpf_program__set_type(prog, prog_type[i]);
1316 bpf_program__set_expected_attach_type(prog,
1317 prog_attach_type[i]);
1318 i++;
1319 }
1320
1321 i = bpf_object__load(obj);
1322 i = 0;
1323 bpf_object__for_each_program(prog, obj) {
1324 prog_fd[i] = bpf_program__fd(prog);
1325 i++;
1326 }
1327
1328 for (i = 0; i < sizeof(map_fd)/sizeof(int); i++) {
1329 maps[i] = bpf_object__find_map_by_name(obj, map_names[i]);
1330 map_fd[i] = bpf_map__fd(maps[i]);
1331 if (map_fd[i] < 0) {
1332 fprintf(stderr, "load_bpf_file: (%i) %s\n",
1333 map_fd[i], strerror(errno));
1334 return -1;
1335 }
1336 }
1337
1338 return 0;
1339}
1340
b8b394fa 1341static int __test_suite(char *bpf_file)
16962b24
JF		 1342{
1343 int cg_fd, err;
1344
b8b394fa 1345 err = populate_progs(bpf_file);
16962b24
JF		 1346 if (err < 0) {
1347 fprintf(stderr, "ERROR: (%i) load bpf failed\n", err);
1348 return err;
1349 }
1350
1351 if (setup_cgroup_environment()) {
1352 fprintf(stderr, "ERROR: cgroup env failed\n");
1353 return -EINVAL;
1354 }
1355
1356 cg_fd = create_and_get_cgroup(CG_PATH);
1357 if (cg_fd < 0) {
1358 fprintf(stderr,
1359 "ERROR: (%i) open cg path failed: %s\n",
1360 cg_fd, optarg);
1361 return cg_fd;
1362 }
1363
035b37ff
PB		 1364 if (join_cgroup(CG_PATH)) {
1365 fprintf(stderr, "ERROR: failed to join cgroup\n");
1366 return -EINVAL;
1367 }
1368
16962b24
JF		 1369 /* Tests basic commands and APIs with range of iov values */
1370 txmsg_start = txmsg_end = 0;
1371 err = test_txmsg(cg_fd);
1372 if (err)
1373 goto out;
1374
1375 /* Tests interesting combinations of APIs used together */
1376 err = test_mixed(cg_fd);
1377 if (err)
1378 goto out;
1379
1380 /* Tests pull_data API using start/end API */
1381 err = test_start_end(cg_fd);
1382 if (err)
1383 goto out;
1384
1385out:
1386 printf("Summary: %i PASSED %i FAILED\n", passed, failed);
b8b394fa 1387 cleanup_cgroup_environment();
16962b24
JF		 1388 close(cg_fd);
1389 return err;
1390}
1391
b8b394fa
JF		 1392static int test_suite(void)
1393{
1394 int err;
1395
1396 err = __test_suite(BPF_SOCKMAP_FILENAME);
1397 if (err)
1398 goto out;
1399 err = __test_suite(BPF_SOCKHASH_FILENAME);
1400out:
1401 return err;
1402}
1403
16962b24
JF		 1404int main(int argc, char **argv)
1405{
1406 struct rlimit r = {10 * 1024 * 1024, RLIM_INFINITY};
1407 int iov_count = 1, length = 1024, rate = 1;
1408 struct sockmap_options options = {0};
1409 int opt, longindex, err, cg_fd = 0;
b8b394fa 1410 char *bpf_file = BPF_SOCKMAP_FILENAME;
16962b24
JF		 1411 int test = PING_PONG;
1412
1413 if (setrlimit(RLIMIT_MEMLOCK, &r)) {
1414 perror("setrlimit(RLIMIT_MEMLOCK)");
1415 return 1;
1416 }
1417
1418 if (argc < 2)
1419 return test_suite();
1420
1421 while ((opt = getopt_long(argc, argv, ":dhvc:r:i:l:t:",
1422 long_options, &longindex)) != -1) {
1423 switch (opt) {
1424 case 's':
1425 txmsg_start = atoi(optarg);
1426 break;
1427 case 'e':
1428 txmsg_end = atoi(optarg);
1429 break;
1430 case 'a':
1431 txmsg_apply = atoi(optarg);
1432 break;
1433 case 'k':
1434 txmsg_cork = atoi(optarg);
1435 break;
1436 case 'c':
1437 cg_fd = open(optarg, O_DIRECTORY, O_RDONLY);
1438 if (cg_fd < 0) {
1439 fprintf(stderr,
1440 "ERROR: (%i) open cg path failed: %s\n",
1441 cg_fd, optarg);
1442 return cg_fd;
1443 }
1444 break;
1445 case 'r':
1446 rate = atoi(optarg);
1447 break;
1448 case 'v':
1449 options.verbose = 1;
1450 break;
1451 case 'i':
1452 iov_count = atoi(optarg);
1453 break;
1454 case 'l':
1455 length = atoi(optarg);
1456 break;
1457 case 'd':
1458 options.data_test = true;
1459 break;
1460 case 't':
1461 if (strcmp(optarg, "ping") == 0) {
1462 test = PING_PONG;
1463 } else if (strcmp(optarg, "sendmsg") == 0) {
1464 test = SENDMSG;
1465 } else if (strcmp(optarg, "base") == 0) {
1466 test = BASE;
1467 } else if (strcmp(optarg, "base_sendpage") == 0) {
1468 test = BASE_SENDPAGE;
1469 } else if (strcmp(optarg, "sendpage") == 0) {
1470 test = SENDPAGE;
1471 } else {
1472 usage(argv);
1473 return -1;
1474 }
1475 break;
1476 case 0:
1477 break;
1478 case 'h':
1479 default:
1480 usage(argv);
1481 return -1;
1482 }
1483 }
1484
1485 if (!cg_fd) {
1486 fprintf(stderr, "%s requires cgroup option: --cgroup <path>\n",
1487 argv[0]);
1488 return -1;
1489 }
1490
b8b394fa 1491 err = populate_progs(bpf_file);
16962b24
JF		 1492 if (err) {
1493 fprintf(stderr, "populate program: (%s) %s\n",
1494 bpf_file, strerror(errno));
1495 return 1;
1496 }
1497 running = 1;
1498
1499 /* catch SIGINT */
1500 signal(SIGINT, running_handler);
1501
1502 options.iov_count = iov_count;
1503 options.iov_length = length;
1504 options.rate = rate;
1505
1506 err = run_options(&options, cg_fd, test);
1507 close(cg_fd);
1508 return err;
1509}
1510
1511void running_handler(int a)
1512{
1513 running = 0;
1514}
Linux 6.x block layer and io_uring tree(s)