projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
selftests/bpf: test_sockmap, join cgroup in selftest mode
[linux-2.6-block.git] / tools / testing / selftests / bpf / test_sockmap.c
CommitLineData
16962b24
JF		 1// SPDX-License-Identifier: GPL-2.0
2// Copyright (c) 2017-2018 Covalent IO, Inc. http://covalent.io
3#include <stdio.h>
4#include <stdlib.h>
5#include <sys/socket.h>
6#include <sys/ioctl.h>
7#include <sys/select.h>
8#include <netinet/in.h>
9#include <arpa/inet.h>
10#include <unistd.h>
11#include <string.h>
12#include <errno.h>
13#include <sys/ioctl.h>
14#include <stdbool.h>
15#include <signal.h>
16#include <fcntl.h>
17#include <sys/wait.h>
18#include <time.h>
19#include <sched.h>
20
21#include <sys/time.h>
22#include <sys/resource.h>
23#include <sys/types.h>
24#include <sys/sendfile.h>
25
26#include <linux/netlink.h>
27#include <linux/socket.h>
28#include <linux/sock_diag.h>
29#include <linux/bpf.h>
30#include <linux/if_link.h>
31#include <assert.h>
32#include <libgen.h>
33
34#include <getopt.h>
35
36#include <bpf/bpf.h>
37#include <bpf/libbpf.h>
38
39#include "bpf_util.h"
40#include "bpf_rlimit.h"
41#include "cgroup_helpers.h"
42
43int running;
44static void running_handler(int a);
45
46/* randomly selected ports for testing on lo */
47#define S1_PORT 10000
48#define S2_PORT 10001
49
b8b394fa
JF		 50#define BPF_SOCKMAP_FILENAME "test_sockmap_kern.o"
51#define BPF_SOCKHASH_FILENAME "test_sockhash_kern.o"
16962b24
JF		 52#define CG_PATH "/sockmap"
53
54/* global sockets */
55int s1, s2, c1, c2, p1, p2;
56int test_cnt;
57int passed;
58int failed;
59int map_fd[8];
60struct bpf_map *maps[8];
61int prog_fd[11];
62
63int txmsg_pass;
64int txmsg_noisy;
65int txmsg_redir;
66int txmsg_redir_noisy;
67int txmsg_drop;
68int txmsg_apply;
69int txmsg_cork;
70int txmsg_start;
71int txmsg_end;
72int txmsg_ingress;
73int txmsg_skb;
74
75static const struct option long_options[] = {
76 {"help", no_argument, NULL, 'h' },
77 {"cgroup", required_argument, NULL, 'c' },
78 {"rate", required_argument, NULL, 'r' },
79 {"verbose", no_argument, NULL, 'v' },
80 {"iov_count", required_argument, NULL, 'i' },
81 {"length", required_argument, NULL, 'l' },
82 {"test", required_argument, NULL, 't' },
83 {"data_test", no_argument, NULL, 'd' },
84 {"txmsg", no_argument, &txmsg_pass, 1 },
85 {"txmsg_noisy", no_argument, &txmsg_noisy, 1 },
86 {"txmsg_redir", no_argument, &txmsg_redir, 1 },
87 {"txmsg_redir_noisy", no_argument, &txmsg_redir_noisy, 1},
88 {"txmsg_drop", no_argument, &txmsg_drop, 1 },
89 {"txmsg_apply", required_argument, NULL, 'a'},
90 {"txmsg_cork", required_argument, NULL, 'k'},
91 {"txmsg_start", required_argument, NULL, 's'},
92 {"txmsg_end", required_argument, NULL, 'e'},
93 {"txmsg_ingress", no_argument, &txmsg_ingress, 1 },
94 {"txmsg_skb", no_argument, &txmsg_skb, 1 },
95 {0, 0, NULL, 0 }
96};
97
98static void usage(char *argv[])
99{
100 int i;
101
102 printf(" Usage: %s --cgroup <cgroup_path>\n", argv[0]);
103 printf(" options:\n");
104 for (i = 0; long_options[i].name != 0; i++) {
105 printf(" --%-12s", long_options[i].name);
106 if (long_options[i].flag != NULL)
107 printf(" flag (internal value:%d)\n",
108 *long_options[i].flag);
109 else
110 printf(" -%c\n", long_options[i].val);
111 }
112 printf("\n");
113}
114
115static int sockmap_init_sockets(int verbose)
116{
117 int i, err, one = 1;
118 struct sockaddr_in addr;
119 int *fds[4] = {&s1, &s2, &c1, &c2};
120
121 s1 = s2 = p1 = p2 = c1 = c2 = 0;
122
123 /* Init sockets */
124 for (i = 0; i < 4; i++) {
125 *fds[i] = socket(AF_INET, SOCK_STREAM, 0);
126 if (*fds[i] < 0) {
127 perror("socket s1 failed()");
128 return errno;
129 }
130 }
131
132 /* Allow reuse */
133 for (i = 0; i < 2; i++) {
134 err = setsockopt(*fds[i], SOL_SOCKET, SO_REUSEADDR,
135 (char *)&one, sizeof(one));
136 if (err) {
137 perror("setsockopt failed()");
138 return errno;
139 }
140 }
141
142 /* Non-blocking sockets */
143 for (i = 0; i < 2; i++) {
144 err = ioctl(*fds[i], FIONBIO, (char *)&one);
145 if (err < 0) {
146 perror("ioctl s1 failed()");
147 return errno;
148 }
149 }
150
151 /* Bind server sockets */
152 memset(&addr, 0, sizeof(struct sockaddr_in));
153 addr.sin_family = AF_INET;
154 addr.sin_addr.s_addr = inet_addr("127.0.0.1");
155
156 addr.sin_port = htons(S1_PORT);
157 err = bind(s1, (struct sockaddr *)&addr, sizeof(addr));
158 if (err < 0) {
159 perror("bind s1 failed()\n");
160 return errno;
161 }
162
163 addr.sin_port = htons(S2_PORT);
164 err = bind(s2, (struct sockaddr *)&addr, sizeof(addr));
165 if (err < 0) {
166 perror("bind s2 failed()\n");
167 return errno;
168 }
169
170 /* Listen server sockets */
171 addr.sin_port = htons(S1_PORT);
172 err = listen(s1, 32);
173 if (err < 0) {
174 perror("listen s1 failed()\n");
175 return errno;
176 }
177
178 addr.sin_port = htons(S2_PORT);
179 err = listen(s2, 32);
180 if (err < 0) {
181 perror("listen s1 failed()\n");
182 return errno;
183 }
184
185 /* Initiate Connect */
186 addr.sin_port = htons(S1_PORT);
187 err = connect(c1, (struct sockaddr *)&addr, sizeof(addr));
188 if (err < 0 && errno != EINPROGRESS) {
189 perror("connect c1 failed()\n");
190 return errno;
191 }
192
193 addr.sin_port = htons(S2_PORT);
194 err = connect(c2, (struct sockaddr *)&addr, sizeof(addr));
195 if (err < 0 && errno != EINPROGRESS) {
196 perror("connect c2 failed()\n");
197 return errno;
198 } else if (err < 0) {
199 err = 0;
200 }
201
202 /* Accept Connecrtions */
203 p1 = accept(s1, NULL, NULL);
204 if (p1 < 0) {
205 perror("accept s1 failed()\n");
206 return errno;
207 }
208
209 p2 = accept(s2, NULL, NULL);
210 if (p2 < 0) {
211 perror("accept s1 failed()\n");
212 return errno;
213 }
214
215 if (verbose) {
216 printf("connected sockets: c1 <-> p1, c2 <-> p2\n");
217 printf("cgroups binding: c1(%i) <-> s1(%i) - - - c2(%i) <-> s2(%i)\n",
218 c1, s1, c2, s2);
219 }
220 return 0;
221}
222
223struct msg_stats {
224 size_t bytes_sent;
225 size_t bytes_recvd;
226 struct timespec start;
227 struct timespec end;
228};
229
230struct sockmap_options {
231 int verbose;
232 bool base;
233 bool sendpage;
234 bool data_test;
235 bool drop_expected;
236 int iov_count;
237 int iov_length;
238 int rate;
239};
240
241static int msg_loop_sendpage(int fd, int iov_length, int cnt,
242 struct msg_stats *s,
243 struct sockmap_options *opt)
244{
245 bool drop = opt->drop_expected;
246 unsigned char k = 0;
247 FILE *file;
248 int i, fp;
249
250 file = fopen(".sendpage_tst.tmp", "w+");
251 for (i = 0; i < iov_length * cnt; i++, k++)
252 fwrite(&k, sizeof(char), 1, file);
253 fflush(file);
254 fseek(file, 0, SEEK_SET);
255 fclose(file);
256
257 fp = open(".sendpage_tst.tmp", O_RDONLY);
258 clock_gettime(CLOCK_MONOTONIC, &s->start);
259 for (i = 0; i < cnt; i++) {
260 int sent = sendfile(fd, fp, NULL, iov_length);
261
262 if (!drop && sent < 0) {
263 perror("send loop error:");
264 close(fp);
265 return sent;
266 } else if (drop && sent >= 0) {
267 printf("sendpage loop error expected: %i\n", sent);
268 close(fp);
269 return -EIO;
270 }
271
272 if (sent > 0)
273 s->bytes_sent += sent;
274 }
275 clock_gettime(CLOCK_MONOTONIC, &s->end);
276 close(fp);
277 return 0;
278}
279
280static int msg_loop(int fd, int iov_count, int iov_length, int cnt,
281 struct msg_stats *s, bool tx,
282 struct sockmap_options *opt)
283{
284 struct msghdr msg = {0};
285 int err, i, flags = MSG_NOSIGNAL;
286 struct iovec *iov;
287 unsigned char k;
288 bool data_test = opt->data_test;
289 bool drop = opt->drop_expected;
290
291 iov = calloc(iov_count, sizeof(struct iovec));
292 if (!iov)
293 return errno;
294
295 k = 0;
296 for (i = 0; i < iov_count; i++) {
297 unsigned char *d = calloc(iov_length, sizeof(char));
298
299 if (!d) {
300 fprintf(stderr, "iov_count %i/%i OOM\n", i, iov_count);
301 goto out_errno;
302 }
303 iov[i].iov_base = d;
304 iov[i].iov_len = iov_length;
305
306 if (data_test && tx) {
307 int j;
308
309 for (j = 0; j < iov_length; j++)
310 d[j] = k++;
311 }
312 }
313
314 msg.msg_iov = iov;
315 msg.msg_iovlen = iov_count;
316 k = 0;
317
318 if (tx) {
319 clock_gettime(CLOCK_MONOTONIC, &s->start);
320 for (i = 0; i < cnt; i++) {
321 int sent = sendmsg(fd, &msg, flags);
322
323 if (!drop && sent < 0) {
324 perror("send loop error:");
325 goto out_errno;
326 } else if (drop && sent >= 0) {
327 printf("send loop error expected: %i\n", sent);
328 errno = -EIO;
329 goto out_errno;
330 }
331 if (sent > 0)
332 s->bytes_sent += sent;
333 }
334 clock_gettime(CLOCK_MONOTONIC, &s->end);
335 } else {
336 int slct, recv, max_fd = fd;
337 int fd_flags = O_NONBLOCK;
338 struct timeval timeout;
339 float total_bytes;
340 fd_set w;
341
342 fcntl(fd, fd_flags);
343 total_bytes = (float)iov_count * (float)iov_length * (float)cnt;
344 err = clock_gettime(CLOCK_MONOTONIC, &s->start);
345 if (err < 0)
346 perror("recv start time: ");
347 while (s->bytes_recvd < total_bytes) {
a18fda1a
JF		 348 timeout.tv_sec = 0;
349 timeout.tv_usec = 10;
16962b24
JF		 350
351 /* FD sets */
352 FD_ZERO(&w);
353 FD_SET(fd, &w);
354
355 slct = select(max_fd + 1, &w, NULL, NULL, &timeout);
356 if (slct == -1) {
357 perror("select()");
358 clock_gettime(CLOCK_MONOTONIC, &s->end);
359 goto out_errno;
360 } else if (!slct) {
361 if (opt->verbose)
362 fprintf(stderr, "unexpected timeout\n");
363 errno = -EIO;
364 clock_gettime(CLOCK_MONOTONIC, &s->end);
365 goto out_errno;
366 }
367
368 recv = recvmsg(fd, &msg, flags);
369 if (recv < 0) {
370 if (errno != EWOULDBLOCK) {
371 clock_gettime(CLOCK_MONOTONIC, &s->end);
372 perror("recv failed()\n");
373 goto out_errno;
374 }
375 }
376
377 s->bytes_recvd += recv;
378
379 if (data_test) {
380 int j;
381
382 for (i = 0; i < msg.msg_iovlen; i++) {
383 unsigned char *d = iov[i].iov_base;
384
385 for (j = 0;
386 j < iov[i].iov_len && recv; j++) {
387 if (d[j] != k++) {
388 errno = -EIO;
389 fprintf(stderr,
390 "detected data corruption @iov[%i]:%i %02x != %02x, %02x ?= %02x\n",
391 i, j, d[j], k - 1, d[j+1], k + 1);
392 goto out_errno;
393 }
394 recv--;
395 }
396 }
397 }
398 }
399 clock_gettime(CLOCK_MONOTONIC, &s->end);
400 }
401
402 for (i = 0; i < iov_count; i++)
403 free(iov[i].iov_base);
404 free(iov);
405 return 0;
406out_errno:
407 for (i = 0; i < iov_count; i++)
408 free(iov[i].iov_base);
409 free(iov);
410 return errno;
411}
412
413static float giga = 1000000000;
414
415static inline float sentBps(struct msg_stats s)
416{
417 return s.bytes_sent / (s.end.tv_sec - s.start.tv_sec);
418}
419
420static inline float recvdBps(struct msg_stats s)
421{
422 return s.bytes_recvd / (s.end.tv_sec - s.start.tv_sec);
423}
424
425static int sendmsg_test(struct sockmap_options *opt)
426{
427 float sent_Bps = 0, recvd_Bps = 0;
428 int rx_fd, txpid, rxpid, err = 0;
429 struct msg_stats s = {0};
430 int iov_count = opt->iov_count;
431 int iov_buf = opt->iov_length;
16edddfe 432 int rx_status, tx_status;
16962b24 433 int cnt = opt->rate;
16962b24
JF		 434
435 errno = 0;
436
437 if (opt->base)
438 rx_fd = p1;
439 else
440 rx_fd = p2;
441
442 rxpid = fork();
443 if (rxpid == 0) {
444 if (opt->drop_expected)
16edddfe 445 exit(0);
16962b24
JF		 446
447 if (opt->sendpage)
448 iov_count = 1;
449 err = msg_loop(rx_fd, iov_count, iov_buf,
450 cnt, &s, false, opt);
451 if (err && opt->verbose)
452 fprintf(stderr,
453 "msg_loop_rx: iov_count %i iov_buf %i cnt %i err %i\n",
454 iov_count, iov_buf, cnt, err);
455 shutdown(p2, SHUT_RDWR);
456 shutdown(p1, SHUT_RDWR);
457 if (s.end.tv_sec - s.start.tv_sec) {
458 sent_Bps = sentBps(s);
459 recvd_Bps = recvdBps(s);
460 }
461 if (opt->verbose)
462 fprintf(stdout,
463 "rx_sendmsg: TX: %zuB %fB/s %fGB/s RX: %zuB %fB/s %fGB/s\n",
464 s.bytes_sent, sent_Bps, sent_Bps/giga,
465 s.bytes_recvd, recvd_Bps, recvd_Bps/giga);
16edddfe
PB		 466 if (err && txmsg_cork)
467 err = 0;
468 exit(err ? 1 : 0);
16962b24
JF		 469 } else if (rxpid == -1) {
470 perror("msg_loop_rx: ");
471 return errno;
472 }
473
474 txpid = fork();
475 if (txpid == 0) {
476 if (opt->sendpage)
477 err = msg_loop_sendpage(c1, iov_buf, cnt, &s, opt);
478 else
479 err = msg_loop(c1, iov_count, iov_buf,
480 cnt, &s, true, opt);
481
482 if (err)
483 fprintf(stderr,
484 "msg_loop_tx: iov_count %i iov_buf %i cnt %i err %i\n",
485 iov_count, iov_buf, cnt, err);
486 shutdown(c1, SHUT_RDWR);
487 if (s.end.tv_sec - s.start.tv_sec) {
488 sent_Bps = sentBps(s);
489 recvd_Bps = recvdBps(s);
490 }
491 if (opt->verbose)
492 fprintf(stdout,
493 "tx_sendmsg: TX: %zuB %fB/s %f GB/s RX: %zuB %fB/s %fGB/s\n",
494 s.bytes_sent, sent_Bps, sent_Bps/giga,
495 s.bytes_recvd, recvd_Bps, recvd_Bps/giga);
16edddfe 496 exit(err ? 1 : 0);
16962b24
JF		 497 } else if (txpid == -1) {
498 perror("msg_loop_tx: ");
499 return errno;
500 }
501
16edddfe
PB		 502 assert(waitpid(rxpid, &rx_status, 0) == rxpid);
503 assert(waitpid(txpid, &tx_status, 0) == txpid);
504 if (WIFEXITED(rx_status)) {
505 err = WEXITSTATUS(rx_status);
506 if (err) {
507 fprintf(stderr, "rx thread exited with err %d. ", err);
508 goto out;
509 }
510 }
511 if (WIFEXITED(tx_status)) {
512 err = WEXITSTATUS(tx_status);
513 if (err)
514 fprintf(stderr, "tx thread exited with err %d. ", err);
515 }
516out:
16962b24
JF		 517 return err;
518}
519
520static int forever_ping_pong(int rate, struct sockmap_options *opt)
521{
522 struct timeval timeout;
523 char buf[1024] = {0};
524 int sc;
525
526 timeout.tv_sec = 10;
527 timeout.tv_usec = 0;
528
529 /* Ping/Pong data from client to server */
530 sc = send(c1, buf, sizeof(buf), 0);
531 if (sc < 0) {
532 perror("send failed()\n");
533 return sc;
534 }
535
536 do {
537 int s, rc, i, max_fd = p2;
538 fd_set w;
539
540 /* FD sets */
541 FD_ZERO(&w);
542 FD_SET(c1, &w);
543 FD_SET(c2, &w);
544 FD_SET(p1, &w);
545 FD_SET(p2, &w);
546
547 s = select(max_fd + 1, &w, NULL, NULL, &timeout);
548 if (s == -1) {
549 perror("select()");
550 break;
551 } else if (!s) {
552 fprintf(stderr, "unexpected timeout\n");
553 break;
554 }
555
556 for (i = 0; i <= max_fd && s > 0; ++i) {
557 if (!FD_ISSET(i, &w))
558 continue;
559
560 s--;
561
562 rc = recv(i, buf, sizeof(buf), 0);
563 if (rc < 0) {
564 if (errno != EWOULDBLOCK) {
565 perror("recv failed()\n");
566 return rc;
567 }
568 }
569
570 if (rc == 0) {
571 close(i);
572 break;
573 }
574
575 sc = send(i, buf, rc, 0);
576 if (sc < 0) {
577 perror("send failed()\n");
578 return sc;
579 }
580 }
581
582 if (rate)
583 sleep(rate);
584
585 if (opt->verbose) {
586 printf(".");
587 fflush(stdout);
588
589 }
590 } while (running);
591
592 return 0;
593}
594
595enum {
596 PING_PONG,
597 SENDMSG,
598 BASE,
599 BASE_SENDPAGE,
600 SENDPAGE,
601};
602
603static int run_options(struct sockmap_options *options, int cg_fd, int test)
604{
605 int i, key, next_key, err, tx_prog_fd = -1, zero = 0;
606
607 /* If base test skip BPF setup */
608 if (test == BASE || test == BASE_SENDPAGE)
609 goto run;
610
611 /* Attach programs to sockmap */
612 err = bpf_prog_attach(prog_fd[0], map_fd[0],
613 BPF_SK_SKB_STREAM_PARSER, 0);
614 if (err) {
615 fprintf(stderr,
616 "ERROR: bpf_prog_attach (sockmap %i->%i): %d (%s)\n",
617 prog_fd[0], map_fd[0], err, strerror(errno));
618 return err;
619 }
620
621 err = bpf_prog_attach(prog_fd[1], map_fd[0],
622 BPF_SK_SKB_STREAM_VERDICT, 0);
623 if (err) {
624 fprintf(stderr, "ERROR: bpf_prog_attach (sockmap): %d (%s)\n",
625 err, strerror(errno));
626 return err;
627 }
628
629 /* Attach to cgroups */
630 err = bpf_prog_attach(prog_fd[2], cg_fd, BPF_CGROUP_SOCK_OPS, 0);
631 if (err) {
632 fprintf(stderr, "ERROR: bpf_prog_attach (groups): %d (%s)\n",
633 err, strerror(errno));
634 return err;
635 }
636
637run:
638 err = sockmap_init_sockets(options->verbose);
639 if (err) {
640 fprintf(stderr, "ERROR: test socket failed: %d\n", err);
641 goto out;
642 }
643
644 /* Attach txmsg program to sockmap */
645 if (txmsg_pass)
646 tx_prog_fd = prog_fd[3];
647 else if (txmsg_noisy)
648 tx_prog_fd = prog_fd[4];
649 else if (txmsg_redir)
650 tx_prog_fd = prog_fd[5];
651 else if (txmsg_redir_noisy)
652 tx_prog_fd = prog_fd[6];
653 else if (txmsg_drop)
654 tx_prog_fd = prog_fd[9];
655 /* apply and cork must be last */
656 else if (txmsg_apply)
657 tx_prog_fd = prog_fd[7];
658 else if (txmsg_cork)
659 tx_prog_fd = prog_fd[8];
660 else
661 tx_prog_fd = 0;
662
663 if (tx_prog_fd) {
664 int redir_fd, i = 0;
665
666 err = bpf_prog_attach(tx_prog_fd,
667 map_fd[1], BPF_SK_MSG_VERDICT, 0);
668 if (err) {
669 fprintf(stderr,
670 "ERROR: bpf_prog_attach (txmsg): %d (%s)\n",
671 err, strerror(errno));
672 goto out;
673 }
674
675 err = bpf_map_update_elem(map_fd[1], &i, &c1, BPF_ANY);
676 if (err) {
677 fprintf(stderr,
678 "ERROR: bpf_map_update_elem (txmsg): %d (%s\n",
679 err, strerror(errno));
680 goto out;
681 }
682
683 if (txmsg_redir || txmsg_redir_noisy)
684 redir_fd = c2;
685 else
686 redir_fd = c1;
687
688 err = bpf_map_update_elem(map_fd[2], &i, &redir_fd, BPF_ANY);
689 if (err) {
690 fprintf(stderr,
691 "ERROR: bpf_map_update_elem (txmsg): %d (%s\n",
692 err, strerror(errno));
693 goto out;
694 }
695
696 if (txmsg_apply) {
697 err = bpf_map_update_elem(map_fd[3],
698 &i, &txmsg_apply, BPF_ANY);
699 if (err) {
700 fprintf(stderr,
701 "ERROR: bpf_map_update_elem (apply_bytes): %d (%s\n",
702 err, strerror(errno));
703 goto out;
704 }
705 }
706
707 if (txmsg_cork) {
708 err = bpf_map_update_elem(map_fd[4],
709 &i, &txmsg_cork, BPF_ANY);
710 if (err) {
711 fprintf(stderr,
712 "ERROR: bpf_map_update_elem (cork_bytes): %d (%s\n",
713 err, strerror(errno));
714 goto out;
715 }
716 }
717
718 if (txmsg_start) {
719 err = bpf_map_update_elem(map_fd[5],
720 &i, &txmsg_start, BPF_ANY);
721 if (err) {
722 fprintf(stderr,
723 "ERROR: bpf_map_update_elem (txmsg_start): %d (%s)\n",
724 err, strerror(errno));
725 goto out;
726 }
727 }
728
729 if (txmsg_end) {
730 i = 1;
731 err = bpf_map_update_elem(map_fd[5],
732 &i, &txmsg_end, BPF_ANY);
733 if (err) {
734 fprintf(stderr,
735 "ERROR: bpf_map_update_elem (txmsg_end): %d (%s)\n",
736 err, strerror(errno));
737 goto out;
738 }
739 }
740
741 if (txmsg_ingress) {
742 int in = BPF_F_INGRESS;
743
744 i = 0;
745 err = bpf_map_update_elem(map_fd[6], &i, &in, BPF_ANY);
746 if (err) {
747 fprintf(stderr,
748 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
749 err, strerror(errno));
750 }
751 i = 1;
752 err = bpf_map_update_elem(map_fd[1], &i, &p1, BPF_ANY);
753 if (err) {
754 fprintf(stderr,
755 "ERROR: bpf_map_update_elem (p1 txmsg): %d (%s)\n",
756 err, strerror(errno));
757 }
758 err = bpf_map_update_elem(map_fd[2], &i, &p1, BPF_ANY);
759 if (err) {
760 fprintf(stderr,
761 "ERROR: bpf_map_update_elem (p1 redir): %d (%s)\n",
762 err, strerror(errno));
763 }
764
765 i = 2;
766 err = bpf_map_update_elem(map_fd[2], &i, &p2, BPF_ANY);
767 if (err) {
768 fprintf(stderr,
769 "ERROR: bpf_map_update_elem (p2 txmsg): %d (%s)\n",
770 err, strerror(errno));
771 }
772 }
773
774 if (txmsg_skb) {
775 int skb_fd = (test == SENDMSG || test == SENDPAGE) ?
776 p2 : p1;
777 int ingress = BPF_F_INGRESS;
778
779 i = 0;
780 err = bpf_map_update_elem(map_fd[7],
781 &i, &ingress, BPF_ANY);
782 if (err) {
783 fprintf(stderr,
784 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
785 err, strerror(errno));
786 }
787
788 i = 3;
789 err = bpf_map_update_elem(map_fd[0],
790 &i, &skb_fd, BPF_ANY);
791 if (err) {
792 fprintf(stderr,
793 "ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n",
794 err, strerror(errno));
795 }
796 }
797 }
798
799 if (txmsg_drop)
800 options->drop_expected = true;
801
802 if (test == PING_PONG)
803 err = forever_ping_pong(options->rate, options);
804 else if (test == SENDMSG) {
805 options->base = false;
806 options->sendpage = false;
807 err = sendmsg_test(options);
808 } else if (test == SENDPAGE) {
809 options->base = false;
810 options->sendpage = true;
811 err = sendmsg_test(options);
812 } else if (test == BASE) {
813 options->base = true;
814 options->sendpage = false;
815 err = sendmsg_test(options);
816 } else if (test == BASE_SENDPAGE) {
817 options->base = true;
818 options->sendpage = true;
819 err = sendmsg_test(options);
820 } else
821 fprintf(stderr, "unknown test\n");
822out:
823 /* Detatch and zero all the maps */
824 bpf_prog_detach2(prog_fd[2], cg_fd, BPF_CGROUP_SOCK_OPS);
825 bpf_prog_detach2(prog_fd[0], map_fd[0], BPF_SK_SKB_STREAM_PARSER);
826 bpf_prog_detach2(prog_fd[1], map_fd[0], BPF_SK_SKB_STREAM_VERDICT);
827 if (tx_prog_fd >= 0)
828 bpf_prog_detach2(tx_prog_fd, map_fd[1], BPF_SK_MSG_VERDICT);
829
830 for (i = 0; i < 8; i++) {
831 key = next_key = 0;
832 bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY);
833 while (bpf_map_get_next_key(map_fd[i], &key, &next_key) == 0) {
834 bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY);
835 key = next_key;
836 }
837 }
838
839 close(s1);
840 close(s2);
841 close(p1);
842 close(p2);
843 close(c1);
844 close(c2);
845 return err;
846}
847
848static char *test_to_str(int test)
849{
850 switch (test) {
851 case SENDMSG:
852 return "sendmsg";
853 case SENDPAGE:
854 return "sendpage";
855 }
856 return "unknown";
857}
858
859#define OPTSTRING 60
860static void test_options(char *options)
861{
862 memset(options, 0, OPTSTRING);
863
864 if (txmsg_pass)
865 strncat(options, "pass,", OPTSTRING);
866 if (txmsg_noisy)
867 strncat(options, "pass_noisy,", OPTSTRING);
868 if (txmsg_redir)
869 strncat(options, "redir,", OPTSTRING);
870 if (txmsg_redir_noisy)
871 strncat(options, "redir_noisy,", OPTSTRING);
872 if (txmsg_drop)
873 strncat(options, "drop,", OPTSTRING);
874 if (txmsg_apply)
875 strncat(options, "apply,", OPTSTRING);
876 if (txmsg_cork)
877 strncat(options, "cork,", OPTSTRING);
878 if (txmsg_start)
879 strncat(options, "start,", OPTSTRING);
880 if (txmsg_end)
881 strncat(options, "end,", OPTSTRING);
882 if (txmsg_ingress)
883 strncat(options, "ingress,", OPTSTRING);
884 if (txmsg_skb)
885 strncat(options, "skb,", OPTSTRING);
886}
887
888static int __test_exec(int cgrp, int test, struct sockmap_options *opt)
889{
890 char *options = calloc(60, sizeof(char));
891 int err;
892
893 if (test == SENDPAGE)
894 opt->sendpage = true;
895 else
896 opt->sendpage = false;
897
898 if (txmsg_drop)
899 opt->drop_expected = true;
900 else
901 opt->drop_expected = false;
902
903 test_options(options);
904
905 fprintf(stdout,
906 "[TEST %i]: (%i, %i, %i, %s, %s): ",
907 test_cnt, opt->rate, opt->iov_count, opt->iov_length,
908 test_to_str(test), options);
909 fflush(stdout);
910 err = run_options(opt, cgrp, test);
911 fprintf(stdout, "%s\n", !err ? "PASS" : "FAILED");
912 test_cnt++;
913 !err ? passed++ : failed++;
914 free(options);
915 return err;
916}
917
918static int test_exec(int cgrp, struct sockmap_options *opt)
919{
920 int err = __test_exec(cgrp, SENDMSG, opt);
921
16962b24
JF		 922 if (err)
923 goto out;
924
925 err = __test_exec(cgrp, SENDPAGE, opt);
16962b24
JF		 926out:
927 return err;
928}
929
930static int test_loop(int cgrp)
931{
932 struct sockmap_options opt;
933
934 int err, i, l, r;
935
936 opt.verbose = 0;
937 opt.base = false;
938 opt.sendpage = false;
939 opt.data_test = false;
940 opt.drop_expected = false;
941 opt.iov_count = 0;
942 opt.iov_length = 0;
943 opt.rate = 0;
944
a18fda1a
JF		 945 r = 1;
946 for (i = 1; i < 100; i += 33) {
947 for (l = 1; l < 100; l += 33) {
948 opt.rate = r;
949 opt.iov_count = i;
950 opt.iov_length = l;
951 err = test_exec(cgrp, &opt);
952 if (err)
953 goto out;
16962b24
JF		 954 }
955 }
a18fda1a 956 sched_yield();
16962b24
JF		 957out:
958 return err;
959}
960
961static int test_txmsg(int cgrp)
962{
963 int err;
964
965 txmsg_pass = txmsg_noisy = txmsg_redir_noisy = txmsg_drop = 0;
966 txmsg_apply = txmsg_cork = 0;
967 txmsg_ingress = txmsg_skb = 0;
968
969 txmsg_pass = 1;
970 err = test_loop(cgrp);
971 txmsg_pass = 0;
972 if (err)
973 goto out;
974
975 txmsg_redir = 1;
976 err = test_loop(cgrp);
977 txmsg_redir = 0;
978 if (err)
979 goto out;
980
981 txmsg_drop = 1;
982 err = test_loop(cgrp);
983 txmsg_drop = 0;
984 if (err)
985 goto out;
986
987 txmsg_redir = 1;
988 txmsg_ingress = 1;
989 err = test_loop(cgrp);
990 txmsg_redir = 0;
991 txmsg_ingress = 0;
992 if (err)
993 goto out;
994out:
995 txmsg_pass = 0;
996 txmsg_redir = 0;
997 txmsg_drop = 0;
998 return err;
999}
1000
1001static int test_send(struct sockmap_options *opt, int cgrp)
1002{
1003 int err;
1004
1005 opt->iov_length = 1;
1006 opt->iov_count = 1;
1007 opt->rate = 1;
1008 err = test_exec(cgrp, opt);
1009 if (err)
1010 goto out;
1011
1012 opt->iov_length = 1;
1013 opt->iov_count = 1024;
1014 opt->rate = 1;
1015 err = test_exec(cgrp, opt);
1016 if (err)
1017 goto out;
1018
1019 opt->iov_length = 1024;
1020 opt->iov_count = 1;
1021 opt->rate = 1;
1022 err = test_exec(cgrp, opt);
1023 if (err)
1024 goto out;
1025
1026 opt->iov_length = 1;
1027 opt->iov_count = 1;
1028 opt->rate = 1024;
1029 err = test_exec(cgrp, opt);
1030 if (err)
1031 goto out;
1032
1033 opt->iov_length = 256;
1034 opt->iov_count = 1024;
1035 opt->rate = 10;
1036 err = test_exec(cgrp, opt);
1037 if (err)
1038 goto out;
1039
1040 opt->rate = 100;
1041 opt->iov_count = 1;
1042 opt->iov_length = 5;
1043 err = test_exec(cgrp, opt);
1044 if (err)
1045 goto out;
1046out:
a18fda1a 1047 sched_yield();
16962b24
JF		 1048 return err;
1049}
1050
1051static int test_mixed(int cgrp)
1052{
1053 struct sockmap_options opt = {0};
1054 int err;
1055
1056 txmsg_pass = txmsg_noisy = txmsg_redir_noisy = txmsg_drop = 0;
1057 txmsg_apply = txmsg_cork = 0;
1058 txmsg_start = txmsg_end = 0;
1059 /* Test small and large iov_count values with pass/redir/apply/cork */
1060 txmsg_pass = 1;
1061 txmsg_redir = 0;
1062 txmsg_apply = 1;
1063 txmsg_cork = 0;
1064 err = test_send(&opt, cgrp);
1065 if (err)
1066 goto out;
1067
1068 txmsg_pass = 1;
1069 txmsg_redir = 0;
1070 txmsg_apply = 0;
1071 txmsg_cork = 1;
1072 err = test_send(&opt, cgrp);
1073 if (err)
1074 goto out;
1075
1076 txmsg_pass = 1;
1077 txmsg_redir = 0;
1078 txmsg_apply = 1;
1079 txmsg_cork = 1;
1080 err = test_send(&opt, cgrp);
1081 if (err)
1082 goto out;
1083
1084 txmsg_pass = 1;
1085 txmsg_redir = 0;
1086 txmsg_apply = 1024;
1087 txmsg_cork = 0;
1088 err = test_send(&opt, cgrp);
1089 if (err)
1090 goto out;
1091
1092 txmsg_pass = 1;
1093 txmsg_redir = 0;
1094 txmsg_apply = 0;
1095 txmsg_cork = 1024;
1096 err = test_send(&opt, cgrp);
1097 if (err)
1098 goto out;
1099
1100 txmsg_pass = 1;
1101 txmsg_redir = 0;
1102 txmsg_apply = 1024;
1103 txmsg_cork = 1024;
1104 err = test_send(&opt, cgrp);
1105 if (err)
1106 goto out;
1107
1108 txmsg_pass = 1;
1109 txmsg_redir = 0;
1110 txmsg_cork = 4096;
1111 txmsg_apply = 4096;
1112 err = test_send(&opt, cgrp);
1113 if (err)
1114 goto out;
1115
1116 txmsg_pass = 0;
1117 txmsg_redir = 1;
1118 txmsg_apply = 1;
1119 txmsg_cork = 0;
1120 err = test_send(&opt, cgrp);
1121 if (err)
1122 goto out;
1123
1124 txmsg_pass = 0;
1125 txmsg_redir = 1;
1126 txmsg_apply = 0;
1127 txmsg_cork = 1;
1128 err = test_send(&opt, cgrp);
1129 if (err)
1130 goto out;
1131
1132 txmsg_pass = 0;
1133 txmsg_redir = 1;
1134 txmsg_apply = 1024;
1135 txmsg_cork = 0;
1136 err = test_send(&opt, cgrp);
1137 if (err)
1138 goto out;
1139
1140 txmsg_pass = 0;
1141 txmsg_redir = 1;
1142 txmsg_apply = 0;
1143 txmsg_cork = 1024;
1144 err = test_send(&opt, cgrp);
1145 if (err)
1146 goto out;
1147
1148 txmsg_pass = 0;
1149 txmsg_redir = 1;
1150 txmsg_apply = 1024;
1151 txmsg_cork = 1024;
1152 err = test_send(&opt, cgrp);
1153 if (err)
1154 goto out;
1155
1156 txmsg_pass = 0;
1157 txmsg_redir = 1;
1158 txmsg_cork = 4096;
1159 txmsg_apply = 4096;
1160 err = test_send(&opt, cgrp);
1161 if (err)
1162 goto out;
1163out:
1164 return err;
1165}
1166
1167static int test_start_end(int cgrp)
1168{
1169 struct sockmap_options opt = {0};
1170 int err, i;
1171
1172 /* Test basic start/end with lots of iov_count and iov_lengths */
1173 txmsg_start = 1;
1174 txmsg_end = 2;
1175 err = test_txmsg(cgrp);
1176 if (err)
1177 goto out;
1178
1179 /* Test start/end with cork */
1180 opt.rate = 16;
1181 opt.iov_count = 1;
1182 opt.iov_length = 100;
1183 txmsg_cork = 1600;
1184
a18fda1a 1185 for (i = 99; i <= 1600; i += 500) {
16962b24
JF		 1186 txmsg_start = 0;
1187 txmsg_end = i;
1188 err = test_exec(cgrp, &opt);
1189 if (err)
1190 goto out;
1191 }
1192
1193 /* Test start/end with cork but pull data in middle */
a18fda1a 1194 for (i = 199; i <= 1600; i += 500) {
16962b24
JF		 1195 txmsg_start = 100;
1196 txmsg_end = i;
1197 err = test_exec(cgrp, &opt);
1198 if (err)
1199 goto out;
1200 }
1201
1202 /* Test start/end with cork pulling last sg entry */
1203 txmsg_start = 1500;
1204 txmsg_end = 1600;
1205 err = test_exec(cgrp, &opt);
1206 if (err)
1207 goto out;
1208
1209 /* Test start/end pull of single byte in last page */
1210 txmsg_start = 1111;
1211 txmsg_end = 1112;
1212 err = test_exec(cgrp, &opt);
1213 if (err)
1214 goto out;
1215
1216 /* Test start/end with end < start */
1217 txmsg_start = 1111;
1218 txmsg_end = 0;
1219 err = test_exec(cgrp, &opt);
1220 if (err)
1221 goto out;
1222
1223 /* Test start/end with end > data */
1224 txmsg_start = 0;
1225 txmsg_end = 1601;
1226 err = test_exec(cgrp, &opt);
1227 if (err)
1228 goto out;
1229
1230 /* Test start/end with start > data */
1231 txmsg_start = 1601;
1232 txmsg_end = 1600;
1233 err = test_exec(cgrp, &opt);
1234
1235out:
1236 txmsg_start = 0;
1237 txmsg_end = 0;
a18fda1a 1238 sched_yield();
16962b24
JF		 1239 return err;
1240}
1241
1242char *map_names[] = {
1243 "sock_map",
1244 "sock_map_txmsg",
1245 "sock_map_redir",
1246 "sock_apply_bytes",
1247 "sock_cork_bytes",
1248 "sock_pull_bytes",
1249 "sock_redir_flags",
1250 "sock_skb_opts",
1251};
1252
1253int prog_attach_type[] = {
1254 BPF_SK_SKB_STREAM_PARSER,
1255 BPF_SK_SKB_STREAM_VERDICT,
1256 BPF_CGROUP_SOCK_OPS,
1257 BPF_SK_MSG_VERDICT,
1258 BPF_SK_MSG_VERDICT,
1259 BPF_SK_MSG_VERDICT,
1260 BPF_SK_MSG_VERDICT,
1261 BPF_SK_MSG_VERDICT,
1262 BPF_SK_MSG_VERDICT,
1263 BPF_SK_MSG_VERDICT,
1264};
1265
1266int prog_type[] = {
1267 BPF_PROG_TYPE_SK_SKB,
1268 BPF_PROG_TYPE_SK_SKB,
1269 BPF_PROG_TYPE_SOCK_OPS,
1270 BPF_PROG_TYPE_SK_MSG,
1271 BPF_PROG_TYPE_SK_MSG,
1272 BPF_PROG_TYPE_SK_MSG,
1273 BPF_PROG_TYPE_SK_MSG,
1274 BPF_PROG_TYPE_SK_MSG,
1275 BPF_PROG_TYPE_SK_MSG,
1276 BPF_PROG_TYPE_SK_MSG,
1277};
1278
b8b394fa 1279static int populate_progs(char *bpf_file)
16962b24 1280{
16962b24
JF		 1281 struct bpf_program *prog;
1282 struct bpf_object *obj;
1283 int i = 0;
1284 long err;
1285
1286 obj = bpf_object__open(bpf_file);
1287 err = libbpf_get_error(obj);
1288 if (err) {
1289 char err_buf[256];
1290
1291 libbpf_strerror(err, err_buf, sizeof(err_buf));
1292 printf("Unable to load eBPF objects in file '%s' : %s\n",
1293 bpf_file, err_buf);
1294 return -1;
1295 }
1296
1297 bpf_object__for_each_program(prog, obj) {
1298 bpf_program__set_type(prog, prog_type[i]);
1299 bpf_program__set_expected_attach_type(prog,
1300 prog_attach_type[i]);
1301 i++;
1302 }
1303
1304 i = bpf_object__load(obj);
1305 i = 0;
1306 bpf_object__for_each_program(prog, obj) {
1307 prog_fd[i] = bpf_program__fd(prog);
1308 i++;
1309 }
1310
1311 for (i = 0; i < sizeof(map_fd)/sizeof(int); i++) {
1312 maps[i] = bpf_object__find_map_by_name(obj, map_names[i]);
1313 map_fd[i] = bpf_map__fd(maps[i]);
1314 if (map_fd[i] < 0) {
1315 fprintf(stderr, "load_bpf_file: (%i) %s\n",
1316 map_fd[i], strerror(errno));
1317 return -1;
1318 }
1319 }
1320
1321 return 0;
1322}
1323
b8b394fa 1324static int __test_suite(char *bpf_file)
16962b24
JF		 1325{
1326 int cg_fd, err;
1327
b8b394fa 1328 err = populate_progs(bpf_file);
16962b24
JF		 1329 if (err < 0) {
1330 fprintf(stderr, "ERROR: (%i) load bpf failed\n", err);
1331 return err;
1332 }
1333
1334 if (setup_cgroup_environment()) {
1335 fprintf(stderr, "ERROR: cgroup env failed\n");
1336 return -EINVAL;
1337 }
1338
1339 cg_fd = create_and_get_cgroup(CG_PATH);
1340 if (cg_fd < 0) {
1341 fprintf(stderr,
1342 "ERROR: (%i) open cg path failed: %s\n",
1343 cg_fd, optarg);
1344 return cg_fd;
1345 }
1346
035b37ff
PB		 1347 if (join_cgroup(CG_PATH)) {
1348 fprintf(stderr, "ERROR: failed to join cgroup\n");
1349 return -EINVAL;
1350 }
1351
16962b24
JF		 1352 /* Tests basic commands and APIs with range of iov values */
1353 txmsg_start = txmsg_end = 0;
1354 err = test_txmsg(cg_fd);
1355 if (err)
1356 goto out;
1357
1358 /* Tests interesting combinations of APIs used together */
1359 err = test_mixed(cg_fd);
1360 if (err)
1361 goto out;
1362
1363 /* Tests pull_data API using start/end API */
1364 err = test_start_end(cg_fd);
1365 if (err)
1366 goto out;
1367
1368out:
1369 printf("Summary: %i PASSED %i FAILED\n", passed, failed);
b8b394fa 1370 cleanup_cgroup_environment();
16962b24
JF		 1371 close(cg_fd);
1372 return err;
1373}
1374
b8b394fa
JF		 1375static int test_suite(void)
1376{
1377 int err;
1378
1379 err = __test_suite(BPF_SOCKMAP_FILENAME);
1380 if (err)
1381 goto out;
1382 err = __test_suite(BPF_SOCKHASH_FILENAME);
1383out:
1384 return err;
1385}
1386
16962b24
JF		 1387int main(int argc, char **argv)
1388{
1389 struct rlimit r = {10 * 1024 * 1024, RLIM_INFINITY};
1390 int iov_count = 1, length = 1024, rate = 1;
1391 struct sockmap_options options = {0};
1392 int opt, longindex, err, cg_fd = 0;
b8b394fa 1393 char *bpf_file = BPF_SOCKMAP_FILENAME;
16962b24
JF		 1394 int test = PING_PONG;
1395
1396 if (setrlimit(RLIMIT_MEMLOCK, &r)) {
1397 perror("setrlimit(RLIMIT_MEMLOCK)");
1398 return 1;
1399 }
1400
1401 if (argc < 2)
1402 return test_suite();
1403
1404 while ((opt = getopt_long(argc, argv, ":dhvc:r:i:l:t:",
1405 long_options, &longindex)) != -1) {
1406 switch (opt) {
1407 case 's':
1408 txmsg_start = atoi(optarg);
1409 break;
1410 case 'e':
1411 txmsg_end = atoi(optarg);
1412 break;
1413 case 'a':
1414 txmsg_apply = atoi(optarg);
1415 break;
1416 case 'k':
1417 txmsg_cork = atoi(optarg);
1418 break;
1419 case 'c':
1420 cg_fd = open(optarg, O_DIRECTORY, O_RDONLY);
1421 if (cg_fd < 0) {
1422 fprintf(stderr,
1423 "ERROR: (%i) open cg path failed: %s\n",
1424 cg_fd, optarg);
1425 return cg_fd;
1426 }
1427 break;
1428 case 'r':
1429 rate = atoi(optarg);
1430 break;
1431 case 'v':
1432 options.verbose = 1;
1433 break;
1434 case 'i':
1435 iov_count = atoi(optarg);
1436 break;
1437 case 'l':
1438 length = atoi(optarg);
1439 break;
1440 case 'd':
1441 options.data_test = true;
1442 break;
1443 case 't':
1444 if (strcmp(optarg, "ping") == 0) {
1445 test = PING_PONG;
1446 } else if (strcmp(optarg, "sendmsg") == 0) {
1447 test = SENDMSG;
1448 } else if (strcmp(optarg, "base") == 0) {
1449 test = BASE;
1450 } else if (strcmp(optarg, "base_sendpage") == 0) {
1451 test = BASE_SENDPAGE;
1452 } else if (strcmp(optarg, "sendpage") == 0) {
1453 test = SENDPAGE;
1454 } else {
1455 usage(argv);
1456 return -1;
1457 }
1458 break;
1459 case 0:
1460 break;
1461 case 'h':
1462 default:
1463 usage(argv);
1464 return -1;
1465 }
1466 }
1467
1468 if (!cg_fd) {
1469 fprintf(stderr, "%s requires cgroup option: --cgroup <path>\n",
1470 argv[0]);
1471 return -1;
1472 }
1473
b8b394fa 1474 err = populate_progs(bpf_file);
16962b24
JF		 1475 if (err) {
1476 fprintf(stderr, "populate program: (%s) %s\n",
1477 bpf_file, strerror(errno));
1478 return 1;
1479 }
1480 running = 1;
1481
1482 /* catch SIGINT */
1483 signal(SIGINT, running_handler);
1484
1485 options.iov_count = iov_count;
1486 options.iov_length = length;
1487 options.rate = rate;
1488
1489 err = run_options(&options, cg_fd, test);
1490 close(cg_fd);
1491 return err;
1492}
1493
1494void running_handler(int a)
1495{
1496 running = 0;
1497}
Linux 6.x block layer and io_uring tree(s)