1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2017 - 2018 Intel Corporation. */
4 #include <asm/barrier.h>
9 #include <linux/compiler.h>
10 #include <linux/if_link.h>
11 #include <linux/if_xdp.h>
12 #include <linux/if_ether.h>
14 #include <net/ethernet.h>
24 #include <sys/resource.h>
25 #include <sys/socket.h>
26 #include <sys/types.h>
47 #define NUM_FRAMES (4 * 1024)
50 #define DEBUG_HEXDUMP 0
55 static unsigned long prev_time;
63 static enum benchmark_type opt_bench = BENCH_RXDROP;
64 static u32 opt_xdp_flags = XDP_FLAGS_UPDATE_IF_NOEXIST;
65 static const char *opt_if = "";
66 static int opt_ifindex;
69 static int opt_interval = 1;
70 static u32 opt_xdp_bind_flags = XDP_USE_NEED_WAKEUP;
71 static u32 opt_umem_flags;
72 static int opt_unaligned_chunks;
73 static int opt_mmap_flags;
74 static u32 opt_xdp_bind_flags;
75 static int opt_xsk_frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE;
76 static int opt_timeout = 1000;
77 static bool opt_need_wakeup = true;
78 static u32 opt_num_xsks = 1;
81 struct xsk_umem_info {
82 struct xsk_ring_prod fq;
83 struct xsk_ring_cons cq;
84 struct xsk_umem *umem;
88 struct xsk_socket_info {
89 struct xsk_ring_cons rx;
90 struct xsk_ring_prod tx;
91 struct xsk_umem_info *umem;
92 struct xsk_socket *xsk;
93 unsigned long rx_npkts;
94 unsigned long tx_npkts;
95 unsigned long prev_rx_npkts;
96 unsigned long prev_tx_npkts;
100 static int num_socks;
101 struct xsk_socket_info *xsks[MAX_SOCKS];
103 static unsigned long get_nsecs(void)
107 clock_gettime(CLOCK_MONOTONIC, &ts);
108 return ts.tv_sec * 1000000000UL + ts.tv_nsec;
111 static void print_benchmark(bool running)
113 const char *bench_str = "INVALID";
115 if (opt_bench == BENCH_RXDROP)
116 bench_str = "rxdrop";
117 else if (opt_bench == BENCH_TXONLY)
118 bench_str = "txonly";
119 else if (opt_bench == BENCH_L2FWD)
122 printf("%s:%d %s ", opt_if, opt_queue, bench_str);
123 if (opt_xdp_flags & XDP_FLAGS_SKB_MODE)
125 else if (opt_xdp_flags & XDP_FLAGS_DRV_MODE)
134 printf("running...");
139 static void dump_stats(void)
141 unsigned long now = get_nsecs();
142 long dt = now - prev_time;
147 for (i = 0; i < num_socks && xsks[i]; i++) {
148 char *fmt = "%-15s %'-11.0f %'-11lu\n";
149 double rx_pps, tx_pps;
151 rx_pps = (xsks[i]->rx_npkts - xsks[i]->prev_rx_npkts) *
153 tx_pps = (xsks[i]->tx_npkts - xsks[i]->prev_tx_npkts) *
156 printf("\n sock%d@", i);
157 print_benchmark(false);
160 printf("%-15s %-11s %-11s %-11.2f\n", "", "pps", "pkts",
162 printf(fmt, "rx", rx_pps, xsks[i]->rx_npkts);
163 printf(fmt, "tx", tx_pps, xsks[i]->tx_npkts);
165 xsks[i]->prev_rx_npkts = xsks[i]->rx_npkts;
166 xsks[i]->prev_tx_npkts = xsks[i]->tx_npkts;
170 static void *poller(void *arg)
181 static void remove_xdp_program(void)
183 u32 curr_prog_id = 0;
185 if (bpf_get_link_xdp_id(opt_ifindex, &curr_prog_id, opt_xdp_flags)) {
186 printf("bpf_get_link_xdp_id failed\n");
189 if (prog_id == curr_prog_id)
190 bpf_set_link_xdp_fd(opt_ifindex, -1, opt_xdp_flags);
191 else if (!curr_prog_id)
192 printf("couldn't find a prog id on a given interface\n");
194 printf("program on interface changed, not removing\n");
197 static void int_exit(int sig)
199 struct xsk_umem *umem = xsks[0]->umem->umem;
203 for (i = 0; i < num_socks; i++)
204 xsk_socket__delete(xsks[i]->xsk);
205 (void)xsk_umem__delete(umem);
206 remove_xdp_program();
211 static void __exit_with_error(int error, const char *file, const char *func,
214 fprintf(stderr, "%s:%s:%i: errno: %d/\"%s\"\n", file, func,
215 line, error, strerror(error));
217 remove_xdp_program();
221 #define exit_with_error(error) __exit_with_error(error, __FILE__, __func__, \
224 static const char pkt_data[] =
225 "\x3c\xfd\xfe\x9e\x7f\x71\xec\xb1\xd7\x98\x3a\xc0\x08\x00\x45\x00"
226 "\x00\x2e\x00\x00\x00\x00\x40\x11\x88\x97\x05\x08\x07\x08\xc8\x14"
227 "\x1e\x04\x10\x92\x10\x92\x00\x1a\x6d\xa3\x34\x33\x1f\x69\x40\x6b"
228 "\x54\x59\xb6\x14\x2d\x11\x44\xbf\xaf\xd9\xbe\xaa";
230 static void swap_mac_addresses(void *data)
232 struct ether_header *eth = (struct ether_header *)data;
233 struct ether_addr *src_addr = (struct ether_addr *)ð->ether_shost;
234 struct ether_addr *dst_addr = (struct ether_addr *)ð->ether_dhost;
235 struct ether_addr tmp;
238 *src_addr = *dst_addr;
242 static void hex_dump(void *pkt, size_t length, u64 addr)
244 const unsigned char *address = (unsigned char *)pkt;
245 const unsigned char *line = address;
246 size_t line_size = 32;
254 sprintf(buf, "addr=%llu", addr);
255 printf("length = %zu\n", length);
256 printf("%s | ", buf);
257 while (length-- > 0) {
258 printf("%02X ", *address++);
259 if (!(++i % line_size) || (length == 0 && i % line_size)) {
261 while (i++ % line_size)
264 printf(" | "); /* right close */
265 while (line < address) {
267 printf("%c", (c < 33 || c == 255) ? 0x2E : c);
271 printf("%s | ", buf);
277 static size_t gen_eth_frame(struct xsk_umem_info *umem, u64 addr)
279 memcpy(xsk_umem__get_data(umem->buffer, addr), pkt_data,
280 sizeof(pkt_data) - 1);
281 return sizeof(pkt_data) - 1;
284 static struct xsk_umem_info *xsk_configure_umem(void *buffer, u64 size)
286 struct xsk_umem_info *umem;
287 struct xsk_umem_config cfg = {
288 .fill_size = XSK_RING_PROD__DEFAULT_NUM_DESCS,
289 .comp_size = XSK_RING_CONS__DEFAULT_NUM_DESCS,
290 .frame_size = opt_xsk_frame_size,
291 .frame_headroom = XSK_UMEM__DEFAULT_FRAME_HEADROOM,
292 .flags = opt_umem_flags
296 umem = calloc(1, sizeof(*umem));
298 exit_with_error(errno);
300 ret = xsk_umem__create(&umem->umem, buffer, size, &umem->fq, &umem->cq,
303 exit_with_error(-ret);
305 umem->buffer = buffer;
309 static void xsk_populate_fill_ring(struct xsk_umem_info *umem)
314 ret = xsk_ring_prod__reserve(&umem->fq,
315 XSK_RING_PROD__DEFAULT_NUM_DESCS, &idx);
316 if (ret != XSK_RING_PROD__DEFAULT_NUM_DESCS)
317 exit_with_error(-ret);
318 for (i = 0; i < XSK_RING_PROD__DEFAULT_NUM_DESCS; i++)
319 *xsk_ring_prod__fill_addr(&umem->fq, idx++) =
320 i * opt_xsk_frame_size;
321 xsk_ring_prod__submit(&umem->fq, XSK_RING_PROD__DEFAULT_NUM_DESCS);
324 static struct xsk_socket_info *xsk_configure_socket(struct xsk_umem_info *umem,
327 struct xsk_socket_config cfg;
328 struct xsk_socket_info *xsk;
329 struct xsk_ring_cons *rxr;
330 struct xsk_ring_prod *txr;
333 xsk = calloc(1, sizeof(*xsk));
335 exit_with_error(errno);
338 cfg.rx_size = XSK_RING_CONS__DEFAULT_NUM_DESCS;
339 cfg.tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS;
340 if (opt_num_xsks > 1)
341 cfg.libbpf_flags = XSK_LIBBPF_FLAGS__INHIBIT_PROG_LOAD;
343 cfg.libbpf_flags = 0;
344 cfg.xdp_flags = opt_xdp_flags;
345 cfg.bind_flags = opt_xdp_bind_flags;
347 rxr = rx ? &xsk->rx : NULL;
348 txr = tx ? &xsk->tx : NULL;
349 ret = xsk_socket__create(&xsk->xsk, opt_if, opt_queue, umem->umem,
352 exit_with_error(-ret);
354 ret = bpf_get_link_xdp_id(opt_ifindex, &prog_id, opt_xdp_flags);
356 exit_with_error(-ret);
361 static struct option long_options[] = {
362 {"rxdrop", no_argument, 0, 'r'},
363 {"txonly", no_argument, 0, 't'},
364 {"l2fwd", no_argument, 0, 'l'},
365 {"interface", required_argument, 0, 'i'},
366 {"queue", required_argument, 0, 'q'},
367 {"poll", no_argument, 0, 'p'},
368 {"xdp-skb", no_argument, 0, 'S'},
369 {"xdp-native", no_argument, 0, 'N'},
370 {"interval", required_argument, 0, 'n'},
371 {"zero-copy", no_argument, 0, 'z'},
372 {"copy", no_argument, 0, 'c'},
373 {"frame-size", required_argument, 0, 'f'},
374 {"no-need-wakeup", no_argument, 0, 'm'},
375 {"unaligned", no_argument, 0, 'u'},
376 {"shared-umem", no_argument, 0, 'M'},
377 {"force", no_argument, 0, 'F'},
381 static void usage(const char *prog)
384 " Usage: %s [OPTIONS]\n"
386 " -r, --rxdrop Discard all incoming packets (default)\n"
387 " -t, --txonly Only send packets\n"
388 " -l, --l2fwd MAC swap L2 forwarding\n"
389 " -i, --interface=n Run on interface n\n"
390 " -q, --queue=n Use queue n (default 0)\n"
391 " -p, --poll Use poll syscall\n"
392 " -S, --xdp-skb=n Use XDP skb-mod\n"
393 " -N, --xdp-native=n Enforce XDP native mode\n"
394 " -n, --interval=n Specify statistics update interval (default 1 sec).\n"
395 " -z, --zero-copy Force zero-copy mode.\n"
396 " -c, --copy Force copy mode.\n"
397 " -m, --no-need-wakeup Turn off use of driver need wakeup flag.\n"
398 " -f, --frame-size=n Set the frame size (must be a power of two in aligned mode, default is %d).\n"
399 " -u, --unaligned Enable unaligned chunk placement\n"
400 " -M, --shared-umem Enable XDP_SHARED_UMEM\n"
401 " -F, --force Force loading the XDP prog\n"
403 fprintf(stderr, str, prog, XSK_UMEM__DEFAULT_FRAME_SIZE);
407 static void parse_command_line(int argc, char **argv)
414 c = getopt_long(argc, argv, "Frtli:q:psSNn:czf:muM",
415 long_options, &option_index);
421 opt_bench = BENCH_RXDROP;
424 opt_bench = BENCH_TXONLY;
427 opt_bench = BENCH_L2FWD;
433 opt_queue = atoi(optarg);
439 opt_xdp_flags |= XDP_FLAGS_SKB_MODE;
440 opt_xdp_bind_flags |= XDP_COPY;
443 opt_xdp_flags |= XDP_FLAGS_DRV_MODE;
446 opt_interval = atoi(optarg);
449 opt_xdp_bind_flags |= XDP_ZEROCOPY;
452 opt_xdp_bind_flags |= XDP_COPY;
455 opt_umem_flags |= XDP_UMEM_UNALIGNED_CHUNK_FLAG;
456 opt_unaligned_chunks = 1;
457 opt_mmap_flags = MAP_HUGETLB;
460 opt_xdp_flags &= ~XDP_FLAGS_UPDATE_IF_NOEXIST;
463 opt_xsk_frame_size = atoi(optarg);
466 opt_need_wakeup = false;
467 opt_xdp_bind_flags &= ~XDP_USE_NEED_WAKEUP;
470 opt_num_xsks = MAX_SOCKS;
473 usage(basename(argv[0]));
477 opt_ifindex = if_nametoindex(opt_if);
479 fprintf(stderr, "ERROR: interface \"%s\" does not exist\n",
481 usage(basename(argv[0]));
484 if ((opt_xsk_frame_size & (opt_xsk_frame_size - 1)) &&
485 !opt_unaligned_chunks) {
486 fprintf(stderr, "--frame-size=%d is not a power of two\n",
488 usage(basename(argv[0]));
492 static void kick_tx(struct xsk_socket_info *xsk)
496 ret = sendto(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, 0);
497 if (ret >= 0 || errno == ENOBUFS || errno == EAGAIN || errno == EBUSY)
499 exit_with_error(errno);
502 static inline void complete_tx_l2fwd(struct xsk_socket_info *xsk,
505 struct xsk_umem_info *umem = xsk->umem;
506 u32 idx_cq = 0, idx_fq = 0;
510 if (!xsk->outstanding_tx)
513 if (!opt_need_wakeup || xsk_ring_prod__needs_wakeup(&xsk->tx))
516 ndescs = (xsk->outstanding_tx > BATCH_SIZE) ? BATCH_SIZE :
519 /* re-add completed Tx buffers */
520 rcvd = xsk_ring_cons__peek(&umem->cq, ndescs, &idx_cq);
525 ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq);
526 while (ret != rcvd) {
528 exit_with_error(-ret);
529 if (xsk_ring_prod__needs_wakeup(&umem->fq))
530 ret = poll(fds, num_socks, opt_timeout);
531 ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq);
534 for (i = 0; i < rcvd; i++)
535 *xsk_ring_prod__fill_addr(&umem->fq, idx_fq++) =
536 *xsk_ring_cons__comp_addr(&umem->cq, idx_cq++);
538 xsk_ring_prod__submit(&xsk->umem->fq, rcvd);
539 xsk_ring_cons__release(&xsk->umem->cq, rcvd);
540 xsk->outstanding_tx -= rcvd;
541 xsk->tx_npkts += rcvd;
545 static inline void complete_tx_only(struct xsk_socket_info *xsk)
550 if (!xsk->outstanding_tx)
553 if (!opt_need_wakeup || xsk_ring_prod__needs_wakeup(&xsk->tx))
556 rcvd = xsk_ring_cons__peek(&xsk->umem->cq, BATCH_SIZE, &idx);
558 xsk_ring_cons__release(&xsk->umem->cq, rcvd);
559 xsk->outstanding_tx -= rcvd;
560 xsk->tx_npkts += rcvd;
564 static void rx_drop(struct xsk_socket_info *xsk, struct pollfd *fds)
566 unsigned int rcvd, i;
567 u32 idx_rx = 0, idx_fq = 0;
570 rcvd = xsk_ring_cons__peek(&xsk->rx, BATCH_SIZE, &idx_rx);
572 if (xsk_ring_prod__needs_wakeup(&xsk->umem->fq))
573 ret = poll(fds, num_socks, opt_timeout);
577 ret = xsk_ring_prod__reserve(&xsk->umem->fq, rcvd, &idx_fq);
578 while (ret != rcvd) {
580 exit_with_error(-ret);
581 if (xsk_ring_prod__needs_wakeup(&xsk->umem->fq))
582 ret = poll(fds, num_socks, opt_timeout);
583 ret = xsk_ring_prod__reserve(&xsk->umem->fq, rcvd, &idx_fq);
586 for (i = 0; i < rcvd; i++) {
587 u64 addr = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx)->addr;
588 u32 len = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx++)->len;
589 u64 orig = xsk_umem__extract_addr(addr);
591 addr = xsk_umem__add_offset_to_addr(addr);
592 char *pkt = xsk_umem__get_data(xsk->umem->buffer, addr);
594 hex_dump(pkt, len, addr);
595 *xsk_ring_prod__fill_addr(&xsk->umem->fq, idx_fq++) = orig;
598 xsk_ring_prod__submit(&xsk->umem->fq, rcvd);
599 xsk_ring_cons__release(&xsk->rx, rcvd);
600 xsk->rx_npkts += rcvd;
603 static void rx_drop_all(void)
605 struct pollfd fds[MAX_SOCKS] = {};
608 for (i = 0; i < num_socks; i++) {
609 fds[i].fd = xsk_socket__fd(xsks[i]->xsk);
610 fds[i].events = POLLIN;
615 ret = poll(fds, num_socks, opt_timeout);
620 for (i = 0; i < num_socks; i++)
621 rx_drop(xsks[i], fds);
625 static void tx_only(struct xsk_socket_info *xsk, u32 frame_nb)
629 if (xsk_ring_prod__reserve(&xsk->tx, BATCH_SIZE, &idx) == BATCH_SIZE) {
632 for (i = 0; i < BATCH_SIZE; i++) {
633 xsk_ring_prod__tx_desc(&xsk->tx, idx + i)->addr =
634 (frame_nb + i) << XSK_UMEM__DEFAULT_FRAME_SHIFT;
635 xsk_ring_prod__tx_desc(&xsk->tx, idx + i)->len =
636 sizeof(pkt_data) - 1;
639 xsk_ring_prod__submit(&xsk->tx, BATCH_SIZE);
640 xsk->outstanding_tx += BATCH_SIZE;
641 frame_nb += BATCH_SIZE;
642 frame_nb %= NUM_FRAMES;
645 complete_tx_only(xsk);
648 static void tx_only_all(void)
650 struct pollfd fds[MAX_SOCKS] = {};
651 u32 frame_nb[MAX_SOCKS] = {};
654 for (i = 0; i < num_socks; i++) {
655 fds[0].fd = xsk_socket__fd(xsks[i]->xsk);
656 fds[0].events = POLLOUT;
661 ret = poll(fds, num_socks, opt_timeout);
665 if (!(fds[0].revents & POLLOUT))
669 for (i = 0; i < num_socks; i++)
670 tx_only(xsks[i], frame_nb[i]);
674 static void l2fwd(struct xsk_socket_info *xsk, struct pollfd *fds)
676 unsigned int rcvd, i;
677 u32 idx_rx = 0, idx_tx = 0;
680 complete_tx_l2fwd(xsk, fds);
682 rcvd = xsk_ring_cons__peek(&xsk->rx, BATCH_SIZE, &idx_rx);
684 if (xsk_ring_prod__needs_wakeup(&xsk->umem->fq))
685 ret = poll(fds, num_socks, opt_timeout);
689 ret = xsk_ring_prod__reserve(&xsk->tx, rcvd, &idx_tx);
690 while (ret != rcvd) {
692 exit_with_error(-ret);
693 if (xsk_ring_prod__needs_wakeup(&xsk->tx))
695 ret = xsk_ring_prod__reserve(&xsk->tx, rcvd, &idx_tx);
698 for (i = 0; i < rcvd; i++) {
699 u64 addr = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx)->addr;
700 u32 len = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx++)->len;
703 addr = xsk_umem__add_offset_to_addr(addr);
704 char *pkt = xsk_umem__get_data(xsk->umem->buffer, addr);
706 swap_mac_addresses(pkt);
708 hex_dump(pkt, len, addr);
709 xsk_ring_prod__tx_desc(&xsk->tx, idx_tx)->addr = orig;
710 xsk_ring_prod__tx_desc(&xsk->tx, idx_tx++)->len = len;
713 xsk_ring_prod__submit(&xsk->tx, rcvd);
714 xsk_ring_cons__release(&xsk->rx, rcvd);
716 xsk->rx_npkts += rcvd;
717 xsk->outstanding_tx += rcvd;
720 static void l2fwd_all(void)
722 struct pollfd fds[MAX_SOCKS] = {};
725 for (i = 0; i < num_socks; i++) {
726 fds[i].fd = xsk_socket__fd(xsks[i]->xsk);
727 fds[i].events = POLLOUT | POLLIN;
732 ret = poll(fds, num_socks, opt_timeout);
737 for (i = 0; i < num_socks; i++)
742 static void load_xdp_program(char **argv, struct bpf_object **obj)
744 struct bpf_prog_load_attr prog_load_attr = {
745 .prog_type = BPF_PROG_TYPE_XDP,
747 char xdp_filename[256];
750 snprintf(xdp_filename, sizeof(xdp_filename), "%s_kern.o", argv[0]);
751 prog_load_attr.file = xdp_filename;
753 if (bpf_prog_load_xattr(&prog_load_attr, obj, &prog_fd))
756 fprintf(stderr, "ERROR: no program found: %s\n",
761 if (bpf_set_link_xdp_fd(opt_ifindex, prog_fd, opt_xdp_flags) < 0) {
762 fprintf(stderr, "ERROR: link set xdp fd failed\n");
767 static void enter_xsks_into_map(struct bpf_object *obj)
772 map = bpf_object__find_map_by_name(obj, "xsks_map");
773 xsks_map = bpf_map__fd(map);
775 fprintf(stderr, "ERROR: no xsks map found: %s\n",
780 for (i = 0; i < num_socks; i++) {
781 int fd = xsk_socket__fd(xsks[i]->xsk);
785 ret = bpf_map_update_elem(xsks_map, &key, &fd, 0);
787 fprintf(stderr, "ERROR: bpf_map_update_elem %d\n", i);
793 int main(int argc, char **argv)
795 struct rlimit r = {RLIM_INFINITY, RLIM_INFINITY};
796 bool rx = false, tx = false;
797 struct xsk_umem_info *umem;
798 struct bpf_object *obj;
803 parse_command_line(argc, argv);
805 if (setrlimit(RLIMIT_MEMLOCK, &r)) {
806 fprintf(stderr, "ERROR: setrlimit(RLIMIT_MEMLOCK) \"%s\"\n",
811 if (opt_num_xsks > 1)
812 load_xdp_program(argv, &obj);
814 /* Reserve memory for the umem. Use hugepages if unaligned chunk mode */
815 bufs = mmap(NULL, NUM_FRAMES * opt_xsk_frame_size,
816 PROT_READ | PROT_WRITE,
817 MAP_PRIVATE | MAP_ANONYMOUS | opt_mmap_flags, -1, 0);
818 if (bufs == MAP_FAILED) {
819 printf("ERROR: mmap failed\n");
823 /* Create sockets... */
824 umem = xsk_configure_umem(bufs, NUM_FRAMES * opt_xsk_frame_size);
825 if (opt_bench == BENCH_RXDROP || opt_bench == BENCH_L2FWD) {
827 xsk_populate_fill_ring(umem);
829 if (opt_bench == BENCH_L2FWD || opt_bench == BENCH_TXONLY)
831 for (i = 0; i < opt_num_xsks; i++)
832 xsks[num_socks++] = xsk_configure_socket(umem, rx, tx);
834 if (opt_bench == BENCH_TXONLY)
835 for (i = 0; i < NUM_FRAMES; i++)
836 gen_eth_frame(umem, i * opt_xsk_frame_size);
838 if (opt_num_xsks > 1 && opt_bench != BENCH_TXONLY)
839 enter_xsks_into_map(obj);
841 signal(SIGINT, int_exit);
842 signal(SIGTERM, int_exit);
843 signal(SIGABRT, int_exit);
845 setlocale(LC_ALL, "");
847 ret = pthread_create(&pt, NULL, poller, NULL);
849 exit_with_error(ret);
851 prev_time = get_nsecs();
853 if (opt_bench == BENCH_RXDROP)
855 else if (opt_bench == BENCH_TXONLY)