Commit | Line | Data |
---|---|---|
b4b8faa1 | 1 | // SPDX-License-Identifier: GPL-2.0 |
dac09149 | 2 | /* Copyright(c) 2017 - 2018 Intel Corporation. */ |
b4b8faa1 | 3 | |
248c7f9c | 4 | #include <asm/barrier.h> |
b4b8faa1 MK |
5 | #include <errno.h> |
6 | #include <getopt.h> | |
7 | #include <libgen.h> | |
8 | #include <linux/bpf.h> | |
248c7f9c | 9 | #include <linux/compiler.h> |
b4b8faa1 MK |
10 | #include <linux/if_link.h> |
11 | #include <linux/if_xdp.h> | |
12 | #include <linux/if_ether.h> | |
248c7f9c MK |
13 | #include <locale.h> |
14 | #include <net/ethernet.h> | |
b4b8faa1 | 15 | #include <net/if.h> |
248c7f9c MK |
16 | #include <poll.h> |
17 | #include <pthread.h> | |
b4b8faa1 MK |
18 | #include <signal.h> |
19 | #include <stdbool.h> | |
20 | #include <stdio.h> | |
21 | #include <stdlib.h> | |
22 | #include <string.h> | |
248c7f9c | 23 | #include <sys/mman.h> |
b4b8faa1 MK |
24 | #include <sys/resource.h> |
25 | #include <sys/socket.h> | |
248c7f9c | 26 | #include <sys/types.h> |
b4b8faa1 MK |
27 | #include <time.h> |
28 | #include <unistd.h> | |
b4b8faa1 | 29 | |
6748182c | 30 | #include "bpf/libbpf.h" |
248c7f9c | 31 | #include "bpf/xsk.h" |
2bf3e2ef | 32 | #include <bpf/bpf.h> |
b4b8faa1 | 33 | |
b4b8faa1 MK |
34 | #ifndef SOL_XDP |
35 | #define SOL_XDP 283 | |
36 | #endif | |
37 | ||
38 | #ifndef AF_XDP | |
39 | #define AF_XDP 44 | |
40 | #endif | |
41 | ||
42 | #ifndef PF_XDP | |
43 | #define PF_XDP AF_XDP | |
44 | #endif | |
45 | ||
248c7f9c MK |
46 | #define NUM_FRAMES (4 * 1024) |
47 | #define BATCH_SIZE 64 | |
b4b8faa1 MK |
48 | |
49 | #define DEBUG_HEXDUMP 0 | |
248c7f9c | 50 | #define MAX_SOCKS 8 |
b4b8faa1 | 51 | |
a412ef54 | 52 | typedef __u64 u64; |
b4b8faa1 MK |
53 | typedef __u32 u32; |
54 | ||
55 | static unsigned long prev_time; | |
56 | ||
57 | enum benchmark_type { | |
58 | BENCH_RXDROP = 0, | |
59 | BENCH_TXONLY = 1, | |
60 | BENCH_L2FWD = 2, | |
61 | }; | |
62 | ||
63 | static enum benchmark_type opt_bench = BENCH_RXDROP; | |
743e568c | 64 | static u32 opt_xdp_flags = XDP_FLAGS_UPDATE_IF_NOEXIST; |
b4b8faa1 MK |
65 | static const char *opt_if = ""; |
66 | static int opt_ifindex; | |
67 | static int opt_queue; | |
68 | static int opt_poll; | |
b4b8faa1 | 69 | static int opt_interval = 1; |
9f5232cc | 70 | static u32 opt_xdp_bind_flags; |
3b7a8ec2 | 71 | static __u32 prog_id; |
b4b8faa1 | 72 | |
248c7f9c MK |
73 | struct xsk_umem_info { |
74 | struct xsk_ring_prod fq; | |
75 | struct xsk_ring_cons cq; | |
76 | struct xsk_umem *umem; | |
77 | void *buffer; | |
b4b8faa1 MK |
78 | }; |
79 | ||
248c7f9c MK |
80 | struct xsk_socket_info { |
81 | struct xsk_ring_cons rx; | |
82 | struct xsk_ring_prod tx; | |
83 | struct xsk_umem_info *umem; | |
84 | struct xsk_socket *xsk; | |
b4b8faa1 MK |
85 | unsigned long rx_npkts; |
86 | unsigned long tx_npkts; | |
87 | unsigned long prev_rx_npkts; | |
88 | unsigned long prev_tx_npkts; | |
248c7f9c | 89 | u32 outstanding_tx; |
b4b8faa1 MK |
90 | }; |
91 | ||
b4b8faa1 | 92 | static int num_socks; |
248c7f9c | 93 | struct xsk_socket_info *xsks[MAX_SOCKS]; |
b4b8faa1 MK |
94 | |
95 | static unsigned long get_nsecs(void) | |
96 | { | |
97 | struct timespec ts; | |
98 | ||
99 | clock_gettime(CLOCK_MONOTONIC, &ts); | |
100 | return ts.tv_sec * 1000000000UL + ts.tv_nsec; | |
101 | } | |
102 | ||
248c7f9c | 103 | static void print_benchmark(bool running) |
b4b8faa1 | 104 | { |
248c7f9c | 105 | const char *bench_str = "INVALID"; |
b4b8faa1 | 106 | |
248c7f9c MK |
107 | if (opt_bench == BENCH_RXDROP) |
108 | bench_str = "rxdrop"; | |
109 | else if (opt_bench == BENCH_TXONLY) | |
110 | bench_str = "txonly"; | |
111 | else if (opt_bench == BENCH_L2FWD) | |
112 | bench_str = "l2fwd"; | |
b4b8faa1 | 113 | |
248c7f9c MK |
114 | printf("%s:%d %s ", opt_if, opt_queue, bench_str); |
115 | if (opt_xdp_flags & XDP_FLAGS_SKB_MODE) | |
116 | printf("xdp-skb "); | |
117 | else if (opt_xdp_flags & XDP_FLAGS_DRV_MODE) | |
118 | printf("xdp-drv "); | |
119 | else | |
120 | printf(" "); | |
b4b8faa1 | 121 | |
248c7f9c MK |
122 | if (opt_poll) |
123 | printf("poll() "); | |
b4b8faa1 | 124 | |
248c7f9c MK |
125 | if (running) { |
126 | printf("running..."); | |
127 | fflush(stdout); | |
b4b8faa1 | 128 | } |
b4b8faa1 MK |
129 | } |
130 | ||
248c7f9c | 131 | static void dump_stats(void) |
b4b8faa1 | 132 | { |
248c7f9c MK |
133 | unsigned long now = get_nsecs(); |
134 | long dt = now - prev_time; | |
135 | int i; | |
b4b8faa1 | 136 | |
248c7f9c | 137 | prev_time = now; |
b4b8faa1 | 138 | |
248c7f9c MK |
139 | for (i = 0; i < num_socks && xsks[i]; i++) { |
140 | char *fmt = "%-15s %'-11.0f %'-11lu\n"; | |
141 | double rx_pps, tx_pps; | |
b4b8faa1 | 142 | |
248c7f9c MK |
143 | rx_pps = (xsks[i]->rx_npkts - xsks[i]->prev_rx_npkts) * |
144 | 1000000000. / dt; | |
145 | tx_pps = (xsks[i]->tx_npkts - xsks[i]->prev_tx_npkts) * | |
146 | 1000000000. / dt; | |
b4b8faa1 | 147 | |
248c7f9c MK |
148 | printf("\n sock%d@", i); |
149 | print_benchmark(false); | |
150 | printf("\n"); | |
b4b8faa1 | 151 | |
248c7f9c MK |
152 | printf("%-15s %-11s %-11s %-11.2f\n", "", "pps", "pkts", |
153 | dt / 1000000000.); | |
154 | printf(fmt, "rx", rx_pps, xsks[i]->rx_npkts); | |
155 | printf(fmt, "tx", tx_pps, xsks[i]->tx_npkts); | |
b4b8faa1 | 156 | |
248c7f9c MK |
157 | xsks[i]->prev_rx_npkts = xsks[i]->rx_npkts; |
158 | xsks[i]->prev_tx_npkts = xsks[i]->tx_npkts; | |
b4b8faa1 | 159 | } |
b4b8faa1 MK |
160 | } |
161 | ||
248c7f9c | 162 | static void *poller(void *arg) |
b4b8faa1 | 163 | { |
248c7f9c MK |
164 | (void)arg; |
165 | for (;;) { | |
166 | sleep(opt_interval); | |
167 | dump_stats(); | |
b4b8faa1 MK |
168 | } |
169 | ||
248c7f9c | 170 | return NULL; |
b4b8faa1 MK |
171 | } |
172 | ||
248c7f9c | 173 | static void remove_xdp_program(void) |
b4b8faa1 | 174 | { |
248c7f9c | 175 | __u32 curr_prog_id = 0; |
b4b8faa1 | 176 | |
248c7f9c MK |
177 | if (bpf_get_link_xdp_id(opt_ifindex, &curr_prog_id, opt_xdp_flags)) { |
178 | printf("bpf_get_link_xdp_id failed\n"); | |
179 | exit(EXIT_FAILURE); | |
b4b8faa1 | 180 | } |
248c7f9c MK |
181 | if (prog_id == curr_prog_id) |
182 | bpf_set_link_xdp_fd(opt_ifindex, -1, opt_xdp_flags); | |
183 | else if (!curr_prog_id) | |
184 | printf("couldn't find a prog id on a given interface\n"); | |
185 | else | |
186 | printf("program on interface changed, not removing\n"); | |
b4b8faa1 MK |
187 | } |
188 | ||
248c7f9c | 189 | static void int_exit(int sig) |
b4b8faa1 | 190 | { |
248c7f9c | 191 | struct xsk_umem *umem = xsks[0]->umem->umem; |
b4b8faa1 | 192 | |
248c7f9c | 193 | (void)sig; |
b4b8faa1 | 194 | |
248c7f9c MK |
195 | dump_stats(); |
196 | xsk_socket__delete(xsks[0]->xsk); | |
197 | (void)xsk_umem__delete(umem); | |
198 | remove_xdp_program(); | |
b4b8faa1 | 199 | |
248c7f9c | 200 | exit(EXIT_SUCCESS); |
b4b8faa1 MK |
201 | } |
202 | ||
248c7f9c MK |
203 | static void __exit_with_error(int error, const char *file, const char *func, |
204 | int line) | |
b4b8faa1 | 205 | { |
248c7f9c MK |
206 | fprintf(stderr, "%s:%s:%i: errno: %d/\"%s\"\n", file, func, |
207 | line, error, strerror(error)); | |
208 | dump_stats(); | |
209 | remove_xdp_program(); | |
210 | exit(EXIT_FAILURE); | |
b4b8faa1 MK |
211 | } |
212 | ||
248c7f9c MK |
213 | #define exit_with_error(error) __exit_with_error(error, __FILE__, __func__, \ |
214 | __LINE__) | |
b4b8faa1 | 215 | |
248c7f9c MK |
216 | static const char pkt_data[] = |
217 | "\x3c\xfd\xfe\x9e\x7f\x71\xec\xb1\xd7\x98\x3a\xc0\x08\x00\x45\x00" | |
218 | "\x00\x2e\x00\x00\x00\x00\x40\x11\x88\x97\x05\x08\x07\x08\xc8\x14" | |
219 | "\x1e\x04\x10\x92\x10\x92\x00\x1a\x6d\xa3\x34\x33\x1f\x69\x40\x6b" | |
220 | "\x54\x59\xb6\x14\x2d\x11\x44\xbf\xaf\xd9\xbe\xaa"; | |
b4b8faa1 MK |
221 | |
222 | static void swap_mac_addresses(void *data) | |
223 | { | |
224 | struct ether_header *eth = (struct ether_header *)data; | |
225 | struct ether_addr *src_addr = (struct ether_addr *)ð->ether_shost; | |
226 | struct ether_addr *dst_addr = (struct ether_addr *)ð->ether_dhost; | |
227 | struct ether_addr tmp; | |
228 | ||
229 | tmp = *src_addr; | |
230 | *src_addr = *dst_addr; | |
231 | *dst_addr = tmp; | |
232 | } | |
233 | ||
a412ef54 | 234 | static void hex_dump(void *pkt, size_t length, u64 addr) |
b4b8faa1 | 235 | { |
b4b8faa1 MK |
236 | const unsigned char *address = (unsigned char *)pkt; |
237 | const unsigned char *line = address; | |
238 | size_t line_size = 32; | |
239 | unsigned char c; | |
a412ef54 BT |
240 | char buf[32]; |
241 | int i = 0; | |
b4b8faa1 | 242 | |
a412ef54 BT |
243 | if (!DEBUG_HEXDUMP) |
244 | return; | |
245 | ||
246 | sprintf(buf, "addr=%llu", addr); | |
b4b8faa1 | 247 | printf("length = %zu\n", length); |
a412ef54 | 248 | printf("%s | ", buf); |
b4b8faa1 MK |
249 | while (length-- > 0) { |
250 | printf("%02X ", *address++); | |
251 | if (!(++i % line_size) || (length == 0 && i % line_size)) { | |
252 | if (length == 0) { | |
253 | while (i++ % line_size) | |
254 | printf("__ "); | |
255 | } | |
256 | printf(" | "); /* right close */ | |
257 | while (line < address) { | |
258 | c = *line++; | |
259 | printf("%c", (c < 33 || c == 255) ? 0x2E : c); | |
260 | } | |
261 | printf("\n"); | |
262 | if (length > 0) | |
a412ef54 | 263 | printf("%s | ", buf); |
b4b8faa1 MK |
264 | } |
265 | } | |
266 | printf("\n"); | |
267 | } | |
b4b8faa1 | 268 | |
248c7f9c | 269 | static size_t gen_eth_frame(struct xsk_umem_info *umem, u64 addr) |
b4b8faa1 | 270 | { |
248c7f9c MK |
271 | memcpy(xsk_umem__get_data(umem->buffer, addr), pkt_data, |
272 | sizeof(pkt_data) - 1); | |
b4b8faa1 MK |
273 | return sizeof(pkt_data) - 1; |
274 | } | |
275 | ||
248c7f9c | 276 | static struct xsk_umem_info *xsk_configure_umem(void *buffer, u64 size) |
b4b8faa1 | 277 | { |
248c7f9c MK |
278 | struct xsk_umem_info *umem; |
279 | int ret; | |
b4b8faa1 MK |
280 | |
281 | umem = calloc(1, sizeof(*umem)); | |
248c7f9c MK |
282 | if (!umem) |
283 | exit_with_error(errno); | |
b4b8faa1 | 284 | |
248c7f9c MK |
285 | ret = xsk_umem__create(&umem->umem, buffer, size, &umem->fq, &umem->cq, |
286 | NULL); | |
287 | if (ret) | |
288 | exit_with_error(-ret); | |
b4b8faa1 | 289 | |
248c7f9c | 290 | umem->buffer = buffer; |
b4b8faa1 MK |
291 | return umem; |
292 | } | |
293 | ||
248c7f9c | 294 | static struct xsk_socket_info *xsk_configure_socket(struct xsk_umem_info *umem) |
b4b8faa1 | 295 | { |
248c7f9c MK |
296 | struct xsk_socket_config cfg; |
297 | struct xsk_socket_info *xsk; | |
298 | int ret; | |
299 | u32 idx; | |
300 | int i; | |
b4b8faa1 MK |
301 | |
302 | xsk = calloc(1, sizeof(*xsk)); | |
248c7f9c MK |
303 | if (!xsk) |
304 | exit_with_error(errno); | |
305 | ||
306 | xsk->umem = umem; | |
307 | cfg.rx_size = XSK_RING_CONS__DEFAULT_NUM_DESCS; | |
308 | cfg.tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS; | |
309 | cfg.libbpf_flags = 0; | |
310 | cfg.xdp_flags = opt_xdp_flags; | |
311 | cfg.bind_flags = opt_xdp_bind_flags; | |
312 | ret = xsk_socket__create(&xsk->xsk, opt_if, opt_queue, umem->umem, | |
313 | &xsk->rx, &xsk->tx, &cfg); | |
314 | if (ret) | |
315 | exit_with_error(-ret); | |
316 | ||
317 | ret = bpf_get_link_xdp_id(opt_ifindex, &prog_id, opt_xdp_flags); | |
318 | if (ret) | |
319 | exit_with_error(-ret); | |
320 | ||
321 | ret = xsk_ring_prod__reserve(&xsk->umem->fq, | |
322 | XSK_RING_PROD__DEFAULT_NUM_DESCS, | |
323 | &idx); | |
324 | if (ret != XSK_RING_PROD__DEFAULT_NUM_DESCS) | |
325 | exit_with_error(-ret); | |
326 | for (i = 0; | |
327 | i < XSK_RING_PROD__DEFAULT_NUM_DESCS * | |
328 | XSK_UMEM__DEFAULT_FRAME_SIZE; | |
329 | i += XSK_UMEM__DEFAULT_FRAME_SIZE) | |
330 | *xsk_ring_prod__fill_addr(&xsk->umem->fq, idx++) = i; | |
331 | xsk_ring_prod__submit(&xsk->umem->fq, | |
332 | XSK_RING_PROD__DEFAULT_NUM_DESCS); | |
b4b8faa1 MK |
333 | |
334 | return xsk; | |
335 | } | |
336 | ||
b4b8faa1 MK |
337 | static struct option long_options[] = { |
338 | {"rxdrop", no_argument, 0, 'r'}, | |
339 | {"txonly", no_argument, 0, 't'}, | |
340 | {"l2fwd", no_argument, 0, 'l'}, | |
341 | {"interface", required_argument, 0, 'i'}, | |
342 | {"queue", required_argument, 0, 'q'}, | |
343 | {"poll", no_argument, 0, 'p'}, | |
b4b8faa1 MK |
344 | {"xdp-skb", no_argument, 0, 'S'}, |
345 | {"xdp-native", no_argument, 0, 'N'}, | |
346 | {"interval", required_argument, 0, 'n'}, | |
58c50ae4 BT |
347 | {"zero-copy", no_argument, 0, 'z'}, |
348 | {"copy", no_argument, 0, 'c'}, | |
b4b8faa1 MK |
349 | {0, 0, 0, 0} |
350 | }; | |
351 | ||
352 | static void usage(const char *prog) | |
353 | { | |
354 | const char *str = | |
355 | " Usage: %s [OPTIONS]\n" | |
356 | " Options:\n" | |
357 | " -r, --rxdrop Discard all incoming packets (default)\n" | |
358 | " -t, --txonly Only send packets\n" | |
359 | " -l, --l2fwd MAC swap L2 forwarding\n" | |
360 | " -i, --interface=n Run on interface n\n" | |
361 | " -q, --queue=n Use queue n (default 0)\n" | |
362 | " -p, --poll Use poll syscall\n" | |
b4b8faa1 MK |
363 | " -S, --xdp-skb=n Use XDP skb-mod\n" |
364 | " -N, --xdp-native=n Enfore XDP native mode\n" | |
365 | " -n, --interval=n Specify statistics update interval (default 1 sec).\n" | |
58c50ae4 BT |
366 | " -z, --zero-copy Force zero-copy mode.\n" |
367 | " -c, --copy Force copy mode.\n" | |
b4b8faa1 MK |
368 | "\n"; |
369 | fprintf(stderr, str, prog); | |
370 | exit(EXIT_FAILURE); | |
371 | } | |
372 | ||
373 | static void parse_command_line(int argc, char **argv) | |
374 | { | |
375 | int option_index, c; | |
376 | ||
377 | opterr = 0; | |
378 | ||
379 | for (;;) { | |
743e568c | 380 | c = getopt_long(argc, argv, "Frtli:q:psSNn:cz", long_options, |
b4b8faa1 MK |
381 | &option_index); |
382 | if (c == -1) | |
383 | break; | |
384 | ||
385 | switch (c) { | |
386 | case 'r': | |
387 | opt_bench = BENCH_RXDROP; | |
388 | break; | |
389 | case 't': | |
390 | opt_bench = BENCH_TXONLY; | |
391 | break; | |
392 | case 'l': | |
393 | opt_bench = BENCH_L2FWD; | |
394 | break; | |
395 | case 'i': | |
396 | opt_if = optarg; | |
397 | break; | |
398 | case 'q': | |
399 | opt_queue = atoi(optarg); | |
400 | break; | |
b4b8faa1 MK |
401 | case 'p': |
402 | opt_poll = 1; | |
403 | break; | |
404 | case 'S': | |
405 | opt_xdp_flags |= XDP_FLAGS_SKB_MODE; | |
9f5232cc | 406 | opt_xdp_bind_flags |= XDP_COPY; |
b4b8faa1 MK |
407 | break; |
408 | case 'N': | |
409 | opt_xdp_flags |= XDP_FLAGS_DRV_MODE; | |
410 | break; | |
411 | case 'n': | |
412 | opt_interval = atoi(optarg); | |
413 | break; | |
58c50ae4 BT |
414 | case 'z': |
415 | opt_xdp_bind_flags |= XDP_ZEROCOPY; | |
416 | break; | |
417 | case 'c': | |
418 | opt_xdp_bind_flags |= XDP_COPY; | |
419 | break; | |
743e568c MF |
420 | case 'F': |
421 | opt_xdp_flags &= ~XDP_FLAGS_UPDATE_IF_NOEXIST; | |
422 | break; | |
b4b8faa1 MK |
423 | default: |
424 | usage(basename(argv[0])); | |
425 | } | |
426 | } | |
427 | ||
428 | opt_ifindex = if_nametoindex(opt_if); | |
429 | if (!opt_ifindex) { | |
430 | fprintf(stderr, "ERROR: interface \"%s\" does not exist\n", | |
431 | opt_if); | |
432 | usage(basename(argv[0])); | |
433 | } | |
248c7f9c | 434 | |
b4b8faa1 MK |
435 | } |
436 | ||
248c7f9c | 437 | static void kick_tx(struct xsk_socket_info *xsk) |
b4b8faa1 MK |
438 | { |
439 | int ret; | |
440 | ||
248c7f9c | 441 | ret = sendto(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, 0); |
c03079c9 | 442 | if (ret >= 0 || errno == ENOBUFS || errno == EAGAIN || errno == EBUSY) |
b4b8faa1 | 443 | return; |
248c7f9c | 444 | exit_with_error(errno); |
b4b8faa1 MK |
445 | } |
446 | ||
248c7f9c | 447 | static inline void complete_tx_l2fwd(struct xsk_socket_info *xsk) |
b4b8faa1 | 448 | { |
b74e21ab | 449 | u32 idx_cq = 0, idx_fq = 0; |
b4b8faa1 MK |
450 | unsigned int rcvd; |
451 | size_t ndescs; | |
452 | ||
453 | if (!xsk->outstanding_tx) | |
454 | return; | |
455 | ||
248c7f9c | 456 | kick_tx(xsk); |
b4b8faa1 | 457 | ndescs = (xsk->outstanding_tx > BATCH_SIZE) ? BATCH_SIZE : |
248c7f9c | 458 | xsk->outstanding_tx; |
b4b8faa1 MK |
459 | |
460 | /* re-add completed Tx buffers */ | |
248c7f9c | 461 | rcvd = xsk_ring_cons__peek(&xsk->umem->cq, ndescs, &idx_cq); |
b4b8faa1 | 462 | if (rcvd > 0) { |
248c7f9c MK |
463 | unsigned int i; |
464 | int ret; | |
465 | ||
466 | ret = xsk_ring_prod__reserve(&xsk->umem->fq, rcvd, &idx_fq); | |
467 | while (ret != rcvd) { | |
468 | if (ret < 0) | |
469 | exit_with_error(-ret); | |
470 | ret = xsk_ring_prod__reserve(&xsk->umem->fq, rcvd, | |
471 | &idx_fq); | |
472 | } | |
473 | for (i = 0; i < rcvd; i++) | |
474 | *xsk_ring_prod__fill_addr(&xsk->umem->fq, idx_fq++) = | |
475 | *xsk_ring_cons__comp_addr(&xsk->umem->cq, | |
476 | idx_cq++); | |
477 | ||
478 | xsk_ring_prod__submit(&xsk->umem->fq, rcvd); | |
479 | xsk_ring_cons__release(&xsk->umem->cq, rcvd); | |
b4b8faa1 MK |
480 | xsk->outstanding_tx -= rcvd; |
481 | xsk->tx_npkts += rcvd; | |
482 | } | |
483 | } | |
484 | ||
248c7f9c | 485 | static inline void complete_tx_only(struct xsk_socket_info *xsk) |
b4b8faa1 | 486 | { |
b4b8faa1 | 487 | unsigned int rcvd; |
248c7f9c | 488 | u32 idx; |
b4b8faa1 MK |
489 | |
490 | if (!xsk->outstanding_tx) | |
491 | return; | |
492 | ||
248c7f9c | 493 | kick_tx(xsk); |
b4b8faa1 | 494 | |
248c7f9c | 495 | rcvd = xsk_ring_cons__peek(&xsk->umem->cq, BATCH_SIZE, &idx); |
b4b8faa1 | 496 | if (rcvd > 0) { |
248c7f9c | 497 | xsk_ring_cons__release(&xsk->umem->cq, rcvd); |
b4b8faa1 MK |
498 | xsk->outstanding_tx -= rcvd; |
499 | xsk->tx_npkts += rcvd; | |
500 | } | |
501 | } | |
502 | ||
248c7f9c | 503 | static void rx_drop(struct xsk_socket_info *xsk) |
b4b8faa1 | 504 | { |
b4b8faa1 | 505 | unsigned int rcvd, i; |
b74e21ab | 506 | u32 idx_rx = 0, idx_fq = 0; |
248c7f9c | 507 | int ret; |
b4b8faa1 | 508 | |
248c7f9c | 509 | rcvd = xsk_ring_cons__peek(&xsk->rx, BATCH_SIZE, &idx_rx); |
b4b8faa1 MK |
510 | if (!rcvd) |
511 | return; | |
512 | ||
248c7f9c MK |
513 | ret = xsk_ring_prod__reserve(&xsk->umem->fq, rcvd, &idx_fq); |
514 | while (ret != rcvd) { | |
515 | if (ret < 0) | |
516 | exit_with_error(-ret); | |
517 | ret = xsk_ring_prod__reserve(&xsk->umem->fq, rcvd, &idx_fq); | |
518 | } | |
519 | ||
b4b8faa1 | 520 | for (i = 0; i < rcvd; i++) { |
248c7f9c MK |
521 | u64 addr = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx)->addr; |
522 | u32 len = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx++)->len; | |
523 | char *pkt = xsk_umem__get_data(xsk->umem->buffer, addr); | |
b4b8faa1 | 524 | |
248c7f9c MK |
525 | hex_dump(pkt, len, addr); |
526 | *xsk_ring_prod__fill_addr(&xsk->umem->fq, idx_fq++) = addr; | |
b4b8faa1 MK |
527 | } |
528 | ||
248c7f9c MK |
529 | xsk_ring_prod__submit(&xsk->umem->fq, rcvd); |
530 | xsk_ring_cons__release(&xsk->rx, rcvd); | |
b4b8faa1 | 531 | xsk->rx_npkts += rcvd; |
b4b8faa1 MK |
532 | } |
533 | ||
534 | static void rx_drop_all(void) | |
535 | { | |
536 | struct pollfd fds[MAX_SOCKS + 1]; | |
537 | int i, ret, timeout, nfds = 1; | |
538 | ||
539 | memset(fds, 0, sizeof(fds)); | |
540 | ||
541 | for (i = 0; i < num_socks; i++) { | |
248c7f9c | 542 | fds[i].fd = xsk_socket__fd(xsks[i]->xsk); |
b4b8faa1 MK |
543 | fds[i].events = POLLIN; |
544 | timeout = 1000; /* 1sn */ | |
545 | } | |
546 | ||
547 | for (;;) { | |
548 | if (opt_poll) { | |
549 | ret = poll(fds, nfds, timeout); | |
550 | if (ret <= 0) | |
551 | continue; | |
552 | } | |
553 | ||
554 | for (i = 0; i < num_socks; i++) | |
555 | rx_drop(xsks[i]); | |
556 | } | |
557 | } | |
558 | ||
248c7f9c | 559 | static void tx_only(struct xsk_socket_info *xsk) |
b4b8faa1 MK |
560 | { |
561 | int timeout, ret, nfds = 1; | |
562 | struct pollfd fds[nfds + 1]; | |
248c7f9c | 563 | u32 idx, frame_nb = 0; |
b4b8faa1 MK |
564 | |
565 | memset(fds, 0, sizeof(fds)); | |
248c7f9c | 566 | fds[0].fd = xsk_socket__fd(xsk->xsk); |
b4b8faa1 MK |
567 | fds[0].events = POLLOUT; |
568 | timeout = 1000; /* 1sn */ | |
569 | ||
570 | for (;;) { | |
571 | if (opt_poll) { | |
572 | ret = poll(fds, nfds, timeout); | |
573 | if (ret <= 0) | |
574 | continue; | |
575 | ||
248c7f9c | 576 | if (!(fds[0].revents & POLLOUT)) |
b4b8faa1 MK |
577 | continue; |
578 | } | |
579 | ||
248c7f9c MK |
580 | if (xsk_ring_prod__reserve(&xsk->tx, BATCH_SIZE, &idx) == |
581 | BATCH_SIZE) { | |
582 | unsigned int i; | |
b4b8faa1 | 583 | |
248c7f9c MK |
584 | for (i = 0; i < BATCH_SIZE; i++) { |
585 | xsk_ring_prod__tx_desc(&xsk->tx, idx + i)->addr | |
586 | = (frame_nb + i) << | |
587 | XSK_UMEM__DEFAULT_FRAME_SHIFT; | |
588 | xsk_ring_prod__tx_desc(&xsk->tx, idx + i)->len = | |
589 | sizeof(pkt_data) - 1; | |
590 | } | |
591 | ||
592 | xsk_ring_prod__submit(&xsk->tx, BATCH_SIZE); | |
b4b8faa1 | 593 | xsk->outstanding_tx += BATCH_SIZE; |
248c7f9c MK |
594 | frame_nb += BATCH_SIZE; |
595 | frame_nb %= NUM_FRAMES; | |
b4b8faa1 MK |
596 | } |
597 | ||
598 | complete_tx_only(xsk); | |
599 | } | |
600 | } | |
601 | ||
248c7f9c | 602 | static void l2fwd(struct xsk_socket_info *xsk) |
b4b8faa1 MK |
603 | { |
604 | for (;;) { | |
b4b8faa1 | 605 | unsigned int rcvd, i; |
b74e21ab | 606 | u32 idx_rx = 0, idx_tx = 0; |
b4b8faa1 MK |
607 | int ret; |
608 | ||
609 | for (;;) { | |
610 | complete_tx_l2fwd(xsk); | |
611 | ||
248c7f9c MK |
612 | rcvd = xsk_ring_cons__peek(&xsk->rx, BATCH_SIZE, |
613 | &idx_rx); | |
b4b8faa1 MK |
614 | if (rcvd > 0) |
615 | break; | |
616 | } | |
617 | ||
248c7f9c MK |
618 | ret = xsk_ring_prod__reserve(&xsk->tx, rcvd, &idx_tx); |
619 | while (ret != rcvd) { | |
620 | if (ret < 0) | |
621 | exit_with_error(-ret); | |
622 | ret = xsk_ring_prod__reserve(&xsk->tx, rcvd, &idx_tx); | |
623 | } | |
624 | ||
b4b8faa1 | 625 | for (i = 0; i < rcvd; i++) { |
248c7f9c MK |
626 | u64 addr = xsk_ring_cons__rx_desc(&xsk->rx, |
627 | idx_rx)->addr; | |
628 | u32 len = xsk_ring_cons__rx_desc(&xsk->rx, | |
629 | idx_rx++)->len; | |
630 | char *pkt = xsk_umem__get_data(xsk->umem->buffer, addr); | |
b4b8faa1 MK |
631 | |
632 | swap_mac_addresses(pkt); | |
b4b8faa1 | 633 | |
248c7f9c MK |
634 | hex_dump(pkt, len, addr); |
635 | xsk_ring_prod__tx_desc(&xsk->tx, idx_tx)->addr = addr; | |
636 | xsk_ring_prod__tx_desc(&xsk->tx, idx_tx++)->len = len; | |
b4b8faa1 MK |
637 | } |
638 | ||
248c7f9c MK |
639 | xsk_ring_prod__submit(&xsk->tx, rcvd); |
640 | xsk_ring_cons__release(&xsk->rx, rcvd); | |
b4b8faa1 | 641 | |
248c7f9c | 642 | xsk->rx_npkts += rcvd; |
b4b8faa1 MK |
643 | xsk->outstanding_tx += rcvd; |
644 | } | |
645 | } | |
646 | ||
647 | int main(int argc, char **argv) | |
648 | { | |
649 | struct rlimit r = {RLIM_INFINITY, RLIM_INFINITY}; | |
248c7f9c | 650 | struct xsk_umem_info *umem; |
b4b8faa1 | 651 | pthread_t pt; |
248c7f9c MK |
652 | void *bufs; |
653 | int ret; | |
b4b8faa1 MK |
654 | |
655 | parse_command_line(argc, argv); | |
656 | ||
657 | if (setrlimit(RLIMIT_MEMLOCK, &r)) { | |
658 | fprintf(stderr, "ERROR: setrlimit(RLIMIT_MEMLOCK) \"%s\"\n", | |
659 | strerror(errno)); | |
660 | exit(EXIT_FAILURE); | |
661 | } | |
662 | ||
248c7f9c MK |
663 | ret = posix_memalign(&bufs, getpagesize(), /* PAGE_SIZE aligned */ |
664 | NUM_FRAMES * XSK_UMEM__DEFAULT_FRAME_SIZE); | |
665 | if (ret) | |
666 | exit_with_error(ret); | |
3b7a8ec2 | 667 | |
248c7f9c MK |
668 | /* Create sockets... */ |
669 | umem = xsk_configure_umem(bufs, | |
670 | NUM_FRAMES * XSK_UMEM__DEFAULT_FRAME_SIZE); | |
671 | xsks[num_socks++] = xsk_configure_socket(umem); | |
b4b8faa1 | 672 | |
248c7f9c MK |
673 | if (opt_bench == BENCH_TXONLY) { |
674 | int i; | |
b4b8faa1 | 675 | |
248c7f9c MK |
676 | for (i = 0; i < NUM_FRAMES * XSK_UMEM__DEFAULT_FRAME_SIZE; |
677 | i += XSK_UMEM__DEFAULT_FRAME_SIZE) | |
678 | (void)gen_eth_frame(umem, i); | |
b4b8faa1 MK |
679 | } |
680 | ||
681 | signal(SIGINT, int_exit); | |
682 | signal(SIGTERM, int_exit); | |
683 | signal(SIGABRT, int_exit); | |
684 | ||
685 | setlocale(LC_ALL, ""); | |
686 | ||
687 | ret = pthread_create(&pt, NULL, poller, NULL); | |
248c7f9c MK |
688 | if (ret) |
689 | exit_with_error(ret); | |
b4b8faa1 MK |
690 | |
691 | prev_time = get_nsecs(); | |
692 | ||
693 | if (opt_bench == BENCH_RXDROP) | |
694 | rx_drop_all(); | |
695 | else if (opt_bench == BENCH_TXONLY) | |
696 | tx_only(xsks[0]); | |
697 | else | |
698 | l2fwd(xsks[0]); | |
699 | ||
700 | return 0; | |
701 | } |