Commit | Line | Data |
---|---|---|
b4b8faa1 | 1 | // SPDX-License-Identifier: GPL-2.0 |
dac09149 | 2 | /* Copyright(c) 2017 - 2018 Intel Corporation. */ |
b4b8faa1 | 3 | |
248c7f9c | 4 | #include <asm/barrier.h> |
b4b8faa1 MK |
5 | #include <errno.h> |
6 | #include <getopt.h> | |
7 | #include <libgen.h> | |
8 | #include <linux/bpf.h> | |
248c7f9c | 9 | #include <linux/compiler.h> |
b4b8faa1 MK |
10 | #include <linux/if_link.h> |
11 | #include <linux/if_xdp.h> | |
12 | #include <linux/if_ether.h> | |
248c7f9c MK |
13 | #include <locale.h> |
14 | #include <net/ethernet.h> | |
b4b8faa1 | 15 | #include <net/if.h> |
248c7f9c MK |
16 | #include <poll.h> |
17 | #include <pthread.h> | |
b4b8faa1 MK |
18 | #include <signal.h> |
19 | #include <stdbool.h> | |
20 | #include <stdio.h> | |
21 | #include <stdlib.h> | |
22 | #include <string.h> | |
248c7f9c | 23 | #include <sys/mman.h> |
b4b8faa1 MK |
24 | #include <sys/resource.h> |
25 | #include <sys/socket.h> | |
248c7f9c | 26 | #include <sys/types.h> |
b4b8faa1 MK |
27 | #include <time.h> |
28 | #include <unistd.h> | |
b4b8faa1 | 29 | |
4d18f6de DL |
30 | #include "libbpf.h" |
31 | #include "xsk.h" | |
2bf3e2ef | 32 | #include <bpf/bpf.h> |
b4b8faa1 | 33 | |
b4b8faa1 MK |
34 | #ifndef SOL_XDP |
35 | #define SOL_XDP 283 | |
36 | #endif | |
37 | ||
38 | #ifndef AF_XDP | |
39 | #define AF_XDP 44 | |
40 | #endif | |
41 | ||
42 | #ifndef PF_XDP | |
43 | #define PF_XDP AF_XDP | |
44 | #endif | |
45 | ||
248c7f9c MK |
46 | #define NUM_FRAMES (4 * 1024) |
47 | #define BATCH_SIZE 64 | |
b4b8faa1 MK |
48 | |
49 | #define DEBUG_HEXDUMP 0 | |
248c7f9c | 50 | #define MAX_SOCKS 8 |
b4b8faa1 | 51 | |
a412ef54 | 52 | typedef __u64 u64; |
b4b8faa1 MK |
53 | typedef __u32 u32; |
54 | ||
55 | static unsigned long prev_time; | |
56 | ||
57 | enum benchmark_type { | |
58 | BENCH_RXDROP = 0, | |
59 | BENCH_TXONLY = 1, | |
60 | BENCH_L2FWD = 2, | |
61 | }; | |
62 | ||
63 | static enum benchmark_type opt_bench = BENCH_RXDROP; | |
743e568c | 64 | static u32 opt_xdp_flags = XDP_FLAGS_UPDATE_IF_NOEXIST; |
b4b8faa1 MK |
65 | static const char *opt_if = ""; |
66 | static int opt_ifindex; | |
67 | static int opt_queue; | |
68 | static int opt_poll; | |
b4b8faa1 | 69 | static int opt_interval = 1; |
9f5232cc | 70 | static u32 opt_xdp_bind_flags; |
123e8da1 | 71 | static int opt_xsk_frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE; |
3b7a8ec2 | 72 | static __u32 prog_id; |
b4b8faa1 | 73 | |
248c7f9c MK |
74 | struct xsk_umem_info { |
75 | struct xsk_ring_prod fq; | |
76 | struct xsk_ring_cons cq; | |
77 | struct xsk_umem *umem; | |
78 | void *buffer; | |
b4b8faa1 MK |
79 | }; |
80 | ||
248c7f9c MK |
81 | struct xsk_socket_info { |
82 | struct xsk_ring_cons rx; | |
83 | struct xsk_ring_prod tx; | |
84 | struct xsk_umem_info *umem; | |
85 | struct xsk_socket *xsk; | |
b4b8faa1 MK |
86 | unsigned long rx_npkts; |
87 | unsigned long tx_npkts; | |
88 | unsigned long prev_rx_npkts; | |
89 | unsigned long prev_tx_npkts; | |
248c7f9c | 90 | u32 outstanding_tx; |
b4b8faa1 MK |
91 | }; |
92 | ||
b4b8faa1 | 93 | static int num_socks; |
248c7f9c | 94 | struct xsk_socket_info *xsks[MAX_SOCKS]; |
b4b8faa1 MK |
95 | |
96 | static unsigned long get_nsecs(void) | |
97 | { | |
98 | struct timespec ts; | |
99 | ||
100 | clock_gettime(CLOCK_MONOTONIC, &ts); | |
101 | return ts.tv_sec * 1000000000UL + ts.tv_nsec; | |
102 | } | |
103 | ||
248c7f9c | 104 | static void print_benchmark(bool running) |
b4b8faa1 | 105 | { |
248c7f9c | 106 | const char *bench_str = "INVALID"; |
b4b8faa1 | 107 | |
248c7f9c MK |
108 | if (opt_bench == BENCH_RXDROP) |
109 | bench_str = "rxdrop"; | |
110 | else if (opt_bench == BENCH_TXONLY) | |
111 | bench_str = "txonly"; | |
112 | else if (opt_bench == BENCH_L2FWD) | |
113 | bench_str = "l2fwd"; | |
b4b8faa1 | 114 | |
248c7f9c MK |
115 | printf("%s:%d %s ", opt_if, opt_queue, bench_str); |
116 | if (opt_xdp_flags & XDP_FLAGS_SKB_MODE) | |
117 | printf("xdp-skb "); | |
118 | else if (opt_xdp_flags & XDP_FLAGS_DRV_MODE) | |
119 | printf("xdp-drv "); | |
120 | else | |
121 | printf(" "); | |
b4b8faa1 | 122 | |
248c7f9c MK |
123 | if (opt_poll) |
124 | printf("poll() "); | |
b4b8faa1 | 125 | |
248c7f9c MK |
126 | if (running) { |
127 | printf("running..."); | |
128 | fflush(stdout); | |
b4b8faa1 | 129 | } |
b4b8faa1 MK |
130 | } |
131 | ||
248c7f9c | 132 | static void dump_stats(void) |
b4b8faa1 | 133 | { |
248c7f9c MK |
134 | unsigned long now = get_nsecs(); |
135 | long dt = now - prev_time; | |
136 | int i; | |
b4b8faa1 | 137 | |
248c7f9c | 138 | prev_time = now; |
b4b8faa1 | 139 | |
248c7f9c MK |
140 | for (i = 0; i < num_socks && xsks[i]; i++) { |
141 | char *fmt = "%-15s %'-11.0f %'-11lu\n"; | |
142 | double rx_pps, tx_pps; | |
b4b8faa1 | 143 | |
248c7f9c MK |
144 | rx_pps = (xsks[i]->rx_npkts - xsks[i]->prev_rx_npkts) * |
145 | 1000000000. / dt; | |
146 | tx_pps = (xsks[i]->tx_npkts - xsks[i]->prev_tx_npkts) * | |
147 | 1000000000. / dt; | |
b4b8faa1 | 148 | |
248c7f9c MK |
149 | printf("\n sock%d@", i); |
150 | print_benchmark(false); | |
151 | printf("\n"); | |
b4b8faa1 | 152 | |
248c7f9c MK |
153 | printf("%-15s %-11s %-11s %-11.2f\n", "", "pps", "pkts", |
154 | dt / 1000000000.); | |
155 | printf(fmt, "rx", rx_pps, xsks[i]->rx_npkts); | |
156 | printf(fmt, "tx", tx_pps, xsks[i]->tx_npkts); | |
b4b8faa1 | 157 | |
248c7f9c MK |
158 | xsks[i]->prev_rx_npkts = xsks[i]->rx_npkts; |
159 | xsks[i]->prev_tx_npkts = xsks[i]->tx_npkts; | |
b4b8faa1 | 160 | } |
b4b8faa1 MK |
161 | } |
162 | ||
248c7f9c | 163 | static void *poller(void *arg) |
b4b8faa1 | 164 | { |
248c7f9c MK |
165 | (void)arg; |
166 | for (;;) { | |
167 | sleep(opt_interval); | |
168 | dump_stats(); | |
b4b8faa1 MK |
169 | } |
170 | ||
248c7f9c | 171 | return NULL; |
b4b8faa1 MK |
172 | } |
173 | ||
248c7f9c | 174 | static void remove_xdp_program(void) |
b4b8faa1 | 175 | { |
248c7f9c | 176 | __u32 curr_prog_id = 0; |
b4b8faa1 | 177 | |
248c7f9c MK |
178 | if (bpf_get_link_xdp_id(opt_ifindex, &curr_prog_id, opt_xdp_flags)) { |
179 | printf("bpf_get_link_xdp_id failed\n"); | |
180 | exit(EXIT_FAILURE); | |
b4b8faa1 | 181 | } |
248c7f9c MK |
182 | if (prog_id == curr_prog_id) |
183 | bpf_set_link_xdp_fd(opt_ifindex, -1, opt_xdp_flags); | |
184 | else if (!curr_prog_id) | |
185 | printf("couldn't find a prog id on a given interface\n"); | |
186 | else | |
187 | printf("program on interface changed, not removing\n"); | |
b4b8faa1 MK |
188 | } |
189 | ||
248c7f9c | 190 | static void int_exit(int sig) |
b4b8faa1 | 191 | { |
248c7f9c | 192 | struct xsk_umem *umem = xsks[0]->umem->umem; |
b4b8faa1 | 193 | |
248c7f9c | 194 | (void)sig; |
b4b8faa1 | 195 | |
248c7f9c MK |
196 | dump_stats(); |
197 | xsk_socket__delete(xsks[0]->xsk); | |
198 | (void)xsk_umem__delete(umem); | |
199 | remove_xdp_program(); | |
b4b8faa1 | 200 | |
248c7f9c | 201 | exit(EXIT_SUCCESS); |
b4b8faa1 MK |
202 | } |
203 | ||
248c7f9c MK |
204 | static void __exit_with_error(int error, const char *file, const char *func, |
205 | int line) | |
b4b8faa1 | 206 | { |
248c7f9c MK |
207 | fprintf(stderr, "%s:%s:%i: errno: %d/\"%s\"\n", file, func, |
208 | line, error, strerror(error)); | |
209 | dump_stats(); | |
210 | remove_xdp_program(); | |
211 | exit(EXIT_FAILURE); | |
b4b8faa1 MK |
212 | } |
213 | ||
248c7f9c MK |
214 | #define exit_with_error(error) __exit_with_error(error, __FILE__, __func__, \ |
215 | __LINE__) | |
b4b8faa1 | 216 | |
248c7f9c MK |
217 | static const char pkt_data[] = |
218 | "\x3c\xfd\xfe\x9e\x7f\x71\xec\xb1\xd7\x98\x3a\xc0\x08\x00\x45\x00" | |
219 | "\x00\x2e\x00\x00\x00\x00\x40\x11\x88\x97\x05\x08\x07\x08\xc8\x14" | |
220 | "\x1e\x04\x10\x92\x10\x92\x00\x1a\x6d\xa3\x34\x33\x1f\x69\x40\x6b" | |
221 | "\x54\x59\xb6\x14\x2d\x11\x44\xbf\xaf\xd9\xbe\xaa"; | |
b4b8faa1 MK |
222 | |
223 | static void swap_mac_addresses(void *data) | |
224 | { | |
225 | struct ether_header *eth = (struct ether_header *)data; | |
226 | struct ether_addr *src_addr = (struct ether_addr *)ð->ether_shost; | |
227 | struct ether_addr *dst_addr = (struct ether_addr *)ð->ether_dhost; | |
228 | struct ether_addr tmp; | |
229 | ||
230 | tmp = *src_addr; | |
231 | *src_addr = *dst_addr; | |
232 | *dst_addr = tmp; | |
233 | } | |
234 | ||
a412ef54 | 235 | static void hex_dump(void *pkt, size_t length, u64 addr) |
b4b8faa1 | 236 | { |
b4b8faa1 MK |
237 | const unsigned char *address = (unsigned char *)pkt; |
238 | const unsigned char *line = address; | |
239 | size_t line_size = 32; | |
240 | unsigned char c; | |
a412ef54 BT |
241 | char buf[32]; |
242 | int i = 0; | |
b4b8faa1 | 243 | |
a412ef54 BT |
244 | if (!DEBUG_HEXDUMP) |
245 | return; | |
246 | ||
247 | sprintf(buf, "addr=%llu", addr); | |
b4b8faa1 | 248 | printf("length = %zu\n", length); |
a412ef54 | 249 | printf("%s | ", buf); |
b4b8faa1 MK |
250 | while (length-- > 0) { |
251 | printf("%02X ", *address++); | |
252 | if (!(++i % line_size) || (length == 0 && i % line_size)) { | |
253 | if (length == 0) { | |
254 | while (i++ % line_size) | |
255 | printf("__ "); | |
256 | } | |
257 | printf(" | "); /* right close */ | |
258 | while (line < address) { | |
259 | c = *line++; | |
260 | printf("%c", (c < 33 || c == 255) ? 0x2E : c); | |
261 | } | |
262 | printf("\n"); | |
263 | if (length > 0) | |
a412ef54 | 264 | printf("%s | ", buf); |
b4b8faa1 MK |
265 | } |
266 | } | |
267 | printf("\n"); | |
268 | } | |
b4b8faa1 | 269 | |
248c7f9c | 270 | static size_t gen_eth_frame(struct xsk_umem_info *umem, u64 addr) |
b4b8faa1 | 271 | { |
248c7f9c MK |
272 | memcpy(xsk_umem__get_data(umem->buffer, addr), pkt_data, |
273 | sizeof(pkt_data) - 1); | |
b4b8faa1 MK |
274 | return sizeof(pkt_data) - 1; |
275 | } | |
276 | ||
248c7f9c | 277 | static struct xsk_umem_info *xsk_configure_umem(void *buffer, u64 size) |
b4b8faa1 | 278 | { |
248c7f9c | 279 | struct xsk_umem_info *umem; |
123e8da1 MM |
280 | struct xsk_umem_config cfg = { |
281 | .fill_size = XSK_RING_PROD__DEFAULT_NUM_DESCS, | |
282 | .comp_size = XSK_RING_CONS__DEFAULT_NUM_DESCS, | |
283 | .frame_size = opt_xsk_frame_size, | |
284 | .frame_headroom = XSK_UMEM__DEFAULT_FRAME_HEADROOM, | |
285 | }; | |
248c7f9c | 286 | int ret; |
b4b8faa1 MK |
287 | |
288 | umem = calloc(1, sizeof(*umem)); | |
248c7f9c MK |
289 | if (!umem) |
290 | exit_with_error(errno); | |
b4b8faa1 | 291 | |
248c7f9c | 292 | ret = xsk_umem__create(&umem->umem, buffer, size, &umem->fq, &umem->cq, |
123e8da1 | 293 | &cfg); |
248c7f9c MK |
294 | if (ret) |
295 | exit_with_error(-ret); | |
b4b8faa1 | 296 | |
248c7f9c | 297 | umem->buffer = buffer; |
b4b8faa1 MK |
298 | return umem; |
299 | } | |
300 | ||
248c7f9c | 301 | static struct xsk_socket_info *xsk_configure_socket(struct xsk_umem_info *umem) |
b4b8faa1 | 302 | { |
248c7f9c MK |
303 | struct xsk_socket_config cfg; |
304 | struct xsk_socket_info *xsk; | |
305 | int ret; | |
306 | u32 idx; | |
307 | int i; | |
b4b8faa1 MK |
308 | |
309 | xsk = calloc(1, sizeof(*xsk)); | |
248c7f9c MK |
310 | if (!xsk) |
311 | exit_with_error(errno); | |
312 | ||
313 | xsk->umem = umem; | |
314 | cfg.rx_size = XSK_RING_CONS__DEFAULT_NUM_DESCS; | |
315 | cfg.tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS; | |
316 | cfg.libbpf_flags = 0; | |
317 | cfg.xdp_flags = opt_xdp_flags; | |
318 | cfg.bind_flags = opt_xdp_bind_flags; | |
319 | ret = xsk_socket__create(&xsk->xsk, opt_if, opt_queue, umem->umem, | |
320 | &xsk->rx, &xsk->tx, &cfg); | |
321 | if (ret) | |
322 | exit_with_error(-ret); | |
323 | ||
324 | ret = bpf_get_link_xdp_id(opt_ifindex, &prog_id, opt_xdp_flags); | |
325 | if (ret) | |
326 | exit_with_error(-ret); | |
327 | ||
328 | ret = xsk_ring_prod__reserve(&xsk->umem->fq, | |
329 | XSK_RING_PROD__DEFAULT_NUM_DESCS, | |
330 | &idx); | |
331 | if (ret != XSK_RING_PROD__DEFAULT_NUM_DESCS) | |
332 | exit_with_error(-ret); | |
123e8da1 MM |
333 | for (i = 0; i < XSK_RING_PROD__DEFAULT_NUM_DESCS; i++) |
334 | *xsk_ring_prod__fill_addr(&xsk->umem->fq, idx++) = | |
335 | i * opt_xsk_frame_size; | |
248c7f9c MK |
336 | xsk_ring_prod__submit(&xsk->umem->fq, |
337 | XSK_RING_PROD__DEFAULT_NUM_DESCS); | |
b4b8faa1 MK |
338 | |
339 | return xsk; | |
340 | } | |
341 | ||
b4b8faa1 MK |
342 | static struct option long_options[] = { |
343 | {"rxdrop", no_argument, 0, 'r'}, | |
344 | {"txonly", no_argument, 0, 't'}, | |
345 | {"l2fwd", no_argument, 0, 'l'}, | |
346 | {"interface", required_argument, 0, 'i'}, | |
347 | {"queue", required_argument, 0, 'q'}, | |
348 | {"poll", no_argument, 0, 'p'}, | |
b4b8faa1 MK |
349 | {"xdp-skb", no_argument, 0, 'S'}, |
350 | {"xdp-native", no_argument, 0, 'N'}, | |
351 | {"interval", required_argument, 0, 'n'}, | |
58c50ae4 BT |
352 | {"zero-copy", no_argument, 0, 'z'}, |
353 | {"copy", no_argument, 0, 'c'}, | |
123e8da1 | 354 | {"frame-size", required_argument, 0, 'f'}, |
b4b8faa1 MK |
355 | {0, 0, 0, 0} |
356 | }; | |
357 | ||
358 | static void usage(const char *prog) | |
359 | { | |
360 | const char *str = | |
361 | " Usage: %s [OPTIONS]\n" | |
362 | " Options:\n" | |
363 | " -r, --rxdrop Discard all incoming packets (default)\n" | |
364 | " -t, --txonly Only send packets\n" | |
365 | " -l, --l2fwd MAC swap L2 forwarding\n" | |
366 | " -i, --interface=n Run on interface n\n" | |
367 | " -q, --queue=n Use queue n (default 0)\n" | |
368 | " -p, --poll Use poll syscall\n" | |
b4b8faa1 MK |
369 | " -S, --xdp-skb=n Use XDP skb-mod\n" |
370 | " -N, --xdp-native=n Enfore XDP native mode\n" | |
371 | " -n, --interval=n Specify statistics update interval (default 1 sec).\n" | |
58c50ae4 BT |
372 | " -z, --zero-copy Force zero-copy mode.\n" |
373 | " -c, --copy Force copy mode.\n" | |
123e8da1 | 374 | " -f, --frame-size=n Set the frame size (must be a power of two, default is %d).\n" |
b4b8faa1 | 375 | "\n"; |
123e8da1 | 376 | fprintf(stderr, str, prog, XSK_UMEM__DEFAULT_FRAME_SIZE); |
b4b8faa1 MK |
377 | exit(EXIT_FAILURE); |
378 | } | |
379 | ||
380 | static void parse_command_line(int argc, char **argv) | |
381 | { | |
382 | int option_index, c; | |
383 | ||
384 | opterr = 0; | |
385 | ||
386 | for (;;) { | |
123e8da1 | 387 | c = getopt_long(argc, argv, "Frtli:q:psSNn:czf:", long_options, |
b4b8faa1 MK |
388 | &option_index); |
389 | if (c == -1) | |
390 | break; | |
391 | ||
392 | switch (c) { | |
393 | case 'r': | |
394 | opt_bench = BENCH_RXDROP; | |
395 | break; | |
396 | case 't': | |
397 | opt_bench = BENCH_TXONLY; | |
398 | break; | |
399 | case 'l': | |
400 | opt_bench = BENCH_L2FWD; | |
401 | break; | |
402 | case 'i': | |
403 | opt_if = optarg; | |
404 | break; | |
405 | case 'q': | |
406 | opt_queue = atoi(optarg); | |
407 | break; | |
b4b8faa1 MK |
408 | case 'p': |
409 | opt_poll = 1; | |
410 | break; | |
411 | case 'S': | |
412 | opt_xdp_flags |= XDP_FLAGS_SKB_MODE; | |
9f5232cc | 413 | opt_xdp_bind_flags |= XDP_COPY; |
b4b8faa1 MK |
414 | break; |
415 | case 'N': | |
416 | opt_xdp_flags |= XDP_FLAGS_DRV_MODE; | |
417 | break; | |
418 | case 'n': | |
419 | opt_interval = atoi(optarg); | |
420 | break; | |
58c50ae4 BT |
421 | case 'z': |
422 | opt_xdp_bind_flags |= XDP_ZEROCOPY; | |
423 | break; | |
424 | case 'c': | |
425 | opt_xdp_bind_flags |= XDP_COPY; | |
426 | break; | |
743e568c MF |
427 | case 'F': |
428 | opt_xdp_flags &= ~XDP_FLAGS_UPDATE_IF_NOEXIST; | |
429 | break; | |
123e8da1 MM |
430 | case 'f': |
431 | opt_xsk_frame_size = atoi(optarg); | |
432 | break; | |
b4b8faa1 MK |
433 | default: |
434 | usage(basename(argv[0])); | |
435 | } | |
436 | } | |
437 | ||
438 | opt_ifindex = if_nametoindex(opt_if); | |
439 | if (!opt_ifindex) { | |
440 | fprintf(stderr, "ERROR: interface \"%s\" does not exist\n", | |
441 | opt_if); | |
442 | usage(basename(argv[0])); | |
443 | } | |
248c7f9c | 444 | |
123e8da1 MM |
445 | if (opt_xsk_frame_size & (opt_xsk_frame_size - 1)) { |
446 | fprintf(stderr, "--frame-size=%d is not a power of two\n", | |
447 | opt_xsk_frame_size); | |
448 | usage(basename(argv[0])); | |
449 | } | |
b4b8faa1 MK |
450 | } |
451 | ||
248c7f9c | 452 | static void kick_tx(struct xsk_socket_info *xsk) |
b4b8faa1 MK |
453 | { |
454 | int ret; | |
455 | ||
248c7f9c | 456 | ret = sendto(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, 0); |
c03079c9 | 457 | if (ret >= 0 || errno == ENOBUFS || errno == EAGAIN || errno == EBUSY) |
b4b8faa1 | 458 | return; |
248c7f9c | 459 | exit_with_error(errno); |
b4b8faa1 MK |
460 | } |
461 | ||
248c7f9c | 462 | static inline void complete_tx_l2fwd(struct xsk_socket_info *xsk) |
b4b8faa1 | 463 | { |
b74e21ab | 464 | u32 idx_cq = 0, idx_fq = 0; |
b4b8faa1 MK |
465 | unsigned int rcvd; |
466 | size_t ndescs; | |
467 | ||
468 | if (!xsk->outstanding_tx) | |
469 | return; | |
470 | ||
248c7f9c | 471 | kick_tx(xsk); |
b4b8faa1 | 472 | ndescs = (xsk->outstanding_tx > BATCH_SIZE) ? BATCH_SIZE : |
248c7f9c | 473 | xsk->outstanding_tx; |
b4b8faa1 MK |
474 | |
475 | /* re-add completed Tx buffers */ | |
248c7f9c | 476 | rcvd = xsk_ring_cons__peek(&xsk->umem->cq, ndescs, &idx_cq); |
b4b8faa1 | 477 | if (rcvd > 0) { |
248c7f9c MK |
478 | unsigned int i; |
479 | int ret; | |
480 | ||
481 | ret = xsk_ring_prod__reserve(&xsk->umem->fq, rcvd, &idx_fq); | |
482 | while (ret != rcvd) { | |
483 | if (ret < 0) | |
484 | exit_with_error(-ret); | |
485 | ret = xsk_ring_prod__reserve(&xsk->umem->fq, rcvd, | |
486 | &idx_fq); | |
487 | } | |
488 | for (i = 0; i < rcvd; i++) | |
489 | *xsk_ring_prod__fill_addr(&xsk->umem->fq, idx_fq++) = | |
490 | *xsk_ring_cons__comp_addr(&xsk->umem->cq, | |
491 | idx_cq++); | |
492 | ||
493 | xsk_ring_prod__submit(&xsk->umem->fq, rcvd); | |
494 | xsk_ring_cons__release(&xsk->umem->cq, rcvd); | |
b4b8faa1 MK |
495 | xsk->outstanding_tx -= rcvd; |
496 | xsk->tx_npkts += rcvd; | |
497 | } | |
498 | } | |
499 | ||
248c7f9c | 500 | static inline void complete_tx_only(struct xsk_socket_info *xsk) |
b4b8faa1 | 501 | { |
b4b8faa1 | 502 | unsigned int rcvd; |
248c7f9c | 503 | u32 idx; |
b4b8faa1 MK |
504 | |
505 | if (!xsk->outstanding_tx) | |
506 | return; | |
507 | ||
248c7f9c | 508 | kick_tx(xsk); |
b4b8faa1 | 509 | |
248c7f9c | 510 | rcvd = xsk_ring_cons__peek(&xsk->umem->cq, BATCH_SIZE, &idx); |
b4b8faa1 | 511 | if (rcvd > 0) { |
248c7f9c | 512 | xsk_ring_cons__release(&xsk->umem->cq, rcvd); |
b4b8faa1 MK |
513 | xsk->outstanding_tx -= rcvd; |
514 | xsk->tx_npkts += rcvd; | |
515 | } | |
516 | } | |
517 | ||
248c7f9c | 518 | static void rx_drop(struct xsk_socket_info *xsk) |
b4b8faa1 | 519 | { |
b4b8faa1 | 520 | unsigned int rcvd, i; |
b74e21ab | 521 | u32 idx_rx = 0, idx_fq = 0; |
248c7f9c | 522 | int ret; |
b4b8faa1 | 523 | |
248c7f9c | 524 | rcvd = xsk_ring_cons__peek(&xsk->rx, BATCH_SIZE, &idx_rx); |
b4b8faa1 MK |
525 | if (!rcvd) |
526 | return; | |
527 | ||
248c7f9c MK |
528 | ret = xsk_ring_prod__reserve(&xsk->umem->fq, rcvd, &idx_fq); |
529 | while (ret != rcvd) { | |
530 | if (ret < 0) | |
531 | exit_with_error(-ret); | |
532 | ret = xsk_ring_prod__reserve(&xsk->umem->fq, rcvd, &idx_fq); | |
533 | } | |
534 | ||
b4b8faa1 | 535 | for (i = 0; i < rcvd; i++) { |
248c7f9c MK |
536 | u64 addr = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx)->addr; |
537 | u32 len = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx++)->len; | |
538 | char *pkt = xsk_umem__get_data(xsk->umem->buffer, addr); | |
b4b8faa1 | 539 | |
248c7f9c MK |
540 | hex_dump(pkt, len, addr); |
541 | *xsk_ring_prod__fill_addr(&xsk->umem->fq, idx_fq++) = addr; | |
b4b8faa1 MK |
542 | } |
543 | ||
248c7f9c MK |
544 | xsk_ring_prod__submit(&xsk->umem->fq, rcvd); |
545 | xsk_ring_cons__release(&xsk->rx, rcvd); | |
b4b8faa1 | 546 | xsk->rx_npkts += rcvd; |
b4b8faa1 MK |
547 | } |
548 | ||
549 | static void rx_drop_all(void) | |
550 | { | |
551 | struct pollfd fds[MAX_SOCKS + 1]; | |
552 | int i, ret, timeout, nfds = 1; | |
553 | ||
554 | memset(fds, 0, sizeof(fds)); | |
555 | ||
556 | for (i = 0; i < num_socks; i++) { | |
248c7f9c | 557 | fds[i].fd = xsk_socket__fd(xsks[i]->xsk); |
b4b8faa1 MK |
558 | fds[i].events = POLLIN; |
559 | timeout = 1000; /* 1sn */ | |
560 | } | |
561 | ||
562 | for (;;) { | |
563 | if (opt_poll) { | |
564 | ret = poll(fds, nfds, timeout); | |
565 | if (ret <= 0) | |
566 | continue; | |
567 | } | |
568 | ||
569 | for (i = 0; i < num_socks; i++) | |
570 | rx_drop(xsks[i]); | |
571 | } | |
572 | } | |
573 | ||
248c7f9c | 574 | static void tx_only(struct xsk_socket_info *xsk) |
b4b8faa1 MK |
575 | { |
576 | int timeout, ret, nfds = 1; | |
577 | struct pollfd fds[nfds + 1]; | |
248c7f9c | 578 | u32 idx, frame_nb = 0; |
b4b8faa1 MK |
579 | |
580 | memset(fds, 0, sizeof(fds)); | |
248c7f9c | 581 | fds[0].fd = xsk_socket__fd(xsk->xsk); |
b4b8faa1 MK |
582 | fds[0].events = POLLOUT; |
583 | timeout = 1000; /* 1sn */ | |
584 | ||
585 | for (;;) { | |
586 | if (opt_poll) { | |
587 | ret = poll(fds, nfds, timeout); | |
588 | if (ret <= 0) | |
589 | continue; | |
590 | ||
248c7f9c | 591 | if (!(fds[0].revents & POLLOUT)) |
b4b8faa1 MK |
592 | continue; |
593 | } | |
594 | ||
248c7f9c MK |
595 | if (xsk_ring_prod__reserve(&xsk->tx, BATCH_SIZE, &idx) == |
596 | BATCH_SIZE) { | |
597 | unsigned int i; | |
b4b8faa1 | 598 | |
248c7f9c MK |
599 | for (i = 0; i < BATCH_SIZE; i++) { |
600 | xsk_ring_prod__tx_desc(&xsk->tx, idx + i)->addr | |
123e8da1 | 601 | = (frame_nb + i) * opt_xsk_frame_size; |
248c7f9c MK |
602 | xsk_ring_prod__tx_desc(&xsk->tx, idx + i)->len = |
603 | sizeof(pkt_data) - 1; | |
604 | } | |
605 | ||
606 | xsk_ring_prod__submit(&xsk->tx, BATCH_SIZE); | |
b4b8faa1 | 607 | xsk->outstanding_tx += BATCH_SIZE; |
248c7f9c MK |
608 | frame_nb += BATCH_SIZE; |
609 | frame_nb %= NUM_FRAMES; | |
b4b8faa1 MK |
610 | } |
611 | ||
612 | complete_tx_only(xsk); | |
613 | } | |
614 | } | |
615 | ||
248c7f9c | 616 | static void l2fwd(struct xsk_socket_info *xsk) |
b4b8faa1 MK |
617 | { |
618 | for (;;) { | |
b4b8faa1 | 619 | unsigned int rcvd, i; |
b74e21ab | 620 | u32 idx_rx = 0, idx_tx = 0; |
b4b8faa1 MK |
621 | int ret; |
622 | ||
623 | for (;;) { | |
624 | complete_tx_l2fwd(xsk); | |
625 | ||
248c7f9c MK |
626 | rcvd = xsk_ring_cons__peek(&xsk->rx, BATCH_SIZE, |
627 | &idx_rx); | |
b4b8faa1 MK |
628 | if (rcvd > 0) |
629 | break; | |
630 | } | |
631 | ||
248c7f9c MK |
632 | ret = xsk_ring_prod__reserve(&xsk->tx, rcvd, &idx_tx); |
633 | while (ret != rcvd) { | |
634 | if (ret < 0) | |
635 | exit_with_error(-ret); | |
636 | ret = xsk_ring_prod__reserve(&xsk->tx, rcvd, &idx_tx); | |
637 | } | |
638 | ||
b4b8faa1 | 639 | for (i = 0; i < rcvd; i++) { |
248c7f9c MK |
640 | u64 addr = xsk_ring_cons__rx_desc(&xsk->rx, |
641 | idx_rx)->addr; | |
642 | u32 len = xsk_ring_cons__rx_desc(&xsk->rx, | |
643 | idx_rx++)->len; | |
644 | char *pkt = xsk_umem__get_data(xsk->umem->buffer, addr); | |
b4b8faa1 MK |
645 | |
646 | swap_mac_addresses(pkt); | |
b4b8faa1 | 647 | |
248c7f9c MK |
648 | hex_dump(pkt, len, addr); |
649 | xsk_ring_prod__tx_desc(&xsk->tx, idx_tx)->addr = addr; | |
650 | xsk_ring_prod__tx_desc(&xsk->tx, idx_tx++)->len = len; | |
b4b8faa1 MK |
651 | } |
652 | ||
248c7f9c MK |
653 | xsk_ring_prod__submit(&xsk->tx, rcvd); |
654 | xsk_ring_cons__release(&xsk->rx, rcvd); | |
b4b8faa1 | 655 | |
248c7f9c | 656 | xsk->rx_npkts += rcvd; |
b4b8faa1 MK |
657 | xsk->outstanding_tx += rcvd; |
658 | } | |
659 | } | |
660 | ||
661 | int main(int argc, char **argv) | |
662 | { | |
663 | struct rlimit r = {RLIM_INFINITY, RLIM_INFINITY}; | |
248c7f9c | 664 | struct xsk_umem_info *umem; |
b4b8faa1 | 665 | pthread_t pt; |
248c7f9c MK |
666 | void *bufs; |
667 | int ret; | |
b4b8faa1 MK |
668 | |
669 | parse_command_line(argc, argv); | |
670 | ||
671 | if (setrlimit(RLIMIT_MEMLOCK, &r)) { | |
672 | fprintf(stderr, "ERROR: setrlimit(RLIMIT_MEMLOCK) \"%s\"\n", | |
673 | strerror(errno)); | |
674 | exit(EXIT_FAILURE); | |
675 | } | |
676 | ||
248c7f9c | 677 | ret = posix_memalign(&bufs, getpagesize(), /* PAGE_SIZE aligned */ |
123e8da1 | 678 | NUM_FRAMES * opt_xsk_frame_size); |
248c7f9c MK |
679 | if (ret) |
680 | exit_with_error(ret); | |
3b7a8ec2 | 681 | |
248c7f9c | 682 | /* Create sockets... */ |
123e8da1 | 683 | umem = xsk_configure_umem(bufs, NUM_FRAMES * opt_xsk_frame_size); |
248c7f9c | 684 | xsks[num_socks++] = xsk_configure_socket(umem); |
b4b8faa1 | 685 | |
248c7f9c MK |
686 | if (opt_bench == BENCH_TXONLY) { |
687 | int i; | |
b4b8faa1 | 688 | |
123e8da1 MM |
689 | for (i = 0; i < NUM_FRAMES; i++) |
690 | (void)gen_eth_frame(umem, i * opt_xsk_frame_size); | |
b4b8faa1 MK |
691 | } |
692 | ||
693 | signal(SIGINT, int_exit); | |
694 | signal(SIGTERM, int_exit); | |
695 | signal(SIGABRT, int_exit); | |
696 | ||
697 | setlocale(LC_ALL, ""); | |
698 | ||
699 | ret = pthread_create(&pt, NULL, poller, NULL); | |
248c7f9c MK |
700 | if (ret) |
701 | exit_with_error(ret); | |
b4b8faa1 MK |
702 | |
703 | prev_time = get_nsecs(); | |
704 | ||
705 | if (opt_bench == BENCH_RXDROP) | |
706 | rx_drop_all(); | |
707 | else if (opt_bench == BENCH_TXONLY) | |
708 | tx_only(xsks[0]); | |
709 | else | |
710 | l2fwd(xsks[0]); | |
711 | ||
712 | return 0; | |
713 | } |