Commit | Line | Data |
---|---|---|
b4b8faa1 | 1 | // SPDX-License-Identifier: GPL-2.0 |
dac09149 | 2 | /* Copyright(c) 2017 - 2018 Intel Corporation. */ |
b4b8faa1 | 3 | |
b4b8faa1 MK |
4 | #include <errno.h> |
5 | #include <getopt.h> | |
6 | #include <libgen.h> | |
7 | #include <linux/bpf.h> | |
8 | #include <linux/if_link.h> | |
9 | #include <linux/if_xdp.h> | |
10 | #include <linux/if_ether.h> | |
4a3c23ae | 11 | #include <linux/ip.h> |
67ed3755 | 12 | #include <linux/limits.h> |
4a3c23ae JJ |
13 | #include <linux/udp.h> |
14 | #include <arpa/inet.h> | |
248c7f9c MK |
15 | #include <locale.h> |
16 | #include <net/ethernet.h> | |
6440a6c2 | 17 | #include <netinet/ether.h> |
b4b8faa1 | 18 | #include <net/if.h> |
248c7f9c MK |
19 | #include <poll.h> |
20 | #include <pthread.h> | |
b4b8faa1 MK |
21 | #include <signal.h> |
22 | #include <stdbool.h> | |
23 | #include <stdio.h> | |
24 | #include <stdlib.h> | |
25 | #include <string.h> | |
3627d970 | 26 | #include <sys/capability.h> |
248c7f9c | 27 | #include <sys/mman.h> |
b4b8faa1 | 28 | #include <sys/socket.h> |
248c7f9c | 29 | #include <sys/types.h> |
3627d970 | 30 | #include <sys/un.h> |
b4b8faa1 MK |
31 | #include <time.h> |
32 | #include <unistd.h> | |
fa24d0b1 | 33 | #include <sched.h> |
b4b8faa1 | 34 | |
7cf245a3 THJ |
35 | #include <bpf/libbpf.h> |
36 | #include <bpf/xsk.h> | |
2bf3e2ef | 37 | #include <bpf/bpf.h> |
7cf245a3 | 38 | #include "xdpsock.h" |
b4b8faa1 | 39 | |
c58f9815 AN |
40 | /* libbpf APIs for AF_XDP are deprecated starting from v0.7 */ |
41 | #pragma GCC diagnostic ignored "-Wdeprecated-declarations" | |
42 | ||
b4b8faa1 MK |
43 | #ifndef SOL_XDP |
44 | #define SOL_XDP 283 | |
45 | #endif | |
46 | ||
47 | #ifndef AF_XDP | |
48 | #define AF_XDP 44 | |
49 | #endif | |
50 | ||
51 | #ifndef PF_XDP | |
52 | #define PF_XDP AF_XDP | |
53 | #endif | |
54 | ||
248c7f9c | 55 | #define NUM_FRAMES (4 * 1024) |
4a3c23ae | 56 | #define MIN_PKT_SIZE 64 |
b4b8faa1 MK |
57 | |
58 | #define DEBUG_HEXDUMP 0 | |
59 | ||
2741a049 OBL |
60 | #define VLAN_PRIO_MASK 0xe000 /* Priority Code Point */ |
61 | #define VLAN_PRIO_SHIFT 13 | |
62 | #define VLAN_VID_MASK 0x0fff /* VLAN Identifier */ | |
63 | #define VLAN_VID__DEFAULT 1 | |
64 | #define VLAN_PRI__DEFAULT 0 | |
65 | ||
fa0d27a1 OBL |
66 | #define NSEC_PER_SEC 1000000000UL |
67 | #define NSEC_PER_USEC 1000 | |
68 | ||
fa24d0b1 OBL |
69 | #define SCHED_PRI__DEFAULT 0 |
70 | ||
a412ef54 | 71 | typedef __u64 u64; |
b4b8faa1 | 72 | typedef __u32 u32; |
4a3c23ae JJ |
73 | typedef __u16 u16; |
74 | typedef __u8 u8; | |
b4b8faa1 MK |
75 | |
76 | static unsigned long prev_time; | |
fa0d27a1 OBL |
77 | static long tx_cycle_diff_min; |
78 | static long tx_cycle_diff_max; | |
79 | static double tx_cycle_diff_ave; | |
80 | static long tx_cycle_cnt; | |
b4b8faa1 MK |
81 | |
82 | enum benchmark_type { | |
83 | BENCH_RXDROP = 0, | |
84 | BENCH_TXONLY = 1, | |
85 | BENCH_L2FWD = 2, | |
86 | }; | |
87 | ||
88 | static enum benchmark_type opt_bench = BENCH_RXDROP; | |
743e568c | 89 | static u32 opt_xdp_flags = XDP_FLAGS_UPDATE_IF_NOEXIST; |
b4b8faa1 MK |
90 | static const char *opt_if = ""; |
91 | static int opt_ifindex; | |
92 | static int opt_queue; | |
d3f11b01 JJ |
93 | static unsigned long opt_duration; |
94 | static unsigned long start_time; | |
95 | static bool benchmark_done; | |
cd9e72b6 | 96 | static u32 opt_batch_size = 64; |
ece6e969 | 97 | static int opt_pkt_count; |
4a3c23ae | 98 | static u16 opt_pkt_size = MIN_PKT_SIZE; |
46e3268e | 99 | static u32 opt_pkt_fill_pattern = 0x12345678; |
2741a049 OBL |
100 | static bool opt_vlan_tag; |
101 | static u16 opt_pkt_vlan_id = VLAN_VID__DEFAULT; | |
102 | static u16 opt_pkt_vlan_pri = VLAN_PRI__DEFAULT; | |
6440a6c2 OBL |
103 | static struct ether_addr opt_txdmac = {{ 0x3c, 0xfd, 0xfe, |
104 | 0x9e, 0x7f, 0x71 }}; | |
105 | static struct ether_addr opt_txsmac = {{ 0xec, 0xb1, 0xd7, | |
106 | 0x98, 0x3a, 0xc0 }}; | |
b36c3206 | 107 | static bool opt_extra_stats; |
74e00676 | 108 | static bool opt_quiet; |
60dc609d | 109 | static bool opt_app_stats; |
67ed3755 CL |
110 | static const char *opt_irq_str = ""; |
111 | static u32 irq_no; | |
112 | static int irqs_at_init = -1; | |
eb68db45 | 113 | static u32 sequence; |
b4b8faa1 | 114 | static int opt_poll; |
b4b8faa1 | 115 | static int opt_interval = 1; |
8121e789 | 116 | static int opt_retries = 3; |
46738f73 | 117 | static u32 opt_xdp_bind_flags = XDP_USE_NEED_WAKEUP; |
c543f546 KL |
118 | static u32 opt_umem_flags; |
119 | static int opt_unaligned_chunks; | |
3945b37a | 120 | static int opt_mmap_flags; |
123e8da1 | 121 | static int opt_xsk_frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE; |
46738f73 MK |
122 | static int opt_timeout = 1000; |
123 | static bool opt_need_wakeup = true; | |
2e5d72c1 | 124 | static u32 opt_num_xsks = 1; |
2620e92a | 125 | static u32 prog_id; |
b35fc148 | 126 | static bool opt_busy_poll; |
3627d970 | 127 | static bool opt_reduced_cap; |
5a388254 | 128 | static clockid_t opt_clock = CLOCK_MONOTONIC; |
fa0d27a1 | 129 | static unsigned long opt_tx_cycle_ns; |
fa24d0b1 OBL |
130 | static int opt_schpolicy = SCHED_OTHER; |
131 | static int opt_schprio = SCHED_PRI__DEFAULT; | |
eb68db45 | 132 | static bool opt_tstamp; |
b4b8faa1 | 133 | |
2741a049 OBL |
134 | struct vlan_ethhdr { |
135 | unsigned char h_dest[6]; | |
136 | unsigned char h_source[6]; | |
137 | __be16 h_vlan_proto; | |
138 | __be16 h_vlan_TCI; | |
139 | __be16 h_vlan_encapsulated_proto; | |
140 | }; | |
141 | ||
eb68db45 OBL |
142 | #define PKTGEN_MAGIC 0xbe9be955 |
143 | struct pktgen_hdr { | |
144 | __be32 pgh_magic; | |
145 | __be32 seq_num; | |
146 | __be32 tv_sec; | |
147 | __be32 tv_usec; | |
148 | }; | |
149 | ||
2e8806f0 | 150 | struct xsk_ring_stats { |
b4b8faa1 MK |
151 | unsigned long rx_npkts; |
152 | unsigned long tx_npkts; | |
b36c3206 CL |
153 | unsigned long rx_dropped_npkts; |
154 | unsigned long rx_invalid_npkts; | |
155 | unsigned long tx_invalid_npkts; | |
156 | unsigned long rx_full_npkts; | |
157 | unsigned long rx_fill_empty_npkts; | |
158 | unsigned long tx_empty_npkts; | |
b4b8faa1 MK |
159 | unsigned long prev_rx_npkts; |
160 | unsigned long prev_tx_npkts; | |
b36c3206 CL |
161 | unsigned long prev_rx_dropped_npkts; |
162 | unsigned long prev_rx_invalid_npkts; | |
163 | unsigned long prev_tx_invalid_npkts; | |
164 | unsigned long prev_rx_full_npkts; | |
165 | unsigned long prev_rx_fill_empty_npkts; | |
166 | unsigned long prev_tx_empty_npkts; | |
2e8806f0 CL |
167 | }; |
168 | ||
67ed3755 CL |
169 | struct xsk_driver_stats { |
170 | unsigned long intrs; | |
171 | unsigned long prev_intrs; | |
172 | }; | |
173 | ||
60dc609d CL |
174 | struct xsk_app_stats { |
175 | unsigned long rx_empty_polls; | |
176 | unsigned long fill_fail_polls; | |
177 | unsigned long copy_tx_sendtos; | |
178 | unsigned long tx_wakeup_sendtos; | |
179 | unsigned long opt_polls; | |
180 | unsigned long prev_rx_empty_polls; | |
181 | unsigned long prev_fill_fail_polls; | |
182 | unsigned long prev_copy_tx_sendtos; | |
183 | unsigned long prev_tx_wakeup_sendtos; | |
184 | unsigned long prev_opt_polls; | |
185 | }; | |
186 | ||
2e8806f0 CL |
187 | struct xsk_umem_info { |
188 | struct xsk_ring_prod fq; | |
189 | struct xsk_ring_cons cq; | |
190 | struct xsk_umem *umem; | |
191 | void *buffer; | |
192 | }; | |
193 | ||
194 | struct xsk_socket_info { | |
195 | struct xsk_ring_cons rx; | |
196 | struct xsk_ring_prod tx; | |
197 | struct xsk_umem_info *umem; | |
198 | struct xsk_socket *xsk; | |
199 | struct xsk_ring_stats ring_stats; | |
60dc609d | 200 | struct xsk_app_stats app_stats; |
67ed3755 | 201 | struct xsk_driver_stats drv_stats; |
248c7f9c | 202 | u32 outstanding_tx; |
b4b8faa1 MK |
203 | }; |
204 | ||
5a388254 OBL |
205 | static const struct clockid_map { |
206 | const char *name; | |
207 | clockid_t clockid; | |
208 | } clockids_map[] = { | |
209 | { "REALTIME", CLOCK_REALTIME }, | |
210 | { "TAI", CLOCK_TAI }, | |
211 | { "BOOTTIME", CLOCK_BOOTTIME }, | |
212 | { "MONOTONIC", CLOCK_MONOTONIC }, | |
213 | { NULL } | |
214 | }; | |
215 | ||
fa24d0b1 OBL |
216 | static const struct sched_map { |
217 | const char *name; | |
218 | int policy; | |
219 | } schmap[] = { | |
220 | { "OTHER", SCHED_OTHER }, | |
221 | { "FIFO", SCHED_FIFO }, | |
222 | { NULL } | |
223 | }; | |
224 | ||
b4b8faa1 | 225 | static int num_socks; |
248c7f9c | 226 | struct xsk_socket_info *xsks[MAX_SOCKS]; |
3627d970 | 227 | int sock; |
b4b8faa1 | 228 | |
5a388254 OBL |
229 | static int get_clockid(clockid_t *id, const char *name) |
230 | { | |
231 | const struct clockid_map *clk; | |
232 | ||
233 | for (clk = clockids_map; clk->name; clk++) { | |
234 | if (strcasecmp(clk->name, name) == 0) { | |
235 | *id = clk->clockid; | |
236 | return 0; | |
237 | } | |
238 | } | |
239 | ||
240 | return -1; | |
241 | } | |
242 | ||
fa24d0b1 OBL |
243 | static int get_schpolicy(int *policy, const char *name) |
244 | { | |
245 | const struct sched_map *sch; | |
246 | ||
247 | for (sch = schmap; sch->name; sch++) { | |
248 | if (strcasecmp(sch->name, name) == 0) { | |
249 | *policy = sch->policy; | |
250 | return 0; | |
251 | } | |
252 | } | |
253 | ||
254 | return -1; | |
255 | } | |
256 | ||
b4b8faa1 MK |
257 | static unsigned long get_nsecs(void) |
258 | { | |
259 | struct timespec ts; | |
260 | ||
5a388254 | 261 | clock_gettime(opt_clock, &ts); |
b4b8faa1 MK |
262 | return ts.tv_sec * 1000000000UL + ts.tv_nsec; |
263 | } | |
264 | ||
248c7f9c | 265 | static void print_benchmark(bool running) |
b4b8faa1 | 266 | { |
248c7f9c | 267 | const char *bench_str = "INVALID"; |
b4b8faa1 | 268 | |
248c7f9c MK |
269 | if (opt_bench == BENCH_RXDROP) |
270 | bench_str = "rxdrop"; | |
271 | else if (opt_bench == BENCH_TXONLY) | |
272 | bench_str = "txonly"; | |
273 | else if (opt_bench == BENCH_L2FWD) | |
274 | bench_str = "l2fwd"; | |
b4b8faa1 | 275 | |
248c7f9c MK |
276 | printf("%s:%d %s ", opt_if, opt_queue, bench_str); |
277 | if (opt_xdp_flags & XDP_FLAGS_SKB_MODE) | |
278 | printf("xdp-skb "); | |
279 | else if (opt_xdp_flags & XDP_FLAGS_DRV_MODE) | |
280 | printf("xdp-drv "); | |
281 | else | |
282 | printf(" "); | |
b4b8faa1 | 283 | |
248c7f9c MK |
284 | if (opt_poll) |
285 | printf("poll() "); | |
b4b8faa1 | 286 | |
248c7f9c MK |
287 | if (running) { |
288 | printf("running..."); | |
289 | fflush(stdout); | |
b4b8faa1 | 290 | } |
b4b8faa1 MK |
291 | } |
292 | ||
b36c3206 CL |
293 | static int xsk_get_xdp_stats(int fd, struct xsk_socket_info *xsk) |
294 | { | |
295 | struct xdp_statistics stats; | |
296 | socklen_t optlen; | |
297 | int err; | |
298 | ||
299 | optlen = sizeof(stats); | |
300 | err = getsockopt(fd, SOL_XDP, XDP_STATISTICS, &stats, &optlen); | |
301 | if (err) | |
302 | return err; | |
303 | ||
304 | if (optlen == sizeof(struct xdp_statistics)) { | |
2e8806f0 CL |
305 | xsk->ring_stats.rx_dropped_npkts = stats.rx_dropped; |
306 | xsk->ring_stats.rx_invalid_npkts = stats.rx_invalid_descs; | |
307 | xsk->ring_stats.tx_invalid_npkts = stats.tx_invalid_descs; | |
308 | xsk->ring_stats.rx_full_npkts = stats.rx_ring_full; | |
309 | xsk->ring_stats.rx_fill_empty_npkts = stats.rx_fill_ring_empty_descs; | |
310 | xsk->ring_stats.tx_empty_npkts = stats.tx_ring_empty_descs; | |
b36c3206 CL |
311 | return 0; |
312 | } | |
313 | ||
314 | return -EINVAL; | |
315 | } | |
316 | ||
60dc609d CL |
317 | static void dump_app_stats(long dt) |
318 | { | |
319 | int i; | |
320 | ||
321 | for (i = 0; i < num_socks && xsks[i]; i++) { | |
322 | char *fmt = "%-18s %'-14.0f %'-14lu\n"; | |
323 | double rx_empty_polls_ps, fill_fail_polls_ps, copy_tx_sendtos_ps, | |
324 | tx_wakeup_sendtos_ps, opt_polls_ps; | |
325 | ||
326 | rx_empty_polls_ps = (xsks[i]->app_stats.rx_empty_polls - | |
327 | xsks[i]->app_stats.prev_rx_empty_polls) * 1000000000. / dt; | |
328 | fill_fail_polls_ps = (xsks[i]->app_stats.fill_fail_polls - | |
329 | xsks[i]->app_stats.prev_fill_fail_polls) * 1000000000. / dt; | |
330 | copy_tx_sendtos_ps = (xsks[i]->app_stats.copy_tx_sendtos - | |
331 | xsks[i]->app_stats.prev_copy_tx_sendtos) * 1000000000. / dt; | |
332 | tx_wakeup_sendtos_ps = (xsks[i]->app_stats.tx_wakeup_sendtos - | |
333 | xsks[i]->app_stats.prev_tx_wakeup_sendtos) | |
334 | * 1000000000. / dt; | |
335 | opt_polls_ps = (xsks[i]->app_stats.opt_polls - | |
336 | xsks[i]->app_stats.prev_opt_polls) * 1000000000. / dt; | |
337 | ||
338 | printf("\n%-18s %-14s %-14s\n", "", "calls/s", "count"); | |
339 | printf(fmt, "rx empty polls", rx_empty_polls_ps, xsks[i]->app_stats.rx_empty_polls); | |
340 | printf(fmt, "fill fail polls", fill_fail_polls_ps, | |
341 | xsks[i]->app_stats.fill_fail_polls); | |
342 | printf(fmt, "copy tx sendtos", copy_tx_sendtos_ps, | |
343 | xsks[i]->app_stats.copy_tx_sendtos); | |
344 | printf(fmt, "tx wakeup sendtos", tx_wakeup_sendtos_ps, | |
345 | xsks[i]->app_stats.tx_wakeup_sendtos); | |
346 | printf(fmt, "opt polls", opt_polls_ps, xsks[i]->app_stats.opt_polls); | |
347 | ||
348 | xsks[i]->app_stats.prev_rx_empty_polls = xsks[i]->app_stats.rx_empty_polls; | |
349 | xsks[i]->app_stats.prev_fill_fail_polls = xsks[i]->app_stats.fill_fail_polls; | |
350 | xsks[i]->app_stats.prev_copy_tx_sendtos = xsks[i]->app_stats.copy_tx_sendtos; | |
351 | xsks[i]->app_stats.prev_tx_wakeup_sendtos = xsks[i]->app_stats.tx_wakeup_sendtos; | |
352 | xsks[i]->app_stats.prev_opt_polls = xsks[i]->app_stats.opt_polls; | |
353 | } | |
fa0d27a1 OBL |
354 | |
355 | if (opt_tx_cycle_ns) { | |
356 | printf("\n%-18s %-10s %-10s %-10s %-10s %-10s\n", | |
357 | "", "period", "min", "ave", "max", "cycle"); | |
358 | printf("%-18s %-10lu %-10lu %-10lu %-10lu %-10lu\n", | |
359 | "Cyclic TX", opt_tx_cycle_ns, tx_cycle_diff_min, | |
360 | (long)(tx_cycle_diff_ave / tx_cycle_cnt), | |
361 | tx_cycle_diff_max, tx_cycle_cnt); | |
362 | } | |
60dc609d CL |
363 | } |
364 | ||
67ed3755 CL |
365 | static bool get_interrupt_number(void) |
366 | { | |
367 | FILE *f_int_proc; | |
368 | char line[4096]; | |
369 | bool found = false; | |
370 | ||
371 | f_int_proc = fopen("/proc/interrupts", "r"); | |
372 | if (f_int_proc == NULL) { | |
373 | printf("Failed to open /proc/interrupts.\n"); | |
374 | return found; | |
375 | } | |
376 | ||
377 | while (!feof(f_int_proc) && !found) { | |
378 | /* Make sure to read a full line at a time */ | |
379 | if (fgets(line, sizeof(line), f_int_proc) == NULL || | |
380 | line[strlen(line) - 1] != '\n') { | |
381 | printf("Error reading from interrupts file\n"); | |
382 | break; | |
383 | } | |
384 | ||
385 | /* Extract interrupt number from line */ | |
386 | if (strstr(line, opt_irq_str) != NULL) { | |
387 | irq_no = atoi(line); | |
388 | found = true; | |
389 | break; | |
390 | } | |
391 | } | |
392 | ||
393 | fclose(f_int_proc); | |
394 | ||
395 | return found; | |
396 | } | |
397 | ||
398 | static int get_irqs(void) | |
399 | { | |
400 | char count_path[PATH_MAX]; | |
401 | int total_intrs = -1; | |
402 | FILE *f_count_proc; | |
403 | char line[4096]; | |
404 | ||
405 | snprintf(count_path, sizeof(count_path), | |
406 | "/sys/kernel/irq/%i/per_cpu_count", irq_no); | |
407 | f_count_proc = fopen(count_path, "r"); | |
408 | if (f_count_proc == NULL) { | |
409 | printf("Failed to open %s\n", count_path); | |
410 | return total_intrs; | |
411 | } | |
412 | ||
413 | if (fgets(line, sizeof(line), f_count_proc) == NULL || | |
414 | line[strlen(line) - 1] != '\n') { | |
415 | printf("Error reading from %s\n", count_path); | |
416 | } else { | |
417 | static const char com[2] = ","; | |
418 | char *token; | |
419 | ||
420 | total_intrs = 0; | |
421 | token = strtok(line, com); | |
422 | while (token != NULL) { | |
423 | /* sum up interrupts across all cores */ | |
424 | total_intrs += atoi(token); | |
425 | token = strtok(NULL, com); | |
426 | } | |
427 | } | |
428 | ||
429 | fclose(f_count_proc); | |
430 | ||
431 | return total_intrs; | |
432 | } | |
433 | ||
434 | static void dump_driver_stats(long dt) | |
435 | { | |
436 | int i; | |
437 | ||
438 | for (i = 0; i < num_socks && xsks[i]; i++) { | |
439 | char *fmt = "%-18s %'-14.0f %'-14lu\n"; | |
440 | double intrs_ps; | |
441 | int n_ints = get_irqs(); | |
442 | ||
443 | if (n_ints < 0) { | |
444 | printf("error getting intr info for intr %i\n", irq_no); | |
445 | return; | |
446 | } | |
447 | xsks[i]->drv_stats.intrs = n_ints - irqs_at_init; | |
448 | ||
449 | intrs_ps = (xsks[i]->drv_stats.intrs - xsks[i]->drv_stats.prev_intrs) * | |
450 | 1000000000. / dt; | |
451 | ||
452 | printf("\n%-18s %-14s %-14s\n", "", "intrs/s", "count"); | |
453 | printf(fmt, "irqs", intrs_ps, xsks[i]->drv_stats.intrs); | |
454 | ||
455 | xsks[i]->drv_stats.prev_intrs = xsks[i]->drv_stats.intrs; | |
456 | } | |
457 | } | |
458 | ||
248c7f9c | 459 | static void dump_stats(void) |
b4b8faa1 | 460 | { |
248c7f9c MK |
461 | unsigned long now = get_nsecs(); |
462 | long dt = now - prev_time; | |
463 | int i; | |
b4b8faa1 | 464 | |
248c7f9c | 465 | prev_time = now; |
b4b8faa1 | 466 | |
248c7f9c | 467 | for (i = 0; i < num_socks && xsks[i]; i++) { |
60dc609d | 468 | char *fmt = "%-18s %'-14.0f %'-14lu\n"; |
b36c3206 CL |
469 | double rx_pps, tx_pps, dropped_pps, rx_invalid_pps, full_pps, fill_empty_pps, |
470 | tx_invalid_pps, tx_empty_pps; | |
b4b8faa1 | 471 | |
2e8806f0 | 472 | rx_pps = (xsks[i]->ring_stats.rx_npkts - xsks[i]->ring_stats.prev_rx_npkts) * |
248c7f9c | 473 | 1000000000. / dt; |
2e8806f0 | 474 | tx_pps = (xsks[i]->ring_stats.tx_npkts - xsks[i]->ring_stats.prev_tx_npkts) * |
248c7f9c | 475 | 1000000000. / dt; |
b4b8faa1 | 476 | |
248c7f9c MK |
477 | printf("\n sock%d@", i); |
478 | print_benchmark(false); | |
479 | printf("\n"); | |
b4b8faa1 | 480 | |
60dc609d | 481 | printf("%-18s %-14s %-14s %-14.2f\n", "", "pps", "pkts", |
248c7f9c | 482 | dt / 1000000000.); |
2e8806f0 CL |
483 | printf(fmt, "rx", rx_pps, xsks[i]->ring_stats.rx_npkts); |
484 | printf(fmt, "tx", tx_pps, xsks[i]->ring_stats.tx_npkts); | |
b4b8faa1 | 485 | |
2e8806f0 CL |
486 | xsks[i]->ring_stats.prev_rx_npkts = xsks[i]->ring_stats.rx_npkts; |
487 | xsks[i]->ring_stats.prev_tx_npkts = xsks[i]->ring_stats.tx_npkts; | |
b36c3206 CL |
488 | |
489 | if (opt_extra_stats) { | |
490 | if (!xsk_get_xdp_stats(xsk_socket__fd(xsks[i]->xsk), xsks[i])) { | |
2e8806f0 CL |
491 | dropped_pps = (xsks[i]->ring_stats.rx_dropped_npkts - |
492 | xsks[i]->ring_stats.prev_rx_dropped_npkts) * | |
493 | 1000000000. / dt; | |
494 | rx_invalid_pps = (xsks[i]->ring_stats.rx_invalid_npkts - | |
495 | xsks[i]->ring_stats.prev_rx_invalid_npkts) * | |
496 | 1000000000. / dt; | |
497 | tx_invalid_pps = (xsks[i]->ring_stats.tx_invalid_npkts - | |
498 | xsks[i]->ring_stats.prev_tx_invalid_npkts) * | |
499 | 1000000000. / dt; | |
500 | full_pps = (xsks[i]->ring_stats.rx_full_npkts - | |
501 | xsks[i]->ring_stats.prev_rx_full_npkts) * | |
502 | 1000000000. / dt; | |
503 | fill_empty_pps = (xsks[i]->ring_stats.rx_fill_empty_npkts - | |
504 | xsks[i]->ring_stats.prev_rx_fill_empty_npkts) * | |
505 | 1000000000. / dt; | |
506 | tx_empty_pps = (xsks[i]->ring_stats.tx_empty_npkts - | |
507 | xsks[i]->ring_stats.prev_tx_empty_npkts) * | |
508 | 1000000000. / dt; | |
b36c3206 CL |
509 | |
510 | printf(fmt, "rx dropped", dropped_pps, | |
2e8806f0 | 511 | xsks[i]->ring_stats.rx_dropped_npkts); |
b36c3206 | 512 | printf(fmt, "rx invalid", rx_invalid_pps, |
2e8806f0 | 513 | xsks[i]->ring_stats.rx_invalid_npkts); |
b36c3206 | 514 | printf(fmt, "tx invalid", tx_invalid_pps, |
2e8806f0 | 515 | xsks[i]->ring_stats.tx_invalid_npkts); |
b36c3206 | 516 | printf(fmt, "rx queue full", full_pps, |
2e8806f0 | 517 | xsks[i]->ring_stats.rx_full_npkts); |
b36c3206 | 518 | printf(fmt, "fill ring empty", fill_empty_pps, |
2e8806f0 | 519 | xsks[i]->ring_stats.rx_fill_empty_npkts); |
b36c3206 | 520 | printf(fmt, "tx ring empty", tx_empty_pps, |
2e8806f0 CL |
521 | xsks[i]->ring_stats.tx_empty_npkts); |
522 | ||
523 | xsks[i]->ring_stats.prev_rx_dropped_npkts = | |
524 | xsks[i]->ring_stats.rx_dropped_npkts; | |
525 | xsks[i]->ring_stats.prev_rx_invalid_npkts = | |
526 | xsks[i]->ring_stats.rx_invalid_npkts; | |
527 | xsks[i]->ring_stats.prev_tx_invalid_npkts = | |
528 | xsks[i]->ring_stats.tx_invalid_npkts; | |
529 | xsks[i]->ring_stats.prev_rx_full_npkts = | |
530 | xsks[i]->ring_stats.rx_full_npkts; | |
531 | xsks[i]->ring_stats.prev_rx_fill_empty_npkts = | |
532 | xsks[i]->ring_stats.rx_fill_empty_npkts; | |
533 | xsks[i]->ring_stats.prev_tx_empty_npkts = | |
534 | xsks[i]->ring_stats.tx_empty_npkts; | |
b36c3206 CL |
535 | } else { |
536 | printf("%-15s\n", "Error retrieving extra stats"); | |
537 | } | |
538 | } | |
b4b8faa1 | 539 | } |
60dc609d CL |
540 | |
541 | if (opt_app_stats) | |
542 | dump_app_stats(dt); | |
67ed3755 CL |
543 | if (irq_no) |
544 | dump_driver_stats(dt); | |
b4b8faa1 MK |
545 | } |
546 | ||
d3f11b01 JJ |
547 | static bool is_benchmark_done(void) |
548 | { | |
549 | if (opt_duration > 0) { | |
550 | unsigned long dt = (get_nsecs() - start_time); | |
551 | ||
552 | if (dt >= opt_duration) | |
553 | benchmark_done = true; | |
554 | } | |
555 | return benchmark_done; | |
556 | } | |
557 | ||
248c7f9c | 558 | static void *poller(void *arg) |
b4b8faa1 | 559 | { |
248c7f9c | 560 | (void)arg; |
d3f11b01 | 561 | while (!is_benchmark_done()) { |
248c7f9c MK |
562 | sleep(opt_interval); |
563 | dump_stats(); | |
b4b8faa1 MK |
564 | } |
565 | ||
248c7f9c | 566 | return NULL; |
b4b8faa1 MK |
567 | } |
568 | ||
2620e92a WH |
569 | static void remove_xdp_program(void) |
570 | { | |
571 | u32 curr_prog_id = 0; | |
572 | ||
d4e34bfc AN |
573 | if (bpf_xdp_query_id(opt_ifindex, opt_xdp_flags, &curr_prog_id)) { |
574 | printf("bpf_xdp_query_id failed\n"); | |
2620e92a WH |
575 | exit(EXIT_FAILURE); |
576 | } | |
577 | ||
578 | if (prog_id == curr_prog_id) | |
d4e34bfc | 579 | bpf_xdp_detach(opt_ifindex, opt_xdp_flags, NULL); |
2620e92a WH |
580 | else if (!curr_prog_id) |
581 | printf("couldn't find a prog id on a given interface\n"); | |
582 | else | |
583 | printf("program on interface changed, not removing\n"); | |
584 | } | |
585 | ||
c9d27c9e | 586 | static void int_exit(int sig) |
b4b8faa1 | 587 | { |
c9d27c9e | 588 | benchmark_done = true; |
b4b8faa1 MK |
589 | } |
590 | ||
c9d27c9e MF |
591 | static void __exit_with_error(int error, const char *file, const char *func, |
592 | int line) | |
69525588 | 593 | { |
c9d27c9e MF |
594 | fprintf(stderr, "%s:%s:%i: errno: %d/\"%s\"\n", file, func, |
595 | line, error, strerror(error)); | |
2620e92a WH |
596 | |
597 | if (opt_num_xsks > 1) | |
598 | remove_xdp_program(); | |
c9d27c9e | 599 | exit(EXIT_FAILURE); |
69525588 JJ |
600 | } |
601 | ||
c9d27c9e MF |
602 | #define exit_with_error(error) __exit_with_error(error, __FILE__, __func__, __LINE__) |
603 | ||
69525588 | 604 | static void xdpsock_cleanup(void) |
b4b8faa1 | 605 | { |
248c7f9c | 606 | struct xsk_umem *umem = xsks[0]->umem->umem; |
c9d27c9e | 607 | int i, cmd = CLOSE_CONN; |
b4b8faa1 | 608 | |
248c7f9c | 609 | dump_stats(); |
2e5d72c1 MK |
610 | for (i = 0; i < num_socks; i++) |
611 | xsk_socket__delete(xsks[i]->xsk); | |
248c7f9c | 612 | (void)xsk_umem__delete(umem); |
b4b8faa1 | 613 | |
c9d27c9e MF |
614 | if (opt_reduced_cap) { |
615 | if (write(sock, &cmd, sizeof(int)) < 0) | |
616 | exit_with_error(errno); | |
617 | } | |
2620e92a WH |
618 | |
619 | if (opt_num_xsks > 1) | |
620 | remove_xdp_program(); | |
b4b8faa1 MK |
621 | } |
622 | ||
b4b8faa1 MK |
623 | static void swap_mac_addresses(void *data) |
624 | { | |
625 | struct ether_header *eth = (struct ether_header *)data; | |
626 | struct ether_addr *src_addr = (struct ether_addr *)ð->ether_shost; | |
627 | struct ether_addr *dst_addr = (struct ether_addr *)ð->ether_dhost; | |
628 | struct ether_addr tmp; | |
629 | ||
630 | tmp = *src_addr; | |
631 | *src_addr = *dst_addr; | |
632 | *dst_addr = tmp; | |
633 | } | |
634 | ||
a412ef54 | 635 | static void hex_dump(void *pkt, size_t length, u64 addr) |
b4b8faa1 | 636 | { |
b4b8faa1 MK |
637 | const unsigned char *address = (unsigned char *)pkt; |
638 | const unsigned char *line = address; | |
639 | size_t line_size = 32; | |
640 | unsigned char c; | |
a412ef54 BT |
641 | char buf[32]; |
642 | int i = 0; | |
b4b8faa1 | 643 | |
a412ef54 BT |
644 | if (!DEBUG_HEXDUMP) |
645 | return; | |
646 | ||
647 | sprintf(buf, "addr=%llu", addr); | |
b4b8faa1 | 648 | printf("length = %zu\n", length); |
a412ef54 | 649 | printf("%s | ", buf); |
b4b8faa1 MK |
650 | while (length-- > 0) { |
651 | printf("%02X ", *address++); | |
652 | if (!(++i % line_size) || (length == 0 && i % line_size)) { | |
653 | if (length == 0) { | |
654 | while (i++ % line_size) | |
655 | printf("__ "); | |
656 | } | |
657 | printf(" | "); /* right close */ | |
658 | while (line < address) { | |
659 | c = *line++; | |
660 | printf("%c", (c < 33 || c == 255) ? 0x2E : c); | |
661 | } | |
662 | printf("\n"); | |
663 | if (length > 0) | |
a412ef54 | 664 | printf("%s | ", buf); |
b4b8faa1 MK |
665 | } |
666 | } | |
667 | printf("\n"); | |
668 | } | |
b4b8faa1 | 669 | |
4a3c23ae JJ |
670 | static void *memset32_htonl(void *dest, u32 val, u32 size) |
671 | { | |
672 | u32 *ptr = (u32 *)dest; | |
673 | int i; | |
674 | ||
675 | val = htonl(val); | |
676 | ||
677 | for (i = 0; i < (size & (~0x3)); i += 4) | |
678 | ptr[i >> 2] = val; | |
679 | ||
680 | for (; i < size; i++) | |
681 | ((char *)dest)[i] = ((char *)&val)[i & 3]; | |
682 | ||
683 | return dest; | |
684 | } | |
685 | ||
686 | /* | |
687 | * This function code has been taken from | |
688 | * Linux kernel lib/checksum.c | |
689 | */ | |
690 | static inline unsigned short from32to16(unsigned int x) | |
691 | { | |
692 | /* add up 16-bit and 16-bit for 16+c bit */ | |
693 | x = (x & 0xffff) + (x >> 16); | |
694 | /* add up carry.. */ | |
695 | x = (x & 0xffff) + (x >> 16); | |
696 | return x; | |
697 | } | |
698 | ||
699 | /* | |
700 | * This function code has been taken from | |
701 | * Linux kernel lib/checksum.c | |
702 | */ | |
703 | static unsigned int do_csum(const unsigned char *buff, int len) | |
704 | { | |
705 | unsigned int result = 0; | |
706 | int odd; | |
707 | ||
708 | if (len <= 0) | |
709 | goto out; | |
710 | odd = 1 & (unsigned long)buff; | |
711 | if (odd) { | |
712 | #ifdef __LITTLE_ENDIAN | |
713 | result += (*buff << 8); | |
714 | #else | |
715 | result = *buff; | |
716 | #endif | |
717 | len--; | |
718 | buff++; | |
719 | } | |
720 | if (len >= 2) { | |
721 | if (2 & (unsigned long)buff) { | |
722 | result += *(unsigned short *)buff; | |
723 | len -= 2; | |
724 | buff += 2; | |
725 | } | |
726 | if (len >= 4) { | |
727 | const unsigned char *end = buff + | |
728 | ((unsigned int)len & ~3); | |
729 | unsigned int carry = 0; | |
730 | ||
731 | do { | |
732 | unsigned int w = *(unsigned int *)buff; | |
733 | ||
734 | buff += 4; | |
735 | result += carry; | |
736 | result += w; | |
737 | carry = (w > result); | |
738 | } while (buff < end); | |
739 | result += carry; | |
740 | result = (result & 0xffff) + (result >> 16); | |
741 | } | |
742 | if (len & 2) { | |
743 | result += *(unsigned short *)buff; | |
744 | buff += 2; | |
745 | } | |
746 | } | |
747 | if (len & 1) | |
748 | #ifdef __LITTLE_ENDIAN | |
749 | result += *buff; | |
750 | #else | |
751 | result += (*buff << 8); | |
752 | #endif | |
753 | result = from32to16(result); | |
754 | if (odd) | |
755 | result = ((result >> 8) & 0xff) | ((result & 0xff) << 8); | |
756 | out: | |
757 | return result; | |
758 | } | |
759 | ||
4a3c23ae JJ |
760 | /* |
761 | * This is a version of ip_compute_csum() optimized for IP headers, | |
762 | * which always checksum on 4 octet boundaries. | |
763 | * This function code has been taken from | |
764 | * Linux kernel lib/checksum.c | |
765 | */ | |
f4700a62 | 766 | static inline __sum16 ip_fast_csum(const void *iph, unsigned int ihl) |
4a3c23ae | 767 | { |
29f24c43 | 768 | return (__sum16)~do_csum(iph, ihl * 4); |
4a3c23ae JJ |
769 | } |
770 | ||
771 | /* | |
772 | * Fold a partial checksum | |
773 | * This function code has been taken from | |
774 | * Linux kernel include/asm-generic/checksum.h | |
775 | */ | |
776 | static inline __sum16 csum_fold(__wsum csum) | |
777 | { | |
29f24c43 | 778 | u32 sum = (u32)csum; |
4a3c23ae JJ |
779 | |
780 | sum = (sum & 0xffff) + (sum >> 16); | |
781 | sum = (sum & 0xffff) + (sum >> 16); | |
29f24c43 | 782 | return (__sum16)~sum; |
4a3c23ae JJ |
783 | } |
784 | ||
785 | /* | |
786 | * This function code has been taken from | |
787 | * Linux kernel lib/checksum.c | |
788 | */ | |
789 | static inline u32 from64to32(u64 x) | |
790 | { | |
791 | /* add up 32-bit and 32-bit for 32+c bit */ | |
792 | x = (x & 0xffffffff) + (x >> 32); | |
793 | /* add up carry.. */ | |
794 | x = (x & 0xffffffff) + (x >> 32); | |
795 | return (u32)x; | |
796 | } | |
797 | ||
798 | __wsum csum_tcpudp_nofold(__be32 saddr, __be32 daddr, | |
799 | __u32 len, __u8 proto, __wsum sum); | |
800 | ||
801 | /* | |
802 | * This function code has been taken from | |
803 | * Linux kernel lib/checksum.c | |
804 | */ | |
805 | __wsum csum_tcpudp_nofold(__be32 saddr, __be32 daddr, | |
806 | __u32 len, __u8 proto, __wsum sum) | |
807 | { | |
29f24c43 | 808 | unsigned long long s = (u32)sum; |
4a3c23ae | 809 | |
29f24c43 NS |
810 | s += (u32)saddr; |
811 | s += (u32)daddr; | |
4a3c23ae JJ |
812 | #ifdef __BIG_ENDIAN__ |
813 | s += proto + len; | |
814 | #else | |
815 | s += (proto + len) << 8; | |
816 | #endif | |
29f24c43 | 817 | return (__wsum)from64to32(s); |
4a3c23ae JJ |
818 | } |
819 | ||
820 | /* | |
821 | * This function has been taken from | |
822 | * Linux kernel include/asm-generic/checksum.h | |
823 | */ | |
824 | static inline __sum16 | |
825 | csum_tcpudp_magic(__be32 saddr, __be32 daddr, __u32 len, | |
826 | __u8 proto, __wsum sum) | |
827 | { | |
828 | return csum_fold(csum_tcpudp_nofold(saddr, daddr, len, proto, sum)); | |
829 | } | |
830 | ||
831 | static inline u16 udp_csum(u32 saddr, u32 daddr, u32 len, | |
832 | u8 proto, u16 *udp_pkt) | |
833 | { | |
834 | u32 csum = 0; | |
835 | u32 cnt = 0; | |
836 | ||
837 | /* udp hdr and data */ | |
838 | for (; cnt < len; cnt += 2) | |
839 | csum += udp_pkt[cnt >> 1]; | |
840 | ||
841 | return csum_tcpudp_magic(saddr, daddr, len, proto, csum); | |
842 | } | |
843 | ||
844 | #define ETH_FCS_SIZE 4 | |
845 | ||
2741a049 OBL |
846 | #define ETH_HDR_SIZE (opt_vlan_tag ? sizeof(struct vlan_ethhdr) : \ |
847 | sizeof(struct ethhdr)) | |
eb68db45 | 848 | #define PKTGEN_HDR_SIZE (opt_tstamp ? sizeof(struct pktgen_hdr) : 0) |
2741a049 | 849 | #define PKT_HDR_SIZE (ETH_HDR_SIZE + sizeof(struct iphdr) + \ |
eb68db45 OBL |
850 | sizeof(struct udphdr) + PKTGEN_HDR_SIZE) |
851 | #define PKTGEN_HDR_OFFSET (ETH_HDR_SIZE + sizeof(struct iphdr) + \ | |
852 | sizeof(struct udphdr)) | |
853 | #define PKTGEN_SIZE_MIN (PKTGEN_HDR_OFFSET + sizeof(struct pktgen_hdr) + \ | |
854 | ETH_FCS_SIZE) | |
4a3c23ae JJ |
855 | |
856 | #define PKT_SIZE (opt_pkt_size - ETH_FCS_SIZE) | |
2741a049 | 857 | #define IP_PKT_SIZE (PKT_SIZE - ETH_HDR_SIZE) |
4a3c23ae | 858 | #define UDP_PKT_SIZE (IP_PKT_SIZE - sizeof(struct iphdr)) |
eb68db45 OBL |
859 | #define UDP_PKT_DATA_SIZE (UDP_PKT_SIZE - \ |
860 | (sizeof(struct udphdr) + PKTGEN_HDR_SIZE)) | |
4a3c23ae JJ |
861 | |
862 | static u8 pkt_data[XSK_UMEM__DEFAULT_FRAME_SIZE]; | |
863 | ||
864 | static void gen_eth_hdr_data(void) | |
865 | { | |
eb68db45 | 866 | struct pktgen_hdr *pktgen_hdr; |
2741a049 OBL |
867 | struct udphdr *udp_hdr; |
868 | struct iphdr *ip_hdr; | |
869 | ||
870 | if (opt_vlan_tag) { | |
871 | struct vlan_ethhdr *veth_hdr = (struct vlan_ethhdr *)pkt_data; | |
872 | u16 vlan_tci = 0; | |
873 | ||
874 | udp_hdr = (struct udphdr *)(pkt_data + | |
875 | sizeof(struct vlan_ethhdr) + | |
876 | sizeof(struct iphdr)); | |
877 | ip_hdr = (struct iphdr *)(pkt_data + | |
878 | sizeof(struct vlan_ethhdr)); | |
eb68db45 OBL |
879 | pktgen_hdr = (struct pktgen_hdr *)(pkt_data + |
880 | sizeof(struct vlan_ethhdr) + | |
881 | sizeof(struct iphdr) + | |
882 | sizeof(struct udphdr)); | |
2741a049 | 883 | /* ethernet & VLAN header */ |
6440a6c2 OBL |
884 | memcpy(veth_hdr->h_dest, &opt_txdmac, ETH_ALEN); |
885 | memcpy(veth_hdr->h_source, &opt_txsmac, ETH_ALEN); | |
2741a049 OBL |
886 | veth_hdr->h_vlan_proto = htons(ETH_P_8021Q); |
887 | vlan_tci = opt_pkt_vlan_id & VLAN_VID_MASK; | |
888 | vlan_tci |= (opt_pkt_vlan_pri << VLAN_PRIO_SHIFT) & VLAN_PRIO_MASK; | |
889 | veth_hdr->h_vlan_TCI = htons(vlan_tci); | |
890 | veth_hdr->h_vlan_encapsulated_proto = htons(ETH_P_IP); | |
891 | } else { | |
892 | struct ethhdr *eth_hdr = (struct ethhdr *)pkt_data; | |
893 | ||
894 | udp_hdr = (struct udphdr *)(pkt_data + | |
895 | sizeof(struct ethhdr) + | |
896 | sizeof(struct iphdr)); | |
897 | ip_hdr = (struct iphdr *)(pkt_data + | |
898 | sizeof(struct ethhdr)); | |
eb68db45 OBL |
899 | pktgen_hdr = (struct pktgen_hdr *)(pkt_data + |
900 | sizeof(struct ethhdr) + | |
901 | sizeof(struct iphdr) + | |
902 | sizeof(struct udphdr)); | |
2741a049 | 903 | /* ethernet header */ |
6440a6c2 OBL |
904 | memcpy(eth_hdr->h_dest, &opt_txdmac, ETH_ALEN); |
905 | memcpy(eth_hdr->h_source, &opt_txsmac, ETH_ALEN); | |
2741a049 OBL |
906 | eth_hdr->h_proto = htons(ETH_P_IP); |
907 | } | |
908 | ||
4a3c23ae JJ |
909 | |
910 | /* IP header */ | |
911 | ip_hdr->version = IPVERSION; | |
912 | ip_hdr->ihl = 0x5; /* 20 byte header */ | |
913 | ip_hdr->tos = 0x0; | |
914 | ip_hdr->tot_len = htons(IP_PKT_SIZE); | |
915 | ip_hdr->id = 0; | |
916 | ip_hdr->frag_off = 0; | |
917 | ip_hdr->ttl = IPDEFTTL; | |
918 | ip_hdr->protocol = IPPROTO_UDP; | |
919 | ip_hdr->saddr = htonl(0x0a0a0a10); | |
920 | ip_hdr->daddr = htonl(0x0a0a0a20); | |
921 | ||
922 | /* IP header checksum */ | |
923 | ip_hdr->check = 0; | |
924 | ip_hdr->check = ip_fast_csum((const void *)ip_hdr, ip_hdr->ihl); | |
925 | ||
926 | /* UDP header */ | |
927 | udp_hdr->source = htons(0x1000); | |
928 | udp_hdr->dest = htons(0x1000); | |
929 | udp_hdr->len = htons(UDP_PKT_SIZE); | |
930 | ||
eb68db45 OBL |
931 | if (opt_tstamp) |
932 | pktgen_hdr->pgh_magic = htonl(PKTGEN_MAGIC); | |
933 | ||
4a3c23ae | 934 | /* UDP data */ |
46e3268e | 935 | memset32_htonl(pkt_data + PKT_HDR_SIZE, opt_pkt_fill_pattern, |
4a3c23ae JJ |
936 | UDP_PKT_DATA_SIZE); |
937 | ||
938 | /* UDP header checksum */ | |
939 | udp_hdr->check = 0; | |
940 | udp_hdr->check = udp_csum(ip_hdr->saddr, ip_hdr->daddr, UDP_PKT_SIZE, | |
941 | IPPROTO_UDP, (u16 *)udp_hdr); | |
942 | } | |
943 | ||
cd9e72b6 | 944 | static void gen_eth_frame(struct xsk_umem_info *umem, u64 addr) |
b4b8faa1 | 945 | { |
248c7f9c | 946 | memcpy(xsk_umem__get_data(umem->buffer, addr), pkt_data, |
4a3c23ae | 947 | PKT_SIZE); |
b4b8faa1 MK |
948 | } |
949 | ||
248c7f9c | 950 | static struct xsk_umem_info *xsk_configure_umem(void *buffer, u64 size) |
b4b8faa1 | 951 | { |
248c7f9c | 952 | struct xsk_umem_info *umem; |
123e8da1 | 953 | struct xsk_umem_config cfg = { |
c8a039a4 MK |
954 | /* We recommend that you set the fill ring size >= HW RX ring size + |
955 | * AF_XDP RX ring size. Make sure you fill up the fill ring | |
956 | * with buffers at regular intervals, and you will with this setting | |
957 | * avoid allocation failures in the driver. These are usually quite | |
958 | * expensive since drivers have not been written to assume that | |
959 | * allocation failures are common. For regular sockets, kernel | |
960 | * allocated memory is used that only runs out in OOM situations | |
961 | * that should be rare. | |
962 | */ | |
963 | .fill_size = XSK_RING_PROD__DEFAULT_NUM_DESCS * 2, | |
123e8da1 MM |
964 | .comp_size = XSK_RING_CONS__DEFAULT_NUM_DESCS, |
965 | .frame_size = opt_xsk_frame_size, | |
966 | .frame_headroom = XSK_UMEM__DEFAULT_FRAME_HEADROOM, | |
c543f546 | 967 | .flags = opt_umem_flags |
123e8da1 | 968 | }; |
661842c4 | 969 | int ret; |
b4b8faa1 MK |
970 | |
971 | umem = calloc(1, sizeof(*umem)); | |
248c7f9c MK |
972 | if (!umem) |
973 | exit_with_error(errno); | |
b4b8faa1 | 974 | |
248c7f9c | 975 | ret = xsk_umem__create(&umem->umem, buffer, size, &umem->fq, &umem->cq, |
123e8da1 | 976 | &cfg); |
248c7f9c MK |
977 | if (ret) |
978 | exit_with_error(-ret); | |
b4b8faa1 | 979 | |
661842c4 MK |
980 | umem->buffer = buffer; |
981 | return umem; | |
982 | } | |
983 | ||
984 | static void xsk_populate_fill_ring(struct xsk_umem_info *umem) | |
985 | { | |
986 | int ret, i; | |
987 | u32 idx; | |
988 | ||
2e5d72c1 | 989 | ret = xsk_ring_prod__reserve(&umem->fq, |
c8a039a4 MK |
990 | XSK_RING_PROD__DEFAULT_NUM_DESCS * 2, &idx); |
991 | if (ret != XSK_RING_PROD__DEFAULT_NUM_DESCS * 2) | |
2e5d72c1 | 992 | exit_with_error(-ret); |
c8a039a4 | 993 | for (i = 0; i < XSK_RING_PROD__DEFAULT_NUM_DESCS * 2; i++) |
2e5d72c1 MK |
994 | *xsk_ring_prod__fill_addr(&umem->fq, idx++) = |
995 | i * opt_xsk_frame_size; | |
c8a039a4 | 996 | xsk_ring_prod__submit(&umem->fq, XSK_RING_PROD__DEFAULT_NUM_DESCS * 2); |
b4b8faa1 MK |
997 | } |
998 | ||
661842c4 MK |
999 | static struct xsk_socket_info *xsk_configure_socket(struct xsk_umem_info *umem, |
1000 | bool rx, bool tx) | |
b4b8faa1 | 1001 | { |
248c7f9c MK |
1002 | struct xsk_socket_config cfg; |
1003 | struct xsk_socket_info *xsk; | |
661842c4 MK |
1004 | struct xsk_ring_cons *rxr; |
1005 | struct xsk_ring_prod *txr; | |
248c7f9c | 1006 | int ret; |
b4b8faa1 MK |
1007 | |
1008 | xsk = calloc(1, sizeof(*xsk)); | |
248c7f9c MK |
1009 | if (!xsk) |
1010 | exit_with_error(errno); | |
1011 | ||
1012 | xsk->umem = umem; | |
1013 | cfg.rx_size = XSK_RING_CONS__DEFAULT_NUM_DESCS; | |
1014 | cfg.tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS; | |
3627d970 | 1015 | if (opt_num_xsks > 1 || opt_reduced_cap) |
2e5d72c1 MK |
1016 | cfg.libbpf_flags = XSK_LIBBPF_FLAGS__INHIBIT_PROG_LOAD; |
1017 | else | |
1018 | cfg.libbpf_flags = 0; | |
248c7f9c MK |
1019 | cfg.xdp_flags = opt_xdp_flags; |
1020 | cfg.bind_flags = opt_xdp_bind_flags; | |
2e5d72c1 | 1021 | |
661842c4 MK |
1022 | rxr = rx ? &xsk->rx : NULL; |
1023 | txr = tx ? &xsk->tx : NULL; | |
1024 | ret = xsk_socket__create(&xsk->xsk, opt_if, opt_queue, umem->umem, | |
1025 | rxr, txr, &cfg); | |
248c7f9c MK |
1026 | if (ret) |
1027 | exit_with_error(-ret); | |
1028 | ||
d4e34bfc | 1029 | ret = bpf_xdp_query_id(opt_ifindex, opt_xdp_flags, &prog_id); |
2620e92a WH |
1030 | if (ret) |
1031 | exit_with_error(-ret); | |
1032 | ||
60dc609d CL |
1033 | xsk->app_stats.rx_empty_polls = 0; |
1034 | xsk->app_stats.fill_fail_polls = 0; | |
1035 | xsk->app_stats.copy_tx_sendtos = 0; | |
1036 | xsk->app_stats.tx_wakeup_sendtos = 0; | |
1037 | xsk->app_stats.opt_polls = 0; | |
1038 | xsk->app_stats.prev_rx_empty_polls = 0; | |
1039 | xsk->app_stats.prev_fill_fail_polls = 0; | |
1040 | xsk->app_stats.prev_copy_tx_sendtos = 0; | |
1041 | xsk->app_stats.prev_tx_wakeup_sendtos = 0; | |
1042 | xsk->app_stats.prev_opt_polls = 0; | |
1043 | ||
b4b8faa1 MK |
1044 | return xsk; |
1045 | } | |
1046 | ||
b4b8faa1 MK |
1047 | static struct option long_options[] = { |
1048 | {"rxdrop", no_argument, 0, 'r'}, | |
1049 | {"txonly", no_argument, 0, 't'}, | |
1050 | {"l2fwd", no_argument, 0, 'l'}, | |
1051 | {"interface", required_argument, 0, 'i'}, | |
1052 | {"queue", required_argument, 0, 'q'}, | |
1053 | {"poll", no_argument, 0, 'p'}, | |
b4b8faa1 MK |
1054 | {"xdp-skb", no_argument, 0, 'S'}, |
1055 | {"xdp-native", no_argument, 0, 'N'}, | |
1056 | {"interval", required_argument, 0, 'n'}, | |
8121e789 | 1057 | {"retries", required_argument, 0, 'O'}, |
58c50ae4 BT |
1058 | {"zero-copy", no_argument, 0, 'z'}, |
1059 | {"copy", no_argument, 0, 'c'}, | |
123e8da1 | 1060 | {"frame-size", required_argument, 0, 'f'}, |
46738f73 | 1061 | {"no-need-wakeup", no_argument, 0, 'm'}, |
c543f546 | 1062 | {"unaligned", no_argument, 0, 'u'}, |
2e5d72c1 | 1063 | {"shared-umem", no_argument, 0, 'M'}, |
b3133329 | 1064 | {"force", no_argument, 0, 'F'}, |
d3f11b01 | 1065 | {"duration", required_argument, 0, 'd'}, |
5a388254 | 1066 | {"clock", required_argument, 0, 'w'}, |
cd9e72b6 | 1067 | {"batch-size", required_argument, 0, 'b'}, |
ece6e969 | 1068 | {"tx-pkt-count", required_argument, 0, 'C'}, |
4a3c23ae | 1069 | {"tx-pkt-size", required_argument, 0, 's'}, |
46e3268e | 1070 | {"tx-pkt-pattern", required_argument, 0, 'P'}, |
2741a049 OBL |
1071 | {"tx-vlan", no_argument, 0, 'V'}, |
1072 | {"tx-vlan-id", required_argument, 0, 'J'}, | |
1073 | {"tx-vlan-pri", required_argument, 0, 'K'}, | |
6440a6c2 OBL |
1074 | {"tx-dmac", required_argument, 0, 'G'}, |
1075 | {"tx-smac", required_argument, 0, 'H'}, | |
fa0d27a1 | 1076 | {"tx-cycle", required_argument, 0, 'T'}, |
eb68db45 | 1077 | {"tstamp", no_argument, 0, 'y'}, |
fa24d0b1 OBL |
1078 | {"policy", required_argument, 0, 'W'}, |
1079 | {"schpri", required_argument, 0, 'U'}, | |
b36c3206 | 1080 | {"extra-stats", no_argument, 0, 'x'}, |
74e00676 | 1081 | {"quiet", no_argument, 0, 'Q'}, |
60dc609d | 1082 | {"app-stats", no_argument, 0, 'a'}, |
67ed3755 | 1083 | {"irq-string", no_argument, 0, 'I'}, |
b35fc148 | 1084 | {"busy-poll", no_argument, 0, 'B'}, |
3627d970 | 1085 | {"reduce-cap", no_argument, 0, 'R'}, |
b4b8faa1 MK |
1086 | {0, 0, 0, 0} |
1087 | }; | |
1088 | ||
1089 | static void usage(const char *prog) | |
1090 | { | |
1091 | const char *str = | |
1092 | " Usage: %s [OPTIONS]\n" | |
1093 | " Options:\n" | |
1094 | " -r, --rxdrop Discard all incoming packets (default)\n" | |
1095 | " -t, --txonly Only send packets\n" | |
1096 | " -l, --l2fwd MAC swap L2 forwarding\n" | |
1097 | " -i, --interface=n Run on interface n\n" | |
1098 | " -q, --queue=n Use queue n (default 0)\n" | |
1099 | " -p, --poll Use poll syscall\n" | |
b4b8faa1 | 1100 | " -S, --xdp-skb=n Use XDP skb-mod\n" |
4564a8bb | 1101 | " -N, --xdp-native=n Enforce XDP native mode\n" |
b4b8faa1 | 1102 | " -n, --interval=n Specify statistics update interval (default 1 sec).\n" |
8121e789 | 1103 | " -O, --retries=n Specify time-out retries (1s interval) attempt (default 3).\n" |
58c50ae4 BT |
1104 | " -z, --zero-copy Force zero-copy mode.\n" |
1105 | " -c, --copy Force copy mode.\n" | |
46738f73 | 1106 | " -m, --no-need-wakeup Turn off use of driver need wakeup flag.\n" |
c543f546 KL |
1107 | " -f, --frame-size=n Set the frame size (must be a power of two in aligned mode, default is %d).\n" |
1108 | " -u, --unaligned Enable unaligned chunk placement\n" | |
3627d970 | 1109 | " -M, --shared-umem Enable XDP_SHARED_UMEM (cannot be used with -R)\n" |
b3133329 | 1110 | " -F, --force Force loading the XDP prog\n" |
d3f11b01 JJ |
1111 | " -d, --duration=n Duration in secs to run command.\n" |
1112 | " Default: forever.\n" | |
5a388254 | 1113 | " -w, --clock=CLOCK Clock NAME (default MONOTONIC).\n" |
cd9e72b6 JJ |
1114 | " -b, --batch-size=n Batch size for sending or receiving\n" |
1115 | " packets. Default: %d\n" | |
ece6e969 JJ |
1116 | " -C, --tx-pkt-count=n Number of packets to send.\n" |
1117 | " Default: Continuous packets.\n" | |
4a3c23ae JJ |
1118 | " -s, --tx-pkt-size=n Transmit packet size.\n" |
1119 | " (Default: %d bytes)\n" | |
1120 | " Min size: %d, Max size %d.\n" | |
46e3268e | 1121 | " -P, --tx-pkt-pattern=nPacket fill pattern. Default: 0x%x\n" |
2741a049 OBL |
1122 | " -V, --tx-vlan Send VLAN tagged packets (For -t|--txonly)\n" |
1123 | " -J, --tx-vlan-id=n Tx VLAN ID [1-4095]. Default: %d (For -V|--tx-vlan)\n" | |
1124 | " -K, --tx-vlan-pri=n Tx VLAN Priority [0-7]. Default: %d (For -V|--tx-vlan)\n" | |
6440a6c2 OBL |
1125 | " -G, --tx-dmac=<MAC> Dest MAC addr of TX frame in aa:bb:cc:dd:ee:ff format (For -V|--tx-vlan)\n" |
1126 | " -H, --tx-smac=<MAC> Src MAC addr of TX frame in aa:bb:cc:dd:ee:ff format (For -V|--tx-vlan)\n" | |
fa0d27a1 | 1127 | " -T, --tx-cycle=n Tx cycle time in micro-seconds (For -t|--txonly).\n" |
eb68db45 | 1128 | " -y, --tstamp Add time-stamp to packet (For -t|--txonly).\n" |
fa24d0b1 OBL |
1129 | " -W, --policy=POLICY Schedule policy. Default: SCHED_OTHER\n" |
1130 | " -U, --schpri=n Schedule priority. Default: %d\n" | |
b36c3206 | 1131 | " -x, --extra-stats Display extra statistics.\n" |
74e00676 | 1132 | " -Q, --quiet Do not display any stats.\n" |
60dc609d | 1133 | " -a, --app-stats Display application (syscall) statistics.\n" |
67ed3755 | 1134 | " -I, --irq-string Display driver interrupt statistics for interface associated with irq-string.\n" |
b35fc148 | 1135 | " -B, --busy-poll Busy poll.\n" |
3627d970 | 1136 | " -R, --reduce-cap Use reduced capabilities (cannot be used with -M)\n" |
b4b8faa1 | 1137 | "\n"; |
cd9e72b6 | 1138 | fprintf(stderr, str, prog, XSK_UMEM__DEFAULT_FRAME_SIZE, |
4a3c23ae | 1139 | opt_batch_size, MIN_PKT_SIZE, MIN_PKT_SIZE, |
2741a049 | 1140 | XSK_UMEM__DEFAULT_FRAME_SIZE, opt_pkt_fill_pattern, |
fa24d0b1 OBL |
1141 | VLAN_VID__DEFAULT, VLAN_PRI__DEFAULT, |
1142 | SCHED_PRI__DEFAULT); | |
4a3c23ae | 1143 | |
b4b8faa1 MK |
1144 | exit(EXIT_FAILURE); |
1145 | } | |
1146 | ||
1147 | static void parse_command_line(int argc, char **argv) | |
1148 | { | |
1149 | int option_index, c; | |
1150 | ||
1151 | opterr = 0; | |
1152 | ||
1153 | for (;;) { | |
fa24d0b1 | 1154 | c = getopt_long(argc, argv, |
eb68db45 | 1155 | "Frtli:q:pSNn:w:O:czf:muMd:b:C:s:P:VJ:K:G:H:T:yW:U:xQaI:BR", |
46738f73 | 1156 | long_options, &option_index); |
b4b8faa1 MK |
1157 | if (c == -1) |
1158 | break; | |
1159 | ||
1160 | switch (c) { | |
1161 | case 'r': | |
1162 | opt_bench = BENCH_RXDROP; | |
1163 | break; | |
1164 | case 't': | |
1165 | opt_bench = BENCH_TXONLY; | |
1166 | break; | |
1167 | case 'l': | |
1168 | opt_bench = BENCH_L2FWD; | |
1169 | break; | |
1170 | case 'i': | |
1171 | opt_if = optarg; | |
1172 | break; | |
1173 | case 'q': | |
1174 | opt_queue = atoi(optarg); | |
1175 | break; | |
b4b8faa1 MK |
1176 | case 'p': |
1177 | opt_poll = 1; | |
1178 | break; | |
1179 | case 'S': | |
1180 | opt_xdp_flags |= XDP_FLAGS_SKB_MODE; | |
9f5232cc | 1181 | opt_xdp_bind_flags |= XDP_COPY; |
b4b8faa1 MK |
1182 | break; |
1183 | case 'N': | |
d50ecc46 | 1184 | /* default, set below */ |
b4b8faa1 MK |
1185 | break; |
1186 | case 'n': | |
1187 | opt_interval = atoi(optarg); | |
1188 | break; | |
5a388254 OBL |
1189 | case 'w': |
1190 | if (get_clockid(&opt_clock, optarg)) { | |
1191 | fprintf(stderr, | |
1192 | "ERROR: Invalid clock %s. Default to CLOCK_MONOTONIC.\n", | |
1193 | optarg); | |
1194 | opt_clock = CLOCK_MONOTONIC; | |
1195 | } | |
1196 | break; | |
8121e789 OBL |
1197 | case 'O': |
1198 | opt_retries = atoi(optarg); | |
1199 | break; | |
58c50ae4 BT |
1200 | case 'z': |
1201 | opt_xdp_bind_flags |= XDP_ZEROCOPY; | |
1202 | break; | |
1203 | case 'c': | |
1204 | opt_xdp_bind_flags |= XDP_COPY; | |
1205 | break; | |
c543f546 KL |
1206 | case 'u': |
1207 | opt_umem_flags |= XDP_UMEM_UNALIGNED_CHUNK_FLAG; | |
1208 | opt_unaligned_chunks = 1; | |
3945b37a | 1209 | opt_mmap_flags = MAP_HUGETLB; |
c543f546 | 1210 | break; |
743e568c MF |
1211 | case 'F': |
1212 | opt_xdp_flags &= ~XDP_FLAGS_UPDATE_IF_NOEXIST; | |
1213 | break; | |
123e8da1 MM |
1214 | case 'f': |
1215 | opt_xsk_frame_size = atoi(optarg); | |
2e5d72c1 | 1216 | break; |
46738f73 MK |
1217 | case 'm': |
1218 | opt_need_wakeup = false; | |
1219 | opt_xdp_bind_flags &= ~XDP_USE_NEED_WAKEUP; | |
123e8da1 | 1220 | break; |
2e5d72c1 MK |
1221 | case 'M': |
1222 | opt_num_xsks = MAX_SOCKS; | |
1223 | break; | |
d3f11b01 JJ |
1224 | case 'd': |
1225 | opt_duration = atoi(optarg); | |
1226 | opt_duration *= 1000000000; | |
1227 | break; | |
cd9e72b6 JJ |
1228 | case 'b': |
1229 | opt_batch_size = atoi(optarg); | |
1230 | break; | |
ece6e969 JJ |
1231 | case 'C': |
1232 | opt_pkt_count = atoi(optarg); | |
1233 | break; | |
4a3c23ae JJ |
1234 | case 's': |
1235 | opt_pkt_size = atoi(optarg); | |
1236 | if (opt_pkt_size > (XSK_UMEM__DEFAULT_FRAME_SIZE) || | |
1237 | opt_pkt_size < MIN_PKT_SIZE) { | |
1238 | fprintf(stderr, | |
1239 | "ERROR: Invalid frame size %d\n", | |
1240 | opt_pkt_size); | |
1241 | usage(basename(argv[0])); | |
1242 | } | |
1243 | break; | |
46e3268e JJ |
1244 | case 'P': |
1245 | opt_pkt_fill_pattern = strtol(optarg, NULL, 16); | |
1246 | break; | |
2741a049 OBL |
1247 | case 'V': |
1248 | opt_vlan_tag = true; | |
1249 | break; | |
1250 | case 'J': | |
1251 | opt_pkt_vlan_id = atoi(optarg); | |
1252 | break; | |
1253 | case 'K': | |
1254 | opt_pkt_vlan_pri = atoi(optarg); | |
1255 | break; | |
6440a6c2 OBL |
1256 | case 'G': |
1257 | if (!ether_aton_r(optarg, | |
1258 | (struct ether_addr *)&opt_txdmac)) { | |
1259 | fprintf(stderr, "Invalid dmac address:%s\n", | |
1260 | optarg); | |
1261 | usage(basename(argv[0])); | |
1262 | } | |
1263 | break; | |
1264 | case 'H': | |
1265 | if (!ether_aton_r(optarg, | |
1266 | (struct ether_addr *)&opt_txsmac)) { | |
1267 | fprintf(stderr, "Invalid smac address:%s\n", | |
1268 | optarg); | |
1269 | usage(basename(argv[0])); | |
1270 | } | |
1271 | break; | |
fa0d27a1 OBL |
1272 | case 'T': |
1273 | opt_tx_cycle_ns = atoi(optarg); | |
1274 | opt_tx_cycle_ns *= NSEC_PER_USEC; | |
1275 | break; | |
eb68db45 OBL |
1276 | case 'y': |
1277 | opt_tstamp = 1; | |
1278 | break; | |
fa24d0b1 OBL |
1279 | case 'W': |
1280 | if (get_schpolicy(&opt_schpolicy, optarg)) { | |
1281 | fprintf(stderr, | |
1282 | "ERROR: Invalid policy %s. Default to SCHED_OTHER.\n", | |
1283 | optarg); | |
1284 | opt_schpolicy = SCHED_OTHER; | |
1285 | } | |
1286 | break; | |
1287 | case 'U': | |
1288 | opt_schprio = atoi(optarg); | |
1289 | break; | |
b36c3206 CL |
1290 | case 'x': |
1291 | opt_extra_stats = 1; | |
1292 | break; | |
74e00676 MK |
1293 | case 'Q': |
1294 | opt_quiet = 1; | |
1295 | break; | |
60dc609d CL |
1296 | case 'a': |
1297 | opt_app_stats = 1; | |
67ed3755 CL |
1298 | break; |
1299 | case 'I': | |
1300 | opt_irq_str = optarg; | |
1301 | if (get_interrupt_number()) | |
1302 | irqs_at_init = get_irqs(); | |
1303 | if (irqs_at_init < 0) { | |
1304 | fprintf(stderr, "ERROR: Failed to get irqs for %s\n", opt_irq_str); | |
1305 | usage(basename(argv[0])); | |
1306 | } | |
b35fc148 BT |
1307 | break; |
1308 | case 'B': | |
1309 | opt_busy_poll = 1; | |
60dc609d | 1310 | break; |
3627d970 MD |
1311 | case 'R': |
1312 | opt_reduced_cap = true; | |
1313 | break; | |
b4b8faa1 MK |
1314 | default: |
1315 | usage(basename(argv[0])); | |
1316 | } | |
1317 | } | |
1318 | ||
d50ecc46 THJ |
1319 | if (!(opt_xdp_flags & XDP_FLAGS_SKB_MODE)) |
1320 | opt_xdp_flags |= XDP_FLAGS_DRV_MODE; | |
1321 | ||
b4b8faa1 MK |
1322 | opt_ifindex = if_nametoindex(opt_if); |
1323 | if (!opt_ifindex) { | |
1324 | fprintf(stderr, "ERROR: interface \"%s\" does not exist\n", | |
1325 | opt_if); | |
1326 | usage(basename(argv[0])); | |
1327 | } | |
248c7f9c | 1328 | |
c543f546 KL |
1329 | if ((opt_xsk_frame_size & (opt_xsk_frame_size - 1)) && |
1330 | !opt_unaligned_chunks) { | |
123e8da1 MM |
1331 | fprintf(stderr, "--frame-size=%d is not a power of two\n", |
1332 | opt_xsk_frame_size); | |
1333 | usage(basename(argv[0])); | |
1334 | } | |
3627d970 MD |
1335 | |
1336 | if (opt_reduced_cap && opt_num_xsks > 1) { | |
1337 | fprintf(stderr, "ERROR: -M and -R cannot be used together\n"); | |
1338 | usage(basename(argv[0])); | |
1339 | } | |
b4b8faa1 MK |
1340 | } |
1341 | ||
248c7f9c | 1342 | static void kick_tx(struct xsk_socket_info *xsk) |
b4b8faa1 MK |
1343 | { |
1344 | int ret; | |
1345 | ||
248c7f9c | 1346 | ret = sendto(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, 0); |
8ed47e14 MF |
1347 | if (ret >= 0 || errno == ENOBUFS || errno == EAGAIN || |
1348 | errno == EBUSY || errno == ENETDOWN) | |
b4b8faa1 | 1349 | return; |
248c7f9c | 1350 | exit_with_error(errno); |
b4b8faa1 MK |
1351 | } |
1352 | ||
284cbc61 | 1353 | static inline void complete_tx_l2fwd(struct xsk_socket_info *xsk) |
b4b8faa1 | 1354 | { |
03895e63 | 1355 | struct xsk_umem_info *umem = xsk->umem; |
b74e21ab | 1356 | u32 idx_cq = 0, idx_fq = 0; |
b4b8faa1 MK |
1357 | unsigned int rcvd; |
1358 | size_t ndescs; | |
1359 | ||
1360 | if (!xsk->outstanding_tx) | |
1361 | return; | |
1362 | ||
3131cf66 MK |
1363 | /* In copy mode, Tx is driven by a syscall so we need to use e.g. sendto() to |
1364 | * really send the packets. In zero-copy mode we do not have to do this, since Tx | |
1365 | * is driven by the NAPI loop. So as an optimization, we do not have to call | |
1366 | * sendto() all the time in zero-copy mode for l2fwd. | |
1367 | */ | |
60dc609d CL |
1368 | if (opt_xdp_bind_flags & XDP_COPY) { |
1369 | xsk->app_stats.copy_tx_sendtos++; | |
3131cf66 | 1370 | kick_tx(xsk); |
60dc609d | 1371 | } |
3131cf66 | 1372 | |
cd9e72b6 | 1373 | ndescs = (xsk->outstanding_tx > opt_batch_size) ? opt_batch_size : |
248c7f9c | 1374 | xsk->outstanding_tx; |
b4b8faa1 MK |
1375 | |
1376 | /* re-add completed Tx buffers */ | |
03895e63 | 1377 | rcvd = xsk_ring_cons__peek(&umem->cq, ndescs, &idx_cq); |
b4b8faa1 | 1378 | if (rcvd > 0) { |
248c7f9c MK |
1379 | unsigned int i; |
1380 | int ret; | |
1381 | ||
03895e63 | 1382 | ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq); |
248c7f9c MK |
1383 | while (ret != rcvd) { |
1384 | if (ret < 0) | |
1385 | exit_with_error(-ret); | |
b35fc148 | 1386 | if (opt_busy_poll || xsk_ring_prod__needs_wakeup(&umem->fq)) { |
60dc609d | 1387 | xsk->app_stats.fill_fail_polls++; |
284cbc61 BT |
1388 | recvfrom(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, |
1389 | NULL); | |
60dc609d | 1390 | } |
03895e63 | 1391 | ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq); |
248c7f9c | 1392 | } |
03895e63 | 1393 | |
248c7f9c | 1394 | for (i = 0; i < rcvd; i++) |
03895e63 KL |
1395 | *xsk_ring_prod__fill_addr(&umem->fq, idx_fq++) = |
1396 | *xsk_ring_cons__comp_addr(&umem->cq, idx_cq++); | |
248c7f9c MK |
1397 | |
1398 | xsk_ring_prod__submit(&xsk->umem->fq, rcvd); | |
1399 | xsk_ring_cons__release(&xsk->umem->cq, rcvd); | |
b4b8faa1 | 1400 | xsk->outstanding_tx -= rcvd; |
b4b8faa1 MK |
1401 | } |
1402 | } | |
1403 | ||
ece6e969 JJ |
1404 | static inline void complete_tx_only(struct xsk_socket_info *xsk, |
1405 | int batch_size) | |
b4b8faa1 | 1406 | { |
b4b8faa1 | 1407 | unsigned int rcvd; |
248c7f9c | 1408 | u32 idx; |
b4b8faa1 MK |
1409 | |
1410 | if (!xsk->outstanding_tx) | |
1411 | return; | |
1412 | ||
60dc609d CL |
1413 | if (!opt_need_wakeup || xsk_ring_prod__needs_wakeup(&xsk->tx)) { |
1414 | xsk->app_stats.tx_wakeup_sendtos++; | |
46738f73 | 1415 | kick_tx(xsk); |
60dc609d | 1416 | } |
b4b8faa1 | 1417 | |
ece6e969 | 1418 | rcvd = xsk_ring_cons__peek(&xsk->umem->cq, batch_size, &idx); |
b4b8faa1 | 1419 | if (rcvd > 0) { |
248c7f9c | 1420 | xsk_ring_cons__release(&xsk->umem->cq, rcvd); |
b4b8faa1 | 1421 | xsk->outstanding_tx -= rcvd; |
b4b8faa1 MK |
1422 | } |
1423 | } | |
1424 | ||
f2d27282 | 1425 | static void rx_drop(struct xsk_socket_info *xsk) |
b4b8faa1 | 1426 | { |
b4b8faa1 | 1427 | unsigned int rcvd, i; |
b74e21ab | 1428 | u32 idx_rx = 0, idx_fq = 0; |
248c7f9c | 1429 | int ret; |
b4b8faa1 | 1430 | |
cd9e72b6 | 1431 | rcvd = xsk_ring_cons__peek(&xsk->rx, opt_batch_size, &idx_rx); |
46738f73 | 1432 | if (!rcvd) { |
b35fc148 | 1433 | if (opt_busy_poll || xsk_ring_prod__needs_wakeup(&xsk->umem->fq)) { |
60dc609d | 1434 | xsk->app_stats.rx_empty_polls++; |
f2d27282 | 1435 | recvfrom(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, NULL); |
60dc609d | 1436 | } |
b4b8faa1 | 1437 | return; |
46738f73 | 1438 | } |
b4b8faa1 | 1439 | |
248c7f9c MK |
1440 | ret = xsk_ring_prod__reserve(&xsk->umem->fq, rcvd, &idx_fq); |
1441 | while (ret != rcvd) { | |
1442 | if (ret < 0) | |
1443 | exit_with_error(-ret); | |
b35fc148 | 1444 | if (opt_busy_poll || xsk_ring_prod__needs_wakeup(&xsk->umem->fq)) { |
60dc609d | 1445 | xsk->app_stats.fill_fail_polls++; |
f2d27282 | 1446 | recvfrom(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, NULL); |
60dc609d | 1447 | } |
248c7f9c MK |
1448 | ret = xsk_ring_prod__reserve(&xsk->umem->fq, rcvd, &idx_fq); |
1449 | } | |
1450 | ||
b4b8faa1 | 1451 | for (i = 0; i < rcvd; i++) { |
248c7f9c MK |
1452 | u64 addr = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx)->addr; |
1453 | u32 len = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx++)->len; | |
03895e63 KL |
1454 | u64 orig = xsk_umem__extract_addr(addr); |
1455 | ||
1456 | addr = xsk_umem__add_offset_to_addr(addr); | |
248c7f9c | 1457 | char *pkt = xsk_umem__get_data(xsk->umem->buffer, addr); |
b4b8faa1 | 1458 | |
248c7f9c | 1459 | hex_dump(pkt, len, addr); |
03895e63 | 1460 | *xsk_ring_prod__fill_addr(&xsk->umem->fq, idx_fq++) = orig; |
b4b8faa1 MK |
1461 | } |
1462 | ||
248c7f9c MK |
1463 | xsk_ring_prod__submit(&xsk->umem->fq, rcvd); |
1464 | xsk_ring_cons__release(&xsk->rx, rcvd); | |
2e8806f0 | 1465 | xsk->ring_stats.rx_npkts += rcvd; |
b4b8faa1 MK |
1466 | } |
1467 | ||
1468 | static void rx_drop_all(void) | |
1469 | { | |
2e5d72c1 | 1470 | struct pollfd fds[MAX_SOCKS] = {}; |
46738f73 | 1471 | int i, ret; |
b4b8faa1 | 1472 | |
b4b8faa1 | 1473 | for (i = 0; i < num_socks; i++) { |
248c7f9c | 1474 | fds[i].fd = xsk_socket__fd(xsks[i]->xsk); |
b4b8faa1 | 1475 | fds[i].events = POLLIN; |
b4b8faa1 MK |
1476 | } |
1477 | ||
1478 | for (;;) { | |
1479 | if (opt_poll) { | |
60dc609d CL |
1480 | for (i = 0; i < num_socks; i++) |
1481 | xsks[i]->app_stats.opt_polls++; | |
46738f73 | 1482 | ret = poll(fds, num_socks, opt_timeout); |
b4b8faa1 MK |
1483 | if (ret <= 0) |
1484 | continue; | |
1485 | } | |
1486 | ||
1487 | for (i = 0; i < num_socks; i++) | |
f2d27282 | 1488 | rx_drop(xsks[i]); |
d3f11b01 JJ |
1489 | |
1490 | if (benchmark_done) | |
1491 | break; | |
46738f73 MK |
1492 | } |
1493 | } | |
1494 | ||
eb68db45 OBL |
1495 | static int tx_only(struct xsk_socket_info *xsk, u32 *frame_nb, |
1496 | int batch_size, unsigned long tx_ns) | |
46738f73 | 1497 | { |
eb68db45 | 1498 | u32 idx, tv_sec, tv_usec; |
cd9e72b6 | 1499 | unsigned int i; |
46738f73 | 1500 | |
ece6e969 JJ |
1501 | while (xsk_ring_prod__reserve(&xsk->tx, batch_size, &idx) < |
1502 | batch_size) { | |
1503 | complete_tx_only(xsk, batch_size); | |
092fde0f | 1504 | if (benchmark_done) |
fa0d27a1 | 1505 | return 0; |
cd9e72b6 | 1506 | } |
46738f73 | 1507 | |
eb68db45 OBL |
1508 | if (opt_tstamp) { |
1509 | tv_sec = (u32)(tx_ns / NSEC_PER_SEC); | |
1510 | tv_usec = (u32)((tx_ns % NSEC_PER_SEC) / 1000); | |
1511 | } | |
1512 | ||
ece6e969 | 1513 | for (i = 0; i < batch_size; i++) { |
cd9e72b6 JJ |
1514 | struct xdp_desc *tx_desc = xsk_ring_prod__tx_desc(&xsk->tx, |
1515 | idx + i); | |
3b80d106 | 1516 | tx_desc->addr = (*frame_nb + i) * opt_xsk_frame_size; |
4a3c23ae | 1517 | tx_desc->len = PKT_SIZE; |
eb68db45 OBL |
1518 | |
1519 | if (opt_tstamp) { | |
1520 | struct pktgen_hdr *pktgen_hdr; | |
1521 | u64 addr = tx_desc->addr; | |
1522 | char *pkt; | |
1523 | ||
1524 | pkt = xsk_umem__get_data(xsk->umem->buffer, addr); | |
1525 | pktgen_hdr = (struct pktgen_hdr *)(pkt + PKTGEN_HDR_OFFSET); | |
1526 | ||
1527 | pktgen_hdr->seq_num = htonl(sequence++); | |
1528 | pktgen_hdr->tv_sec = htonl(tv_sec); | |
1529 | pktgen_hdr->tv_usec = htonl(tv_usec); | |
1530 | ||
1531 | hex_dump(pkt, PKT_SIZE, addr); | |
1532 | } | |
b4b8faa1 | 1533 | } |
46738f73 | 1534 | |
ece6e969 | 1535 | xsk_ring_prod__submit(&xsk->tx, batch_size); |
90da4b32 | 1536 | xsk->ring_stats.tx_npkts += batch_size; |
ece6e969 | 1537 | xsk->outstanding_tx += batch_size; |
b69e56cf WJ |
1538 | *frame_nb += batch_size; |
1539 | *frame_nb %= NUM_FRAMES; | |
ece6e969 | 1540 | complete_tx_only(xsk, batch_size); |
fa0d27a1 OBL |
1541 | |
1542 | return batch_size; | |
ece6e969 JJ |
1543 | } |
1544 | ||
1545 | static inline int get_batch_size(int pkt_cnt) | |
1546 | { | |
1547 | if (!opt_pkt_count) | |
1548 | return opt_batch_size; | |
1549 | ||
1550 | if (pkt_cnt + opt_batch_size <= opt_pkt_count) | |
1551 | return opt_batch_size; | |
1552 | ||
1553 | return opt_pkt_count - pkt_cnt; | |
1554 | } | |
1555 | ||
1556 | static void complete_tx_only_all(void) | |
1557 | { | |
1558 | bool pending; | |
1559 | int i; | |
1560 | ||
1561 | do { | |
1562 | pending = false; | |
1563 | for (i = 0; i < num_socks; i++) { | |
1564 | if (xsks[i]->outstanding_tx) { | |
1565 | complete_tx_only(xsks[i], opt_batch_size); | |
1566 | pending = !!xsks[i]->outstanding_tx; | |
1567 | } | |
1568 | } | |
8121e789 OBL |
1569 | sleep(1); |
1570 | } while (pending && opt_retries-- > 0); | |
b4b8faa1 MK |
1571 | } |
1572 | ||
46738f73 | 1573 | static void tx_only_all(void) |
b4b8faa1 | 1574 | { |
2e5d72c1 | 1575 | struct pollfd fds[MAX_SOCKS] = {}; |
46738f73 | 1576 | u32 frame_nb[MAX_SOCKS] = {}; |
fa0d27a1 | 1577 | unsigned long next_tx_ns = 0; |
ece6e969 | 1578 | int pkt_cnt = 0; |
46738f73 | 1579 | int i, ret; |
b4b8faa1 | 1580 | |
fa0d27a1 OBL |
1581 | if (opt_poll && opt_tx_cycle_ns) { |
1582 | fprintf(stderr, | |
1583 | "Error: --poll and --tx-cycles are both set\n"); | |
1584 | return; | |
1585 | } | |
1586 | ||
46738f73 MK |
1587 | for (i = 0; i < num_socks; i++) { |
1588 | fds[0].fd = xsk_socket__fd(xsks[i]->xsk); | |
1589 | fds[0].events = POLLOUT; | |
1590 | } | |
b4b8faa1 | 1591 | |
fa0d27a1 OBL |
1592 | if (opt_tx_cycle_ns) { |
1593 | /* Align Tx time to micro-second boundary */ | |
1594 | next_tx_ns = (get_nsecs() / NSEC_PER_USEC + 1) * | |
1595 | NSEC_PER_USEC; | |
1596 | next_tx_ns += opt_tx_cycle_ns; | |
1597 | ||
1598 | /* Initialize periodic Tx scheduling variance */ | |
1599 | tx_cycle_diff_min = 1000000000; | |
1600 | tx_cycle_diff_max = 0; | |
1601 | tx_cycle_diff_ave = 0.0; | |
1602 | } | |
1603 | ||
ece6e969 JJ |
1604 | while ((opt_pkt_count && pkt_cnt < opt_pkt_count) || !opt_pkt_count) { |
1605 | int batch_size = get_batch_size(pkt_cnt); | |
eb68db45 | 1606 | unsigned long tx_ns = 0; |
fa0d27a1 OBL |
1607 | struct timespec next; |
1608 | int tx_cnt = 0; | |
1609 | long diff; | |
1610 | int err; | |
ece6e969 | 1611 | |
b4b8faa1 | 1612 | if (opt_poll) { |
60dc609d CL |
1613 | for (i = 0; i < num_socks; i++) |
1614 | xsks[i]->app_stats.opt_polls++; | |
46738f73 | 1615 | ret = poll(fds, num_socks, opt_timeout); |
b4b8faa1 MK |
1616 | if (ret <= 0) |
1617 | continue; | |
1618 | ||
248c7f9c | 1619 | if (!(fds[0].revents & POLLOUT)) |
b4b8faa1 MK |
1620 | continue; |
1621 | } | |
1622 | ||
fa0d27a1 OBL |
1623 | if (opt_tx_cycle_ns) { |
1624 | next.tv_sec = next_tx_ns / NSEC_PER_SEC; | |
1625 | next.tv_nsec = next_tx_ns % NSEC_PER_SEC; | |
1626 | err = clock_nanosleep(opt_clock, TIMER_ABSTIME, &next, NULL); | |
1627 | if (err) { | |
1628 | if (err != EINTR) | |
1629 | fprintf(stderr, | |
1630 | "clock_nanosleep failed. Err:%d errno:%d\n", | |
1631 | err, errno); | |
1632 | break; | |
1633 | } | |
1634 | ||
1635 | /* Measure periodic Tx scheduling variance */ | |
eb68db45 OBL |
1636 | tx_ns = get_nsecs(); |
1637 | diff = tx_ns - next_tx_ns; | |
fa0d27a1 OBL |
1638 | if (diff < tx_cycle_diff_min) |
1639 | tx_cycle_diff_min = diff; | |
1640 | ||
1641 | if (diff > tx_cycle_diff_max) | |
1642 | tx_cycle_diff_max = diff; | |
1643 | ||
1644 | tx_cycle_diff_ave += (double)diff; | |
1645 | tx_cycle_cnt++; | |
eb68db45 OBL |
1646 | } else if (opt_tstamp) { |
1647 | tx_ns = get_nsecs(); | |
fa0d27a1 OBL |
1648 | } |
1649 | ||
46738f73 | 1650 | for (i = 0; i < num_socks; i++) |
eb68db45 | 1651 | tx_cnt += tx_only(xsks[i], &frame_nb[i], batch_size, tx_ns); |
ece6e969 | 1652 | |
fa0d27a1 | 1653 | pkt_cnt += tx_cnt; |
d3f11b01 JJ |
1654 | |
1655 | if (benchmark_done) | |
1656 | break; | |
fa0d27a1 OBL |
1657 | |
1658 | if (opt_tx_cycle_ns) | |
1659 | next_tx_ns += opt_tx_cycle_ns; | |
b4b8faa1 | 1660 | } |
ece6e969 JJ |
1661 | |
1662 | if (opt_pkt_count) | |
1663 | complete_tx_only_all(); | |
b4b8faa1 MK |
1664 | } |
1665 | ||
284cbc61 | 1666 | static void l2fwd(struct xsk_socket_info *xsk) |
b4b8faa1 | 1667 | { |
46738f73 MK |
1668 | unsigned int rcvd, i; |
1669 | u32 idx_rx = 0, idx_tx = 0; | |
1670 | int ret; | |
b4b8faa1 | 1671 | |
284cbc61 | 1672 | complete_tx_l2fwd(xsk); |
b4b8faa1 | 1673 | |
cd9e72b6 | 1674 | rcvd = xsk_ring_cons__peek(&xsk->rx, opt_batch_size, &idx_rx); |
46738f73 | 1675 | if (!rcvd) { |
b35fc148 | 1676 | if (opt_busy_poll || xsk_ring_prod__needs_wakeup(&xsk->umem->fq)) { |
60dc609d | 1677 | xsk->app_stats.rx_empty_polls++; |
284cbc61 | 1678 | recvfrom(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, NULL); |
60dc609d | 1679 | } |
46738f73 MK |
1680 | return; |
1681 | } | |
90da4b32 | 1682 | xsk->ring_stats.rx_npkts += rcvd; |
b4b8faa1 | 1683 | |
46738f73 MK |
1684 | ret = xsk_ring_prod__reserve(&xsk->tx, rcvd, &idx_tx); |
1685 | while (ret != rcvd) { | |
1686 | if (ret < 0) | |
1687 | exit_with_error(-ret); | |
284cbc61 | 1688 | complete_tx_l2fwd(xsk); |
b35fc148 | 1689 | if (opt_busy_poll || xsk_ring_prod__needs_wakeup(&xsk->tx)) { |
60dc609d | 1690 | xsk->app_stats.tx_wakeup_sendtos++; |
46738f73 | 1691 | kick_tx(xsk); |
60dc609d | 1692 | } |
248c7f9c | 1693 | ret = xsk_ring_prod__reserve(&xsk->tx, rcvd, &idx_tx); |
46738f73 MK |
1694 | } |
1695 | ||
1696 | for (i = 0; i < rcvd; i++) { | |
1697 | u64 addr = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx)->addr; | |
1698 | u32 len = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx++)->len; | |
5a712e13 | 1699 | u64 orig = addr; |
03895e63 KL |
1700 | |
1701 | addr = xsk_umem__add_offset_to_addr(addr); | |
46738f73 MK |
1702 | char *pkt = xsk_umem__get_data(xsk->umem->buffer, addr); |
1703 | ||
1704 | swap_mac_addresses(pkt); | |
248c7f9c | 1705 | |
46738f73 | 1706 | hex_dump(pkt, len, addr); |
03895e63 | 1707 | xsk_ring_prod__tx_desc(&xsk->tx, idx_tx)->addr = orig; |
46738f73 MK |
1708 | xsk_ring_prod__tx_desc(&xsk->tx, idx_tx++)->len = len; |
1709 | } | |
b4b8faa1 | 1710 | |
46738f73 MK |
1711 | xsk_ring_prod__submit(&xsk->tx, rcvd); |
1712 | xsk_ring_cons__release(&xsk->rx, rcvd); | |
b4b8faa1 | 1713 | |
90da4b32 | 1714 | xsk->ring_stats.tx_npkts += rcvd; |
46738f73 MK |
1715 | xsk->outstanding_tx += rcvd; |
1716 | } | |
1717 | ||
1718 | static void l2fwd_all(void) | |
1719 | { | |
2e5d72c1 | 1720 | struct pollfd fds[MAX_SOCKS] = {}; |
46738f73 MK |
1721 | int i, ret; |
1722 | ||
46738f73 MK |
1723 | for (;;) { |
1724 | if (opt_poll) { | |
284cbc61 BT |
1725 | for (i = 0; i < num_socks; i++) { |
1726 | fds[i].fd = xsk_socket__fd(xsks[i]->xsk); | |
1727 | fds[i].events = POLLOUT | POLLIN; | |
60dc609d | 1728 | xsks[i]->app_stats.opt_polls++; |
284cbc61 | 1729 | } |
46738f73 MK |
1730 | ret = poll(fds, num_socks, opt_timeout); |
1731 | if (ret <= 0) | |
1732 | continue; | |
1733 | } | |
b4b8faa1 | 1734 | |
46738f73 | 1735 | for (i = 0; i < num_socks; i++) |
284cbc61 | 1736 | l2fwd(xsks[i]); |
d3f11b01 JJ |
1737 | |
1738 | if (benchmark_done) | |
1739 | break; | |
b4b8faa1 MK |
1740 | } |
1741 | } | |
1742 | ||
2e5d72c1 MK |
1743 | static void load_xdp_program(char **argv, struct bpf_object **obj) |
1744 | { | |
1745 | struct bpf_prog_load_attr prog_load_attr = { | |
1746 | .prog_type = BPF_PROG_TYPE_XDP, | |
1747 | }; | |
1748 | char xdp_filename[256]; | |
1749 | int prog_fd; | |
1750 | ||
1751 | snprintf(xdp_filename, sizeof(xdp_filename), "%s_kern.o", argv[0]); | |
1752 | prog_load_attr.file = xdp_filename; | |
1753 | ||
1754 | if (bpf_prog_load_xattr(&prog_load_attr, obj, &prog_fd)) | |
1755 | exit(EXIT_FAILURE); | |
1756 | if (prog_fd < 0) { | |
1757 | fprintf(stderr, "ERROR: no program found: %s\n", | |
1758 | strerror(prog_fd)); | |
1759 | exit(EXIT_FAILURE); | |
1760 | } | |
1761 | ||
d4e34bfc | 1762 | if (bpf_xdp_attach(opt_ifindex, prog_fd, opt_xdp_flags, NULL) < 0) { |
2e5d72c1 MK |
1763 | fprintf(stderr, "ERROR: link set xdp fd failed\n"); |
1764 | exit(EXIT_FAILURE); | |
1765 | } | |
1766 | } | |
1767 | ||
1768 | static void enter_xsks_into_map(struct bpf_object *obj) | |
1769 | { | |
1770 | struct bpf_map *map; | |
1771 | int i, xsks_map; | |
1772 | ||
1773 | map = bpf_object__find_map_by_name(obj, "xsks_map"); | |
1774 | xsks_map = bpf_map__fd(map); | |
1775 | if (xsks_map < 0) { | |
1776 | fprintf(stderr, "ERROR: no xsks map found: %s\n", | |
1777 | strerror(xsks_map)); | |
1778 | exit(EXIT_FAILURE); | |
1779 | } | |
1780 | ||
1781 | for (i = 0; i < num_socks; i++) { | |
1782 | int fd = xsk_socket__fd(xsks[i]->xsk); | |
1783 | int key, ret; | |
1784 | ||
1785 | key = i; | |
1786 | ret = bpf_map_update_elem(xsks_map, &key, &fd, 0); | |
1787 | if (ret) { | |
1788 | fprintf(stderr, "ERROR: bpf_map_update_elem %d\n", i); | |
1789 | exit(EXIT_FAILURE); | |
1790 | } | |
1791 | } | |
1792 | } | |
1793 | ||
b35fc148 BT |
1794 | static void apply_setsockopt(struct xsk_socket_info *xsk) |
1795 | { | |
1796 | int sock_opt; | |
1797 | ||
1798 | if (!opt_busy_poll) | |
1799 | return; | |
1800 | ||
1801 | sock_opt = 1; | |
1802 | if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_PREFER_BUSY_POLL, | |
1803 | (void *)&sock_opt, sizeof(sock_opt)) < 0) | |
1804 | exit_with_error(errno); | |
1805 | ||
1806 | sock_opt = 20; | |
1807 | if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_BUSY_POLL, | |
1808 | (void *)&sock_opt, sizeof(sock_opt)) < 0) | |
1809 | exit_with_error(errno); | |
41bf900f BT |
1810 | |
1811 | sock_opt = opt_batch_size; | |
1812 | if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_BUSY_POLL_BUDGET, | |
1813 | (void *)&sock_opt, sizeof(sock_opt)) < 0) | |
1814 | exit_with_error(errno); | |
b35fc148 BT |
1815 | } |
1816 | ||
3627d970 MD |
1817 | static int recv_xsks_map_fd_from_ctrl_node(int sock, int *_fd) |
1818 | { | |
1819 | char cms[CMSG_SPACE(sizeof(int))]; | |
1820 | struct cmsghdr *cmsg; | |
1821 | struct msghdr msg; | |
1822 | struct iovec iov; | |
1823 | int value; | |
1824 | int len; | |
1825 | ||
1826 | iov.iov_base = &value; | |
1827 | iov.iov_len = sizeof(int); | |
1828 | ||
1829 | msg.msg_name = 0; | |
1830 | msg.msg_namelen = 0; | |
1831 | msg.msg_iov = &iov; | |
1832 | msg.msg_iovlen = 1; | |
1833 | msg.msg_flags = 0; | |
1834 | msg.msg_control = (caddr_t)cms; | |
1835 | msg.msg_controllen = sizeof(cms); | |
1836 | ||
1837 | len = recvmsg(sock, &msg, 0); | |
1838 | ||
1839 | if (len < 0) { | |
1840 | fprintf(stderr, "Recvmsg failed length incorrect.\n"); | |
1841 | return -EINVAL; | |
1842 | } | |
1843 | ||
1844 | if (len == 0) { | |
1845 | fprintf(stderr, "Recvmsg failed no data\n"); | |
1846 | return -EINVAL; | |
1847 | } | |
1848 | ||
1849 | cmsg = CMSG_FIRSTHDR(&msg); | |
1850 | *_fd = *(int *)CMSG_DATA(cmsg); | |
1851 | ||
1852 | return 0; | |
1853 | } | |
1854 | ||
1855 | static int | |
1856 | recv_xsks_map_fd(int *xsks_map_fd) | |
1857 | { | |
1858 | struct sockaddr_un server; | |
1859 | int err; | |
1860 | ||
1861 | sock = socket(AF_UNIX, SOCK_STREAM, 0); | |
1862 | if (sock < 0) { | |
1863 | fprintf(stderr, "Error opening socket stream: %s", strerror(errno)); | |
1864 | return errno; | |
1865 | } | |
1866 | ||
1867 | server.sun_family = AF_UNIX; | |
1868 | strcpy(server.sun_path, SOCKET_NAME); | |
1869 | ||
1870 | if (connect(sock, (struct sockaddr *)&server, sizeof(struct sockaddr_un)) < 0) { | |
1871 | close(sock); | |
1872 | fprintf(stderr, "Error connecting stream socket: %s", strerror(errno)); | |
1873 | return errno; | |
1874 | } | |
1875 | ||
1876 | err = recv_xsks_map_fd_from_ctrl_node(sock, xsks_map_fd); | |
1877 | if (err) { | |
2faa7328 | 1878 | fprintf(stderr, "Error %d receiving fd\n", err); |
3627d970 MD |
1879 | return err; |
1880 | } | |
1881 | return 0; | |
1882 | } | |
1883 | ||
b4b8faa1 MK |
1884 | int main(int argc, char **argv) |
1885 | { | |
3627d970 MD |
1886 | struct __user_cap_header_struct hdr = { _LINUX_CAPABILITY_VERSION_3, 0 }; |
1887 | struct __user_cap_data_struct data[2] = { { 0 } }; | |
661842c4 | 1888 | bool rx = false, tx = false; |
fa24d0b1 | 1889 | struct sched_param schparam; |
248c7f9c | 1890 | struct xsk_umem_info *umem; |
2e5d72c1 | 1891 | struct bpf_object *obj; |
3627d970 | 1892 | int xsks_map_fd = 0; |
b4b8faa1 | 1893 | pthread_t pt; |
2e5d72c1 | 1894 | int i, ret; |
248c7f9c | 1895 | void *bufs; |
b4b8faa1 MK |
1896 | |
1897 | parse_command_line(argc, argv); | |
1898 | ||
3627d970 MD |
1899 | if (opt_reduced_cap) { |
1900 | if (capget(&hdr, data) < 0) | |
1901 | fprintf(stderr, "Error getting capabilities\n"); | |
1902 | ||
1903 | data->effective &= CAP_TO_MASK(CAP_NET_RAW); | |
1904 | data->permitted &= CAP_TO_MASK(CAP_NET_RAW); | |
1905 | ||
1906 | if (capset(&hdr, data) < 0) | |
1907 | fprintf(stderr, "Setting capabilities failed\n"); | |
1908 | ||
1909 | if (capget(&hdr, data) < 0) { | |
1910 | fprintf(stderr, "Error getting capabilities\n"); | |
1911 | } else { | |
1912 | fprintf(stderr, "Capabilities EFF %x Caps INH %x Caps Per %x\n", | |
1913 | data[0].effective, data[0].inheritable, data[0].permitted); | |
1914 | fprintf(stderr, "Capabilities EFF %x Caps INH %x Caps Per %x\n", | |
1915 | data[1].effective, data[1].inheritable, data[1].permitted); | |
1916 | } | |
1917 | } else { | |
b25acdaf YS |
1918 | /* Use libbpf 1.0 API mode */ |
1919 | libbpf_set_strict_mode(LIBBPF_STRICT_ALL); | |
3627d970 MD |
1920 | |
1921 | if (opt_num_xsks > 1) | |
1922 | load_xdp_program(argv, &obj); | |
1923 | } | |
2e5d72c1 | 1924 | |
3945b37a KL |
1925 | /* Reserve memory for the umem. Use hugepages if unaligned chunk mode */ |
1926 | bufs = mmap(NULL, NUM_FRAMES * opt_xsk_frame_size, | |
1927 | PROT_READ | PROT_WRITE, | |
1928 | MAP_PRIVATE | MAP_ANONYMOUS | opt_mmap_flags, -1, 0); | |
1929 | if (bufs == MAP_FAILED) { | |
1930 | printf("ERROR: mmap failed\n"); | |
1931 | exit(EXIT_FAILURE); | |
1932 | } | |
2e5d72c1 MK |
1933 | |
1934 | /* Create sockets... */ | |
123e8da1 | 1935 | umem = xsk_configure_umem(bufs, NUM_FRAMES * opt_xsk_frame_size); |
661842c4 MK |
1936 | if (opt_bench == BENCH_RXDROP || opt_bench == BENCH_L2FWD) { |
1937 | rx = true; | |
1938 | xsk_populate_fill_ring(umem); | |
1939 | } | |
1940 | if (opt_bench == BENCH_L2FWD || opt_bench == BENCH_TXONLY) | |
1941 | tx = true; | |
2e5d72c1 | 1942 | for (i = 0; i < opt_num_xsks; i++) |
661842c4 | 1943 | xsks[num_socks++] = xsk_configure_socket(umem, rx, tx); |
b4b8faa1 | 1944 | |
b35fc148 BT |
1945 | for (i = 0; i < opt_num_xsks; i++) |
1946 | apply_setsockopt(xsks[i]); | |
1947 | ||
4a3c23ae | 1948 | if (opt_bench == BENCH_TXONLY) { |
eb68db45 OBL |
1949 | if (opt_tstamp && opt_pkt_size < PKTGEN_SIZE_MIN) |
1950 | opt_pkt_size = PKTGEN_SIZE_MIN; | |
1951 | ||
4a3c23ae JJ |
1952 | gen_eth_hdr_data(); |
1953 | ||
661842c4 MK |
1954 | for (i = 0; i < NUM_FRAMES; i++) |
1955 | gen_eth_frame(umem, i * opt_xsk_frame_size); | |
4a3c23ae | 1956 | } |
b4b8faa1 | 1957 | |
2e5d72c1 MK |
1958 | if (opt_num_xsks > 1 && opt_bench != BENCH_TXONLY) |
1959 | enter_xsks_into_map(obj); | |
b4b8faa1 | 1960 | |
3627d970 MD |
1961 | if (opt_reduced_cap) { |
1962 | ret = recv_xsks_map_fd(&xsks_map_fd); | |
1963 | if (ret) { | |
1964 | fprintf(stderr, "Error %d receiving xsks_map_fd\n", ret); | |
1965 | exit_with_error(ret); | |
1966 | } | |
1967 | if (xsks[0]->xsk) { | |
1968 | ret = xsk_socket__update_xskmap(xsks[0]->xsk, xsks_map_fd); | |
1969 | if (ret) { | |
1970 | fprintf(stderr, "Update of BPF map failed(%d)\n", ret); | |
1971 | exit_with_error(ret); | |
1972 | } | |
1973 | } | |
1974 | } | |
1975 | ||
b4b8faa1 MK |
1976 | signal(SIGINT, int_exit); |
1977 | signal(SIGTERM, int_exit); | |
1978 | signal(SIGABRT, int_exit); | |
1979 | ||
1980 | setlocale(LC_ALL, ""); | |
1981 | ||
8fa42d78 NS |
1982 | prev_time = get_nsecs(); |
1983 | start_time = prev_time; | |
1984 | ||
74e00676 MK |
1985 | if (!opt_quiet) { |
1986 | ret = pthread_create(&pt, NULL, poller, NULL); | |
1987 | if (ret) | |
1988 | exit_with_error(ret); | |
1989 | } | |
b4b8faa1 | 1990 | |
fa24d0b1 OBL |
1991 | /* Configure sched priority for better wake-up accuracy */ |
1992 | memset(&schparam, 0, sizeof(schparam)); | |
1993 | schparam.sched_priority = opt_schprio; | |
1994 | ret = sched_setscheduler(0, opt_schpolicy, &schparam); | |
1995 | if (ret) { | |
1996 | fprintf(stderr, "Error(%d) in setting priority(%d): %s\n", | |
1997 | errno, opt_schprio, strerror(errno)); | |
1998 | goto out; | |
1999 | } | |
2000 | ||
b4b8faa1 MK |
2001 | if (opt_bench == BENCH_RXDROP) |
2002 | rx_drop_all(); | |
2003 | else if (opt_bench == BENCH_TXONLY) | |
46738f73 | 2004 | tx_only_all(); |
b4b8faa1 | 2005 | else |
46738f73 | 2006 | l2fwd_all(); |
b4b8faa1 | 2007 | |
fa24d0b1 | 2008 | out: |
ece6e969 JJ |
2009 | benchmark_done = true; |
2010 | ||
74e00676 MK |
2011 | if (!opt_quiet) |
2012 | pthread_join(pt, NULL); | |
d3f11b01 | 2013 | |
69525588 JJ |
2014 | xdpsock_cleanup(); |
2015 | ||
6bc66998 MF |
2016 | munmap(bufs, NUM_FRAMES * opt_xsk_frame_size); |
2017 | ||
b4b8faa1 MK |
2018 | return 0; |
2019 | } |