1 // SPDX-License-Identifier: GPL-2.0
8 #include <linux/errqueue.h>
9 #include <linux/net_tstamp.h>
10 #include <netinet/if_ether.h>
11 #include <netinet/in.h>
12 #include <netinet/ip.h>
13 #include <netinet/ip6.h>
14 #include <netinet/udp.h>
22 #include <sys/socket.h>
25 #include <sys/types.h>
28 #include "../kselftest.h"
31 #define ETH_MAX_MTU 0xFFFFU
35 #define UDP_SEGMENT 103
39 #define SO_ZEROCOPY 60
42 #ifndef SO_EE_ORIGIN_ZEROCOPY
43 #define SO_EE_ORIGIN_ZEROCOPY 5
47 #define MSG_ZEROCOPY 0x4000000
56 static bool cfg_cache_trash;
57 static int cfg_cpu = -1;
58 static int cfg_connected = true;
59 static int cfg_family = PF_UNSPEC;
60 static uint16_t cfg_mss;
61 static int cfg_payload_len = (1472 * 42);
62 static int cfg_port = 8000;
63 static int cfg_runtime_ms = -1;
65 static bool cfg_segment;
66 static bool cfg_sendmmsg;
68 static uint32_t cfg_tx_ts = SOF_TIMESTAMPING_TX_SOFTWARE;
69 static bool cfg_tx_tstamp;
70 static bool cfg_audit;
71 static bool cfg_verbose;
72 static bool cfg_zerocopy;
73 static int cfg_msg_nr;
74 static uint16_t cfg_gso_size;
75 static unsigned long total_num_msgs;
76 static unsigned long total_num_sends;
77 static unsigned long stat_tx_ts;
78 static unsigned long stat_tx_ts_errors;
79 static unsigned long tstart;
80 static unsigned long tend;
81 static unsigned long stat_zcopies;
83 static socklen_t cfg_alen;
84 static struct sockaddr_storage cfg_dst_addr;
86 static bool interrupted;
87 static char buf[NUM_PKT][ETH_MAX_MTU];
89 static void sigint_handler(int signum)
95 static unsigned long gettimeofday_ms(void)
99 gettimeofday(&tv, NULL);
100 return (tv.tv_sec * 1000) + (tv.tv_usec / 1000);
103 static int set_cpu(int cpu)
109 if (sched_setaffinity(0, sizeof(mask), &mask))
110 error(1, 0, "setaffinity %d", cpu);
115 static void setup_sockaddr(int domain, const char *str_addr, void *sockaddr)
117 struct sockaddr_in6 *addr6 = (void *) sockaddr;
118 struct sockaddr_in *addr4 = (void *) sockaddr;
122 addr4->sin_family = AF_INET;
123 addr4->sin_port = htons(cfg_port);
124 if (inet_pton(AF_INET, str_addr, &(addr4->sin_addr)) != 1)
125 error(1, 0, "ipv4 parse error: %s", str_addr);
128 addr6->sin6_family = AF_INET6;
129 addr6->sin6_port = htons(cfg_port);
130 if (inet_pton(AF_INET6, str_addr, &(addr6->sin6_addr)) != 1)
131 error(1, 0, "ipv6 parse error: %s", str_addr);
134 error(1, 0, "illegal domain");
138 static void flush_cmsg(struct cmsghdr *cmsg)
140 struct sock_extended_err *err;
141 struct scm_timestamping *tss;
146 switch (cmsg->cmsg_level) {
148 if (cmsg->cmsg_type == SO_TIMESTAMPING) {
149 i = (cfg_tx_ts == SOF_TIMESTAMPING_TX_HARDWARE) ? 2 : 0;
150 tss = (struct scm_timestamping *)CMSG_DATA(cmsg);
151 if (tss->ts[i].tv_sec == 0)
154 error(1, 0, "unknown SOL_SOCKET cmsg type=%u\n",
160 switch (cmsg->cmsg_type) {
164 err = (struct sock_extended_err *)CMSG_DATA(cmsg);
165 switch (err->ee_origin) {
166 case SO_EE_ORIGIN_TIMESTAMPING:
167 /* Got a TX timestamp from error queue */
170 case SO_EE_ORIGIN_ICMP:
171 case SO_EE_ORIGIN_ICMP6:
174 "received ICMP error: type=%u, code=%u\n",
175 err->ee_type, err->ee_code);
177 case SO_EE_ORIGIN_ZEROCOPY:
181 /* range of IDs acknowledged */
182 stat_zcopies += hi - lo + 1;
185 case SO_EE_ORIGIN_LOCAL:
188 "received packet with local origin: %u\n",
192 error(0, 1, "received packet with origin: %u",
198 error(0, 1, "unknown IP msg type=%u\n",
204 error(0, 1, "unknown cmsg level=%u\n",
209 static void flush_errqueue_recv(int fd)
211 char control[CMSG_SPACE(sizeof(struct scm_timestamping)) +
212 CMSG_SPACE(sizeof(struct sock_extended_err)) +
213 CMSG_SPACE(sizeof(struct sockaddr_in6))] = {0};
214 struct msghdr msg = {0};
215 struct cmsghdr *cmsg;
219 msg.msg_control = control;
220 msg.msg_controllen = sizeof(control);
221 ret = recvmsg(fd, &msg, MSG_ERRQUEUE);
222 if (ret == -1 && errno == EAGAIN)
225 error(1, errno, "errqueue");
226 if (msg.msg_flags != MSG_ERRQUEUE)
227 error(1, 0, "errqueue: flags 0x%x\n", msg.msg_flags);
229 for (cmsg = CMSG_FIRSTHDR(&msg);
231 cmsg = CMSG_NXTHDR(&msg, cmsg))
238 static void flush_errqueue(int fd, const bool do_poll)
241 struct pollfd fds = {0};
245 ret = poll(&fds, 1, 500);
248 fprintf(stderr, "poll timeout\n");
249 } else if (ret < 0) {
250 error(1, errno, "poll");
254 flush_errqueue_recv(fd);
257 static int send_tcp(int fd, char *data)
259 int ret, done = 0, count = 0;
261 while (done < cfg_payload_len) {
262 ret = send(fd, data + done, cfg_payload_len - done,
263 cfg_zerocopy ? MSG_ZEROCOPY : 0);
265 error(1, errno, "write");
274 static int send_udp(int fd, char *data)
276 int ret, total_len, len, count = 0;
278 total_len = cfg_payload_len;
281 len = total_len < cfg_mss ? total_len : cfg_mss;
283 ret = sendto(fd, data, len, cfg_zerocopy ? MSG_ZEROCOPY : 0,
284 cfg_connected ? NULL : (void *)&cfg_dst_addr,
285 cfg_connected ? 0 : cfg_alen);
287 error(1, errno, "write");
289 error(1, errno, "write: %uB != %uB\n", ret, len);
298 static void send_ts_cmsg(struct cmsghdr *cm)
302 cm->cmsg_level = SOL_SOCKET;
303 cm->cmsg_type = SO_TIMESTAMPING;
304 cm->cmsg_len = CMSG_LEN(sizeof(cfg_tx_ts));
305 valp = (void *)CMSG_DATA(cm);
309 static int send_udp_sendmmsg(int fd, char *data)
311 char control[CMSG_SPACE(sizeof(cfg_tx_ts))] = {0};
312 const int max_nr_msg = ETH_MAX_MTU / ETH_DATA_LEN;
313 struct mmsghdr mmsgs[max_nr_msg];
314 struct iovec iov[max_nr_msg];
315 unsigned int off = 0, left;
316 size_t msg_controllen = 0;
319 memset(mmsgs, 0, sizeof(mmsgs));
322 struct msghdr msg = {0};
323 struct cmsghdr *cmsg;
325 msg.msg_control = control;
326 msg.msg_controllen = sizeof(control);
327 cmsg = CMSG_FIRSTHDR(&msg);
329 msg_controllen += CMSG_SPACE(sizeof(cfg_tx_ts));
332 left = cfg_payload_len;
335 error(1, 0, "sendmmsg: exceeds max_nr_msg");
337 iov[i].iov_base = data + off;
338 iov[i].iov_len = cfg_mss < left ? cfg_mss : left;
340 mmsgs[i].msg_hdr.msg_iov = iov + i;
341 mmsgs[i].msg_hdr.msg_iovlen = 1;
343 mmsgs[i].msg_hdr.msg_name = (void *)&cfg_dst_addr;
344 mmsgs[i].msg_hdr.msg_namelen = cfg_alen;
345 if (msg_controllen) {
346 mmsgs[i].msg_hdr.msg_control = control;
347 mmsgs[i].msg_hdr.msg_controllen = msg_controllen;
350 off += iov[i].iov_len;
351 left -= iov[i].iov_len;
355 ret = sendmmsg(fd, mmsgs, i, cfg_zerocopy ? MSG_ZEROCOPY : 0);
357 error(1, errno, "sendmmsg");
362 static void send_udp_segment_cmsg(struct cmsghdr *cm)
366 cm->cmsg_level = SOL_UDP;
367 cm->cmsg_type = UDP_SEGMENT;
368 cm->cmsg_len = CMSG_LEN(sizeof(cfg_gso_size));
369 valp = (void *)CMSG_DATA(cm);
370 *valp = cfg_gso_size;
373 static int send_udp_segment(int fd, char *data)
375 char control[CMSG_SPACE(sizeof(cfg_gso_size)) +
376 CMSG_SPACE(sizeof(cfg_tx_ts))] = {0};
377 struct msghdr msg = {0};
378 struct iovec iov = {0};
379 size_t msg_controllen;
380 struct cmsghdr *cmsg;
384 iov.iov_len = cfg_payload_len;
389 msg.msg_control = control;
390 msg.msg_controllen = sizeof(control);
391 cmsg = CMSG_FIRSTHDR(&msg);
392 send_udp_segment_cmsg(cmsg);
393 msg_controllen = CMSG_SPACE(sizeof(cfg_mss));
395 cmsg = CMSG_NXTHDR(&msg, cmsg);
397 msg_controllen += CMSG_SPACE(sizeof(cfg_tx_ts));
400 msg.msg_controllen = msg_controllen;
401 msg.msg_name = (void *)&cfg_dst_addr;
402 msg.msg_namelen = cfg_alen;
404 ret = sendmsg(fd, &msg, cfg_zerocopy ? MSG_ZEROCOPY : 0);
406 error(1, errno, "sendmsg");
407 if (ret != iov.iov_len)
408 error(1, 0, "sendmsg: %u != %llu\n", ret,
409 (unsigned long long)iov.iov_len);
414 static void usage(const char *filepath)
416 error(1, 0, "Usage: %s [-46acmHPtTuvz] [-C cpu] [-D dst ip] [-l secs] [-M messagenr] [-p port] [-s sendsize] [-S gsosize]",
420 static void parse_opts(int argc, char **argv)
422 const char *bind_addr = NULL;
426 while ((c = getopt(argc, argv, "46acC:D:Hl:mM:p:s:PS:tTuvz")) != -1) {
429 if (cfg_family != PF_UNSPEC)
430 error(1, 0, "Pass one of -4 or -6");
431 cfg_family = PF_INET;
432 cfg_alen = sizeof(struct sockaddr_in);
435 if (cfg_family != PF_UNSPEC)
436 error(1, 0, "Pass one of -4 or -6");
437 cfg_family = PF_INET6;
438 cfg_alen = sizeof(struct sockaddr_in6);
444 cfg_cache_trash = true;
447 cfg_cpu = strtol(optarg, NULL, 0);
453 cfg_runtime_ms = strtoul(optarg, NULL, 10) * 1000;
459 cfg_msg_nr = strtoul(optarg, NULL, 10);
462 cfg_port = strtoul(optarg, NULL, 0);
468 cfg_payload_len = strtoul(optarg, NULL, 0);
471 cfg_gso_size = strtoul(optarg, NULL, 0);
475 cfg_tx_ts = SOF_TIMESTAMPING_TX_HARDWARE;
476 cfg_tx_tstamp = true;
482 cfg_tx_tstamp = true;
485 cfg_connected = false;
497 bind_addr = cfg_family == PF_INET6 ? "::" : "0.0.0.0";
499 setup_sockaddr(cfg_family, bind_addr, &cfg_dst_addr);
504 if (cfg_family == PF_UNSPEC)
505 error(1, 0, "must pass one of -4 or -6");
506 if (cfg_tcp && !cfg_connected)
507 error(1, 0, "connectionless tcp makes no sense");
508 if (cfg_segment && cfg_sendmmsg)
509 error(1, 0, "cannot combine segment offload and sendmmsg");
510 if (cfg_tx_tstamp && !(cfg_segment || cfg_sendmmsg))
511 error(1, 0, "Options -T and -H require either -S or -m option");
513 if (cfg_family == PF_INET)
514 hdrlen = sizeof(struct iphdr) + sizeof(struct udphdr);
516 hdrlen = sizeof(struct ip6_hdr) + sizeof(struct udphdr);
518 cfg_mss = ETH_DATA_LEN - hdrlen;
519 max_len = ETH_MAX_MTU - hdrlen;
521 cfg_gso_size = cfg_mss;
523 if (cfg_payload_len > max_len)
524 error(1, 0, "payload length %u exceeds max %u",
525 cfg_payload_len, max_len);
528 static void set_pmtu_discover(int fd, bool is_ipv4)
530 int level, name, val;
534 name = IP_MTU_DISCOVER;
535 val = IP_PMTUDISC_DO;
538 name = IPV6_MTU_DISCOVER;
539 val = IPV6_PMTUDISC_DO;
542 if (setsockopt(fd, level, name, &val, sizeof(val)))
543 error(1, errno, "setsockopt path mtu");
546 static void set_tx_timestamping(int fd)
548 int val = SOF_TIMESTAMPING_OPT_CMSG | SOF_TIMESTAMPING_OPT_ID |
549 SOF_TIMESTAMPING_OPT_TSONLY;
551 if (cfg_tx_ts == SOF_TIMESTAMPING_TX_SOFTWARE)
552 val |= SOF_TIMESTAMPING_SOFTWARE;
554 val |= SOF_TIMESTAMPING_RAW_HARDWARE;
556 if (setsockopt(fd, SOL_SOCKET, SO_TIMESTAMPING, &val, sizeof(val)))
557 error(1, errno, "setsockopt tx timestamping");
560 static void print_audit_report(unsigned long num_msgs, unsigned long num_sends)
562 unsigned long tdelta;
564 tdelta = tend - tstart;
568 fprintf(stderr, "Summary over %lu.%03lu seconds...\n",
569 tdelta / 1000, tdelta % 1000);
571 "sum %s tx: %6lu MB/s %10lu calls (%lu/s) %10lu msgs (%lu/s)\n",
572 cfg_tcp ? "tcp" : "udp",
573 ((num_msgs * cfg_payload_len) >> 10) / tdelta,
574 num_sends, num_sends * 1000 / tdelta,
575 num_msgs, num_msgs * 1000 / tdelta);
578 if (stat_tx_ts_errors)
580 "Expected clean TX Timestamps: %9lu msgs received %6lu errors",
581 stat_tx_ts, stat_tx_ts_errors);
582 if (stat_tx_ts != num_sends)
584 "Unexpected number of TX Timestamps: %9lu expected %9lu received",
585 num_sends, stat_tx_ts);
587 "Tx Timestamps: %19lu received %17lu errors\n",
588 stat_tx_ts, stat_tx_ts_errors);
592 if (stat_zcopies != num_sends)
593 error(1, 0, "Unexpected number of Zerocopy completions: %9lu expected %9lu received",
594 num_sends, stat_zcopies);
596 "Zerocopy acks: %19lu\n",
601 static void print_report(unsigned long num_msgs, unsigned long num_sends)
604 "%s tx: %6lu MB/s %8lu calls/s %6lu msg/s\n",
605 cfg_tcp ? "tcp" : "udp",
606 (num_msgs * cfg_payload_len) >> 20,
607 num_sends, num_msgs);
610 total_num_msgs += num_msgs;
611 total_num_sends += num_sends;
615 int main(int argc, char **argv)
617 unsigned long num_msgs, num_sends;
618 unsigned long tnow, treport, tstop;
621 parse_opts(argc, argv);
626 for (i = 0; i < sizeof(buf[0]); i++)
627 buf[0][i] = 'a' + (i % 26);
628 for (i = 1; i < NUM_PKT; i++)
629 memcpy(buf[i], buf[0], sizeof(buf[0]));
631 signal(SIGINT, sigint_handler);
633 fd = socket(cfg_family, cfg_tcp ? SOCK_STREAM : SOCK_DGRAM, 0);
635 error(1, errno, "socket");
640 ret = setsockopt(fd, SOL_SOCKET, SO_ZEROCOPY,
643 if (errno == ENOPROTOOPT || errno == ENOTSUPP) {
644 fprintf(stderr, "SO_ZEROCOPY not supported");
647 error(1, errno, "setsockopt zerocopy");
652 connect(fd, (void *)&cfg_dst_addr, cfg_alen))
653 error(1, errno, "connect");
656 set_pmtu_discover(fd, cfg_family == PF_INET);
659 set_tx_timestamping(fd);
661 num_msgs = num_sends = 0;
662 tnow = gettimeofday_ms();
665 tstop = tnow + cfg_runtime_ms;
666 treport = tnow + 1000;
671 num_sends += send_tcp(fd, buf[i]);
672 else if (cfg_segment)
673 num_sends += send_udp_segment(fd, buf[i]);
674 else if (cfg_sendmmsg)
675 num_sends += send_udp_sendmmsg(fd, buf[i]);
677 num_sends += send_udp(fd, buf[i]);
679 if ((cfg_zerocopy && ((num_msgs & 0xF) == 0)) || cfg_tx_tstamp)
680 flush_errqueue(fd, cfg_poll);
682 if (cfg_msg_nr && num_msgs >= cfg_msg_nr)
685 tnow = gettimeofday_ms();
686 if (tnow >= treport) {
687 print_report(num_msgs, num_sends);
688 num_msgs = num_sends = 0;
689 treport = tnow + 1000;
692 /* cold cache when writing buffer */
694 i = ++i < NUM_PKT ? i : 0;
696 } while (!interrupted && (cfg_runtime_ms == -1 || tnow < tstop));
698 if (cfg_zerocopy || cfg_tx_tstamp)
699 flush_errqueue(fd, true);
702 error(1, errno, "close");
706 total_num_msgs += num_msgs;
707 total_num_sends += num_sends;
708 print_audit_report(total_num_msgs, total_num_sends);