Commit | Line | Data |
---|---|---|
91134d84 MKL |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* Copyright (c) 2018 Facebook */ | |
3 | ||
4 | #include <stdlib.h> | |
5 | #include <unistd.h> | |
6 | #include <stdbool.h> | |
7 | #include <string.h> | |
8 | #include <errno.h> | |
9 | #include <assert.h> | |
10 | #include <fcntl.h> | |
11 | #include <linux/bpf.h> | |
12 | #include <linux/err.h> | |
13 | #include <linux/types.h> | |
14 | #include <linux/if_ether.h> | |
15 | #include <sys/types.h> | |
16 | #include <sys/epoll.h> | |
17 | #include <sys/socket.h> | |
18 | #include <netinet/in.h> | |
19 | #include <bpf/bpf.h> | |
20 | #include <bpf/libbpf.h> | |
21 | #include "bpf_rlimit.h" | |
22 | #include "bpf_util.h" | |
7ee0d4e9 JS |
23 | |
24 | #include "test_progs.h" | |
91134d84 MKL |
25 | #include "test_select_reuseport_common.h" |
26 | ||
7ee0d4e9 | 27 | #define MAX_TEST_NAME 80 |
91134d84 MKL |
28 | #define MIN_TCPHDR_LEN 20 |
29 | #define UDPHDR_LEN 8 | |
30 | ||
31 | #define TCP_SYNCOOKIE_SYSCTL "/proc/sys/net/ipv4/tcp_syncookies" | |
32 | #define TCP_FO_SYSCTL "/proc/sys/net/ipv4/tcp_fastopen" | |
33 | #define REUSEPORT_ARRAY_SIZE 32 | |
34 | ||
35 | static int result_map, tmp_index_ovr_map, linum_map, data_check_map; | |
51bad0f0 | 36 | static __u32 expected_results[NR_RESULTS]; |
91134d84 | 37 | static int sk_fds[REUSEPORT_ARRAY_SIZE]; |
9af6c844 | 38 | static int reuseport_array = -1, outer_map = -1; |
91134d84 | 39 | static int select_by_skb_data_prog; |
7ee0d4e9 | 40 | static int saved_tcp_syncookie = -1; |
91134d84 | 41 | static struct bpf_object *obj; |
7ee0d4e9 | 42 | static int saved_tcp_fo = -1; |
91134d84 MKL |
43 | static __u32 index_zero; |
44 | static int epfd; | |
45 | ||
46 | static union sa46 { | |
47 | struct sockaddr_in6 v6; | |
48 | struct sockaddr_in v4; | |
49 | sa_family_t family; | |
50 | } srv_sa; | |
51 | ||
7ee0d4e9 JS |
52 | #define RET_IF(condition, tag, format...) ({ \ |
53 | if (CHECK_FAIL(condition)) { \ | |
54 | printf(tag " " format); \ | |
55 | return; \ | |
91134d84 MKL |
56 | } \ |
57 | }) | |
58 | ||
9af6c844 | 59 | #define RET_ERR(condition, tag, format...) ({ \ |
7ee0d4e9 JS |
60 | if (CHECK_FAIL(condition)) { \ |
61 | printf(tag " " format); \ | |
9af6c844 JS |
62 | return -1; \ |
63 | } \ | |
64 | }) | |
65 | ||
66 | static int create_maps(void) | |
91134d84 MKL |
67 | { |
68 | struct bpf_create_map_attr attr = {}; | |
69 | ||
70 | /* Creating reuseport_array */ | |
71 | attr.name = "reuseport_array"; | |
72 | attr.map_type = BPF_MAP_TYPE_REUSEPORT_SOCKARRAY; | |
73 | attr.key_size = sizeof(__u32); | |
74 | attr.value_size = sizeof(__u32); | |
75 | attr.max_entries = REUSEPORT_ARRAY_SIZE; | |
76 | ||
77 | reuseport_array = bpf_create_map_xattr(&attr); | |
9af6c844 JS |
78 | RET_ERR(reuseport_array == -1, "creating reuseport_array", |
79 | "reuseport_array:%d errno:%d\n", reuseport_array, errno); | |
91134d84 MKL |
80 | |
81 | /* Creating outer_map */ | |
82 | attr.name = "outer_map"; | |
83 | attr.map_type = BPF_MAP_TYPE_ARRAY_OF_MAPS; | |
84 | attr.key_size = sizeof(__u32); | |
85 | attr.value_size = sizeof(__u32); | |
86 | attr.max_entries = 1; | |
87 | attr.inner_map_fd = reuseport_array; | |
88 | outer_map = bpf_create_map_xattr(&attr); | |
9af6c844 JS |
89 | RET_ERR(outer_map == -1, "creating outer_map", |
90 | "outer_map:%d errno:%d\n", outer_map, errno); | |
91 | ||
92 | return 0; | |
91134d84 MKL |
93 | } |
94 | ||
9af6c844 | 95 | static int prepare_bpf_obj(void) |
91134d84 MKL |
96 | { |
97 | struct bpf_program *prog; | |
98 | struct bpf_map *map; | |
99 | int err; | |
91134d84 | 100 | |
1fbcef92 | 101 | obj = bpf_object__open("test_select_reuseport_kern.o"); |
9af6c844 JS |
102 | RET_ERR(IS_ERR_OR_NULL(obj), "open test_select_reuseport_kern.o", |
103 | "obj:%p PTR_ERR(obj):%ld\n", obj, PTR_ERR(obj)); | |
91134d84 | 104 | |
91134d84 | 105 | map = bpf_object__find_map_by_name(obj, "outer_map"); |
9af6c844 | 106 | RET_ERR(!map, "find outer_map", "!map\n"); |
91134d84 | 107 | err = bpf_map__reuse_fd(map, outer_map); |
9af6c844 | 108 | RET_ERR(err, "reuse outer_map", "err:%d\n", err); |
91134d84 MKL |
109 | |
110 | err = bpf_object__load(obj); | |
9af6c844 | 111 | RET_ERR(err, "load bpf_object", "err:%d\n", err); |
91134d84 | 112 | |
1fbcef92 | 113 | prog = bpf_program__next(NULL, obj); |
9af6c844 | 114 | RET_ERR(!prog, "get first bpf_program", "!prog\n"); |
91134d84 | 115 | select_by_skb_data_prog = bpf_program__fd(prog); |
9af6c844 JS |
116 | RET_ERR(select_by_skb_data_prog == -1, "get prog fd", |
117 | "select_by_skb_data_prog:%d\n", select_by_skb_data_prog); | |
91134d84 MKL |
118 | |
119 | map = bpf_object__find_map_by_name(obj, "result_map"); | |
9af6c844 | 120 | RET_ERR(!map, "find result_map", "!map\n"); |
91134d84 | 121 | result_map = bpf_map__fd(map); |
9af6c844 JS |
122 | RET_ERR(result_map == -1, "get result_map fd", |
123 | "result_map:%d\n", result_map); | |
91134d84 MKL |
124 | |
125 | map = bpf_object__find_map_by_name(obj, "tmp_index_ovr_map"); | |
9af6c844 | 126 | RET_ERR(!map, "find tmp_index_ovr_map\n", "!map"); |
91134d84 | 127 | tmp_index_ovr_map = bpf_map__fd(map); |
9af6c844 JS |
128 | RET_ERR(tmp_index_ovr_map == -1, "get tmp_index_ovr_map fd", |
129 | "tmp_index_ovr_map:%d\n", tmp_index_ovr_map); | |
91134d84 MKL |
130 | |
131 | map = bpf_object__find_map_by_name(obj, "linum_map"); | |
9af6c844 | 132 | RET_ERR(!map, "find linum_map", "!map\n"); |
91134d84 | 133 | linum_map = bpf_map__fd(map); |
9af6c844 JS |
134 | RET_ERR(linum_map == -1, "get linum_map fd", |
135 | "linum_map:%d\n", linum_map); | |
91134d84 MKL |
136 | |
137 | map = bpf_object__find_map_by_name(obj, "data_check_map"); | |
9af6c844 | 138 | RET_ERR(!map, "find data_check_map", "!map\n"); |
91134d84 | 139 | data_check_map = bpf_map__fd(map); |
9af6c844 JS |
140 | RET_ERR(data_check_map == -1, "get data_check_map fd", |
141 | "data_check_map:%d\n", data_check_map); | |
142 | ||
143 | return 0; | |
91134d84 MKL |
144 | } |
145 | ||
146 | static void sa46_init_loopback(union sa46 *sa, sa_family_t family) | |
147 | { | |
148 | memset(sa, 0, sizeof(*sa)); | |
149 | sa->family = family; | |
150 | if (sa->family == AF_INET6) | |
151 | sa->v6.sin6_addr = in6addr_loopback; | |
152 | else | |
153 | sa->v4.sin_addr.s_addr = htonl(INADDR_LOOPBACK); | |
154 | } | |
155 | ||
156 | static void sa46_init_inany(union sa46 *sa, sa_family_t family) | |
157 | { | |
158 | memset(sa, 0, sizeof(*sa)); | |
159 | sa->family = family; | |
160 | if (sa->family == AF_INET6) | |
161 | sa->v6.sin6_addr = in6addr_any; | |
162 | else | |
163 | sa->v4.sin_addr.s_addr = INADDR_ANY; | |
164 | } | |
165 | ||
166 | static int read_int_sysctl(const char *sysctl) | |
167 | { | |
168 | char buf[16]; | |
169 | int fd, ret; | |
170 | ||
171 | fd = open(sysctl, 0); | |
9af6c844 JS |
172 | RET_ERR(fd == -1, "open(sysctl)", |
173 | "sysctl:%s fd:%d errno:%d\n", sysctl, fd, errno); | |
91134d84 MKL |
174 | |
175 | ret = read(fd, buf, sizeof(buf)); | |
9af6c844 JS |
176 | RET_ERR(ret <= 0, "read(sysctl)", |
177 | "sysctl:%s ret:%d errno:%d\n", sysctl, ret, errno); | |
91134d84 | 178 | |
9af6c844 | 179 | close(fd); |
91134d84 MKL |
180 | return atoi(buf); |
181 | } | |
182 | ||
9af6c844 | 183 | static int write_int_sysctl(const char *sysctl, int v) |
91134d84 MKL |
184 | { |
185 | int fd, ret, size; | |
186 | char buf[16]; | |
187 | ||
188 | fd = open(sysctl, O_RDWR); | |
9af6c844 JS |
189 | RET_ERR(fd == -1, "open(sysctl)", |
190 | "sysctl:%s fd:%d errno:%d\n", sysctl, fd, errno); | |
91134d84 MKL |
191 | |
192 | size = snprintf(buf, sizeof(buf), "%d", v); | |
193 | ret = write(fd, buf, size); | |
9af6c844 JS |
194 | RET_ERR(ret != size, "write(sysctl)", |
195 | "sysctl:%s ret:%d size:%d errno:%d\n", | |
196 | sysctl, ret, size, errno); | |
197 | ||
91134d84 | 198 | close(fd); |
9af6c844 | 199 | return 0; |
91134d84 MKL |
200 | } |
201 | ||
202 | static void restore_sysctls(void) | |
203 | { | |
7ee0d4e9 JS |
204 | if (saved_tcp_fo != -1) |
205 | write_int_sysctl(TCP_FO_SYSCTL, saved_tcp_fo); | |
206 | if (saved_tcp_syncookie != -1) | |
207 | write_int_sysctl(TCP_SYNCOOKIE_SYSCTL, saved_tcp_syncookie); | |
91134d84 MKL |
208 | } |
209 | ||
9af6c844 | 210 | static int enable_fastopen(void) |
91134d84 MKL |
211 | { |
212 | int fo; | |
213 | ||
214 | fo = read_int_sysctl(TCP_FO_SYSCTL); | |
9af6c844 JS |
215 | if (fo < 0) |
216 | return -1; | |
217 | ||
218 | return write_int_sysctl(TCP_FO_SYSCTL, fo | 7); | |
91134d84 MKL |
219 | } |
220 | ||
9af6c844 | 221 | static int enable_syncookie(void) |
91134d84 | 222 | { |
9af6c844 | 223 | return write_int_sysctl(TCP_SYNCOOKIE_SYSCTL, 2); |
91134d84 MKL |
224 | } |
225 | ||
9af6c844 | 226 | static int disable_syncookie(void) |
91134d84 | 227 | { |
9af6c844 | 228 | return write_int_sysctl(TCP_SYNCOOKIE_SYSCTL, 0); |
91134d84 MKL |
229 | } |
230 | ||
7ee0d4e9 | 231 | static long get_linum(void) |
91134d84 MKL |
232 | { |
233 | __u32 linum; | |
234 | int err; | |
235 | ||
236 | err = bpf_map_lookup_elem(linum_map, &index_zero, &linum); | |
7ee0d4e9 JS |
237 | RET_ERR(err == -1, "lookup_elem(linum_map)", "err:%d errno:%d\n", |
238 | err, errno); | |
91134d84 MKL |
239 | |
240 | return linum; | |
241 | } | |
242 | ||
243 | static void check_data(int type, sa_family_t family, const struct cmd *cmd, | |
244 | int cli_fd) | |
245 | { | |
246 | struct data_check expected = {}, result; | |
247 | union sa46 cli_sa; | |
248 | socklen_t addrlen; | |
249 | int err; | |
250 | ||
251 | addrlen = sizeof(cli_sa); | |
252 | err = getsockname(cli_fd, (struct sockaddr *)&cli_sa, | |
253 | &addrlen); | |
7ee0d4e9 JS |
254 | RET_IF(err == -1, "getsockname(cli_fd)", "err:%d errno:%d\n", |
255 | err, errno); | |
91134d84 MKL |
256 | |
257 | err = bpf_map_lookup_elem(data_check_map, &index_zero, &result); | |
7ee0d4e9 JS |
258 | RET_IF(err == -1, "lookup_elem(data_check_map)", "err:%d errno:%d\n", |
259 | err, errno); | |
91134d84 MKL |
260 | |
261 | if (type == SOCK_STREAM) { | |
262 | expected.len = MIN_TCPHDR_LEN; | |
263 | expected.ip_protocol = IPPROTO_TCP; | |
264 | } else { | |
265 | expected.len = UDPHDR_LEN; | |
266 | expected.ip_protocol = IPPROTO_UDP; | |
267 | } | |
268 | ||
269 | if (family == AF_INET6) { | |
270 | expected.eth_protocol = htons(ETH_P_IPV6); | |
271 | expected.bind_inany = !srv_sa.v6.sin6_addr.s6_addr32[3] && | |
272 | !srv_sa.v6.sin6_addr.s6_addr32[2] && | |
273 | !srv_sa.v6.sin6_addr.s6_addr32[1] && | |
274 | !srv_sa.v6.sin6_addr.s6_addr32[0]; | |
275 | ||
276 | memcpy(&expected.skb_addrs[0], cli_sa.v6.sin6_addr.s6_addr32, | |
277 | sizeof(cli_sa.v6.sin6_addr)); | |
278 | memcpy(&expected.skb_addrs[4], &in6addr_loopback, | |
279 | sizeof(in6addr_loopback)); | |
280 | expected.skb_ports[0] = cli_sa.v6.sin6_port; | |
281 | expected.skb_ports[1] = srv_sa.v6.sin6_port; | |
282 | } else { | |
283 | expected.eth_protocol = htons(ETH_P_IP); | |
284 | expected.bind_inany = !srv_sa.v4.sin_addr.s_addr; | |
285 | ||
286 | expected.skb_addrs[0] = cli_sa.v4.sin_addr.s_addr; | |
287 | expected.skb_addrs[1] = htonl(INADDR_LOOPBACK); | |
288 | expected.skb_ports[0] = cli_sa.v4.sin_port; | |
289 | expected.skb_ports[1] = srv_sa.v4.sin_port; | |
290 | } | |
291 | ||
292 | if (memcmp(&result, &expected, offsetof(struct data_check, | |
293 | equal_check_end))) { | |
294 | printf("unexpected data_check\n"); | |
295 | printf(" result: (0x%x, %u, %u)\n", | |
296 | result.eth_protocol, result.ip_protocol, | |
297 | result.bind_inany); | |
298 | printf("expected: (0x%x, %u, %u)\n", | |
299 | expected.eth_protocol, expected.ip_protocol, | |
300 | expected.bind_inany); | |
7ee0d4e9 JS |
301 | RET_IF(1, "data_check result != expected", |
302 | "bpf_prog_linum:%ld\n", get_linum()); | |
91134d84 MKL |
303 | } |
304 | ||
7ee0d4e9 JS |
305 | RET_IF(!result.hash, "data_check result.hash empty", |
306 | "result.hash:%u", result.hash); | |
91134d84 MKL |
307 | |
308 | expected.len += cmd ? sizeof(*cmd) : 0; | |
309 | if (type == SOCK_STREAM) | |
7ee0d4e9 JS |
310 | RET_IF(expected.len > result.len, "expected.len > result.len", |
311 | "expected.len:%u result.len:%u bpf_prog_linum:%ld\n", | |
312 | expected.len, result.len, get_linum()); | |
91134d84 | 313 | else |
7ee0d4e9 JS |
314 | RET_IF(expected.len != result.len, "expected.len != result.len", |
315 | "expected.len:%u result.len:%u bpf_prog_linum:%ld\n", | |
316 | expected.len, result.len, get_linum()); | |
91134d84 MKL |
317 | } |
318 | ||
603fba9d LB |
319 | static const char *result_to_str(enum result res) |
320 | { | |
321 | switch (res) { | |
322 | case DROP_ERR_INNER_MAP: | |
323 | return "DROP_ERR_INNER_MAP"; | |
324 | case DROP_ERR_SKB_DATA: | |
325 | return "DROP_ERR_SKB_DATA"; | |
326 | case DROP_ERR_SK_SELECT_REUSEPORT: | |
327 | return "DROP_ERR_SK_SELECT_REUSEPORT"; | |
328 | case DROP_MISC: | |
329 | return "DROP_MISC"; | |
330 | case PASS: | |
331 | return "PASS"; | |
332 | case PASS_ERR_SK_SELECT_REUSEPORT: | |
333 | return "PASS_ERR_SK_SELECT_REUSEPORT"; | |
334 | default: | |
335 | return "UNKNOWN"; | |
336 | } | |
337 | } | |
338 | ||
91134d84 MKL |
339 | static void check_results(void) |
340 | { | |
341 | __u32 results[NR_RESULTS]; | |
342 | __u32 i, broken = 0; | |
343 | int err; | |
344 | ||
345 | for (i = 0; i < NR_RESULTS; i++) { | |
346 | err = bpf_map_lookup_elem(result_map, &i, &results[i]); | |
7ee0d4e9 JS |
347 | RET_IF(err == -1, "lookup_elem(result_map)", |
348 | "i:%u err:%d errno:%d\n", i, err, errno); | |
91134d84 MKL |
349 | } |
350 | ||
351 | for (i = 0; i < NR_RESULTS; i++) { | |
352 | if (results[i] != expected_results[i]) { | |
353 | broken = i; | |
354 | break; | |
355 | } | |
356 | } | |
357 | ||
358 | if (i == NR_RESULTS) | |
359 | return; | |
360 | ||
361 | printf("unexpected result\n"); | |
362 | printf(" result: ["); | |
363 | printf("%u", results[0]); | |
364 | for (i = 1; i < NR_RESULTS; i++) | |
365 | printf(", %u", results[i]); | |
366 | printf("]\n"); | |
367 | ||
368 | printf("expected: ["); | |
369 | printf("%u", expected_results[0]); | |
370 | for (i = 1; i < NR_RESULTS; i++) | |
371 | printf(", %u", expected_results[i]); | |
372 | printf("]\n"); | |
373 | ||
603fba9d LB |
374 | printf("mismatch on %s (bpf_prog_linum:%ld)\n", result_to_str(broken), |
375 | get_linum()); | |
376 | ||
377 | CHECK_FAIL(true); | |
91134d84 MKL |
378 | } |
379 | ||
380 | static int send_data(int type, sa_family_t family, void *data, size_t len, | |
381 | enum result expected) | |
382 | { | |
383 | union sa46 cli_sa; | |
384 | int fd, err; | |
385 | ||
386 | fd = socket(family, type, 0); | |
7ee0d4e9 | 387 | RET_ERR(fd == -1, "socket()", "fd:%d errno:%d\n", fd, errno); |
91134d84 MKL |
388 | |
389 | sa46_init_loopback(&cli_sa, family); | |
390 | err = bind(fd, (struct sockaddr *)&cli_sa, sizeof(cli_sa)); | |
7ee0d4e9 | 391 | RET_ERR(fd == -1, "bind(cli_sa)", "err:%d errno:%d\n", err, errno); |
91134d84 MKL |
392 | |
393 | err = sendto(fd, data, len, MSG_FASTOPEN, (struct sockaddr *)&srv_sa, | |
394 | sizeof(srv_sa)); | |
7ee0d4e9 JS |
395 | RET_ERR(err != len && expected >= PASS, |
396 | "sendto()", "family:%u err:%d errno:%d expected:%d\n", | |
397 | family, err, errno, expected); | |
91134d84 MKL |
398 | |
399 | return fd; | |
400 | } | |
401 | ||
402 | static void do_test(int type, sa_family_t family, struct cmd *cmd, | |
403 | enum result expected) | |
404 | { | |
405 | int nev, srv_fd, cli_fd; | |
406 | struct epoll_event ev; | |
407 | struct cmd rcv_cmd; | |
408 | ssize_t nread; | |
409 | ||
410 | cli_fd = send_data(type, family, cmd, cmd ? sizeof(*cmd) : 0, | |
411 | expected); | |
7ee0d4e9 JS |
412 | if (cli_fd < 0) |
413 | return; | |
91134d84 | 414 | nev = epoll_wait(epfd, &ev, 1, expected >= PASS ? 5 : 0); |
7ee0d4e9 JS |
415 | RET_IF((nev <= 0 && expected >= PASS) || |
416 | (nev > 0 && expected < PASS), | |
417 | "nev <> expected", | |
418 | "nev:%d expected:%d type:%d family:%d data:(%d, %d)\n", | |
419 | nev, expected, type, family, | |
420 | cmd ? cmd->reuseport_index : -1, | |
421 | cmd ? cmd->pass_on_failure : -1); | |
91134d84 MKL |
422 | check_results(); |
423 | check_data(type, family, cmd, cli_fd); | |
424 | ||
425 | if (expected < PASS) | |
426 | return; | |
427 | ||
7ee0d4e9 JS |
428 | RET_IF(expected != PASS_ERR_SK_SELECT_REUSEPORT && |
429 | cmd->reuseport_index != ev.data.u32, | |
430 | "check cmd->reuseport_index", | |
431 | "cmd:(%u, %u) ev.data.u32:%u\n", | |
432 | cmd->pass_on_failure, cmd->reuseport_index, ev.data.u32); | |
91134d84 MKL |
433 | |
434 | srv_fd = sk_fds[ev.data.u32]; | |
435 | if (type == SOCK_STREAM) { | |
436 | int new_fd = accept(srv_fd, NULL, 0); | |
437 | ||
7ee0d4e9 JS |
438 | RET_IF(new_fd == -1, "accept(srv_fd)", |
439 | "ev.data.u32:%u new_fd:%d errno:%d\n", | |
440 | ev.data.u32, new_fd, errno); | |
91134d84 MKL |
441 | |
442 | nread = recv(new_fd, &rcv_cmd, sizeof(rcv_cmd), MSG_DONTWAIT); | |
7ee0d4e9 JS |
443 | RET_IF(nread != sizeof(rcv_cmd), |
444 | "recv(new_fd)", | |
445 | "ev.data.u32:%u nread:%zd sizeof(rcv_cmd):%zu errno:%d\n", | |
446 | ev.data.u32, nread, sizeof(rcv_cmd), errno); | |
91134d84 MKL |
447 | |
448 | close(new_fd); | |
449 | } else { | |
450 | nread = recv(srv_fd, &rcv_cmd, sizeof(rcv_cmd), MSG_DONTWAIT); | |
7ee0d4e9 JS |
451 | RET_IF(nread != sizeof(rcv_cmd), |
452 | "recv(sk_fds)", | |
453 | "ev.data.u32:%u nread:%zd sizeof(rcv_cmd):%zu errno:%d\n", | |
454 | ev.data.u32, nread, sizeof(rcv_cmd), errno); | |
91134d84 MKL |
455 | } |
456 | ||
457 | close(cli_fd); | |
458 | } | |
459 | ||
460 | static void test_err_inner_map(int type, sa_family_t family) | |
461 | { | |
462 | struct cmd cmd = { | |
463 | .reuseport_index = 0, | |
464 | .pass_on_failure = 0, | |
465 | }; | |
466 | ||
91134d84 MKL |
467 | expected_results[DROP_ERR_INNER_MAP]++; |
468 | do_test(type, family, &cmd, DROP_ERR_INNER_MAP); | |
91134d84 MKL |
469 | } |
470 | ||
471 | static void test_err_skb_data(int type, sa_family_t family) | |
472 | { | |
91134d84 MKL |
473 | expected_results[DROP_ERR_SKB_DATA]++; |
474 | do_test(type, family, NULL, DROP_ERR_SKB_DATA); | |
91134d84 MKL |
475 | } |
476 | ||
477 | static void test_err_sk_select_port(int type, sa_family_t family) | |
478 | { | |
479 | struct cmd cmd = { | |
480 | .reuseport_index = REUSEPORT_ARRAY_SIZE, | |
481 | .pass_on_failure = 0, | |
482 | }; | |
483 | ||
91134d84 MKL |
484 | expected_results[DROP_ERR_SK_SELECT_REUSEPORT]++; |
485 | do_test(type, family, &cmd, DROP_ERR_SK_SELECT_REUSEPORT); | |
91134d84 MKL |
486 | } |
487 | ||
488 | static void test_pass(int type, sa_family_t family) | |
489 | { | |
490 | struct cmd cmd; | |
491 | int i; | |
492 | ||
91134d84 MKL |
493 | cmd.pass_on_failure = 0; |
494 | for (i = 0; i < REUSEPORT_ARRAY_SIZE; i++) { | |
495 | expected_results[PASS]++; | |
496 | cmd.reuseport_index = i; | |
497 | do_test(type, family, &cmd, PASS); | |
498 | } | |
91134d84 MKL |
499 | } |
500 | ||
501 | static void test_syncookie(int type, sa_family_t family) | |
502 | { | |
503 | int err, tmp_index = 1; | |
504 | struct cmd cmd = { | |
505 | .reuseport_index = 0, | |
506 | .pass_on_failure = 0, | |
507 | }; | |
508 | ||
eecd618b JS |
509 | if (type != SOCK_STREAM) { |
510 | test__skip(); | |
91134d84 | 511 | return; |
eecd618b | 512 | } |
91134d84 | 513 | |
91134d84 MKL |
514 | /* |
515 | * +1 for TCP-SYN and | |
516 | * +1 for the TCP-ACK (ack the syncookie) | |
517 | */ | |
518 | expected_results[PASS] += 2; | |
519 | enable_syncookie(); | |
520 | /* | |
521 | * Simulate TCP-SYN and TCP-ACK are handled by two different sk: | |
522 | * TCP-SYN: select sk_fds[tmp_index = 1] tmp_index is from the | |
523 | * tmp_index_ovr_map | |
524 | * TCP-ACK: select sk_fds[reuseport_index = 0] reuseport_index | |
525 | * is from the cmd.reuseport_index | |
526 | */ | |
527 | err = bpf_map_update_elem(tmp_index_ovr_map, &index_zero, | |
528 | &tmp_index, BPF_ANY); | |
7ee0d4e9 JS |
529 | RET_IF(err == -1, "update_elem(tmp_index_ovr_map, 0, 1)", |
530 | "err:%d errno:%d\n", err, errno); | |
91134d84 MKL |
531 | do_test(type, family, &cmd, PASS); |
532 | err = bpf_map_lookup_elem(tmp_index_ovr_map, &index_zero, | |
533 | &tmp_index); | |
7ee0d4e9 JS |
534 | RET_IF(err == -1 || tmp_index != -1, |
535 | "lookup_elem(tmp_index_ovr_map)", | |
536 | "err:%d errno:%d tmp_index:%d\n", | |
537 | err, errno, tmp_index); | |
91134d84 | 538 | disable_syncookie(); |
91134d84 MKL |
539 | } |
540 | ||
541 | static void test_pass_on_err(int type, sa_family_t family) | |
542 | { | |
543 | struct cmd cmd = { | |
544 | .reuseport_index = REUSEPORT_ARRAY_SIZE, | |
545 | .pass_on_failure = 1, | |
546 | }; | |
547 | ||
91134d84 MKL |
548 | expected_results[PASS_ERR_SK_SELECT_REUSEPORT] += 1; |
549 | do_test(type, family, &cmd, PASS_ERR_SK_SELECT_REUSEPORT); | |
91134d84 MKL |
550 | } |
551 | ||
d30bd78c MKL |
552 | static void test_detach_bpf(int type, sa_family_t family) |
553 | { | |
554 | #ifdef SO_DETACH_REUSEPORT_BPF | |
555 | __u32 nr_run_before = 0, nr_run_after = 0, tmp, i; | |
556 | struct epoll_event ev; | |
557 | int cli_fd, err, nev; | |
558 | struct cmd cmd = {}; | |
559 | int optvalue = 0; | |
560 | ||
d30bd78c MKL |
561 | err = setsockopt(sk_fds[0], SOL_SOCKET, SO_DETACH_REUSEPORT_BPF, |
562 | &optvalue, sizeof(optvalue)); | |
7ee0d4e9 JS |
563 | RET_IF(err == -1, "setsockopt(SO_DETACH_REUSEPORT_BPF)", |
564 | "err:%d errno:%d\n", err, errno); | |
d30bd78c MKL |
565 | |
566 | err = setsockopt(sk_fds[1], SOL_SOCKET, SO_DETACH_REUSEPORT_BPF, | |
567 | &optvalue, sizeof(optvalue)); | |
7ee0d4e9 JS |
568 | RET_IF(err == 0 || errno != ENOENT, |
569 | "setsockopt(SO_DETACH_REUSEPORT_BPF)", | |
570 | "err:%d errno:%d\n", err, errno); | |
d30bd78c MKL |
571 | |
572 | for (i = 0; i < NR_RESULTS; i++) { | |
573 | err = bpf_map_lookup_elem(result_map, &i, &tmp); | |
7ee0d4e9 JS |
574 | RET_IF(err == -1, "lookup_elem(result_map)", |
575 | "i:%u err:%d errno:%d\n", i, err, errno); | |
d30bd78c MKL |
576 | nr_run_before += tmp; |
577 | } | |
578 | ||
579 | cli_fd = send_data(type, family, &cmd, sizeof(cmd), PASS); | |
7ee0d4e9 JS |
580 | if (cli_fd < 0) |
581 | return; | |
d30bd78c | 582 | nev = epoll_wait(epfd, &ev, 1, 5); |
7ee0d4e9 JS |
583 | RET_IF(nev <= 0, "nev <= 0", |
584 | "nev:%d expected:1 type:%d family:%d data:(0, 0)\n", | |
585 | nev, type, family); | |
d30bd78c MKL |
586 | |
587 | for (i = 0; i < NR_RESULTS; i++) { | |
588 | err = bpf_map_lookup_elem(result_map, &i, &tmp); | |
7ee0d4e9 JS |
589 | RET_IF(err == -1, "lookup_elem(result_map)", |
590 | "i:%u err:%d errno:%d\n", i, err, errno); | |
d30bd78c MKL |
591 | nr_run_after += tmp; |
592 | } | |
593 | ||
7ee0d4e9 JS |
594 | RET_IF(nr_run_before != nr_run_after, |
595 | "nr_run_before != nr_run_after", | |
596 | "nr_run_before:%u nr_run_after:%u\n", | |
597 | nr_run_before, nr_run_after); | |
d30bd78c | 598 | |
d30bd78c MKL |
599 | close(cli_fd); |
600 | #else | |
7ee0d4e9 | 601 | test__skip(); |
d30bd78c MKL |
602 | #endif |
603 | } | |
604 | ||
91134d84 MKL |
605 | static void prepare_sk_fds(int type, sa_family_t family, bool inany) |
606 | { | |
607 | const int first = REUSEPORT_ARRAY_SIZE - 1; | |
608 | int i, err, optval = 1; | |
609 | struct epoll_event ev; | |
610 | socklen_t addrlen; | |
611 | ||
612 | if (inany) | |
613 | sa46_init_inany(&srv_sa, family); | |
614 | else | |
615 | sa46_init_loopback(&srv_sa, family); | |
616 | addrlen = sizeof(srv_sa); | |
617 | ||
618 | /* | |
619 | * The sk_fds[] is filled from the back such that the order | |
620 | * is exactly opposite to the (struct sock_reuseport *)reuse->socks[]. | |
621 | */ | |
622 | for (i = first; i >= 0; i--) { | |
623 | sk_fds[i] = socket(family, type, 0); | |
7ee0d4e9 JS |
624 | RET_IF(sk_fds[i] == -1, "socket()", "sk_fds[%d]:%d errno:%d\n", |
625 | i, sk_fds[i], errno); | |
91134d84 MKL |
626 | err = setsockopt(sk_fds[i], SOL_SOCKET, SO_REUSEPORT, |
627 | &optval, sizeof(optval)); | |
7ee0d4e9 JS |
628 | RET_IF(err == -1, "setsockopt(SO_REUSEPORT)", |
629 | "sk_fds[%d] err:%d errno:%d\n", | |
630 | i, err, errno); | |
91134d84 MKL |
631 | |
632 | if (i == first) { | |
633 | err = setsockopt(sk_fds[i], SOL_SOCKET, | |
634 | SO_ATTACH_REUSEPORT_EBPF, | |
635 | &select_by_skb_data_prog, | |
636 | sizeof(select_by_skb_data_prog)); | |
7ee0d4e9 JS |
637 | RET_IF(err == -1, "setsockopt(SO_ATTACH_REUEPORT_EBPF)", |
638 | "err:%d errno:%d\n", err, errno); | |
91134d84 MKL |
639 | } |
640 | ||
641 | err = bind(sk_fds[i], (struct sockaddr *)&srv_sa, addrlen); | |
7ee0d4e9 JS |
642 | RET_IF(err == -1, "bind()", "sk_fds[%d] err:%d errno:%d\n", |
643 | i, err, errno); | |
91134d84 MKL |
644 | |
645 | if (type == SOCK_STREAM) { | |
646 | err = listen(sk_fds[i], 10); | |
7ee0d4e9 JS |
647 | RET_IF(err == -1, "listen()", |
648 | "sk_fds[%d] err:%d errno:%d\n", | |
649 | i, err, errno); | |
91134d84 MKL |
650 | } |
651 | ||
652 | err = bpf_map_update_elem(reuseport_array, &i, &sk_fds[i], | |
653 | BPF_NOEXIST); | |
7ee0d4e9 JS |
654 | RET_IF(err == -1, "update_elem(reuseport_array)", |
655 | "sk_fds[%d] err:%d errno:%d\n", i, err, errno); | |
91134d84 MKL |
656 | |
657 | if (i == first) { | |
658 | socklen_t addrlen = sizeof(srv_sa); | |
659 | ||
660 | err = getsockname(sk_fds[i], (struct sockaddr *)&srv_sa, | |
661 | &addrlen); | |
7ee0d4e9 JS |
662 | RET_IF(err == -1, "getsockname()", |
663 | "sk_fds[%d] err:%d errno:%d\n", i, err, errno); | |
91134d84 MKL |
664 | } |
665 | } | |
666 | ||
667 | epfd = epoll_create(1); | |
7ee0d4e9 JS |
668 | RET_IF(epfd == -1, "epoll_create(1)", |
669 | "epfd:%d errno:%d\n", epfd, errno); | |
91134d84 MKL |
670 | |
671 | ev.events = EPOLLIN; | |
672 | for (i = 0; i < REUSEPORT_ARRAY_SIZE; i++) { | |
673 | ev.data.u32 = i; | |
674 | err = epoll_ctl(epfd, EPOLL_CTL_ADD, sk_fds[i], &ev); | |
7ee0d4e9 | 675 | RET_IF(err, "epoll_ctl(EPOLL_CTL_ADD)", "sk_fds[%d]\n", i); |
91134d84 MKL |
676 | } |
677 | } | |
678 | ||
ce7cb5f3 JS |
679 | static void setup_per_test(int type, sa_family_t family, bool inany, |
680 | bool no_inner_map) | |
91134d84 MKL |
681 | { |
682 | int ovr = -1, err; | |
683 | ||
684 | prepare_sk_fds(type, family, inany); | |
685 | err = bpf_map_update_elem(tmp_index_ovr_map, &index_zero, &ovr, | |
686 | BPF_ANY); | |
7ee0d4e9 JS |
687 | RET_IF(err == -1, "update_elem(tmp_index_ovr_map, 0, -1)", |
688 | "err:%d errno:%d\n", err, errno); | |
ce7cb5f3 JS |
689 | |
690 | /* Install reuseport_array to outer_map? */ | |
691 | if (no_inner_map) | |
692 | return; | |
693 | ||
694 | err = bpf_map_update_elem(outer_map, &index_zero, &reuseport_array, | |
695 | BPF_ANY); | |
7ee0d4e9 JS |
696 | RET_IF(err == -1, "update_elem(outer_map, 0, reuseport_array)", |
697 | "err:%d errno:%d\n", err, errno); | |
91134d84 MKL |
698 | } |
699 | ||
ce7cb5f3 | 700 | static void cleanup_per_test(bool no_inner_map) |
91134d84 | 701 | { |
51bad0f0 LB |
702 | int i, err, zero = 0; |
703 | ||
704 | memset(expected_results, 0, sizeof(expected_results)); | |
705 | ||
706 | for (i = 0; i < NR_RESULTS; i++) { | |
707 | err = bpf_map_update_elem(result_map, &i, &zero, BPF_ANY); | |
708 | RET_IF(err, "reset elem in result_map", | |
709 | "i:%u err:%d errno:%d\n", i, err, errno); | |
710 | } | |
711 | ||
712 | err = bpf_map_update_elem(linum_map, &zero, &zero, BPF_ANY); | |
713 | RET_IF(err, "reset line number in linum_map", "err:%d errno:%d\n", | |
714 | err, errno); | |
91134d84 MKL |
715 | |
716 | for (i = 0; i < REUSEPORT_ARRAY_SIZE; i++) | |
717 | close(sk_fds[i]); | |
718 | close(epfd); | |
719 | ||
ce7cb5f3 JS |
720 | /* Delete reuseport_array from outer_map? */ |
721 | if (no_inner_map) | |
722 | return; | |
723 | ||
91134d84 | 724 | err = bpf_map_delete_elem(outer_map, &index_zero); |
7ee0d4e9 JS |
725 | RET_IF(err == -1, "delete_elem(outer_map)", |
726 | "err:%d errno:%d\n", err, errno); | |
91134d84 MKL |
727 | } |
728 | ||
729 | static void cleanup(void) | |
730 | { | |
9af6c844 JS |
731 | if (outer_map != -1) |
732 | close(outer_map); | |
733 | if (reuseport_array != -1) | |
734 | close(reuseport_array); | |
735 | if (obj) | |
736 | bpf_object__close(obj); | |
91134d84 MKL |
737 | } |
738 | ||
a9ce4cf4 JS |
739 | static const char *family_str(sa_family_t family) |
740 | { | |
741 | switch (family) { | |
742 | case AF_INET: | |
743 | return "IPv4"; | |
744 | case AF_INET6: | |
745 | return "IPv6"; | |
746 | default: | |
747 | return "unknown"; | |
748 | } | |
749 | } | |
750 | ||
751 | static const char *sotype_str(int sotype) | |
752 | { | |
753 | switch (sotype) { | |
754 | case SOCK_STREAM: | |
755 | return "TCP"; | |
756 | case SOCK_DGRAM: | |
757 | return "UDP"; | |
758 | default: | |
759 | return "unknown"; | |
760 | } | |
761 | } | |
762 | ||
250a91d4 JS |
763 | #define TEST_INIT(fn, ...) { fn, #fn, __VA_ARGS__ } |
764 | ||
7ee0d4e9 | 765 | static void test_config(int sotype, sa_family_t family, bool inany) |
91134d84 | 766 | { |
ce7cb5f3 JS |
767 | const struct test { |
768 | void (*fn)(int sotype, sa_family_t family); | |
250a91d4 | 769 | const char *name; |
ce7cb5f3 JS |
770 | bool no_inner_map; |
771 | } tests[] = { | |
250a91d4 JS |
772 | TEST_INIT(test_err_inner_map, true /* no_inner_map */), |
773 | TEST_INIT(test_err_skb_data), | |
774 | TEST_INIT(test_err_sk_select_port), | |
775 | TEST_INIT(test_pass), | |
776 | TEST_INIT(test_syncookie), | |
777 | TEST_INIT(test_pass_on_err), | |
778 | TEST_INIT(test_detach_bpf), | |
ce7cb5f3 | 779 | }; |
7ee0d4e9 | 780 | char s[MAX_TEST_NAME]; |
ce7cb5f3 | 781 | const struct test *t; |
91134d84 | 782 | |
ce7cb5f3 | 783 | for (t = tests; t < tests + ARRAY_SIZE(tests); t++) { |
7ee0d4e9 JS |
784 | snprintf(s, sizeof(s), "%s/%s %s %s", |
785 | family_str(family), sotype_str(sotype), | |
786 | inany ? "INANY" : "LOOPBACK", t->name); | |
787 | ||
788 | if (!test__start_subtest(s)) | |
789 | continue; | |
790 | ||
791 | setup_per_test(sotype, family, inany, t->no_inner_map); | |
792 | t->fn(sotype, family); | |
ce7cb5f3 JS |
793 | cleanup_per_test(t->no_inner_map); |
794 | } | |
99363382 | 795 | } |
91134d84 | 796 | |
99363382 | 797 | #define BIND_INANY true |
91134d84 | 798 | |
99363382 JS |
799 | static void test_all(void) |
800 | { | |
801 | const struct config { | |
802 | int sotype; | |
803 | sa_family_t family; | |
804 | bool inany; | |
805 | } configs[] = { | |
806 | { SOCK_STREAM, AF_INET }, | |
807 | { SOCK_STREAM, AF_INET, BIND_INANY }, | |
808 | { SOCK_STREAM, AF_INET6 }, | |
809 | { SOCK_STREAM, AF_INET6, BIND_INANY }, | |
810 | { SOCK_DGRAM, AF_INET }, | |
811 | { SOCK_DGRAM, AF_INET6 }, | |
812 | }; | |
813 | const struct config *c; | |
814 | ||
815 | for (c = configs; c < configs + ARRAY_SIZE(configs); c++) | |
816 | test_config(c->sotype, c->family, c->inany); | |
91134d84 MKL |
817 | } |
818 | ||
7ee0d4e9 | 819 | void test_select_reuseport(void) |
91134d84 | 820 | { |
9af6c844 JS |
821 | if (create_maps()) |
822 | goto out; | |
823 | if (prepare_bpf_obj()) | |
824 | goto out; | |
825 | ||
91134d84 | 826 | saved_tcp_fo = read_int_sysctl(TCP_FO_SYSCTL); |
2fe77100 MKL |
827 | if (saved_tcp_fo < 0) |
828 | goto out; | |
91134d84 | 829 | saved_tcp_syncookie = read_int_sysctl(TCP_SYNCOOKIE_SYSCTL); |
2fe77100 | 830 | if (saved_tcp_syncookie < 0) |
9af6c844 | 831 | goto out; |
91134d84 | 832 | |
9af6c844 JS |
833 | if (enable_fastopen()) |
834 | goto out; | |
835 | if (disable_syncookie()) | |
836 | goto out; | |
837 | ||
91134d84 | 838 | test_all(); |
9af6c844 | 839 | out: |
91134d84 | 840 | cleanup(); |
7ee0d4e9 | 841 | restore_sysctls(); |
91134d84 | 842 | } |