1 #include <linux/module.h>
2 #include <linux/errno.h>
3 #include <linux/socket.h>
4 #include <linux/skbuff.h>
7 #include <linux/types.h>
8 #include <linux/kernel.h>
9 #include <net/genetlink.h>
12 #include <net/protocol.h>
14 #include <net/udp_tunnel.h>
16 #include <uapi/linux/fou.h>
17 #include <uapi/linux/genetlink.h>
25 struct udp_offload udp_offloads;
26 struct list_head list;
30 #define FOU_F_REMCSUM_NOPARTIAL BIT(0)
36 struct udp_port_cfg udp_config;
39 static unsigned int fou_net_id;
42 struct list_head fou_list;
43 struct mutex fou_lock;
46 static inline struct fou *fou_from_sock(struct sock *sk)
48 return sk->sk_user_data;
51 static int fou_recv_pull(struct sk_buff *skb, size_t len)
53 struct iphdr *iph = ip_hdr(skb);
55 /* Remove 'len' bytes from the packet (UDP header and
56 * FOU header if present).
58 iph->tot_len = htons(ntohs(iph->tot_len) - len);
60 skb_postpull_rcsum(skb, udp_hdr(skb), len);
61 skb_reset_transport_header(skb);
62 return iptunnel_pull_offloads(skb);
65 static int fou_udp_recv(struct sock *sk, struct sk_buff *skb)
67 struct fou *fou = fou_from_sock(sk);
72 if (fou_recv_pull(skb, sizeof(struct udphdr)))
75 return -fou->protocol;
82 static struct guehdr *gue_remcsum(struct sk_buff *skb, struct guehdr *guehdr,
83 void *data, size_t hdrlen, u8 ipproto,
87 size_t start = ntohs(pd[0]);
88 size_t offset = ntohs(pd[1]);
89 size_t plen = sizeof(struct udphdr) + hdrlen +
90 max_t(size_t, offset + sizeof(u16), start);
92 if (skb->remcsum_offload)
95 if (!pskb_may_pull(skb, plen))
97 guehdr = (struct guehdr *)&udp_hdr(skb)[1];
99 skb_remcsum_process(skb, (void *)guehdr + hdrlen,
100 start, offset, nopartial);
105 static int gue_control_message(struct sk_buff *skb, struct guehdr *guehdr)
112 static int gue_udp_recv(struct sock *sk, struct sk_buff *skb)
114 struct fou *fou = fou_from_sock(sk);
115 size_t len, optlen, hdrlen;
116 struct guehdr *guehdr;
123 len = sizeof(struct udphdr) + sizeof(struct guehdr);
124 if (!pskb_may_pull(skb, len))
127 guehdr = (struct guehdr *)&udp_hdr(skb)[1];
129 optlen = guehdr->hlen << 2;
132 if (!pskb_may_pull(skb, len))
135 /* guehdr may change after pull */
136 guehdr = (struct guehdr *)&udp_hdr(skb)[1];
138 hdrlen = sizeof(struct guehdr) + optlen;
140 if (guehdr->version != 0 || validate_gue_flags(guehdr, optlen))
143 hdrlen = sizeof(struct guehdr) + optlen;
145 ip_hdr(skb)->tot_len = htons(ntohs(ip_hdr(skb)->tot_len) - len);
147 /* Pull csum through the guehdr now . This can be used if
148 * there is a remote checksum offload.
150 skb_postpull_rcsum(skb, udp_hdr(skb), len);
154 if (guehdr->flags & GUE_FLAG_PRIV) {
155 __be32 flags = *(__be32 *)(data + doffset);
157 doffset += GUE_LEN_PRIV;
159 if (flags & GUE_PFLAG_REMCSUM) {
160 guehdr = gue_remcsum(skb, guehdr, data + doffset,
161 hdrlen, guehdr->proto_ctype,
163 FOU_F_REMCSUM_NOPARTIAL));
169 doffset += GUE_PLEN_REMCSUM;
173 if (unlikely(guehdr->control))
174 return gue_control_message(skb, guehdr);
176 __skb_pull(skb, sizeof(struct udphdr) + hdrlen);
177 skb_reset_transport_header(skb);
179 if (iptunnel_pull_offloads(skb))
182 return -guehdr->proto_ctype;
189 static struct sk_buff **fou_gro_receive(struct sk_buff **head,
191 struct udp_offload *uoff)
193 const struct net_offload *ops;
194 struct sk_buff **pp = NULL;
195 u8 proto = NAPI_GRO_CB(skb)->proto;
196 const struct net_offload **offloads;
199 offloads = NAPI_GRO_CB(skb)->is_ipv6 ? inet6_offloads : inet_offloads;
200 ops = rcu_dereference(offloads[proto]);
201 if (!ops || !ops->callbacks.gro_receive)
204 pp = ops->callbacks.gro_receive(head, skb);
212 static int fou_gro_complete(struct sk_buff *skb, int nhoff,
213 struct udp_offload *uoff)
215 const struct net_offload *ops;
216 u8 proto = NAPI_GRO_CB(skb)->proto;
218 const struct net_offload **offloads;
220 udp_tunnel_gro_complete(skb, nhoff);
223 offloads = NAPI_GRO_CB(skb)->is_ipv6 ? inet6_offloads : inet_offloads;
224 ops = rcu_dereference(offloads[proto]);
225 if (WARN_ON(!ops || !ops->callbacks.gro_complete))
228 err = ops->callbacks.gro_complete(skb, nhoff);
236 static struct guehdr *gue_gro_remcsum(struct sk_buff *skb, unsigned int off,
237 struct guehdr *guehdr, void *data,
238 size_t hdrlen, struct gro_remcsum *grc,
242 size_t start = ntohs(pd[0]);
243 size_t offset = ntohs(pd[1]);
245 if (skb->remcsum_offload)
248 if (!NAPI_GRO_CB(skb)->csum_valid)
251 guehdr = skb_gro_remcsum_process(skb, (void *)guehdr, off, hdrlen,
252 start, offset, grc, nopartial);
254 skb->remcsum_offload = 1;
259 static struct sk_buff **gue_gro_receive(struct sk_buff **head,
261 struct udp_offload *uoff)
263 const struct net_offload **offloads;
264 const struct net_offload *ops;
265 struct sk_buff **pp = NULL;
267 struct guehdr *guehdr;
268 size_t len, optlen, hdrlen, off;
272 struct fou *fou = container_of(uoff, struct fou, udp_offloads);
273 struct gro_remcsum grc;
275 skb_gro_remcsum_init(&grc);
277 off = skb_gro_offset(skb);
278 len = off + sizeof(*guehdr);
280 guehdr = skb_gro_header_fast(skb, off);
281 if (skb_gro_header_hard(skb, len)) {
282 guehdr = skb_gro_header_slow(skb, len, off);
283 if (unlikely(!guehdr))
287 optlen = guehdr->hlen << 2;
290 if (skb_gro_header_hard(skb, len)) {
291 guehdr = skb_gro_header_slow(skb, len, off);
292 if (unlikely(!guehdr))
296 if (unlikely(guehdr->control) || guehdr->version != 0 ||
297 validate_gue_flags(guehdr, optlen))
300 hdrlen = sizeof(*guehdr) + optlen;
302 /* Adjust NAPI_GRO_CB(skb)->csum to account for guehdr,
303 * this is needed if there is a remote checkcsum offload.
305 skb_gro_postpull_rcsum(skb, guehdr, hdrlen);
309 if (guehdr->flags & GUE_FLAG_PRIV) {
310 __be32 flags = *(__be32 *)(data + doffset);
312 doffset += GUE_LEN_PRIV;
314 if (flags & GUE_PFLAG_REMCSUM) {
315 guehdr = gue_gro_remcsum(skb, off, guehdr,
316 data + doffset, hdrlen, &grc,
318 FOU_F_REMCSUM_NOPARTIAL));
325 doffset += GUE_PLEN_REMCSUM;
329 skb_gro_pull(skb, hdrlen);
331 for (p = *head; p; p = p->next) {
332 const struct guehdr *guehdr2;
334 if (!NAPI_GRO_CB(p)->same_flow)
337 guehdr2 = (struct guehdr *)(p->data + off);
339 /* Compare base GUE header to be equal (covers
340 * hlen, version, proto_ctype, and flags.
342 if (guehdr->word != guehdr2->word) {
343 NAPI_GRO_CB(p)->same_flow = 0;
347 /* Compare optional fields are the same. */
348 if (guehdr->hlen && memcmp(&guehdr[1], &guehdr2[1],
349 guehdr->hlen << 2)) {
350 NAPI_GRO_CB(p)->same_flow = 0;
356 offloads = NAPI_GRO_CB(skb)->is_ipv6 ? inet6_offloads : inet_offloads;
357 ops = rcu_dereference(offloads[guehdr->proto_ctype]);
358 if (WARN_ON_ONCE(!ops || !ops->callbacks.gro_receive))
361 pp = ops->callbacks.gro_receive(head, skb);
367 NAPI_GRO_CB(skb)->flush |= flush;
368 skb_gro_remcsum_cleanup(skb, &grc);
373 static int gue_gro_complete(struct sk_buff *skb, int nhoff,
374 struct udp_offload *uoff)
376 const struct net_offload **offloads;
377 struct guehdr *guehdr = (struct guehdr *)(skb->data + nhoff);
378 const struct net_offload *ops;
379 unsigned int guehlen;
383 proto = guehdr->proto_ctype;
385 guehlen = sizeof(*guehdr) + (guehdr->hlen << 2);
388 offloads = NAPI_GRO_CB(skb)->is_ipv6 ? inet6_offloads : inet_offloads;
389 ops = rcu_dereference(offloads[proto]);
390 if (WARN_ON(!ops || !ops->callbacks.gro_complete))
393 err = ops->callbacks.gro_complete(skb, nhoff + guehlen);
400 static int fou_add_to_port_list(struct net *net, struct fou *fou)
402 struct fou_net *fn = net_generic(net, fou_net_id);
405 mutex_lock(&fn->fou_lock);
406 list_for_each_entry(fout, &fn->fou_list, list) {
407 if (fou->port == fout->port) {
408 mutex_unlock(&fn->fou_lock);
413 list_add(&fou->list, &fn->fou_list);
414 mutex_unlock(&fn->fou_lock);
419 static void fou_release(struct fou *fou)
421 struct socket *sock = fou->sock;
422 struct sock *sk = sock->sk;
424 if (sk->sk_family == AF_INET)
425 udp_del_offload(&fou->udp_offloads);
426 list_del(&fou->list);
427 udp_tunnel_sock_release(sock);
432 static int fou_encap_init(struct sock *sk, struct fou *fou, struct fou_cfg *cfg)
434 udp_sk(sk)->encap_rcv = fou_udp_recv;
435 fou->protocol = cfg->protocol;
436 fou->udp_offloads.callbacks.gro_receive = fou_gro_receive;
437 fou->udp_offloads.callbacks.gro_complete = fou_gro_complete;
438 fou->udp_offloads.port = cfg->udp_config.local_udp_port;
439 fou->udp_offloads.ipproto = cfg->protocol;
444 static int gue_encap_init(struct sock *sk, struct fou *fou, struct fou_cfg *cfg)
446 udp_sk(sk)->encap_rcv = gue_udp_recv;
447 fou->udp_offloads.callbacks.gro_receive = gue_gro_receive;
448 fou->udp_offloads.callbacks.gro_complete = gue_gro_complete;
449 fou->udp_offloads.port = cfg->udp_config.local_udp_port;
454 static int fou_create(struct net *net, struct fou_cfg *cfg,
455 struct socket **sockp)
457 struct socket *sock = NULL;
458 struct fou *fou = NULL;
462 /* Open UDP socket */
463 err = udp_sock_create(net, &cfg->udp_config, &sock);
467 /* Allocate FOU port structure */
468 fou = kzalloc(sizeof(*fou), GFP_KERNEL);
476 fou->flags = cfg->flags;
477 fou->port = cfg->udp_config.local_udp_port;
479 /* Initial for fou type */
481 case FOU_ENCAP_DIRECT:
482 err = fou_encap_init(sk, fou, cfg);
487 err = gue_encap_init(sk, fou, cfg);
496 fou->type = cfg->type;
498 udp_sk(sk)->encap_type = 1;
501 sk->sk_user_data = fou;
504 inet_inc_convert_csum(sk);
506 sk->sk_allocation = GFP_ATOMIC;
508 if (cfg->udp_config.family == AF_INET) {
509 err = udp_add_offload(net, &fou->udp_offloads);
514 err = fou_add_to_port_list(net, fou);
526 udp_tunnel_sock_release(sock);
531 static int fou_destroy(struct net *net, struct fou_cfg *cfg)
533 struct fou_net *fn = net_generic(net, fou_net_id);
534 __be16 port = cfg->udp_config.local_udp_port;
538 mutex_lock(&fn->fou_lock);
539 list_for_each_entry(fou, &fn->fou_list, list) {
540 if (fou->port == port) {
546 mutex_unlock(&fn->fou_lock);
551 static struct genl_family fou_nl_family = {
552 .id = GENL_ID_GENERATE,
554 .name = FOU_GENL_NAME,
555 .version = FOU_GENL_VERSION,
556 .maxattr = FOU_ATTR_MAX,
560 static struct nla_policy fou_nl_policy[FOU_ATTR_MAX + 1] = {
561 [FOU_ATTR_PORT] = { .type = NLA_U16, },
562 [FOU_ATTR_AF] = { .type = NLA_U8, },
563 [FOU_ATTR_IPPROTO] = { .type = NLA_U8, },
564 [FOU_ATTR_TYPE] = { .type = NLA_U8, },
565 [FOU_ATTR_REMCSUM_NOPARTIAL] = { .type = NLA_FLAG, },
568 static int parse_nl_config(struct genl_info *info,
571 memset(cfg, 0, sizeof(*cfg));
573 cfg->udp_config.family = AF_INET;
575 if (info->attrs[FOU_ATTR_AF]) {
576 u8 family = nla_get_u8(info->attrs[FOU_ATTR_AF]);
578 if (family != AF_INET)
581 cfg->udp_config.family = family;
584 if (info->attrs[FOU_ATTR_PORT]) {
585 __be16 port = nla_get_be16(info->attrs[FOU_ATTR_PORT]);
587 cfg->udp_config.local_udp_port = port;
590 if (info->attrs[FOU_ATTR_IPPROTO])
591 cfg->protocol = nla_get_u8(info->attrs[FOU_ATTR_IPPROTO]);
593 if (info->attrs[FOU_ATTR_TYPE])
594 cfg->type = nla_get_u8(info->attrs[FOU_ATTR_TYPE]);
596 if (info->attrs[FOU_ATTR_REMCSUM_NOPARTIAL])
597 cfg->flags |= FOU_F_REMCSUM_NOPARTIAL;
602 static int fou_nl_cmd_add_port(struct sk_buff *skb, struct genl_info *info)
604 struct net *net = genl_info_net(info);
608 err = parse_nl_config(info, &cfg);
612 return fou_create(net, &cfg, NULL);
615 static int fou_nl_cmd_rm_port(struct sk_buff *skb, struct genl_info *info)
617 struct net *net = genl_info_net(info);
621 err = parse_nl_config(info, &cfg);
625 return fou_destroy(net, &cfg);
628 static int fou_fill_info(struct fou *fou, struct sk_buff *msg)
630 if (nla_put_u8(msg, FOU_ATTR_AF, fou->sock->sk->sk_family) ||
631 nla_put_be16(msg, FOU_ATTR_PORT, fou->port) ||
632 nla_put_u8(msg, FOU_ATTR_IPPROTO, fou->protocol) ||
633 nla_put_u8(msg, FOU_ATTR_TYPE, fou->type))
636 if (fou->flags & FOU_F_REMCSUM_NOPARTIAL)
637 if (nla_put_flag(msg, FOU_ATTR_REMCSUM_NOPARTIAL))
642 static int fou_dump_info(struct fou *fou, u32 portid, u32 seq,
643 u32 flags, struct sk_buff *skb, u8 cmd)
647 hdr = genlmsg_put(skb, portid, seq, &fou_nl_family, flags, cmd);
651 if (fou_fill_info(fou, skb) < 0)
652 goto nla_put_failure;
654 genlmsg_end(skb, hdr);
658 genlmsg_cancel(skb, hdr);
662 static int fou_nl_cmd_get_port(struct sk_buff *skb, struct genl_info *info)
664 struct net *net = genl_info_net(info);
665 struct fou_net *fn = net_generic(net, fou_net_id);
672 ret = parse_nl_config(info, &cfg);
675 port = cfg.udp_config.local_udp_port;
679 msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
684 mutex_lock(&fn->fou_lock);
685 list_for_each_entry(fout, &fn->fou_list, list) {
686 if (port == fout->port) {
687 ret = fou_dump_info(fout, info->snd_portid,
688 info->snd_seq, 0, msg,
693 mutex_unlock(&fn->fou_lock);
697 return genlmsg_reply(msg, info);
704 static int fou_nl_dump(struct sk_buff *skb, struct netlink_callback *cb)
706 struct net *net = sock_net(skb->sk);
707 struct fou_net *fn = net_generic(net, fou_net_id);
711 mutex_lock(&fn->fou_lock);
712 list_for_each_entry(fout, &fn->fou_list, list) {
713 if (idx++ < cb->args[0])
715 ret = fou_dump_info(fout, NETLINK_CB(cb->skb).portid,
716 cb->nlh->nlmsg_seq, NLM_F_MULTI,
721 mutex_unlock(&fn->fou_lock);
727 static const struct genl_ops fou_nl_ops[] = {
730 .doit = fou_nl_cmd_add_port,
731 .policy = fou_nl_policy,
732 .flags = GENL_ADMIN_PERM,
736 .doit = fou_nl_cmd_rm_port,
737 .policy = fou_nl_policy,
738 .flags = GENL_ADMIN_PERM,
742 .doit = fou_nl_cmd_get_port,
743 .dumpit = fou_nl_dump,
744 .policy = fou_nl_policy,
748 size_t fou_encap_hlen(struct ip_tunnel_encap *e)
750 return sizeof(struct udphdr);
752 EXPORT_SYMBOL(fou_encap_hlen);
754 size_t gue_encap_hlen(struct ip_tunnel_encap *e)
757 bool need_priv = false;
759 len = sizeof(struct udphdr) + sizeof(struct guehdr);
761 if (e->flags & TUNNEL_ENCAP_FLAG_REMCSUM) {
762 len += GUE_PLEN_REMCSUM;
766 len += need_priv ? GUE_LEN_PRIV : 0;
770 EXPORT_SYMBOL(gue_encap_hlen);
772 static void fou_build_udp(struct sk_buff *skb, struct ip_tunnel_encap *e,
773 struct flowi4 *fl4, u8 *protocol, __be16 sport)
777 skb_push(skb, sizeof(struct udphdr));
778 skb_reset_transport_header(skb);
784 uh->len = htons(skb->len);
785 udp_set_csum(!(e->flags & TUNNEL_ENCAP_FLAG_CSUM), skb,
786 fl4->saddr, fl4->daddr, skb->len);
788 *protocol = IPPROTO_UDP;
791 int fou_build_header(struct sk_buff *skb, struct ip_tunnel_encap *e,
792 u8 *protocol, struct flowi4 *fl4)
794 int type = e->flags & TUNNEL_ENCAP_FLAG_CSUM ? SKB_GSO_UDP_TUNNEL_CSUM :
798 skb = iptunnel_handle_offloads(skb, type);
803 sport = e->sport ? : udp_flow_src_port(dev_net(skb->dev),
805 fou_build_udp(skb, e, fl4, protocol, sport);
809 EXPORT_SYMBOL(fou_build_header);
811 int gue_build_header(struct sk_buff *skb, struct ip_tunnel_encap *e,
812 u8 *protocol, struct flowi4 *fl4)
814 int type = e->flags & TUNNEL_ENCAP_FLAG_CSUM ? SKB_GSO_UDP_TUNNEL_CSUM :
816 struct guehdr *guehdr;
817 size_t hdrlen, optlen = 0;
820 bool need_priv = false;
822 if ((e->flags & TUNNEL_ENCAP_FLAG_REMCSUM) &&
823 skb->ip_summed == CHECKSUM_PARTIAL) {
824 optlen += GUE_PLEN_REMCSUM;
825 type |= SKB_GSO_TUNNEL_REMCSUM;
829 optlen += need_priv ? GUE_LEN_PRIV : 0;
831 skb = iptunnel_handle_offloads(skb, type);
836 /* Get source port (based on flow hash) before skb_push */
837 sport = e->sport ? : udp_flow_src_port(dev_net(skb->dev),
840 hdrlen = sizeof(struct guehdr) + optlen;
842 skb_push(skb, hdrlen);
844 guehdr = (struct guehdr *)skb->data;
848 guehdr->hlen = optlen >> 2;
850 guehdr->proto_ctype = *protocol;
855 __be32 *flags = data;
857 guehdr->flags |= GUE_FLAG_PRIV;
859 data += GUE_LEN_PRIV;
861 if (type & SKB_GSO_TUNNEL_REMCSUM) {
862 u16 csum_start = skb_checksum_start_offset(skb);
865 if (csum_start < hdrlen)
868 csum_start -= hdrlen;
869 pd[0] = htons(csum_start);
870 pd[1] = htons(csum_start + skb->csum_offset);
872 if (!skb_is_gso(skb)) {
873 skb->ip_summed = CHECKSUM_NONE;
874 skb->encapsulation = 0;
877 *flags |= GUE_PFLAG_REMCSUM;
878 data += GUE_PLEN_REMCSUM;
883 fou_build_udp(skb, e, fl4, protocol, sport);
887 EXPORT_SYMBOL(gue_build_header);
889 #ifdef CONFIG_NET_FOU_IP_TUNNELS
891 static const struct ip_tunnel_encap_ops fou_iptun_ops = {
892 .encap_hlen = fou_encap_hlen,
893 .build_header = fou_build_header,
896 static const struct ip_tunnel_encap_ops gue_iptun_ops = {
897 .encap_hlen = gue_encap_hlen,
898 .build_header = gue_build_header,
901 static int ip_tunnel_encap_add_fou_ops(void)
905 ret = ip_tunnel_encap_add_ops(&fou_iptun_ops, TUNNEL_ENCAP_FOU);
907 pr_err("can't add fou ops\n");
911 ret = ip_tunnel_encap_add_ops(&gue_iptun_ops, TUNNEL_ENCAP_GUE);
913 pr_err("can't add gue ops\n");
914 ip_tunnel_encap_del_ops(&fou_iptun_ops, TUNNEL_ENCAP_FOU);
921 static void ip_tunnel_encap_del_fou_ops(void)
923 ip_tunnel_encap_del_ops(&fou_iptun_ops, TUNNEL_ENCAP_FOU);
924 ip_tunnel_encap_del_ops(&gue_iptun_ops, TUNNEL_ENCAP_GUE);
929 static int ip_tunnel_encap_add_fou_ops(void)
934 static void ip_tunnel_encap_del_fou_ops(void)
940 static __net_init int fou_init_net(struct net *net)
942 struct fou_net *fn = net_generic(net, fou_net_id);
944 INIT_LIST_HEAD(&fn->fou_list);
945 mutex_init(&fn->fou_lock);
949 static __net_exit void fou_exit_net(struct net *net)
951 struct fou_net *fn = net_generic(net, fou_net_id);
952 struct fou *fou, *next;
954 /* Close all the FOU sockets */
955 mutex_lock(&fn->fou_lock);
956 list_for_each_entry_safe(fou, next, &fn->fou_list, list)
958 mutex_unlock(&fn->fou_lock);
961 static struct pernet_operations fou_net_ops = {
962 .init = fou_init_net,
963 .exit = fou_exit_net,
965 .size = sizeof(struct fou_net),
968 static int __init fou_init(void)
972 ret = register_pernet_device(&fou_net_ops);
976 ret = genl_register_family_with_ops(&fou_nl_family,
981 ret = ip_tunnel_encap_add_fou_ops();
985 genl_unregister_family(&fou_nl_family);
987 unregister_pernet_device(&fou_net_ops);
992 static void __exit fou_fini(void)
994 ip_tunnel_encap_del_fou_ops();
995 genl_unregister_family(&fou_nl_family);
996 unregister_pernet_device(&fou_net_ops);
999 module_init(fou_init);
1000 module_exit(fou_fini);
1001 MODULE_AUTHOR("Tom Herbert <therbert@google.com>");
1002 MODULE_LICENSE("GPL");