[linux-2.6-block.git] / net / ipv6 / tcpv6_offload.c

// SPDX-License-Identifier: GPL-2.0-or-later
/*
 *	IPV6 GSO/GRO offload support
 *	Linux INET6 implementation
 *
 *      TCPv6 GSO/GRO support
 */
#include <linux/indirect_call_wrapper.h>
#include <linux/skbuff.h>
#include <net/inet6_hashtables.h>
#include <net/gro.h>
#include <net/protocol.h>
#include <net/tcp.h>
#include <net/ip6_checksum.h>
#include "ip6_offload.h"

static void tcp6_check_fraglist_gro(struct list_head *head, struct sk_buff *skb,
				    struct tcphdr *th)
{
#if IS_ENABLED(CONFIG_IPV6)
	const struct ipv6hdr *hdr;
	struct sk_buff *p;
	struct sock *sk;
	struct net *net;
	int iif, sdif;

	if (likely(!(skb->dev->features & NETIF_F_GRO_FRAGLIST)))
		return;

	p = tcp_gro_lookup(head, th);
	if (p) {
		NAPI_GRO_CB(skb)->is_flist = NAPI_GRO_CB(p)->is_flist;
		return;
	}

	inet6_get_iif_sdif(skb, &iif, &sdif);
	hdr = skb_gro_network_header(skb);
	net = dev_net_rcu(skb->dev);
	sk = __inet6_lookup_established(net, net->ipv4.tcp_death_row.hashinfo,
					&hdr->saddr, th->source,
					&hdr->daddr, ntohs(th->dest),
					iif, sdif);
	NAPI_GRO_CB(skb)->is_flist = !sk;
	if (sk)
		sock_gen_put(sk);
#endif /* IS_ENABLED(CONFIG_IPV6) */
}

INDIRECT_CALLABLE_SCOPE
struct sk_buff *tcp6_gro_receive(struct list_head *head, struct sk_buff *skb)
{
	struct tcphdr *th;

	/* Don't bother verifying checksum if we're going to flush anyway. */
	if (!NAPI_GRO_CB(skb)->flush &&
	    skb_gro_checksum_validate(skb, IPPROTO_TCP,
				      ip6_gro_compute_pseudo))
		goto flush;

	th = tcp_gro_pull_header(skb);
	if (!th)
		goto flush;

	tcp6_check_fraglist_gro(head, skb, th);

	return tcp_gro_receive(head, skb, th);

flush:
	NAPI_GRO_CB(skb)->flush = 1;
	return NULL;
}

INDIRECT_CALLABLE_SCOPE int tcp6_gro_complete(struct sk_buff *skb, int thoff)
{
	const u16 offset = NAPI_GRO_CB(skb)->network_offsets[skb->encapsulation];
	const struct ipv6hdr *iph = (struct ipv6hdr *)(skb->data + offset);
	struct tcphdr *th = tcp_hdr(skb);

	if (unlikely(NAPI_GRO_CB(skb)->is_flist)) {
		skb_shinfo(skb)->gso_type |= SKB_GSO_FRAGLIST | SKB_GSO_TCPV6;
		skb_shinfo(skb)->gso_segs = NAPI_GRO_CB(skb)->count;

		__skb_incr_checksum_unnecessary(skb);

		return 0;
	}

	th->check = ~tcp_v6_check(skb->len - thoff, &iph->saddr,
				  &iph->daddr, 0);
	skb_shinfo(skb)->gso_type |= SKB_GSO_TCPV6;

	tcp_gro_complete(skb);
	return 0;
}

static void __tcpv6_gso_segment_csum(struct sk_buff *seg,
				     struct in6_addr *oldip,
				     const struct in6_addr *newip,
				     __be16 *oldport, __be16 newport)
{
	struct tcphdr *th = tcp_hdr(seg);

	if (!ipv6_addr_equal(oldip, newip)) {
		inet_proto_csum_replace16(&th->check, seg,
					  oldip->s6_addr32,
					  newip->s6_addr32,
					  true);
		*oldip = *newip;
	}

	if (*oldport == newport)
		return;

	inet_proto_csum_replace2(&th->check, seg, *oldport, newport, false);
	*oldport = newport;
}

static struct sk_buff *__tcpv6_gso_segment_list_csum(struct sk_buff *segs)
{
	const struct tcphdr *th;
	const struct ipv6hdr *iph;
	struct sk_buff *seg;
	struct tcphdr *th2;
	struct ipv6hdr *iph2;

	seg = segs;
	th = tcp_hdr(seg);
	iph = ipv6_hdr(seg);
	th2 = tcp_hdr(seg->next);
	iph2 = ipv6_hdr(seg->next);

	if (!(*(const u32 *)&th->source ^ *(const u32 *)&th2->source) &&
	    ipv6_addr_equal(&iph->saddr, &iph2->saddr) &&
	    ipv6_addr_equal(&iph->daddr, &iph2->daddr))
		return segs;

	while ((seg = seg->next)) {
		th2 = tcp_hdr(seg);
		iph2 = ipv6_hdr(seg);

		__tcpv6_gso_segment_csum(seg, &iph2->saddr, &iph->saddr,
					 &th2->source, th->source);
		__tcpv6_gso_segment_csum(seg, &iph2->daddr, &iph->daddr,
					 &th2->dest, th->dest);
	}

	return segs;
}

static struct sk_buff *__tcp6_gso_segment_list(struct sk_buff *skb,
					      netdev_features_t features)
{
	skb = skb_segment_list(skb, features, skb_mac_header_len(skb));
	if (IS_ERR(skb))
		return skb;

	return __tcpv6_gso_segment_list_csum(skb);
}

static struct sk_buff *tcp6_gso_segment(struct sk_buff *skb,
					netdev_features_t features)
{
	struct tcphdr *th;

	if (!(skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6))
		return ERR_PTR(-EINVAL);

	if (!pskb_may_pull(skb, sizeof(*th)))
		return ERR_PTR(-EINVAL);

	if (skb_shinfo(skb)->gso_type & SKB_GSO_FRAGLIST) {
		struct tcphdr *th = tcp_hdr(skb);

		if (skb_pagelen(skb) - th->doff * 4 == skb_shinfo(skb)->gso_size)
			return __tcp6_gso_segment_list(skb, features);

		skb->ip_summed = CHECKSUM_NONE;
	}

	if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) {
		const struct ipv6hdr *ipv6h = ipv6_hdr(skb);
		struct tcphdr *th = tcp_hdr(skb);

		/* Set up pseudo header, usually expect stack to have done
		 * this.
		 */

		th->check = 0;
		skb->ip_summed = CHECKSUM_PARTIAL;
		__tcp_v6_send_check(skb, &ipv6h->saddr, &ipv6h->daddr);
	}

	return tcp_gso_segment(skb, features);
}

int __init tcpv6_offload_init(void)
{
	net_hotdata.tcpv6_offload = (struct net_offload) {
		.callbacks = {
			.gso_segment	=	tcp6_gso_segment,
			.gro_receive	=	tcp6_gro_receive,
			.gro_complete	=	tcp6_gro_complete,
		},
	};
	return inet6_add_offload(&net_hotdata.tcpv6_offload, IPPROTO_TCP);
}
Commit	Line	Data
2874c5fd	1	// SPDX-License-Identifier: GPL-2.0-or-later
8663e02a VY	2	/*
	3	* IPV6 GSO/GRO offload support
	4	* Linux INET6 implementation
	5	*
8663e02a VY	6	* TCPv6 GSO/GRO support
8663e02a VY	7	*/
028e0a47	8	#include <linux/indirect_call_wrapper.h>
8663e02a	9	#include <linux/skbuff.h>
c9d1d23e	10	#include <net/inet6_hashtables.h>
4721031c	11	#include <net/gro.h>
8663e02a VY	12	#include <net/protocol.h>
	13	#include <net/tcp.h>
	14	#include <net/ip6_checksum.h>
	15	#include "ip6_offload.h"
	16
c9d1d23e FF	17	static void tcp6_check_fraglist_gro(struct list_head head, struct sk_buff skb,
	18	struct tcphdr *th)
	19	{
	20	#if IS_ENABLED(CONFIG_IPV6)
	21	const struct ipv6hdr *hdr;
	22	struct sk_buff *p;
	23	struct sock *sk;
	24	struct net *net;
	25	int iif, sdif;
	26
	27	if (likely(!(skb->dev->features & NETIF_F_GRO_FRAGLIST)))
	28	return;
	29
	30	p = tcp_gro_lookup(head, th);
	31	if (p) {
	32	NAPI_GRO_CB(skb)->is_flist = NAPI_GRO_CB(p)->is_flist;
	33	return;
	34	}
	35
	36	inet6_get_iif_sdif(skb, &iif, &sdif);
	37	hdr = skb_gro_network_header(skb);
9b49f57c	38	net = dev_net_rcu(skb->dev);
c9d1d23e FF	39	sk = __inet6_lookup_established(net, net->ipv4.tcp_death_row.hashinfo,
	40	&hdr->saddr, th->source,
	41	&hdr->daddr, ntohs(th->dest),
	42	iif, sdif);
	43	NAPI_GRO_CB(skb)->is_flist = !sk;
	44	if (sk)
f920436a	45	sock_gen_put(sk);
c9d1d23e FF	46	#endif /* IS_ENABLED(CONFIG_IPV6) */
	47	}
	48
028e0a47 PA	49	INDIRECT_CALLABLE_SCOPE
028e0a47 PA	50	struct sk_buff tcp6_gro_receive(struct list_head head, struct sk_buff *skb)
8663e02a	51	{
7516b27c FF	52	struct tcphdr *th;
7516b27c FF	53
cc5c00bb	54	/* Don't bother verifying checksum if we're going to flush anyway. */
149d0774 TH	55	if (!NAPI_GRO_CB(skb)->flush &&
149d0774 TH	56	skb_gro_checksum_validate(skb, IPPROTO_TCP,
7516b27c FF	57	ip6_gro_compute_pseudo))
	58	goto flush;
	59
	60	th = tcp_gro_pull_header(skb);
	61	if (!th)
	62	goto flush;
	63
c9d1d23e FF	64	tcp6_check_fraglist_gro(head, skb, th);
c9d1d23e FF	65
7516b27c	66	return tcp_gro_receive(head, skb, th);
8663e02a	67
7516b27c FF	68	flush:
	69	NAPI_GRO_CB(skb)->flush = 1;
	70	return NULL;
8663e02a VY	71	}
8663e02a VY	72
028e0a47	73	INDIRECT_CALLABLE_SCOPE int tcp6_gro_complete(struct sk_buff *skb, int thoff)
8663e02a	74	{
186b1ea7 RG	75	const u16 offset = NAPI_GRO_CB(skb)->network_offsets[skb->encapsulation];
186b1ea7 RG	76	const struct ipv6hdr iph = (struct ipv6hdr )(skb->data + offset);
8663e02a VY	77	struct tcphdr *th = tcp_hdr(skb);
8663e02a VY	78
8d95dc47 FF	79	if (unlikely(NAPI_GRO_CB(skb)->is_flist)) {
	80	skb_shinfo(skb)->gso_type \|= SKB_GSO_FRAGLIST \| SKB_GSO_TCPV6;
	81	skb_shinfo(skb)->gso_segs = NAPI_GRO_CB(skb)->count;
	82
	83	__skb_incr_checksum_unnecessary(skb);
	84
	85	return 0;
	86	}
	87
299603e8 JC	88	th->check = ~tcp_v6_check(skb->len - thoff, &iph->saddr,
299603e8 JC	89	&iph->daddr, 0);
c3caf119	90	skb_shinfo(skb)->gso_type \|= SKB_GSO_TCPV6;
8663e02a	91
b1f2abcf PP	92	tcp_gro_complete(skb);
b1f2abcf PP	93	return 0;
8663e02a VY	94	}
8663e02a VY	95
bee88cd5	96	static void __tcpv6_gso_segment_csum(struct sk_buff *seg,
daa624d3 FF	97	struct in6_addr *oldip,
daa624d3 FF	98	const struct in6_addr *newip,
bee88cd5 FF	99	__be16 *oldport, __be16 newport)
bee88cd5 FF	100	{
daa624d3 FF	101	struct tcphdr *th = tcp_hdr(seg);
	102
	103	if (!ipv6_addr_equal(oldip, newip)) {
	104	inet_proto_csum_replace16(&th->check, seg,
	105	oldip->s6_addr32,
	106	newip->s6_addr32,
	107	true);
	108	oldip = newip;
	109	}
bee88cd5 FF	110
	111	if (*oldport == newport)
	112	return;
	113
bee88cd5 FF	114	inet_proto_csum_replace2(&th->check, seg, *oldport, newport, false);
	115	*oldport = newport;
	116	}
	117
	118	static struct sk_buff __tcpv6_gso_segment_list_csum(struct sk_buff segs)
	119	{
	120	const struct tcphdr *th;
	121	const struct ipv6hdr *iph;
	122	struct sk_buff *seg;
	123	struct tcphdr *th2;
	124	struct ipv6hdr *iph2;
	125
	126	seg = segs;
	127	th = tcp_hdr(seg);
	128	iph = ipv6_hdr(seg);
	129	th2 = tcp_hdr(seg->next);
	130	iph2 = ipv6_hdr(seg->next);
	131
	132	if (!((const u32 )&th->source ^ (const u32 )&th2->source) &&
	133	ipv6_addr_equal(&iph->saddr, &iph2->saddr) &&
	134	ipv6_addr_equal(&iph->daddr, &iph2->daddr))
	135	return segs;
	136
	137	while ((seg = seg->next)) {
	138	th2 = tcp_hdr(seg);
	139	iph2 = ipv6_hdr(seg);
	140
daa624d3 FF	141	__tcpv6_gso_segment_csum(seg, &iph2->saddr, &iph->saddr,
	142	&th2->source, th->source);
	143	__tcpv6_gso_segment_csum(seg, &iph2->daddr, &iph->daddr,
	144	&th2->dest, th->dest);
bee88cd5 FF	145	}
	146
	147	return segs;
	148	}
	149
	150	static struct sk_buff __tcp6_gso_segment_list(struct sk_buff skb,
	151	netdev_features_t features)
	152	{
	153	skb = skb_segment_list(skb, features, skb_mac_header_len(skb));
	154	if (IS_ERR(skb))
	155	return skb;
	156
	157	return __tcpv6_gso_segment_list_csum(skb);
	158	}
	159
74abc20c ED	160	static struct sk_buff tcp6_gso_segment(struct sk_buff skb,
74abc20c ED	161	netdev_features_t features)
d020f8f7 TH	162	{
	163	struct tcphdr *th;
	164
121d57af WB	165	if (!(skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6))
	166	return ERR_PTR(-EINVAL);
	167
d020f8f7 TH	168	if (!pskb_may_pull(skb, sizeof(*th)))
	169	return ERR_PTR(-EINVAL);
	170
17bd3bd8 FF	171	if (skb_shinfo(skb)->gso_type & SKB_GSO_FRAGLIST) {
	172	struct tcphdr *th = tcp_hdr(skb);
	173
	174	if (skb_pagelen(skb) - th->doff * 4 == skb_shinfo(skb)->gso_size)
	175	return __tcp6_gso_segment_list(skb, features);
	176
	177	skb->ip_summed = CHECKSUM_NONE;
	178	}
bee88cd5	179
d020f8f7 TH	180	if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) {
	181	const struct ipv6hdr *ipv6h = ipv6_hdr(skb);
	182	struct tcphdr *th = tcp_hdr(skb);
	183
	184	/* Set up pseudo header, usually expect stack to have done
	185	* this.
	186	*/
	187
	188	th->check = 0;
	189	skb->ip_summed = CHECKSUM_PARTIAL;
	190	__tcp_v6_send_check(skb, &ipv6h->saddr, &ipv6h->daddr);
	191	}
	192
	193	return tcp_gso_segment(skb, features);
	194	}
8663e02a VY	195
	196	int __init tcpv6_offload_init(void)
	197	{
0139806e ED	198	net_hotdata.tcpv6_offload = (struct net_offload) {
	199	.callbacks = {
	200	.gso_segment = tcp6_gso_segment,
	201	.gro_receive = tcp6_gro_receive,
	202	.gro_complete = tcp6_gro_complete,
	203	},
	204	};
	205	return inet6_add_offload(&net_hotdata.tcpv6_offload, IPPROTO_TCP);
8663e02a	206	}