[linux-2.6-block.git] / net / ipv6 / ip6_offload.c

/*
 *	IPV6 GSO/GRO offload support
 *	Linux INET6 implementation
 *
 *	This program is free software; you can redistribute it and/or
 *      modify it under the terms of the GNU General Public License
 *      as published by the Free Software Foundation; either version
 *      2 of the License, or (at your option) any later version.
 */

#include <linux/kernel.h>
#include <linux/socket.h>
#include <linux/netdevice.h>
#include <linux/skbuff.h>

#include <net/protocol.h>
#include <net/ipv6.h>

#include "ip6_offload.h"

static int ipv6_gso_pull_exthdrs(struct sk_buff *skb, int proto)
{
	const struct net_offload *ops = NULL;

	for (;;) {
		struct ipv6_opt_hdr *opth;
		int len;

		if (proto != NEXTHDR_HOP) {
			ops = rcu_dereference(inet6_offloads[proto]);

			if (unlikely(!ops))
				break;

			if (!(ops->flags & INET6_PROTO_GSO_EXTHDR))
				break;
		}

		if (unlikely(!pskb_may_pull(skb, 8)))
			break;

		opth = (void *)skb->data;
		len = ipv6_optlen(opth);

		if (unlikely(!pskb_may_pull(skb, len)))
			break;

		proto = opth->nexthdr;
		__skb_pull(skb, len);
	}

	return proto;
}

static int ipv6_gso_send_check(struct sk_buff *skb)
{
	const struct ipv6hdr *ipv6h;
	const struct net_offload *ops;
	int err = -EINVAL;

	if (unlikely(!pskb_may_pull(skb, sizeof(*ipv6h))))
		goto out;

	ipv6h = ipv6_hdr(skb);
	__skb_pull(skb, sizeof(*ipv6h));
	err = -EPROTONOSUPPORT;

	rcu_read_lock();
	ops = rcu_dereference(inet6_offloads[
		ipv6_gso_pull_exthdrs(skb, ipv6h->nexthdr)]);

	if (likely(ops && ops->gso_send_check)) {
		skb_reset_transport_header(skb);
		err = ops->gso_send_check(skb);
	}
	rcu_read_unlock();

out:
	return err;
}

static struct sk_buff *ipv6_gso_segment(struct sk_buff *skb,
	netdev_features_t features)
{
	struct sk_buff *segs = ERR_PTR(-EINVAL);
	struct ipv6hdr *ipv6h;
	const struct net_offload *ops;
	int proto;
	struct frag_hdr *fptr;
	unsigned int unfrag_ip6hlen;
	u8 *prevhdr;
	int offset = 0;

	if (!(features & NETIF_F_V6_CSUM))
		features &= ~NETIF_F_SG;

	if (unlikely(skb_shinfo(skb)->gso_type &
		     ~(SKB_GSO_UDP |
		       SKB_GSO_DODGY |
		       SKB_GSO_TCP_ECN |
		       SKB_GSO_TCPV6 |
		       0)))
		goto out;

	if (unlikely(!pskb_may_pull(skb, sizeof(*ipv6h))))
		goto out;

	ipv6h = ipv6_hdr(skb);
	__skb_pull(skb, sizeof(*ipv6h));
	segs = ERR_PTR(-EPROTONOSUPPORT);

	proto = ipv6_gso_pull_exthdrs(skb, ipv6h->nexthdr);
	rcu_read_lock();
	ops = rcu_dereference(inet6_offloads[proto]);
	if (likely(ops && ops->gso_segment)) {
		skb_reset_transport_header(skb);
		segs = ops->gso_segment(skb, features);
	}
	rcu_read_unlock();

	if (IS_ERR(segs))
		goto out;

	for (skb = segs; skb; skb = skb->next) {
		ipv6h = ipv6_hdr(skb);
		ipv6h->payload_len = htons(skb->len - skb->mac_len -
					   sizeof(*ipv6h));
		if (proto == IPPROTO_UDP) {
			unfrag_ip6hlen = ip6_find_1stfragopt(skb, &prevhdr);
			fptr = (struct frag_hdr *)(skb_network_header(skb) +
				unfrag_ip6hlen);
			fptr->frag_off = htons(offset);
			if (skb->next != NULL)
				fptr->frag_off |= htons(IP6_MF);
			offset += (ntohs(ipv6h->payload_len) -
				   sizeof(struct frag_hdr));
		}
	}

out:
	return segs;
}

static struct sk_buff **ipv6_gro_receive(struct sk_buff **head,
					 struct sk_buff *skb)
{
	const struct net_offload *ops;
	struct sk_buff **pp = NULL;
	struct sk_buff *p;
	struct ipv6hdr *iph;
	unsigned int nlen;
	unsigned int hlen;
	unsigned int off;
	int flush = 1;
	int proto;
	__wsum csum;

	off = skb_gro_offset(skb);
	hlen = off + sizeof(*iph);
	iph = skb_gro_header_fast(skb, off);
	if (skb_gro_header_hard(skb, hlen)) {
		iph = skb_gro_header_slow(skb, hlen, off);
		if (unlikely(!iph))
			goto out;
	}

	skb_gro_pull(skb, sizeof(*iph));
	skb_set_transport_header(skb, skb_gro_offset(skb));

	flush += ntohs(iph->payload_len) != skb_gro_len(skb);

	rcu_read_lock();
	proto = iph->nexthdr;
	ops = rcu_dereference(inet6_offloads[proto]);
	if (!ops || !ops->gro_receive) {
		__pskb_pull(skb, skb_gro_offset(skb));
		proto = ipv6_gso_pull_exthdrs(skb, proto);
		skb_gro_pull(skb, -skb_transport_offset(skb));
		skb_reset_transport_header(skb);
		__skb_push(skb, skb_gro_offset(skb));

		ops = rcu_dereference(inet6_offloads[proto]);
		if (!ops || !ops->gro_receive)
			goto out_unlock;

		iph = ipv6_hdr(skb);
	}

	NAPI_GRO_CB(skb)->proto = proto;

	flush--;
	nlen = skb_network_header_len(skb);

	for (p = *head; p; p = p->next) {
		const struct ipv6hdr *iph2;
		__be32 first_word; /* <Version:4><Traffic_Class:8><Flow_Label:20> */

		if (!NAPI_GRO_CB(p)->same_flow)
			continue;

		iph2 = ipv6_hdr(p);
		first_word = *(__be32 *)iph ^ *(__be32 *)iph2 ;

		/* All fields must match except length and Traffic Class. */
		if (nlen != skb_network_header_len(p) ||
		    (first_word & htonl(0xF00FFFFF)) ||
		    memcmp(&iph->nexthdr, &iph2->nexthdr,
			   nlen - offsetof(struct ipv6hdr, nexthdr))) {
			NAPI_GRO_CB(p)->same_flow = 0;
			continue;
		}
		/* flush if Traffic Class fields are different */
		NAPI_GRO_CB(p)->flush |= !!(first_word & htonl(0x0FF00000));
		NAPI_GRO_CB(p)->flush |= flush;
	}

	NAPI_GRO_CB(skb)->flush |= flush;

	csum = skb->csum;
	skb_postpull_rcsum(skb, iph, skb_network_header_len(skb));

	pp = ops->gro_receive(head, skb);

	skb->csum = csum;

out_unlock:
	rcu_read_unlock();

out:
	NAPI_GRO_CB(skb)->flush |= flush;

	return pp;
}

static int ipv6_gro_complete(struct sk_buff *skb)
{
	const struct net_offload *ops;
	struct ipv6hdr *iph = ipv6_hdr(skb);
	int err = -ENOSYS;

	iph->payload_len = htons(skb->len - skb_network_offset(skb) -
				 sizeof(*iph));

	rcu_read_lock();
	ops = rcu_dereference(inet6_offloads[NAPI_GRO_CB(skb)->proto]);
	if (WARN_ON(!ops || !ops->gro_complete))
		goto out_unlock;

	err = ops->gro_complete(skb);

out_unlock:
	rcu_read_unlock();

	return err;
}

static struct packet_offload ipv6_packet_offload __read_mostly = {
	.type = cpu_to_be16(ETH_P_IPV6),
	.gso_send_check = ipv6_gso_send_check,
	.gso_segment = ipv6_gso_segment,
	.gro_receive = ipv6_gro_receive,
	.gro_complete = ipv6_gro_complete,
};

void __init ipv6_offload_init(void)
{
	dev_add_offload(&ipv6_packet_offload);
}

void ipv6_offload_cleanup(void)
{
	dev_remove_offload(&ipv6_packet_offload);
}
Commit	Line	Data
d1da932e VY	1	/*
	2	* IPV6 GSO/GRO offload support
	3	* Linux INET6 implementation
	4	*
	5	* This program is free software; you can redistribute it and/or
	6	* modify it under the terms of the GNU General Public License
	7	* as published by the Free Software Foundation; either version
	8	* 2 of the License, or (at your option) any later version.
	9	*/
	10
	11	#include <linux/kernel.h>
	12	#include <linux/socket.h>
	13	#include <linux/netdevice.h>
	14	#include <linux/skbuff.h>
	15
	16	#include <net/protocol.h>
	17	#include <net/ipv6.h>
	18
	19	#include "ip6_offload.h"
	20
	21	static int ipv6_gso_pull_exthdrs(struct sk_buff *skb, int proto)
	22	{
	23	const struct net_offload *ops = NULL;
	24
	25	for (;;) {
	26	struct ipv6_opt_hdr *opth;
	27	int len;
	28
	29	if (proto != NEXTHDR_HOP) {
	30	ops = rcu_dereference(inet6_offloads[proto]);
	31
	32	if (unlikely(!ops))
	33	break;
	34
	35	if (!(ops->flags & INET6_PROTO_GSO_EXTHDR))
	36	break;
	37	}
	38
	39	if (unlikely(!pskb_may_pull(skb, 8)))
	40	break;
	41
	42	opth = (void *)skb->data;
	43	len = ipv6_optlen(opth);
	44
	45	if (unlikely(!pskb_may_pull(skb, len)))
	46	break;
	47
	48	proto = opth->nexthdr;
	49	__skb_pull(skb, len);
	50	}
	51
	52	return proto;
	53	}
	54
	55	static int ipv6_gso_send_check(struct sk_buff *skb)
	56	{
	57	const struct ipv6hdr *ipv6h;
	58	const struct net_offload *ops;
	59	int err = -EINVAL;
	60
	61	if (unlikely(!pskb_may_pull(skb, sizeof(*ipv6h))))
	62	goto out;
	63
	64	ipv6h = ipv6_hdr(skb);
65	__skb_pull(skb, sizeof(*ipv6h));
66	err = -EPROTONOSUPPORT;
67
68	rcu_read_lock();
69	ops = rcu_dereference(inet6_offloads[
70	ipv6_gso_pull_exthdrs(skb, ipv6h->nexthdr)]);
71
72	if (likely(ops && ops->gso_send_check)) {
73	skb_reset_transport_header(skb);
74	err = ops->gso_send_check(skb);
75	}
76	rcu_read_unlock();
77
78	out:
79	return err;
80	}
81
82	static struct sk_buff ipv6_gso_segment(struct sk_buff skb,
83	netdev_features_t features)
84	{
85	struct sk_buff *segs = ERR_PTR(-EINVAL);
86	struct ipv6hdr *ipv6h;
87	const struct net_offload *ops;
88	int proto;
89	struct frag_hdr *fptr;
90	unsigned int unfrag_ip6hlen;
91	u8 *prevhdr;
92	int offset = 0;
93
94	if (!(features & NETIF_F_V6_CSUM))
95	features &= ~NETIF_F_SG;
96
97	if (unlikely(skb_shinfo(skb)->gso_type &
98	~(SKB_GSO_UDP \|
99	SKB_GSO_DODGY \|
100	SKB_GSO_TCP_ECN \|
101	SKB_GSO_TCPV6 \|
102	0)))
103	goto out;
104
105	if (unlikely(!pskb_may_pull(skb, sizeof(*ipv6h))))
106	goto out;
107
108	ipv6h = ipv6_hdr(skb);
109	__skb_pull(skb, sizeof(*ipv6h));
110	segs = ERR_PTR(-EPROTONOSUPPORT);
111
112	proto = ipv6_gso_pull_exthdrs(skb, ipv6h->nexthdr);
113	rcu_read_lock();
114	ops = rcu_dereference(inet6_offloads[proto]);
115	if (likely(ops && ops->gso_segment)) {
116	skb_reset_transport_header(skb);
117	segs = ops->gso_segment(skb, features);
118	}
119	rcu_read_unlock();
120
121	if (IS_ERR(segs))
122	goto out;
123
124	for (skb = segs; skb; skb = skb->next) {
125	ipv6h = ipv6_hdr(skb);
126	ipv6h->payload_len = htons(skb->len - skb->mac_len -
127	sizeof(*ipv6h));
128	if (proto == IPPROTO_UDP) {
129	unfrag_ip6hlen = ip6_find_1stfragopt(skb, &prevhdr);
130	fptr = (struct frag_hdr *)(skb_network_header(skb) +
131	unfrag_ip6hlen);
132	fptr->frag_off = htons(offset);
133	if (skb->next != NULL)
134	fptr->frag_off \|= htons(IP6_MF);
135	offset += (ntohs(ipv6h->payload_len) -
136	sizeof(struct frag_hdr));
137	}
138	}
139
140	out:
141	return segs;
142	}
143
144	static struct sk_buff ipv6_gro_receive(struct sk_buff head,
145	struct sk_buff *skb)
146	{
147	const struct net_offload *ops;
148	struct sk_buff **pp = NULL;
149	struct sk_buff *p;
150	struct ipv6hdr *iph;
151	unsigned int nlen;
152	unsigned int hlen;
153	unsigned int off;
154	int flush = 1;
155	int proto;
156	__wsum csum;
157
158	off = skb_gro_offset(skb);
159	hlen = off + sizeof(*iph);
160	iph = skb_gro_header_fast(skb, off);
161	if (skb_gro_header_hard(skb, hlen)) {
162	iph = skb_gro_header_slow(skb, hlen, off);
163	if (unlikely(!iph))
164	goto out;
165	}
166
167	skb_gro_pull(skb, sizeof(*iph));
168	skb_set_transport_header(skb, skb_gro_offset(skb));
169
170	flush += ntohs(iph->payload_len) != skb_gro_len(skb);
171
172	rcu_read_lock();
173	proto = iph->nexthdr;
174	ops = rcu_dereference(inet6_offloads[proto]);
175	if (!ops \|\| !ops->gro_receive) {
176	__pskb_pull(skb, skb_gro_offset(skb));
177	proto = ipv6_gso_pull_exthdrs(skb, proto);
178	skb_gro_pull(skb, -skb_transport_offset(skb));
179	skb_reset_transport_header(skb);
180	__skb_push(skb, skb_gro_offset(skb));
181
182	ops = rcu_dereference(inet6_offloads[proto]);
183	if (!ops \|\| !ops->gro_receive)
184	goto out_unlock;
185
186	iph = ipv6_hdr(skb);
187	}
188
189	NAPI_GRO_CB(skb)->proto = proto;
190
191	flush--;
192	nlen = skb_network_header_len(skb);
193
194	for (p = *head; p; p = p->next) {
195	const struct ipv6hdr *iph2;
196	__be32 first_word; /* <Version:4><Traffic_Class:8><Flow_Label:20> */
197
198	if (!NAPI_GRO_CB(p)->same_flow)
199	continue;
200
201	iph2 = ipv6_hdr(p);
202	first_word = (__be32 )iph ^ (__be32 )iph2 ;
203
204	/* All fields must match except length and Traffic Class. */
205	if (nlen != skb_network_header_len(p) \|\|
206	(first_word & htonl(0xF00FFFFF)) \|\|
207	memcmp(&iph->nexthdr, &iph2->nexthdr,
208	nlen - offsetof(struct ipv6hdr, nexthdr))) {
209	NAPI_GRO_CB(p)->same_flow = 0;
210	continue;
211	}
212	/* flush if Traffic Class fields are different */
213	NAPI_GRO_CB(p)->flush \|= !!(first_word & htonl(0x0FF00000));
214	NAPI_GRO_CB(p)->flush \|= flush;
215	}
216
217	NAPI_GRO_CB(skb)->flush \|= flush;
218
219	csum = skb->csum;
220	skb_postpull_rcsum(skb, iph, skb_network_header_len(skb));
221
222	pp = ops->gro_receive(head, skb);
223
224	skb->csum = csum;
225
226	out_unlock:
227	rcu_read_unlock();
228
229	out:
230	NAPI_GRO_CB(skb)->flush \|= flush;
231
232	return pp;
233	}
234
235	static int ipv6_gro_complete(struct sk_buff *skb)
236	{
237	const struct net_offload *ops;
238	struct ipv6hdr *iph = ipv6_hdr(skb);
239	int err = -ENOSYS;
240
241	iph->payload_len = htons(skb->len - skb_network_offset(skb) -
242	sizeof(*iph));
243
244	rcu_read_lock();
245	ops = rcu_dereference(inet6_offloads[NAPI_GRO_CB(skb)->proto]);
246	if (WARN_ON(!ops \|\| !ops->gro_complete))
247	goto out_unlock;
248
249	err = ops->gro_complete(skb);
250
251	out_unlock:
252	rcu_read_unlock();
253
254	return err;
255	}
256
257	static struct packet_offload ipv6_packet_offload __read_mostly = {
258	.type = cpu_to_be16(ETH_P_IPV6),
259	.gso_send_check = ipv6_gso_send_check,
260	.gso_segment = ipv6_gso_segment,
261	.gro_receive = ipv6_gro_receive,
262	.gro_complete = ipv6_gro_complete,
263	};
264
265	void __init ipv6_offload_init(void)
266	{
267	dev_add_offload(&ipv6_packet_offload);
268	}
269
270	void ipv6_offload_cleanup(void)
271	{
272	dev_remove_offload(&ipv6_packet_offload);
273	}