[linux-2.6-block.git] / net / xfrm / xfrm_input.c

// SPDX-License-Identifier: GPL-2.0
/*
 * xfrm_input.c
 *
 * Changes:
 * 	YOSHIFUJI Hideaki @USAGI
 * 		Split up af-specific portion
 *
 */

#include <linux/bottom_half.h>
#include <linux/cache.h>
#include <linux/interrupt.h>
#include <linux/slab.h>
#include <linux/module.h>
#include <linux/netdevice.h>
#include <linux/percpu.h>
#include <net/dst.h>
#include <net/ip.h>
#include <net/xfrm.h>
#include <net/ip_tunnels.h>
#include <net/ip6_tunnel.h>

struct xfrm_trans_tasklet {
	struct tasklet_struct tasklet;
	struct sk_buff_head queue;
};

struct xfrm_trans_cb {
	union {
		struct inet_skb_parm	h4;
#if IS_ENABLED(CONFIG_IPV6)
		struct inet6_skb_parm	h6;
#endif
	} header;
	int (*finish)(struct net *net, struct sock *sk, struct sk_buff *skb);
};

#define XFRM_TRANS_SKB_CB(__skb) ((struct xfrm_trans_cb *)&((__skb)->cb[0]))

static DEFINE_SPINLOCK(xfrm_input_afinfo_lock);
static struct xfrm_input_afinfo const __rcu *xfrm_input_afinfo[AF_INET6 + 1];

static struct gro_cells gro_cells;
static struct net_device xfrm_napi_dev;

static DEFINE_PER_CPU(struct xfrm_trans_tasklet, xfrm_trans_tasklet);

int xfrm_input_register_afinfo(const struct xfrm_input_afinfo *afinfo)
{
	int err = 0;

	if (WARN_ON(afinfo->family >= ARRAY_SIZE(xfrm_input_afinfo)))
		return -EAFNOSUPPORT;

	spin_lock_bh(&xfrm_input_afinfo_lock);
	if (unlikely(xfrm_input_afinfo[afinfo->family] != NULL))
		err = -EEXIST;
	else
		rcu_assign_pointer(xfrm_input_afinfo[afinfo->family], afinfo);
	spin_unlock_bh(&xfrm_input_afinfo_lock);
	return err;
}
EXPORT_SYMBOL(xfrm_input_register_afinfo);

int xfrm_input_unregister_afinfo(const struct xfrm_input_afinfo *afinfo)
{
	int err = 0;

	spin_lock_bh(&xfrm_input_afinfo_lock);
	if (likely(xfrm_input_afinfo[afinfo->family] != NULL)) {
		if (unlikely(xfrm_input_afinfo[afinfo->family] != afinfo))
			err = -EINVAL;
		else
			RCU_INIT_POINTER(xfrm_input_afinfo[afinfo->family], NULL);
	}
	spin_unlock_bh(&xfrm_input_afinfo_lock);
	synchronize_rcu();
	return err;
}
EXPORT_SYMBOL(xfrm_input_unregister_afinfo);

static const struct xfrm_input_afinfo *xfrm_input_get_afinfo(unsigned int family)
{
	const struct xfrm_input_afinfo *afinfo;

	if (WARN_ON_ONCE(family >= ARRAY_SIZE(xfrm_input_afinfo)))
		return NULL;

	rcu_read_lock();
	afinfo = rcu_dereference(xfrm_input_afinfo[family]);
	if (unlikely(!afinfo))
		rcu_read_unlock();
	return afinfo;
}

static int xfrm_rcv_cb(struct sk_buff *skb, unsigned int family, u8 protocol,
		       int err)
{
	int ret;
	const struct xfrm_input_afinfo *afinfo = xfrm_input_get_afinfo(family);

	if (!afinfo)
		return -EAFNOSUPPORT;

	ret = afinfo->callback(skb, protocol, err);
	rcu_read_unlock();

	return ret;
}

struct sec_path *secpath_set(struct sk_buff *skb)
{
	struct sec_path *sp, *tmp = skb_ext_find(skb, SKB_EXT_SEC_PATH);

	sp = skb_ext_add(skb, SKB_EXT_SEC_PATH);
	if (!sp)
		return NULL;

	if (tmp) /* reused existing one (was COW'd if needed) */
		return sp;

	/* allocated new secpath */
	memset(sp->ovec, 0, sizeof(sp->ovec));
	sp->olen = 0;
	sp->len = 0;

	return sp;
}
EXPORT_SYMBOL(secpath_set);

/* Fetch spi and seq from ipsec header */

int xfrm_parse_spi(struct sk_buff *skb, u8 nexthdr, __be32 *spi, __be32 *seq)
{
	int offset, offset_seq;
	int hlen;

	switch (nexthdr) {
	case IPPROTO_AH:
		hlen = sizeof(struct ip_auth_hdr);
		offset = offsetof(struct ip_auth_hdr, spi);
		offset_seq = offsetof(struct ip_auth_hdr, seq_no);
		break;
	case IPPROTO_ESP:
		hlen = sizeof(struct ip_esp_hdr);
		offset = offsetof(struct ip_esp_hdr, spi);
		offset_seq = offsetof(struct ip_esp_hdr, seq_no);
		break;
	case IPPROTO_COMP:
		if (!pskb_may_pull(skb, sizeof(struct ip_comp_hdr)))
			return -EINVAL;
		*spi = htonl(ntohs(*(__be16 *)(skb_transport_header(skb) + 2)));
		*seq = 0;
		return 0;
	default:
		return 1;
	}

	if (!pskb_may_pull(skb, hlen))
		return -EINVAL;

	*spi = *(__be32 *)(skb_transport_header(skb) + offset);
	*seq = *(__be32 *)(skb_transport_header(skb) + offset_seq);
	return 0;
}
EXPORT_SYMBOL(xfrm_parse_spi);

int xfrm_prepare_input(struct xfrm_state *x, struct sk_buff *skb)
{
	struct xfrm_mode *inner_mode = x->inner_mode;
	int err;

	err = x->outer_mode->afinfo->extract_input(x, skb);
	if (err)
		return err;

	if (x->sel.family == AF_UNSPEC) {
		inner_mode = xfrm_ip2inner_mode(x, XFRM_MODE_SKB_CB(skb)->protocol);
		if (inner_mode == NULL)
			return -EAFNOSUPPORT;
	}

	skb->protocol = inner_mode->afinfo->eth_proto;
	return inner_mode->input2(x, skb);
}
EXPORT_SYMBOL(xfrm_prepare_input);

int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type)
{
	struct net *net = dev_net(skb->dev);
	int err;
	__be32 seq;
	__be32 seq_hi;
	struct xfrm_state *x = NULL;
	xfrm_address_t *daddr;
	struct xfrm_mode *inner_mode;
	u32 mark = skb->mark;
	unsigned int family = AF_UNSPEC;
	int decaps = 0;
	int async = 0;
	bool xfrm_gro = false;
	bool crypto_done = false;
	struct xfrm_offload *xo = xfrm_offload(skb);
	struct sec_path *sp;

	if (encap_type < 0) {
		x = xfrm_input_state(skb);

		if (unlikely(x->km.state != XFRM_STATE_VALID)) {
			if (x->km.state == XFRM_STATE_ACQ)
				XFRM_INC_STATS(net, LINUX_MIB_XFRMACQUIREERROR);
			else
				XFRM_INC_STATS(net,
					       LINUX_MIB_XFRMINSTATEINVALID);
			goto drop;
		}

		family = x->outer_mode->afinfo->family;

		/* An encap_type of -1 indicates async resumption. */
		if (encap_type == -1) {
			async = 1;
			seq = XFRM_SKB_CB(skb)->seq.input.low;
			goto resume;
		}

		/* encap_type < -1 indicates a GRO call. */
		encap_type = 0;
		seq = XFRM_SPI_SKB_CB(skb)->seq;

		if (xo && (xo->flags & CRYPTO_DONE)) {
			crypto_done = true;
			family = XFRM_SPI_SKB_CB(skb)->family;

			if (!(xo->status & CRYPTO_SUCCESS)) {
				if (xo->status &
				    (CRYPTO_TRANSPORT_AH_AUTH_FAILED |
				     CRYPTO_TRANSPORT_ESP_AUTH_FAILED |
				     CRYPTO_TUNNEL_AH_AUTH_FAILED |
				     CRYPTO_TUNNEL_ESP_AUTH_FAILED)) {

					xfrm_audit_state_icvfail(x, skb,
								 x->type->proto);
					x->stats.integrity_failed++;
					XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEPROTOERROR);
					goto drop;
				}

				if (xo->status & CRYPTO_INVALID_PROTOCOL) {
					XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEPROTOERROR);
					goto drop;
				}

				XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR);
				goto drop;
			}

			if ((err = xfrm_parse_spi(skb, nexthdr, &spi, &seq)) != 0) {
				XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR);
				goto drop;
			}
		}

		goto lock;
	}

	family = XFRM_SPI_SKB_CB(skb)->family;

	/* if tunnel is present override skb->mark value with tunnel i_key */
	switch (family) {
	case AF_INET:
		if (XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip4)
			mark = be32_to_cpu(XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip4->parms.i_key);
		break;
	case AF_INET6:
		if (XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip6)
			mark = be32_to_cpu(XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip6->parms.i_key);
		break;
	}

	sp = secpath_set(skb);
	if (!sp) {
		XFRM_INC_STATS(net, LINUX_MIB_XFRMINERROR);
		goto drop;
	}

	seq = 0;
	if (!spi && (err = xfrm_parse_spi(skb, nexthdr, &spi, &seq)) != 0) {
		secpath_reset(skb);
		XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR);
		goto drop;
	}

	daddr = (xfrm_address_t *)(skb_network_header(skb) +
				   XFRM_SPI_SKB_CB(skb)->daddroff);
	do {
		sp = skb_sec_path(skb);

		if (sp->len == XFRM_MAX_DEPTH) {
			secpath_reset(skb);
			XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR);
			goto drop;
		}

		x = xfrm_state_lookup(net, mark, daddr, spi, nexthdr, family);
		if (x == NULL) {
			secpath_reset(skb);
			XFRM_INC_STATS(net, LINUX_MIB_XFRMINNOSTATES);
			xfrm_audit_state_notfound(skb, family, spi, seq);
			goto drop;
		}

		skb->mark = xfrm_smark_get(skb->mark, x);

		sp->xvec[sp->len++] = x;

		skb_dst_force(skb);
		if (!skb_dst(skb)) {
			XFRM_INC_STATS(net, LINUX_MIB_XFRMINERROR);
			goto drop;
		}

lock:
		spin_lock(&x->lock);

		if (unlikely(x->km.state != XFRM_STATE_VALID)) {
			if (x->km.state == XFRM_STATE_ACQ)
				XFRM_INC_STATS(net, LINUX_MIB_XFRMACQUIREERROR);
			else
				XFRM_INC_STATS(net,
					       LINUX_MIB_XFRMINSTATEINVALID);
			goto drop_unlock;
		}

		if ((x->encap ? x->encap->encap_type : 0) != encap_type) {
			XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMISMATCH);
			goto drop_unlock;
		}

		if (x->repl->check(x, skb, seq)) {
			XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATESEQERROR);
			goto drop_unlock;
		}

		if (xfrm_state_check_expire(x)) {
			XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEEXPIRED);
			goto drop_unlock;
		}

		spin_unlock(&x->lock);

		if (xfrm_tunnel_check(skb, x, family)) {
			XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMODEERROR);
			goto drop;
		}

		seq_hi = htonl(xfrm_replay_seqhi(x, seq));

		XFRM_SKB_CB(skb)->seq.input.low = seq;
		XFRM_SKB_CB(skb)->seq.input.hi = seq_hi;

		dev_hold(skb->dev);

		if (crypto_done)
			nexthdr = x->type_offload->input_tail(x, skb);
		else
			nexthdr = x->type->input(x, skb);

		if (nexthdr == -EINPROGRESS)
			return 0;
resume:
		dev_put(skb->dev);

		spin_lock(&x->lock);
		if (nexthdr <= 0) {
			if (nexthdr == -EBADMSG) {
				xfrm_audit_state_icvfail(x, skb,
							 x->type->proto);
				x->stats.integrity_failed++;
			}
			XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEPROTOERROR);
			goto drop_unlock;
		}

		/* only the first xfrm gets the encap type */
		encap_type = 0;

		if (async && x->repl->recheck(x, skb, seq)) {
			XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATESEQERROR);
			goto drop_unlock;
		}

		x->repl->advance(x, seq);

		x->curlft.bytes += skb->len;
		x->curlft.packets++;

		spin_unlock(&x->lock);

		XFRM_MODE_SKB_CB(skb)->protocol = nexthdr;

		inner_mode = x->inner_mode;

		if (x->sel.family == AF_UNSPEC) {
			inner_mode = xfrm_ip2inner_mode(x, XFRM_MODE_SKB_CB(skb)->protocol);
			if (inner_mode == NULL) {
				XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMODEERROR);
				goto drop;
			}
		}

		if (inner_mode->input(x, skb)) {
			XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMODEERROR);
			goto drop;
		}

		if (x->outer_mode->flags & XFRM_MODE_FLAG_TUNNEL) {
			decaps = 1;
			break;
		}

		/*
		 * We need the inner address.  However, we only get here for
		 * transport mode so the outer address is identical.
		 */
		daddr = &x->id.daddr;
		family = x->outer_mode->afinfo->family;

		err = xfrm_parse_spi(skb, nexthdr, &spi, &seq);
		if (err < 0) {
			XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR);
			goto drop;
		}
		crypto_done = false;
	} while (!err);

	err = xfrm_rcv_cb(skb, family, x->type->proto, 0);
	if (err)
		goto drop;

	nf_reset(skb);

	if (decaps) {
		sp = skb_sec_path(skb);
		if (sp)
			sp->olen = 0;
		skb_dst_drop(skb);
		gro_cells_receive(&gro_cells, skb);
		return 0;
	} else {
		xo = xfrm_offload(skb);
		if (xo)
			xfrm_gro = xo->flags & XFRM_GRO;

		err = x->inner_mode->afinfo->transport_finish(skb, xfrm_gro || async);
		if (xfrm_gro) {
			sp = skb_sec_path(skb);
			if (sp)
				sp->olen = 0;
			skb_dst_drop(skb);
			gro_cells_receive(&gro_cells, skb);
			return err;
		}

		return err;
	}

drop_unlock:
	spin_unlock(&x->lock);
drop:
	xfrm_rcv_cb(skb, family, x && x->type ? x->type->proto : nexthdr, -1);
	kfree_skb(skb);
	return 0;
}
EXPORT_SYMBOL(xfrm_input);

int xfrm_input_resume(struct sk_buff *skb, int nexthdr)
{
	return xfrm_input(skb, nexthdr, 0, -1);
}
EXPORT_SYMBOL(xfrm_input_resume);

static void xfrm_trans_reinject(unsigned long data)
{
	struct xfrm_trans_tasklet *trans = (void *)data;
	struct sk_buff_head queue;
	struct sk_buff *skb;

	__skb_queue_head_init(&queue);
	skb_queue_splice_init(&trans->queue, &queue);

	while ((skb = __skb_dequeue(&queue)))
		XFRM_TRANS_SKB_CB(skb)->finish(dev_net(skb->dev), NULL, skb);
}

int xfrm_trans_queue(struct sk_buff *skb,
		     int (*finish)(struct net *, struct sock *,
				   struct sk_buff *))
{
	struct xfrm_trans_tasklet *trans;

	trans = this_cpu_ptr(&xfrm_trans_tasklet);

	if (skb_queue_len(&trans->queue) >= netdev_max_backlog)
		return -ENOBUFS;

	XFRM_TRANS_SKB_CB(skb)->finish = finish;
	__skb_queue_tail(&trans->queue, skb);
	tasklet_schedule(&trans->tasklet);
	return 0;
}
EXPORT_SYMBOL(xfrm_trans_queue);

void __init xfrm_input_init(void)
{
	int err;
	int i;

	init_dummy_netdev(&xfrm_napi_dev);
	err = gro_cells_init(&gro_cells, &xfrm_napi_dev);
	if (err)
		gro_cells.cells = NULL;

	for_each_possible_cpu(i) {
		struct xfrm_trans_tasklet *trans;

		trans = &per_cpu(xfrm_trans_tasklet, i);
		__skb_queue_head_init(&trans->queue);
		tasklet_init(&trans->tasklet, xfrm_trans_reinject,
			     (unsigned long)trans);
	}
}
Commit	Line	Data
b2441318	1	// SPDX-License-Identifier: GPL-2.0
1da177e4 LT	2	/*
	3	* xfrm_input.c
	4	*
	5	* Changes:
	6	* YOSHIFUJI Hideaki @USAGI
	7	* Split up af-specific portion
a716c119	8	*
1da177e4 LT	9	*/
1da177e4 LT	10
acf568ee	11	#include <linux/bottom_half.h>
f8c3d0dd	12	#include <linux/cache.h>
acf568ee	13	#include <linux/interrupt.h>
1da177e4 LT	14	#include <linux/slab.h>
1da177e4 LT	15	#include <linux/module.h>
716062fd	16	#include <linux/netdevice.h>
acf568ee	17	#include <linux/percpu.h>
716062fd	18	#include <net/dst.h>
1da177e4 LT	19	#include <net/ip.h>
1da177e4 LT	20	#include <net/xfrm.h>
049f8e2e AD	21	#include <net/ip_tunnels.h>
049f8e2e AD	22	#include <net/ip6_tunnel.h>
1da177e4	23
acf568ee HX	24	struct xfrm_trans_tasklet {
	25	struct tasklet_struct tasklet;
	26	struct sk_buff_head queue;
	27	};
	28
	29	struct xfrm_trans_cb {
9a3fb9fb SK	30	union {
	31	struct inet_skb_parm h4;
	32	#if IS_ENABLED(CONFIG_IPV6)
	33	struct inet6_skb_parm h6;
	34	#endif
	35	} header;
acf568ee HX	36	int (finish)(struct net net, struct sock sk, struct sk_buff skb);
	37	};
	38
	39	#define XFRM_TRANS_SKB_CB(__skb) ((struct xfrm_trans_cb *)&((__skb)->cb[0]))
	40
2f32b51b	41	static DEFINE_SPINLOCK(xfrm_input_afinfo_lock);
960fdfde	42	static struct xfrm_input_afinfo const __rcu *xfrm_input_afinfo[AF_INET6 + 1];
2f32b51b	43
1995876a SK	44	static struct gro_cells gro_cells;
	45	static struct net_device xfrm_napi_dev;
	46
acf568ee HX	47	static DEFINE_PER_CPU(struct xfrm_trans_tasklet, xfrm_trans_tasklet);
acf568ee HX	48
960fdfde	49	int xfrm_input_register_afinfo(const struct xfrm_input_afinfo *afinfo)
2f32b51b SK	50	{
	51	int err = 0;
	52
960fdfde	53	if (WARN_ON(afinfo->family >= ARRAY_SIZE(xfrm_input_afinfo)))
2f32b51b	54	return -EAFNOSUPPORT;
960fdfde	55
2f32b51b SK	56	spin_lock_bh(&xfrm_input_afinfo_lock);
2f32b51b SK	57	if (unlikely(xfrm_input_afinfo[afinfo->family] != NULL))
f31e8d4f	58	err = -EEXIST;
2f32b51b SK	59	else
	60	rcu_assign_pointer(xfrm_input_afinfo[afinfo->family], afinfo);
	61	spin_unlock_bh(&xfrm_input_afinfo_lock);
	62	return err;
	63	}
	64	EXPORT_SYMBOL(xfrm_input_register_afinfo);
	65
960fdfde	66	int xfrm_input_unregister_afinfo(const struct xfrm_input_afinfo *afinfo)
2f32b51b SK	67	{
	68	int err = 0;
	69
2f32b51b SK	70	spin_lock_bh(&xfrm_input_afinfo_lock);
	71	if (likely(xfrm_input_afinfo[afinfo->family] != NULL)) {
	72	if (unlikely(xfrm_input_afinfo[afinfo->family] != afinfo))
	73	err = -EINVAL;
	74	else
	75	RCU_INIT_POINTER(xfrm_input_afinfo[afinfo->family], NULL);
	76	}
	77	spin_unlock_bh(&xfrm_input_afinfo_lock);
	78	synchronize_rcu();
	79	return err;
	80	}
	81	EXPORT_SYMBOL(xfrm_input_unregister_afinfo);
	82
960fdfde	83	static const struct xfrm_input_afinfo *xfrm_input_get_afinfo(unsigned int family)
2f32b51b	84	{
960fdfde	85	const struct xfrm_input_afinfo *afinfo;
2f32b51b	86
960fdfde	87	if (WARN_ON_ONCE(family >= ARRAY_SIZE(xfrm_input_afinfo)))
2f32b51b	88	return NULL;
960fdfde	89
2f32b51b SK	90	rcu_read_lock();
	91	afinfo = rcu_dereference(xfrm_input_afinfo[family]);
	92	if (unlikely(!afinfo))
	93	rcu_read_unlock();
	94	return afinfo;
	95	}
	96
2f32b51b SK	97	static int xfrm_rcv_cb(struct sk_buff *skb, unsigned int family, u8 protocol,
	98	int err)
	99	{
	100	int ret;
960fdfde	101	const struct xfrm_input_afinfo *afinfo = xfrm_input_get_afinfo(family);
2f32b51b SK	102
	103	if (!afinfo)
	104	return -EAFNOSUPPORT;
	105
	106	ret = afinfo->callback(skb, protocol, err);
960fdfde	107	rcu_read_unlock();
2f32b51b SK	108
	109	return ret;
	110	}
	111
4165079b	112	struct sec_path secpath_set(struct sk_buff skb)
1da177e4	113	{
4165079b	114	struct sec_path sp, tmp = skb_ext_find(skb, SKB_EXT_SEC_PATH);
1da177e4	115
4165079b	116	sp = skb_ext_add(skb, SKB_EXT_SEC_PATH);
1da177e4 LT	117	if (!sp)
	118	return NULL;
	119
4165079b FW	120	if (tmp) /* reused existing one (was COW'd if needed) */
4165079b FW	121	return sp;
54ef207a	122
4165079b	123	/* allocated new secpath */
f1193e91	124	memset(sp->ovec, 0, sizeof(sp->ovec));
4165079b FW	125	sp->olen = 0;
4165079b FW	126	sp->len = 0;
0ca64da1 FW	127
0ca64da1 FW	128	return sp;
b0fcee82 SK	129	}
	130	EXPORT_SYMBOL(secpath_set);
	131
1da177e4 LT	132	/* Fetch spi and seq from ipsec header */
1da177e4 LT	133
6067b2ba	134	int xfrm_parse_spi(struct sk_buff skb, u8 nexthdr, __be32 spi, __be32 *seq)
1da177e4 LT	135	{
1da177e4 LT	136	int offset, offset_seq;
44072500	137	int hlen;
1da177e4 LT	138
	139	switch (nexthdr) {
	140	case IPPROTO_AH:
44072500	141	hlen = sizeof(struct ip_auth_hdr);
1da177e4 LT	142	offset = offsetof(struct ip_auth_hdr, spi);
	143	offset_seq = offsetof(struct ip_auth_hdr, seq_no);
	144	break;
	145	case IPPROTO_ESP:
44072500	146	hlen = sizeof(struct ip_esp_hdr);
1da177e4 LT	147	offset = offsetof(struct ip_esp_hdr, spi);
	148	offset_seq = offsetof(struct ip_esp_hdr, seq_no);
	149	break;
	150	case IPPROTO_COMP:
	151	if (!pskb_may_pull(skb, sizeof(struct ip_comp_hdr)))
	152	return -EINVAL;
3e94c2dc	153	spi = htonl(ntohs((__be16 *)(skb_transport_header(skb) + 2)));
1da177e4 LT	154	*seq = 0;
	155	return 0;
	156	default:
	157	return 1;
	158	}
	159
44072500	160	if (!pskb_may_pull(skb, hlen))
1da177e4 LT	161	return -EINVAL;
1da177e4 LT	162
3e94c2dc WC	163	spi = (__be32 *)(skb_transport_header(skb) + offset);
3e94c2dc WC	164	seq = (__be32 *)(skb_transport_header(skb) + offset_seq);
1da177e4 LT	165	return 0;
1da177e4 LT	166	}
1e295370	167	EXPORT_SYMBOL(xfrm_parse_spi);
1da177e4	168
227620e2 HX	169	int xfrm_prepare_input(struct xfrm_state x, struct sk_buff skb)
227620e2 HX	170	{
df9dcb45	171	struct xfrm_mode *inner_mode = x->inner_mode;
227620e2 HX	172	int err;
	173
	174	err = x->outer_mode->afinfo->extract_input(x, skb);
	175	if (err)
	176	return err;
	177
df9dcb45 KM	178	if (x->sel.family == AF_UNSPEC) {
	179	inner_mode = xfrm_ip2inner_mode(x, XFRM_MODE_SKB_CB(skb)->protocol);
	180	if (inner_mode == NULL)
	181	return -EAFNOSUPPORT;
	182	}
	183
	184	skb->protocol = inner_mode->afinfo->eth_proto;
	185	return inner_mode->input2(x, skb);
227620e2 HX	186	}
	187	EXPORT_SYMBOL(xfrm_prepare_input);
	188
716062fd HX	189	int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type)
716062fd HX	190	{
bd235e3c	191	struct net *net = dev_net(skb->dev);
716062fd HX	192	int err;
716062fd HX	193	__be32 seq;
2cd08467	194	__be32 seq_hi;
3328715e	195	struct xfrm_state *x = NULL;
1bf06cd2	196	xfrm_address_t *daddr;
df9dcb45	197	struct xfrm_mode *inner_mode;
049f8e2e	198	u32 mark = skb->mark;
4ce3dbe3	199	unsigned int family = AF_UNSPEC;
716062fd	200	int decaps = 0;
1bf06cd2	201	int async = 0;
7785bba2	202	bool xfrm_gro = false;
d77e38e6 SK	203	bool crypto_done = false;
d77e38e6 SK	204	struct xfrm_offload *xo = xfrm_offload(skb);
0ca64da1	205	struct sec_path *sp;
1bf06cd2	206
1bf06cd2	207	if (encap_type < 0) {
00501121	208	x = xfrm_input_state(skb);
4ce3dbe3 AH	209
	210	if (unlikely(x->km.state != XFRM_STATE_VALID)) {
	211	if (x->km.state == XFRM_STATE_ACQ)
	212	XFRM_INC_STATS(net, LINUX_MIB_XFRMACQUIREERROR);
	213	else
	214	XFRM_INC_STATS(net,
	215	LINUX_MIB_XFRMINSTATEINVALID);
	216	goto drop;
	217	}
	218
3328715e	219	family = x->outer_mode->afinfo->family;
7785bba2 SK	220
	221	/* An encap_type of -1 indicates async resumption. */
	222	if (encap_type == -1) {
	223	async = 1;
	224	seq = XFRM_SKB_CB(skb)->seq.input.low;
	225	goto resume;
	226	}
bcd1f8a4	227
7785bba2 SK	228	/* encap_type < -1 indicates a GRO call. */
	229	encap_type = 0;
	230	seq = XFRM_SPI_SKB_CB(skb)->seq;
716062fd	231
bcd1f8a4 SK	232	if (xo && (xo->flags & CRYPTO_DONE)) {
bcd1f8a4 SK	233	crypto_done = true;
bcd1f8a4 SK	234	family = XFRM_SPI_SKB_CB(skb)->family;
	235
	236	if (!(xo->status & CRYPTO_SUCCESS)) {
	237	if (xo->status &
	238	(CRYPTO_TRANSPORT_AH_AUTH_FAILED \|
	239	CRYPTO_TRANSPORT_ESP_AUTH_FAILED \|
	240	CRYPTO_TUNNEL_AH_AUTH_FAILED \|
	241	CRYPTO_TUNNEL_ESP_AUTH_FAILED)) {
	242
	243	xfrm_audit_state_icvfail(x, skb,
	244	x->type->proto);
	245	x->stats.integrity_failed++;
	246	XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEPROTOERROR);
	247	goto drop;
	248	}
	249
47ebcc0b YK	250	if (xo->status & CRYPTO_INVALID_PROTOCOL) {
	251	XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEPROTOERROR);
	252	goto drop;
	253	}
	254
bcd1f8a4	255	XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR);
d77e38e6 SK	256	goto drop;
	257	}
	258
bcd1f8a4 SK	259	if ((err = xfrm_parse_spi(skb, nexthdr, &spi, &seq)) != 0) {
	260	XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR);
	261	goto drop;
	262	}
d77e38e6 SK	263	}
d77e38e6 SK	264
7785bba2	265	goto lock;
1bf06cd2	266	}
716062fd	267
3328715e SK	268	family = XFRM_SPI_SKB_CB(skb)->family;
3328715e SK	269
049f8e2e	270	/* if tunnel is present override skb->mark value with tunnel i_key */
1625f452 AK	271	switch (family) {
	272	case AF_INET:
	273	if (XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip4)
049f8e2e	274	mark = be32_to_cpu(XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip4->parms.i_key);
1625f452 AK	275	break;
	276	case AF_INET6:
	277	if (XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip6)
049f8e2e	278	mark = be32_to_cpu(XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip6->parms.i_key);
1625f452	279	break;
049f8e2e AD	280	}
049f8e2e AD	281
0ca64da1 FW	282	sp = secpath_set(skb);
0ca64da1 FW	283	if (!sp) {
b0fcee82 SK	284	XFRM_INC_STATS(net, LINUX_MIB_XFRMINERROR);
b0fcee82 SK	285	goto drop;
b2aa5e9d HX	286	}
b2aa5e9d HX	287
716062fd	288	seq = 0;
0aa64774	289	if (!spi && (err = xfrm_parse_spi(skb, nexthdr, &spi, &seq)) != 0) {
f203b76d	290	secpath_reset(skb);
59c9940e	291	XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR);
716062fd	292	goto drop;
0aa64774	293	}
716062fd	294
cb79a180 FW	295	daddr = (xfrm_address_t *)(skb_network_header(skb) +
cb79a180 FW	296	XFRM_SPI_SKB_CB(skb)->daddroff);
716062fd	297	do {
2294be0f FW	298	sp = skb_sec_path(skb);
	299
	300	if (sp->len == XFRM_MAX_DEPTH) {
f203b76d	301	secpath_reset(skb);
59c9940e	302	XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR);
716062fd	303	goto drop;
0aa64774	304	}
716062fd	305
049f8e2e	306	x = xfrm_state_lookup(net, mark, daddr, spi, nexthdr, family);
0aa64774	307	if (x == NULL) {
f203b76d	308	secpath_reset(skb);
59c9940e	309	XFRM_INC_STATS(net, LINUX_MIB_XFRMINNOSTATES);
afeb14b4	310	xfrm_audit_state_notfound(skb, family, spi, seq);
716062fd	311	goto drop;
0aa64774	312	}
716062fd	313
9b42c1f1 SK	314	skb->mark = xfrm_smark_get(skb->mark, x);
9b42c1f1 SK	315
2294be0f	316	sp->xvec[sp->len++] = x;
b2aa5e9d	317
0152eee6 SK	318	skb_dst_force(skb);
	319	if (!skb_dst(skb)) {
	320	XFRM_INC_STATS(net, LINUX_MIB_XFRMINERROR);
	321	goto drop;
	322	}
	323
7785bba2	324	lock:
716062fd	325	spin_lock(&x->lock);
4c4d41f2	326
0aa64774	327	if (unlikely(x->km.state != XFRM_STATE_VALID)) {
dc0565ce LR	328	if (x->km.state == XFRM_STATE_ACQ)
	329	XFRM_INC_STATS(net, LINUX_MIB_XFRMACQUIREERROR);
	330	else
	331	XFRM_INC_STATS(net,
	332	LINUX_MIB_XFRMINSTATEINVALID);
716062fd	333	goto drop_unlock;
0aa64774	334	}
716062fd	335
3de77cf2 DM	336	if ((x->encap ? x->encap->encap_type : 0) != encap_type) {
	337	XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMISMATCH);
	338	goto drop_unlock;
	339	}
	340
36ae0148	341	if (x->repl->check(x, skb, seq)) {
59c9940e	342	XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATESEQERROR);
716062fd	343	goto drop_unlock;
0aa64774	344	}
716062fd	345
0aa64774	346	if (xfrm_state_check_expire(x)) {
59c9940e	347	XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEEXPIRED);
716062fd	348	goto drop_unlock;
0aa64774	349	}
716062fd	350
0ebea8ef HX	351	spin_unlock(&x->lock);
0ebea8ef HX	352
68c11e98 AD	353	if (xfrm_tunnel_check(skb, x, family)) {
	354	XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMODEERROR);
	355	goto drop;
	356	}
	357
2cd08467 SK	358	seq_hi = htonl(xfrm_replay_seqhi(x, seq));
2cd08467 SK	359
1ce3644a	360	XFRM_SKB_CB(skb)->seq.input.low = seq;
2cd08467	361	XFRM_SKB_CB(skb)->seq.input.hi = seq_hi;
1bf06cd2	362
071d36bf	363	dev_hold(skb->dev);
3bc07321	364
d77e38e6 SK	365	if (crypto_done)
	366	nexthdr = x->type_offload->input_tail(x, skb);
	367	else
	368	nexthdr = x->type->input(x, skb);
0ebea8ef	369
1bf06cd2 HX	370	if (nexthdr == -EINPROGRESS)
1bf06cd2 HX	371	return 0;
1bf06cd2	372	resume:
071d36bf	373	dev_put(skb->dev);
071d36bf	374
0ebea8ef	375	spin_lock(&x->lock);
668dc8af	376	if (nexthdr <= 0) {
9dd3245a HX	377	if (nexthdr == -EBADMSG) {
	378	xfrm_audit_state_icvfail(x, skb,
	379	x->type->proto);
668dc8af	380	x->stats.integrity_failed++;
9dd3245a	381	}
59c9940e	382	XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEPROTOERROR);
716062fd	383	goto drop_unlock;
668dc8af	384	}
716062fd	385
716062fd HX	386	/* only the first xfrm gets the encap type */
	387	encap_type = 0;
	388
3b59df46	389	if (async && x->repl->recheck(x, skb, seq)) {
bcf66bf5 SK	390	XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATESEQERROR);
	391	goto drop_unlock;
	392	}
	393
9fdc4883	394	x->repl->advance(x, seq);
716062fd HX	395
	396	x->curlft.bytes += skb->len;
	397	x->curlft.packets++;
	398
	399	spin_unlock(&x->lock);
	400
60d5fcfb HX	401	XFRM_MODE_SKB_CB(skb)->protocol = nexthdr;
60d5fcfb HX	402
df9dcb45 KM	403	inner_mode = x->inner_mode;
	404
	405	if (x->sel.family == AF_UNSPEC) {
	406	inner_mode = xfrm_ip2inner_mode(x, XFRM_MODE_SKB_CB(skb)->protocol);
cb866e32 SK	407	if (inner_mode == NULL) {
cb866e32 SK	408	XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMODEERROR);
df9dcb45	409	goto drop;
cb866e32	410	}
df9dcb45 KM	411	}
	412
	413	if (inner_mode->input(x, skb)) {
59c9940e	414	XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMODEERROR);
716062fd	415	goto drop;
0aa64774	416	}
716062fd HX	417
	418	if (x->outer_mode->flags & XFRM_MODE_FLAG_TUNNEL) {
	419	decaps = 1;
	420	break;
	421	}
	422
1bf06cd2 HX	423	/*
	424	* We need the inner address. However, we only get here for
	425	* transport mode so the outer address is identical.
	426	*/
	427	daddr = &x->id.daddr;
2fcb45b6	428	family = x->outer_mode->afinfo->family;
1bf06cd2	429
716062fd	430	err = xfrm_parse_spi(skb, nexthdr, &spi, &seq);
0aa64774	431	if (err < 0) {
59c9940e	432	XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR);
716062fd	433	goto drop;
0aa64774	434	}
782710e3	435	crypto_done = false;
716062fd HX	436	} while (!err);
716062fd HX	437
3328715e SK	438	err = xfrm_rcv_cb(skb, family, x->type->proto, 0);
	439	if (err)
	440	goto drop;
	441
716062fd HX	442	nf_reset(skb);
	443
	444	if (decaps) {
2294be0f FW	445	sp = skb_sec_path(skb);
	446	if (sp)
	447	sp->olen = 0;
adf30907	448	skb_dst_drop(skb);
1995876a	449	gro_cells_receive(&gro_cells, skb);
716062fd HX	450	return 0;
716062fd HX	451	} else {
7785bba2 SK	452	xo = xfrm_offload(skb);
	453	if (xo)
	454	xfrm_gro = xo->flags & XFRM_GRO;
	455
cfcf99f9	456	err = x->inner_mode->afinfo->transport_finish(skb, xfrm_gro \|\| async);
7785bba2	457	if (xfrm_gro) {
2294be0f FW	458	sp = skb_sec_path(skb);
	459	if (sp)
	460	sp->olen = 0;
7785bba2 SK	461	skb_dst_drop(skb);
	462	gro_cells_receive(&gro_cells, skb);
	463	return err;
	464	}
	465
	466	return err;
716062fd HX	467	}
	468
	469	drop_unlock:
	470	spin_unlock(&x->lock);
716062fd	471	drop:
3328715e	472	xfrm_rcv_cb(skb, family, x && x->type ? x->type->proto : nexthdr, -1);
716062fd HX	473	kfree_skb(skb);
	474	return 0;
	475	}
	476	EXPORT_SYMBOL(xfrm_input);
	477
1bf06cd2 HX	478	int xfrm_input_resume(struct sk_buff *skb, int nexthdr)
	479	{
	480	return xfrm_input(skb, nexthdr, 0, -1);
	481	}
	482	EXPORT_SYMBOL(xfrm_input_resume);
	483
acf568ee HX	484	static void xfrm_trans_reinject(unsigned long data)
	485	{
	486	struct xfrm_trans_tasklet trans = (void )data;
	487	struct sk_buff_head queue;
	488	struct sk_buff *skb;
	489
	490	__skb_queue_head_init(&queue);
	491	skb_queue_splice_init(&trans->queue, &queue);
	492
	493	while ((skb = __skb_dequeue(&queue)))
	494	XFRM_TRANS_SKB_CB(skb)->finish(dev_net(skb->dev), NULL, skb);
	495	}
	496
	497	int xfrm_trans_queue(struct sk_buff *skb,
	498	int (finish)(struct net , struct sock *,
	499	struct sk_buff *))
	500	{
	501	struct xfrm_trans_tasklet *trans;
	502
	503	trans = this_cpu_ptr(&xfrm_trans_tasklet);
	504
	505	if (skb_queue_len(&trans->queue) >= netdev_max_backlog)
	506	return -ENOBUFS;
	507
	508	XFRM_TRANS_SKB_CB(skb)->finish = finish;
d16b46e4	509	__skb_queue_tail(&trans->queue, skb);
acf568ee HX	510	tasklet_schedule(&trans->tasklet);
	511	return 0;
	512	}
	513	EXPORT_SYMBOL(xfrm_trans_queue);
	514
1da177e4 LT	515	void __init xfrm_input_init(void)
1da177e4 LT	516	{
1995876a	517	int err;
acf568ee	518	int i;
1995876a SK	519
	520	init_dummy_netdev(&xfrm_napi_dev);
	521	err = gro_cells_init(&gro_cells, &xfrm_napi_dev);
	522	if (err)
	523	gro_cells.cells = NULL;
	524
acf568ee HX	525	for_each_possible_cpu(i) {
	526	struct xfrm_trans_tasklet *trans;
	527
	528	trans = &per_cpu(xfrm_trans_tasklet, i);
	529	__skb_queue_head_init(&trans->queue);
	530	tasklet_init(&trans->tasklet, xfrm_trans_reinject,
	531	(unsigned long)trans);
	532	}
1da177e4	533	}