[linux-2.6-block.git] / net / xfrm / xfrm_input.c

// SPDX-License-Identifier: GPL-2.0
/*
 * xfrm_input.c
 *
 * Changes:
 * 	YOSHIFUJI Hideaki @USAGI
 * 		Split up af-specific portion
 *
 */

#include <linux/bottom_half.h>
#include <linux/cache.h>
#include <linux/interrupt.h>
#include <linux/slab.h>
#include <linux/module.h>
#include <linux/netdevice.h>
#include <linux/percpu.h>
#include <net/dst.h>
#include <net/ip.h>
#include <net/xfrm.h>
#include <net/ip_tunnels.h>
#include <net/ip6_tunnel.h>

struct xfrm_trans_tasklet {
	struct tasklet_struct tasklet;
	struct sk_buff_head queue;
};

struct xfrm_trans_cb {
	union {
		struct inet_skb_parm	h4;
#if IS_ENABLED(CONFIG_IPV6)
		struct inet6_skb_parm	h6;
#endif
	} header;
	int (*finish)(struct net *net, struct sock *sk, struct sk_buff *skb);
};

#define XFRM_TRANS_SKB_CB(__skb) ((struct xfrm_trans_cb *)&((__skb)->cb[0]))

static DEFINE_SPINLOCK(xfrm_input_afinfo_lock);
static struct xfrm_input_afinfo const __rcu *xfrm_input_afinfo[AF_INET6 + 1];

static struct gro_cells gro_cells;
static struct net_device xfrm_napi_dev;

static DEFINE_PER_CPU(struct xfrm_trans_tasklet, xfrm_trans_tasklet);

int xfrm_input_register_afinfo(const struct xfrm_input_afinfo *afinfo)
{
	int err = 0;

	if (WARN_ON(afinfo->family >= ARRAY_SIZE(xfrm_input_afinfo)))
		return -EAFNOSUPPORT;

	spin_lock_bh(&xfrm_input_afinfo_lock);
	if (unlikely(xfrm_input_afinfo[afinfo->family] != NULL))
		err = -EEXIST;
	else
		rcu_assign_pointer(xfrm_input_afinfo[afinfo->family], afinfo);
	spin_unlock_bh(&xfrm_input_afinfo_lock);
	return err;
}
EXPORT_SYMBOL(xfrm_input_register_afinfo);

int xfrm_input_unregister_afinfo(const struct xfrm_input_afinfo *afinfo)
{
	int err = 0;

	spin_lock_bh(&xfrm_input_afinfo_lock);
	if (likely(xfrm_input_afinfo[afinfo->family] != NULL)) {
		if (unlikely(xfrm_input_afinfo[afinfo->family] != afinfo))
			err = -EINVAL;
		else
			RCU_INIT_POINTER(xfrm_input_afinfo[afinfo->family], NULL);
	}
	spin_unlock_bh(&xfrm_input_afinfo_lock);
	synchronize_rcu();
	return err;
}
EXPORT_SYMBOL(xfrm_input_unregister_afinfo);

static const struct xfrm_input_afinfo *xfrm_input_get_afinfo(unsigned int family)
{
	const struct xfrm_input_afinfo *afinfo;

	if (WARN_ON_ONCE(family >= ARRAY_SIZE(xfrm_input_afinfo)))
		return NULL;

	rcu_read_lock();
	afinfo = rcu_dereference(xfrm_input_afinfo[family]);
	if (unlikely(!afinfo))
		rcu_read_unlock();
	return afinfo;
}

static int xfrm_rcv_cb(struct sk_buff *skb, unsigned int family, u8 protocol,
		       int err)
{
	int ret;
	const struct xfrm_input_afinfo *afinfo = xfrm_input_get_afinfo(family);

	if (!afinfo)
		return -EAFNOSUPPORT;

	ret = afinfo->callback(skb, protocol, err);
	rcu_read_unlock();

	return ret;
}

struct sec_path *secpath_set(struct sk_buff *skb)
{
	struct sec_path *sp, *tmp = skb_ext_find(skb, SKB_EXT_SEC_PATH);

	sp = skb_ext_add(skb, SKB_EXT_SEC_PATH);
	if (!sp)
		return NULL;

	if (tmp) /* reused existing one (was COW'd if needed) */
		return sp;

	/* allocated new secpath */
	memset(sp->ovec, 0, sizeof(sp->ovec));
	sp->olen = 0;
	sp->len = 0;

	return sp;
}
EXPORT_SYMBOL(secpath_set);

/* Fetch spi and seq from ipsec header */

int xfrm_parse_spi(struct sk_buff *skb, u8 nexthdr, __be32 *spi, __be32 *seq)
{
	int offset, offset_seq;
	int hlen;

	switch (nexthdr) {
	case IPPROTO_AH:
		hlen = sizeof(struct ip_auth_hdr);
		offset = offsetof(struct ip_auth_hdr, spi);
		offset_seq = offsetof(struct ip_auth_hdr, seq_no);
		break;
	case IPPROTO_ESP:
		hlen = sizeof(struct ip_esp_hdr);
		offset = offsetof(struct ip_esp_hdr, spi);
		offset_seq = offsetof(struct ip_esp_hdr, seq_no);
		break;
	case IPPROTO_COMP:
		if (!pskb_may_pull(skb, sizeof(struct ip_comp_hdr)))
			return -EINVAL;
		*spi = htonl(ntohs(*(__be16 *)(skb_transport_header(skb) + 2)));
		*seq = 0;
		return 0;
	default:
		return 1;
	}

	if (!pskb_may_pull(skb, hlen))
		return -EINVAL;

	*spi = *(__be32 *)(skb_transport_header(skb) + offset);
	*seq = *(__be32 *)(skb_transport_header(skb) + offset_seq);
	return 0;
}
EXPORT_SYMBOL(xfrm_parse_spi);

int xfrm_prepare_input(struct xfrm_state *x, struct sk_buff *skb)
{
	struct xfrm_mode *inner_mode = x->inner_mode;
	int err;

	err = x->outer_mode->afinfo->extract_input(x, skb);
	if (err)
		return err;

	if (x->sel.family == AF_UNSPEC) {
		inner_mode = xfrm_ip2inner_mode(x, XFRM_MODE_SKB_CB(skb)->protocol);
		if (inner_mode == NULL)
			return -EAFNOSUPPORT;
	}

	skb->protocol = inner_mode->afinfo->eth_proto;
	return inner_mode->input2(x, skb);
}
EXPORT_SYMBOL(xfrm_prepare_input);

int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type)
{
	struct net *net = dev_net(skb->dev);
	int err;
	__be32 seq;
	__be32 seq_hi;
	struct xfrm_state *x = NULL;
	xfrm_address_t *daddr;
	struct xfrm_mode *inner_mode;
	u32 mark = skb->mark;
	unsigned int family = AF_UNSPEC;
	int decaps = 0;
	int async = 0;
	bool xfrm_gro = false;
	bool crypto_done = false;
	struct xfrm_offload *xo = xfrm_offload(skb);
	struct sec_path *sp;

	if (encap_type < 0) {
		x = xfrm_input_state(skb);

		if (unlikely(x->km.state != XFRM_STATE_VALID)) {
			if (x->km.state == XFRM_STATE_ACQ)
				XFRM_INC_STATS(net, LINUX_MIB_XFRMACQUIREERROR);
			else
				XFRM_INC_STATS(net,
					       LINUX_MIB_XFRMINSTATEINVALID);
			goto drop;
		}

		family = x->outer_mode->afinfo->family;

		/* An encap_type of -1 indicates async resumption. */
		if (encap_type == -1) {
			async = 1;
			seq = XFRM_SKB_CB(skb)->seq.input.low;
			goto resume;
		}

		/* encap_type < -1 indicates a GRO call. */
		encap_type = 0;
		seq = XFRM_SPI_SKB_CB(skb)->seq;

		if (xo && (xo->flags & CRYPTO_DONE)) {
			crypto_done = true;
			family = XFRM_SPI_SKB_CB(skb)->family;

			if (!(xo->status & CRYPTO_SUCCESS)) {
				if (xo->status &
				    (CRYPTO_TRANSPORT_AH_AUTH_FAILED |
				     CRYPTO_TRANSPORT_ESP_AUTH_FAILED |
				     CRYPTO_TUNNEL_AH_AUTH_FAILED |
				     CRYPTO_TUNNEL_ESP_AUTH_FAILED)) {

					xfrm_audit_state_icvfail(x, skb,
								 x->type->proto);
					x->stats.integrity_failed++;
					XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEPROTOERROR);
					goto drop;
				}

				if (xo->status & CRYPTO_INVALID_PROTOCOL) {
					XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEPROTOERROR);
					goto drop;
				}

				XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR);
				goto drop;
			}

			if ((err = xfrm_parse_spi(skb, nexthdr, &spi, &seq)) != 0) {
				XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR);
				goto drop;
			}
		}

		goto lock;
	}

	family = XFRM_SPI_SKB_CB(skb)->family;

	/* if tunnel is present override skb->mark value with tunnel i_key */
	switch (family) {
	case AF_INET:
		if (XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip4)
			mark = be32_to_cpu(XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip4->parms.i_key);
		break;
	case AF_INET6:
		if (XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip6)
			mark = be32_to_cpu(XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip6->parms.i_key);
		break;
	}

	sp = secpath_set(skb);
	if (!sp) {
		XFRM_INC_STATS(net, LINUX_MIB_XFRMINERROR);
		goto drop;
	}

	seq = 0;
	if (!spi && (err = xfrm_parse_spi(skb, nexthdr, &spi, &seq)) != 0) {
		secpath_reset(skb);
		XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR);
		goto drop;
	}

	daddr = (xfrm_address_t *)(skb_network_header(skb) +
				   XFRM_SPI_SKB_CB(skb)->daddroff);
	do {
		sp = skb_sec_path(skb);

		if (sp->len == XFRM_MAX_DEPTH) {
			secpath_reset(skb);
			XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR);
			goto drop;
		}

		x = xfrm_state_lookup(net, mark, daddr, spi, nexthdr, family);
		if (x == NULL) {
			secpath_reset(skb);
			XFRM_INC_STATS(net, LINUX_MIB_XFRMINNOSTATES);
			xfrm_audit_state_notfound(skb, family, spi, seq);
			goto drop;
		}

		skb->mark = xfrm_smark_get(skb->mark, x);

		sp->xvec[sp->len++] = x;

lock:
		spin_lock(&x->lock);

		if (unlikely(x->km.state != XFRM_STATE_VALID)) {
			if (x->km.state == XFRM_STATE_ACQ)
				XFRM_INC_STATS(net, LINUX_MIB_XFRMACQUIREERROR);
			else
				XFRM_INC_STATS(net,
					       LINUX_MIB_XFRMINSTATEINVALID);
			goto drop_unlock;
		}

		if ((x->encap ? x->encap->encap_type : 0) != encap_type) {
			XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMISMATCH);
			goto drop_unlock;
		}

		if (x->repl->check(x, skb, seq)) {
			XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATESEQERROR);
			goto drop_unlock;
		}

		if (xfrm_state_check_expire(x)) {
			XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEEXPIRED);
			goto drop_unlock;
		}

		spin_unlock(&x->lock);

		if (xfrm_tunnel_check(skb, x, family)) {
			XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMODEERROR);
			goto drop;
		}

		seq_hi = htonl(xfrm_replay_seqhi(x, seq));

		XFRM_SKB_CB(skb)->seq.input.low = seq;
		XFRM_SKB_CB(skb)->seq.input.hi = seq_hi;

		skb_dst_force(skb);
		dev_hold(skb->dev);

		if (crypto_done)
			nexthdr = x->type_offload->input_tail(x, skb);
		else
			nexthdr = x->type->input(x, skb);

		if (nexthdr == -EINPROGRESS)
			return 0;
resume:
		dev_put(skb->dev);

		spin_lock(&x->lock);
		if (nexthdr <= 0) {
			if (nexthdr == -EBADMSG) {
				xfrm_audit_state_icvfail(x, skb,
							 x->type->proto);
				x->stats.integrity_failed++;
			}
			XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEPROTOERROR);
			goto drop_unlock;
		}

		/* only the first xfrm gets the encap type */
		encap_type = 0;

		if (async && x->repl->recheck(x, skb, seq)) {
			XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATESEQERROR);
			goto drop_unlock;
		}

		x->repl->advance(x, seq);

		x->curlft.bytes += skb->len;
		x->curlft.packets++;

		spin_unlock(&x->lock);

		XFRM_MODE_SKB_CB(skb)->protocol = nexthdr;

		inner_mode = x->inner_mode;

		if (x->sel.family == AF_UNSPEC) {
			inner_mode = xfrm_ip2inner_mode(x, XFRM_MODE_SKB_CB(skb)->protocol);
			if (inner_mode == NULL) {
				XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMODEERROR);
				goto drop;
			}
		}

		if (inner_mode->input(x, skb)) {
			XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMODEERROR);
			goto drop;
		}

		if (x->outer_mode->flags & XFRM_MODE_FLAG_TUNNEL) {
			decaps = 1;
			break;
		}

		/*
		 * We need the inner address.  However, we only get here for
		 * transport mode so the outer address is identical.
		 */
		daddr = &x->id.daddr;
		family = x->outer_mode->afinfo->family;

		err = xfrm_parse_spi(skb, nexthdr, &spi, &seq);
		if (err < 0) {
			XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR);
			goto drop;
		}
		crypto_done = false;
	} while (!err);

	err = xfrm_rcv_cb(skb, family, x->type->proto, 0);
	if (err)
		goto drop;

	nf_reset(skb);

	if (decaps) {
		sp = skb_sec_path(skb);
		if (sp)
			sp->olen = 0;
		skb_dst_drop(skb);
		gro_cells_receive(&gro_cells, skb);
		return 0;
	} else {
		xo = xfrm_offload(skb);
		if (xo)
			xfrm_gro = xo->flags & XFRM_GRO;

		err = x->inner_mode->afinfo->transport_finish(skb, xfrm_gro || async);
		if (xfrm_gro) {
			sp = skb_sec_path(skb);
			if (sp)
				sp->olen = 0;
			skb_dst_drop(skb);
			gro_cells_receive(&gro_cells, skb);
			return err;
		}

		return err;
	}

drop_unlock:
	spin_unlock(&x->lock);
drop:
	xfrm_rcv_cb(skb, family, x && x->type ? x->type->proto : nexthdr, -1);
	kfree_skb(skb);
	return 0;
}
EXPORT_SYMBOL(xfrm_input);

int xfrm_input_resume(struct sk_buff *skb, int nexthdr)
{
	return xfrm_input(skb, nexthdr, 0, -1);
}
EXPORT_SYMBOL(xfrm_input_resume);

static void xfrm_trans_reinject(unsigned long data)
{
	struct xfrm_trans_tasklet *trans = (void *)data;
	struct sk_buff_head queue;
	struct sk_buff *skb;

	__skb_queue_head_init(&queue);
	skb_queue_splice_init(&trans->queue, &queue);

	while ((skb = __skb_dequeue(&queue)))
		XFRM_TRANS_SKB_CB(skb)->finish(dev_net(skb->dev), NULL, skb);
}

int xfrm_trans_queue(struct sk_buff *skb,
		     int (*finish)(struct net *, struct sock *,
				   struct sk_buff *))
{
	struct xfrm_trans_tasklet *trans;

	trans = this_cpu_ptr(&xfrm_trans_tasklet);

	if (skb_queue_len(&trans->queue) >= netdev_max_backlog)
		return -ENOBUFS;

	XFRM_TRANS_SKB_CB(skb)->finish = finish;
	__skb_queue_tail(&trans->queue, skb);
	tasklet_schedule(&trans->tasklet);
	return 0;
}
EXPORT_SYMBOL(xfrm_trans_queue);

void __init xfrm_input_init(void)
{
	int err;
	int i;

	init_dummy_netdev(&xfrm_napi_dev);
	err = gro_cells_init(&gro_cells, &xfrm_napi_dev);
	if (err)
		gro_cells.cells = NULL;

	for_each_possible_cpu(i) {
		struct xfrm_trans_tasklet *trans;

		trans = &per_cpu(xfrm_trans_tasklet, i);
		__skb_queue_head_init(&trans->queue);
		tasklet_init(&trans->tasklet, xfrm_trans_reinject,
			     (unsigned long)trans);
	}
}
Commit	Line	Data
b2441318	1	// SPDX-License-Identifier: GPL-2.0
1da177e4 LT	2	/*
	3	* xfrm_input.c
	4	*
	5	* Changes:
	6	* YOSHIFUJI Hideaki @USAGI
	7	* Split up af-specific portion
a716c119	8	*
1da177e4 LT	9	*/
1da177e4 LT	10
acf568ee	11	#include <linux/bottom_half.h>
f8c3d0dd	12	#include <linux/cache.h>
acf568ee	13	#include <linux/interrupt.h>
1da177e4 LT	14	#include <linux/slab.h>
1da177e4 LT	15	#include <linux/module.h>
716062fd	16	#include <linux/netdevice.h>
acf568ee	17	#include <linux/percpu.h>
716062fd	18	#include <net/dst.h>
1da177e4 LT	19	#include <net/ip.h>
1da177e4 LT	20	#include <net/xfrm.h>
049f8e2e AD	21	#include <net/ip_tunnels.h>
049f8e2e AD	22	#include <net/ip6_tunnel.h>
1da177e4	23
acf568ee HX	24	struct xfrm_trans_tasklet {
	25	struct tasklet_struct tasklet;
	26	struct sk_buff_head queue;
	27	};
	28
	29	struct xfrm_trans_cb {
9a3fb9fb SK	30	union {
	31	struct inet_skb_parm h4;
	32	#if IS_ENABLED(CONFIG_IPV6)
	33	struct inet6_skb_parm h6;
	34	#endif
	35	} header;
acf568ee HX	36	int (finish)(struct net net, struct sock sk, struct sk_buff skb);
	37	};
	38
	39	#define XFRM_TRANS_SKB_CB(__skb) ((struct xfrm_trans_cb *)&((__skb)->cb[0]))
	40
2f32b51b	41	static DEFINE_SPINLOCK(xfrm_input_afinfo_lock);
960fdfde	42	static struct xfrm_input_afinfo const __rcu *xfrm_input_afinfo[AF_INET6 + 1];
2f32b51b	43
1995876a SK	44	static struct gro_cells gro_cells;
	45	static struct net_device xfrm_napi_dev;
	46
acf568ee HX	47	static DEFINE_PER_CPU(struct xfrm_trans_tasklet, xfrm_trans_tasklet);
acf568ee HX	48
960fdfde	49	int xfrm_input_register_afinfo(const struct xfrm_input_afinfo *afinfo)
2f32b51b SK	50	{
	51	int err = 0;
	52
960fdfde	53	if (WARN_ON(afinfo->family >= ARRAY_SIZE(xfrm_input_afinfo)))
2f32b51b	54	return -EAFNOSUPPORT;
960fdfde	55
2f32b51b SK	56	spin_lock_bh(&xfrm_input_afinfo_lock);
2f32b51b SK	57	if (unlikely(xfrm_input_afinfo[afinfo->family] != NULL))
f31e8d4f	58	err = -EEXIST;
2f32b51b SK	59	else
	60	rcu_assign_pointer(xfrm_input_afinfo[afinfo->family], afinfo);
	61	spin_unlock_bh(&xfrm_input_afinfo_lock);
	62	return err;
	63	}
	64	EXPORT_SYMBOL(xfrm_input_register_afinfo);
	65
960fdfde	66	int xfrm_input_unregister_afinfo(const struct xfrm_input_afinfo *afinfo)
2f32b51b SK	67	{
	68	int err = 0;
	69
2f32b51b SK	70	spin_lock_bh(&xfrm_input_afinfo_lock);
	71	if (likely(xfrm_input_afinfo[afinfo->family] != NULL)) {
	72	if (unlikely(xfrm_input_afinfo[afinfo->family] != afinfo))
	73	err = -EINVAL;
	74	else
	75	RCU_INIT_POINTER(xfrm_input_afinfo[afinfo->family], NULL);
	76	}
	77	spin_unlock_bh(&xfrm_input_afinfo_lock);
	78	synchronize_rcu();
	79	return err;
	80	}
	81	EXPORT_SYMBOL(xfrm_input_unregister_afinfo);
	82
960fdfde	83	static const struct xfrm_input_afinfo *xfrm_input_get_afinfo(unsigned int family)
2f32b51b	84	{
960fdfde	85	const struct xfrm_input_afinfo *afinfo;
2f32b51b	86
960fdfde	87	if (WARN_ON_ONCE(family >= ARRAY_SIZE(xfrm_input_afinfo)))
2f32b51b	88	return NULL;
960fdfde	89
2f32b51b SK	90	rcu_read_lock();
	91	afinfo = rcu_dereference(xfrm_input_afinfo[family]);
	92	if (unlikely(!afinfo))
	93	rcu_read_unlock();
	94	return afinfo;
	95	}
	96
2f32b51b SK	97	static int xfrm_rcv_cb(struct sk_buff *skb, unsigned int family, u8 protocol,
	98	int err)
	99	{
	100	int ret;
960fdfde	101	const struct xfrm_input_afinfo *afinfo = xfrm_input_get_afinfo(family);
2f32b51b SK	102
	103	if (!afinfo)
	104	return -EAFNOSUPPORT;
	105
	106	ret = afinfo->callback(skb, protocol, err);
960fdfde	107	rcu_read_unlock();
2f32b51b SK	108
	109	return ret;
	110	}
	111
4165079b	112	struct sec_path secpath_set(struct sk_buff skb)
1da177e4	113	{
4165079b	114	struct sec_path sp, tmp = skb_ext_find(skb, SKB_EXT_SEC_PATH);
1da177e4	115
4165079b	116	sp = skb_ext_add(skb, SKB_EXT_SEC_PATH);
1da177e4 LT	117	if (!sp)
	118	return NULL;
	119
4165079b FW	120	if (tmp) /* reused existing one (was COW'd if needed) */
4165079b FW	121	return sp;
54ef207a	122
4165079b	123	/* allocated new secpath */
f1193e91	124	memset(sp->ovec, 0, sizeof(sp->ovec));
4165079b FW	125	sp->olen = 0;
4165079b FW	126	sp->len = 0;
0ca64da1 FW	127
0ca64da1 FW	128	return sp;
b0fcee82 SK	129	}
	130	EXPORT_SYMBOL(secpath_set);
	131
1da177e4 LT	132	/* Fetch spi and seq from ipsec header */
1da177e4 LT	133
6067b2ba	134	int xfrm_parse_spi(struct sk_buff skb, u8 nexthdr, __be32 spi, __be32 *seq)
1da177e4 LT	135	{
1da177e4 LT	136	int offset, offset_seq;
44072500	137	int hlen;
1da177e4 LT	138
	139	switch (nexthdr) {
	140	case IPPROTO_AH:
44072500	141	hlen = sizeof(struct ip_auth_hdr);
1da177e4 LT	142	offset = offsetof(struct ip_auth_hdr, spi);
	143	offset_seq = offsetof(struct ip_auth_hdr, seq_no);
	144	break;
	145	case IPPROTO_ESP:
44072500	146	hlen = sizeof(struct ip_esp_hdr);
1da177e4 LT	147	offset = offsetof(struct ip_esp_hdr, spi);
	148	offset_seq = offsetof(struct ip_esp_hdr, seq_no);
	149	break;
	150	case IPPROTO_COMP:
	151	if (!pskb_may_pull(skb, sizeof(struct ip_comp_hdr)))
	152	return -EINVAL;
3e94c2dc	153	spi = htonl(ntohs((__be16 *)(skb_transport_header(skb) + 2)));
1da177e4 LT	154	*seq = 0;
	155	return 0;
	156	default:
	157	return 1;
	158	}
	159
44072500	160	if (!pskb_may_pull(skb, hlen))
1da177e4 LT	161	return -EINVAL;
1da177e4 LT	162
3e94c2dc WC	163	spi = (__be32 *)(skb_transport_header(skb) + offset);
3e94c2dc WC	164	seq = (__be32 *)(skb_transport_header(skb) + offset_seq);
1da177e4 LT	165	return 0;
1da177e4 LT	166	}
1e295370	167	EXPORT_SYMBOL(xfrm_parse_spi);
1da177e4	168
227620e2 HX	169	int xfrm_prepare_input(struct xfrm_state x, struct sk_buff skb)
227620e2 HX	170	{
df9dcb45	171	struct xfrm_mode *inner_mode = x->inner_mode;
227620e2 HX	172	int err;
	173
	174	err = x->outer_mode->afinfo->extract_input(x, skb);
	175	if (err)
	176	return err;
	177
df9dcb45 KM	178	if (x->sel.family == AF_UNSPEC) {
	179	inner_mode = xfrm_ip2inner_mode(x, XFRM_MODE_SKB_CB(skb)->protocol);
	180	if (inner_mode == NULL)
	181	return -EAFNOSUPPORT;
	182	}
	183
	184	skb->protocol = inner_mode->afinfo->eth_proto;
	185	return inner_mode->input2(x, skb);
227620e2 HX	186	}
	187	EXPORT_SYMBOL(xfrm_prepare_input);
	188
716062fd HX	189	int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type)
716062fd HX	190	{
bd235e3c	191	struct net *net = dev_net(skb->dev);
716062fd HX	192	int err;
716062fd HX	193	__be32 seq;
2cd08467	194	__be32 seq_hi;
3328715e	195	struct xfrm_state *x = NULL;
1bf06cd2	196	xfrm_address_t *daddr;
df9dcb45	197	struct xfrm_mode *inner_mode;
049f8e2e	198	u32 mark = skb->mark;
4ce3dbe3	199	unsigned int family = AF_UNSPEC;
716062fd	200	int decaps = 0;
1bf06cd2	201	int async = 0;
7785bba2	202	bool xfrm_gro = false;
d77e38e6 SK	203	bool crypto_done = false;
d77e38e6 SK	204	struct xfrm_offload *xo = xfrm_offload(skb);
0ca64da1	205	struct sec_path *sp;
1bf06cd2	206
1bf06cd2	207	if (encap_type < 0) {
00501121	208	x = xfrm_input_state(skb);
4ce3dbe3 AH	209
	210	if (unlikely(x->km.state != XFRM_STATE_VALID)) {
	211	if (x->km.state == XFRM_STATE_ACQ)
	212	XFRM_INC_STATS(net, LINUX_MIB_XFRMACQUIREERROR);
	213	else
	214	XFRM_INC_STATS(net,
	215	LINUX_MIB_XFRMINSTATEINVALID);
	216	goto drop;
	217	}
	218
3328715e	219	family = x->outer_mode->afinfo->family;
7785bba2 SK	220
	221	/* An encap_type of -1 indicates async resumption. */
	222	if (encap_type == -1) {
	223	async = 1;
	224	seq = XFRM_SKB_CB(skb)->seq.input.low;
	225	goto resume;
	226	}
bcd1f8a4	227
7785bba2 SK	228	/* encap_type < -1 indicates a GRO call. */
	229	encap_type = 0;
	230	seq = XFRM_SPI_SKB_CB(skb)->seq;
716062fd	231
bcd1f8a4 SK	232	if (xo && (xo->flags & CRYPTO_DONE)) {
bcd1f8a4 SK	233	crypto_done = true;
bcd1f8a4 SK	234	family = XFRM_SPI_SKB_CB(skb)->family;
	235
	236	if (!(xo->status & CRYPTO_SUCCESS)) {
	237	if (xo->status &
	238	(CRYPTO_TRANSPORT_AH_AUTH_FAILED \|
	239	CRYPTO_TRANSPORT_ESP_AUTH_FAILED \|
	240	CRYPTO_TUNNEL_AH_AUTH_FAILED \|
	241	CRYPTO_TUNNEL_ESP_AUTH_FAILED)) {
	242
	243	xfrm_audit_state_icvfail(x, skb,
	244	x->type->proto);
	245	x->stats.integrity_failed++;
	246	XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEPROTOERROR);
	247	goto drop;
	248	}
	249
47ebcc0b YK	250	if (xo->status & CRYPTO_INVALID_PROTOCOL) {
	251	XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEPROTOERROR);
	252	goto drop;
	253	}
	254
bcd1f8a4	255	XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR);
d77e38e6 SK	256	goto drop;
	257	}
	258
bcd1f8a4 SK	259	if ((err = xfrm_parse_spi(skb, nexthdr, &spi, &seq)) != 0) {
	260	XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR);
	261	goto drop;
	262	}
d77e38e6 SK	263	}
d77e38e6 SK	264
7785bba2	265	goto lock;
1bf06cd2	266	}
716062fd	267
3328715e SK	268	family = XFRM_SPI_SKB_CB(skb)->family;
3328715e SK	269
049f8e2e	270	/* if tunnel is present override skb->mark value with tunnel i_key */
1625f452 AK	271	switch (family) {
	272	case AF_INET:
	273	if (XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip4)
049f8e2e	274	mark = be32_to_cpu(XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip4->parms.i_key);
1625f452 AK	275	break;
	276	case AF_INET6:
	277	if (XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip6)
049f8e2e	278	mark = be32_to_cpu(XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip6->parms.i_key);
1625f452	279	break;
049f8e2e AD	280	}
049f8e2e AD	281
0ca64da1 FW	282	sp = secpath_set(skb);
0ca64da1 FW	283	if (!sp) {
b0fcee82 SK	284	XFRM_INC_STATS(net, LINUX_MIB_XFRMINERROR);
b0fcee82 SK	285	goto drop;
b2aa5e9d HX	286	}
b2aa5e9d HX	287
716062fd	288	seq = 0;
0aa64774	289	if (!spi && (err = xfrm_parse_spi(skb, nexthdr, &spi, &seq)) != 0) {
f203b76d	290	secpath_reset(skb);
59c9940e	291	XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR);
716062fd	292	goto drop;
0aa64774	293	}
716062fd	294
cb79a180 FW	295	daddr = (xfrm_address_t *)(skb_network_header(skb) +
cb79a180 FW	296	XFRM_SPI_SKB_CB(skb)->daddroff);
716062fd	297	do {
2294be0f FW	298	sp = skb_sec_path(skb);
	299
	300	if (sp->len == XFRM_MAX_DEPTH) {
f203b76d	301	secpath_reset(skb);
59c9940e	302	XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR);
716062fd	303	goto drop;
0aa64774	304	}
716062fd	305
049f8e2e	306	x = xfrm_state_lookup(net, mark, daddr, spi, nexthdr, family);
0aa64774	307	if (x == NULL) {
f203b76d	308	secpath_reset(skb);
59c9940e	309	XFRM_INC_STATS(net, LINUX_MIB_XFRMINNOSTATES);
afeb14b4	310	xfrm_audit_state_notfound(skb, family, spi, seq);
716062fd	311	goto drop;
0aa64774	312	}
716062fd	313
9b42c1f1 SK	314	skb->mark = xfrm_smark_get(skb->mark, x);
9b42c1f1 SK	315
2294be0f	316	sp->xvec[sp->len++] = x;
b2aa5e9d	317
7785bba2	318	lock:
716062fd	319	spin_lock(&x->lock);
4c4d41f2	320
0aa64774	321	if (unlikely(x->km.state != XFRM_STATE_VALID)) {
dc0565ce LR	322	if (x->km.state == XFRM_STATE_ACQ)
	323	XFRM_INC_STATS(net, LINUX_MIB_XFRMACQUIREERROR);
	324	else
	325	XFRM_INC_STATS(net,
	326	LINUX_MIB_XFRMINSTATEINVALID);
716062fd	327	goto drop_unlock;
0aa64774	328	}
716062fd	329
3de77cf2 DM	330	if ((x->encap ? x->encap->encap_type : 0) != encap_type) {
	331	XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMISMATCH);
	332	goto drop_unlock;
	333	}
	334
36ae0148	335	if (x->repl->check(x, skb, seq)) {
59c9940e	336	XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATESEQERROR);
716062fd	337	goto drop_unlock;
0aa64774	338	}
716062fd	339
0aa64774	340	if (xfrm_state_check_expire(x)) {
59c9940e	341	XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEEXPIRED);
716062fd	342	goto drop_unlock;
0aa64774	343	}
716062fd	344
0ebea8ef HX	345	spin_unlock(&x->lock);
0ebea8ef HX	346
68c11e98 AD	347	if (xfrm_tunnel_check(skb, x, family)) {
	348	XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMODEERROR);
	349	goto drop;
	350	}
	351
2cd08467 SK	352	seq_hi = htonl(xfrm_replay_seqhi(x, seq));
2cd08467 SK	353
1ce3644a	354	XFRM_SKB_CB(skb)->seq.input.low = seq;
2cd08467	355	XFRM_SKB_CB(skb)->seq.input.hi = seq_hi;
1bf06cd2	356
3bc07321	357	skb_dst_force(skb);
071d36bf	358	dev_hold(skb->dev);
3bc07321	359
d77e38e6 SK	360	if (crypto_done)
	361	nexthdr = x->type_offload->input_tail(x, skb);
	362	else
	363	nexthdr = x->type->input(x, skb);
0ebea8ef	364
1bf06cd2 HX	365	if (nexthdr == -EINPROGRESS)
1bf06cd2 HX	366	return 0;
1bf06cd2	367	resume:
071d36bf	368	dev_put(skb->dev);
071d36bf	369
0ebea8ef	370	spin_lock(&x->lock);
668dc8af	371	if (nexthdr <= 0) {
9dd3245a HX	372	if (nexthdr == -EBADMSG) {
	373	xfrm_audit_state_icvfail(x, skb,
	374	x->type->proto);
668dc8af	375	x->stats.integrity_failed++;
9dd3245a	376	}
59c9940e	377	XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEPROTOERROR);
716062fd	378	goto drop_unlock;
668dc8af	379	}
716062fd	380
716062fd HX	381	/* only the first xfrm gets the encap type */
	382	encap_type = 0;
	383
3b59df46	384	if (async && x->repl->recheck(x, skb, seq)) {
bcf66bf5 SK	385	XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATESEQERROR);
	386	goto drop_unlock;
	387	}
	388
9fdc4883	389	x->repl->advance(x, seq);
716062fd HX	390
	391	x->curlft.bytes += skb->len;
	392	x->curlft.packets++;
	393
	394	spin_unlock(&x->lock);
	395
60d5fcfb HX	396	XFRM_MODE_SKB_CB(skb)->protocol = nexthdr;
60d5fcfb HX	397
df9dcb45 KM	398	inner_mode = x->inner_mode;
	399
	400	if (x->sel.family == AF_UNSPEC) {
	401	inner_mode = xfrm_ip2inner_mode(x, XFRM_MODE_SKB_CB(skb)->protocol);
cb866e32 SK	402	if (inner_mode == NULL) {
cb866e32 SK	403	XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMODEERROR);
df9dcb45	404	goto drop;
cb866e32	405	}
df9dcb45 KM	406	}
	407
	408	if (inner_mode->input(x, skb)) {
59c9940e	409	XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMODEERROR);
716062fd	410	goto drop;
0aa64774	411	}
716062fd HX	412
	413	if (x->outer_mode->flags & XFRM_MODE_FLAG_TUNNEL) {
	414	decaps = 1;
	415	break;
	416	}
	417
1bf06cd2 HX	418	/*
	419	* We need the inner address. However, we only get here for
	420	* transport mode so the outer address is identical.
	421	*/
	422	daddr = &x->id.daddr;
2fcb45b6	423	family = x->outer_mode->afinfo->family;
1bf06cd2	424
716062fd	425	err = xfrm_parse_spi(skb, nexthdr, &spi, &seq);
0aa64774	426	if (err < 0) {
59c9940e	427	XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR);
716062fd	428	goto drop;
0aa64774	429	}
782710e3	430	crypto_done = false;
716062fd HX	431	} while (!err);
716062fd HX	432
3328715e SK	433	err = xfrm_rcv_cb(skb, family, x->type->proto, 0);
	434	if (err)
	435	goto drop;
	436
716062fd HX	437	nf_reset(skb);
	438
	439	if (decaps) {
2294be0f FW	440	sp = skb_sec_path(skb);
	441	if (sp)
	442	sp->olen = 0;
adf30907	443	skb_dst_drop(skb);
1995876a	444	gro_cells_receive(&gro_cells, skb);
716062fd HX	445	return 0;
716062fd HX	446	} else {
7785bba2 SK	447	xo = xfrm_offload(skb);
	448	if (xo)
	449	xfrm_gro = xo->flags & XFRM_GRO;
	450
cfcf99f9	451	err = x->inner_mode->afinfo->transport_finish(skb, xfrm_gro \|\| async);
7785bba2	452	if (xfrm_gro) {
2294be0f FW	453	sp = skb_sec_path(skb);
	454	if (sp)
	455	sp->olen = 0;
7785bba2 SK	456	skb_dst_drop(skb);
	457	gro_cells_receive(&gro_cells, skb);
	458	return err;
	459	}
	460
	461	return err;
716062fd HX	462	}
	463
	464	drop_unlock:
	465	spin_unlock(&x->lock);
716062fd	466	drop:
3328715e	467	xfrm_rcv_cb(skb, family, x && x->type ? x->type->proto : nexthdr, -1);
716062fd HX	468	kfree_skb(skb);
	469	return 0;
	470	}
	471	EXPORT_SYMBOL(xfrm_input);
	472
1bf06cd2 HX	473	int xfrm_input_resume(struct sk_buff *skb, int nexthdr)
	474	{
	475	return xfrm_input(skb, nexthdr, 0, -1);
	476	}
	477	EXPORT_SYMBOL(xfrm_input_resume);
	478
acf568ee HX	479	static void xfrm_trans_reinject(unsigned long data)
	480	{
	481	struct xfrm_trans_tasklet trans = (void )data;
	482	struct sk_buff_head queue;
	483	struct sk_buff *skb;
	484
	485	__skb_queue_head_init(&queue);
	486	skb_queue_splice_init(&trans->queue, &queue);
	487
	488	while ((skb = __skb_dequeue(&queue)))
	489	XFRM_TRANS_SKB_CB(skb)->finish(dev_net(skb->dev), NULL, skb);
	490	}
	491
	492	int xfrm_trans_queue(struct sk_buff *skb,
	493	int (finish)(struct net , struct sock *,
	494	struct sk_buff *))
	495	{
	496	struct xfrm_trans_tasklet *trans;
	497
	498	trans = this_cpu_ptr(&xfrm_trans_tasklet);
	499
	500	if (skb_queue_len(&trans->queue) >= netdev_max_backlog)
	501	return -ENOBUFS;
	502
	503	XFRM_TRANS_SKB_CB(skb)->finish = finish;
d16b46e4	504	__skb_queue_tail(&trans->queue, skb);
acf568ee HX	505	tasklet_schedule(&trans->tasklet);
	506	return 0;
	507	}
	508	EXPORT_SYMBOL(xfrm_trans_queue);
	509
1da177e4 LT	510	void __init xfrm_input_init(void)
1da177e4 LT	511	{
1995876a	512	int err;
acf568ee	513	int i;
1995876a SK	514
	515	init_dummy_netdev(&xfrm_napi_dev);
	516	err = gro_cells_init(&gro_cells, &xfrm_napi_dev);
	517	if (err)
	518	gro_cells.cells = NULL;
	519
acf568ee HX	520	for_each_possible_cpu(i) {
	521	struct xfrm_trans_tasklet *trans;
	522
	523	trans = &per_cpu(xfrm_trans_tasklet, i);
	524	__skb_queue_head_init(&trans->queue);
	525	tasklet_init(&trans->tasklet, xfrm_trans_reinject,
	526	(unsigned long)trans);
	527	}
1da177e4	528	}