[linux-block.git] / net / tls / tls_strp.c

// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (c) 2016 Tom Herbert <tom@herbertland.com> */

#include <linux/skbuff.h>
#include <linux/workqueue.h>
#include <net/strparser.h>
#include <net/tcp.h>
#include <net/sock.h>
#include <net/tls.h>

#include "tls.h"

static struct workqueue_struct *tls_strp_wq;

static void tls_strp_abort_strp(struct tls_strparser *strp, int err)
{
	if (strp->stopped)
		return;

	strp->stopped = 1;

	/* Report an error on the lower socket */
	strp->sk->sk_err = -err;
	sk_error_report(strp->sk);
}

static void tls_strp_anchor_free(struct tls_strparser *strp)
{
	struct skb_shared_info *shinfo = skb_shinfo(strp->anchor);

	DEBUG_NET_WARN_ON_ONCE(atomic_read(&shinfo->dataref) != 1);
	shinfo->frag_list = NULL;
	consume_skb(strp->anchor);
	strp->anchor = NULL;
}

/* Create a new skb with the contents of input copied to its page frags */
static struct sk_buff *tls_strp_msg_make_copy(struct tls_strparser *strp)
{
	struct strp_msg *rxm;
	struct sk_buff *skb;
	int i, err, offset;

	skb = alloc_skb_with_frags(0, strp->stm.full_len, TLS_PAGE_ORDER,
				   &err, strp->sk->sk_allocation);
	if (!skb)
		return NULL;

	offset = strp->stm.offset;
	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
		skb_frag_t *frag = &skb_shinfo(skb)->frags[i];

		WARN_ON_ONCE(skb_copy_bits(strp->anchor, offset,
					   skb_frag_address(frag),
					   skb_frag_size(frag)));
		offset += skb_frag_size(frag);
	}

	skb_copy_header(skb, strp->anchor);
	rxm = strp_msg(skb);
	rxm->offset = 0;
	return skb;
}

/* Steal the input skb, input msg is invalid after calling this function */
struct sk_buff *tls_strp_msg_detach(struct tls_sw_context_rx *ctx)
{
	struct tls_strparser *strp = &ctx->strp;

#ifdef CONFIG_TLS_DEVICE
	DEBUG_NET_WARN_ON_ONCE(!strp->anchor->decrypted);
#else
	/* This function turns an input into an output,
	 * that can only happen if we have offload.
	 */
	WARN_ON(1);
#endif

	if (strp->copy_mode) {
		struct sk_buff *skb;

		/* Replace anchor with an empty skb, this is a little
		 * dangerous but __tls_cur_msg() warns on empty skbs
		 * so hopefully we'll catch abuses.
		 */
		skb = alloc_skb(0, strp->sk->sk_allocation);
		if (!skb)
			return NULL;

		swap(strp->anchor, skb);
		return skb;
	}

	return tls_strp_msg_make_copy(strp);
}

/* Force the input skb to be in copy mode. The data ownership remains
 * with the input skb itself (meaning unpause will wipe it) but it can
 * be modified.
 */
int tls_strp_msg_cow(struct tls_sw_context_rx *ctx)
{
	struct tls_strparser *strp = &ctx->strp;
	struct sk_buff *skb;

	if (strp->copy_mode)
		return 0;

	skb = tls_strp_msg_make_copy(strp);
	if (!skb)
		return -ENOMEM;

	tls_strp_anchor_free(strp);
	strp->anchor = skb;

	tcp_read_done(strp->sk, strp->stm.full_len);
	strp->copy_mode = 1;

	return 0;
}

/* Make a clone (in the skb sense) of the input msg to keep a reference
 * to the underlying data. The reference-holding skbs get placed on
 * @dst.
 */
int tls_strp_msg_hold(struct tls_strparser *strp, struct sk_buff_head *dst)
{
	struct skb_shared_info *shinfo = skb_shinfo(strp->anchor);

	if (strp->copy_mode) {
		struct sk_buff *skb;

		WARN_ON_ONCE(!shinfo->nr_frags);

		/* We can't skb_clone() the anchor, it gets wiped by unpause */
		skb = alloc_skb(0, strp->sk->sk_allocation);
		if (!skb)
			return -ENOMEM;

		__skb_queue_tail(dst, strp->anchor);
		strp->anchor = skb;
	} else {
		struct sk_buff *iter, *clone;
		int chunk, len, offset;

		offset = strp->stm.offset;
		len = strp->stm.full_len;
		iter = shinfo->frag_list;

		while (len > 0) {
			if (iter->len <= offset) {
				offset -= iter->len;
				goto next;
			}

			chunk = iter->len - offset;
			offset = 0;

			clone = skb_clone(iter, strp->sk->sk_allocation);
			if (!clone)
				return -ENOMEM;
			__skb_queue_tail(dst, clone);

			len -= chunk;
next:
			iter = iter->next;
		}
	}

	return 0;
}

static void tls_strp_flush_anchor_copy(struct tls_strparser *strp)
{
	struct skb_shared_info *shinfo = skb_shinfo(strp->anchor);
	int i;

	DEBUG_NET_WARN_ON_ONCE(atomic_read(&shinfo->dataref) != 1);

	for (i = 0; i < shinfo->nr_frags; i++)
		__skb_frag_unref(&shinfo->frags[i], false);
	shinfo->nr_frags = 0;
	strp->copy_mode = 0;
}

static int tls_strp_copyin(read_descriptor_t *desc, struct sk_buff *in_skb,
			   unsigned int offset, size_t in_len)
{
	struct tls_strparser *strp = (struct tls_strparser *)desc->arg.data;
	struct sk_buff *skb;
	skb_frag_t *frag;
	size_t len, chunk;
	int sz;

	if (strp->msg_ready)
		return 0;

	skb = strp->anchor;
	frag = &skb_shinfo(skb)->frags[skb->len / PAGE_SIZE];

	len = in_len;
	/* First make sure we got the header */
	if (!strp->stm.full_len) {
		/* Assume one page is more than enough for headers */
		chunk =	min_t(size_t, len, PAGE_SIZE - skb_frag_size(frag));
		WARN_ON_ONCE(skb_copy_bits(in_skb, offset,
					   skb_frag_address(frag) +
					   skb_frag_size(frag),
					   chunk));

		sz = tls_rx_msg_size(strp, strp->anchor);
		if (sz < 0) {
			desc->error = sz;
			return 0;
		}

		/* We may have over-read, sz == 0 is guaranteed under-read */
		if (sz > 0)
			chunk =	min_t(size_t, chunk, sz - skb->len);

		skb->len += chunk;
		skb->data_len += chunk;
		skb_frag_size_add(frag, chunk);
		frag++;
		len -= chunk;
		offset += chunk;

		strp->stm.full_len = sz;
		if (!strp->stm.full_len)
			goto read_done;
	}

	/* Load up more data */
	while (len && strp->stm.full_len > skb->len) {
		chunk =	min_t(size_t, len, strp->stm.full_len - skb->len);
		chunk = min_t(size_t, chunk, PAGE_SIZE - skb_frag_size(frag));
		WARN_ON_ONCE(skb_copy_bits(in_skb, offset,
					   skb_frag_address(frag) +
					   skb_frag_size(frag),
					   chunk));

		skb->len += chunk;
		skb->data_len += chunk;
		skb_frag_size_add(frag, chunk);
		frag++;
		len -= chunk;
		offset += chunk;
	}

	if (strp->stm.full_len == skb->len) {
		desc->count = 0;

		strp->msg_ready = 1;
		tls_rx_msg_ready(strp);
	}

read_done:
	return in_len - len;
}

static int tls_strp_read_copyin(struct tls_strparser *strp)
{
	struct socket *sock = strp->sk->sk_socket;
	read_descriptor_t desc;

	desc.arg.data = strp;
	desc.error = 0;
	desc.count = 1; /* give more than one skb per call */

	/* sk should be locked here, so okay to do read_sock */
	sock->ops->read_sock(strp->sk, &desc, tls_strp_copyin);

	return desc.error;
}

static int tls_strp_read_copy(struct tls_strparser *strp, bool qshort)
{
	struct skb_shared_info *shinfo;
	struct page *page;
	int need_spc, len;

	/* If the rbuf is small or rcv window has collapsed to 0 we need
	 * to read the data out. Otherwise the connection will stall.
	 * Without pressure threshold of INT_MAX will never be ready.
	 */
	if (likely(qshort && !tcp_epollin_ready(strp->sk, INT_MAX)))
		return 0;

	shinfo = skb_shinfo(strp->anchor);
	shinfo->frag_list = NULL;

	/* If we don't know the length go max plus page for cipher overhead */
	need_spc = strp->stm.full_len ?: TLS_MAX_PAYLOAD_SIZE + PAGE_SIZE;

	for (len = need_spc; len > 0; len -= PAGE_SIZE) {
		page = alloc_page(strp->sk->sk_allocation);
		if (!page) {
			tls_strp_flush_anchor_copy(strp);
			return -ENOMEM;
		}

		skb_fill_page_desc(strp->anchor, shinfo->nr_frags++,
				   page, 0, 0);
	}

	strp->copy_mode = 1;
	strp->stm.offset = 0;

	strp->anchor->len = 0;
	strp->anchor->data_len = 0;
	strp->anchor->truesize = round_up(need_spc, PAGE_SIZE);

	tls_strp_read_copyin(strp);

	return 0;
}

static bool tls_strp_check_no_dup(struct tls_strparser *strp)
{
	unsigned int len = strp->stm.offset + strp->stm.full_len;
	struct sk_buff *skb;
	u32 seq;

	skb = skb_shinfo(strp->anchor)->frag_list;
	seq = TCP_SKB_CB(skb)->seq;

	while (skb->len < len) {
		seq += skb->len;
		len -= skb->len;
		skb = skb->next;

		if (TCP_SKB_CB(skb)->seq != seq)
			return false;
	}

	return true;
}

static void tls_strp_load_anchor_with_queue(struct tls_strparser *strp, int len)
{
	struct tcp_sock *tp = tcp_sk(strp->sk);
	struct sk_buff *first;
	u32 offset;

	first = tcp_recv_skb(strp->sk, tp->copied_seq, &offset);
	if (WARN_ON_ONCE(!first))
		return;

	/* Bestow the state onto the anchor */
	strp->anchor->len = offset + len;
	strp->anchor->data_len = offset + len;
	strp->anchor->truesize = offset + len;

	skb_shinfo(strp->anchor)->frag_list = first;

	skb_copy_header(strp->anchor, first);
	strp->anchor->destructor = NULL;

	strp->stm.offset = offset;
}

void tls_strp_msg_load(struct tls_strparser *strp, bool force_refresh)
{
	struct strp_msg *rxm;
	struct tls_msg *tlm;

	DEBUG_NET_WARN_ON_ONCE(!strp->msg_ready);
	DEBUG_NET_WARN_ON_ONCE(!strp->stm.full_len);

	if (!strp->copy_mode && force_refresh) {
		if (WARN_ON(tcp_inq(strp->sk) < strp->stm.full_len))
			return;

		tls_strp_load_anchor_with_queue(strp, strp->stm.full_len);
	}

	rxm = strp_msg(strp->anchor);
	rxm->full_len	= strp->stm.full_len;
	rxm->offset	= strp->stm.offset;
	tlm = tls_msg(strp->anchor);
	tlm->control	= strp->mark;
}

/* Called with lock held on lower socket */
static int tls_strp_read_sock(struct tls_strparser *strp)
{
	int sz, inq;

	inq = tcp_inq(strp->sk);
	if (inq < 1)
		return 0;

	if (unlikely(strp->copy_mode))
		return tls_strp_read_copyin(strp);

	if (inq < strp->stm.full_len)
		return tls_strp_read_copy(strp, true);

	if (!strp->stm.full_len) {
		tls_strp_load_anchor_with_queue(strp, inq);

		sz = tls_rx_msg_size(strp, strp->anchor);
		if (sz < 0) {
			tls_strp_abort_strp(strp, sz);
			return sz;
		}

		strp->stm.full_len = sz;

		if (!strp->stm.full_len || inq < strp->stm.full_len)
			return tls_strp_read_copy(strp, true);
	}

	if (!tls_strp_check_no_dup(strp))
		return tls_strp_read_copy(strp, false);

	strp->msg_ready = 1;
	tls_rx_msg_ready(strp);

	return 0;
}

void tls_strp_check_rcv(struct tls_strparser *strp)
{
	if (unlikely(strp->stopped) || strp->msg_ready)
		return;

	if (tls_strp_read_sock(strp) == -ENOMEM)
		queue_work(tls_strp_wq, &strp->work);
}

/* Lower sock lock held */
void tls_strp_data_ready(struct tls_strparser *strp)
{
	/* This check is needed to synchronize with do_tls_strp_work.
	 * do_tls_strp_work acquires a process lock (lock_sock) whereas
	 * the lock held here is bh_lock_sock. The two locks can be
	 * held by different threads at the same time, but bh_lock_sock
	 * allows a thread in BH context to safely check if the process
	 * lock is held. In this case, if the lock is held, queue work.
	 */
	if (sock_owned_by_user_nocheck(strp->sk)) {
		queue_work(tls_strp_wq, &strp->work);
		return;
	}

	tls_strp_check_rcv(strp);
}

static void tls_strp_work(struct work_struct *w)
{
	struct tls_strparser *strp =
		container_of(w, struct tls_strparser, work);

	lock_sock(strp->sk);
	tls_strp_check_rcv(strp);
	release_sock(strp->sk);
}

void tls_strp_msg_done(struct tls_strparser *strp)
{
	WARN_ON(!strp->stm.full_len);

	if (likely(!strp->copy_mode))
		tcp_read_done(strp->sk, strp->stm.full_len);
	else
		tls_strp_flush_anchor_copy(strp);

	strp->msg_ready = 0;
	memset(&strp->stm, 0, sizeof(strp->stm));

	tls_strp_check_rcv(strp);
}

void tls_strp_stop(struct tls_strparser *strp)
{
	strp->stopped = 1;
}

int tls_strp_init(struct tls_strparser *strp, struct sock *sk)
{
	memset(strp, 0, sizeof(*strp));

	strp->sk = sk;

	strp->anchor = alloc_skb(0, GFP_KERNEL);
	if (!strp->anchor)
		return -ENOMEM;

	INIT_WORK(&strp->work, tls_strp_work);

	return 0;
}

/* strp must already be stopped so that tls_strp_recv will no longer be called.
 * Note that tls_strp_done is not called with the lower socket held.
 */
void tls_strp_done(struct tls_strparser *strp)
{
	WARN_ON(!strp->stopped);

	cancel_work_sync(&strp->work);
	tls_strp_anchor_free(strp);
}

int __init tls_strp_dev_init(void)
{
	tls_strp_wq = create_workqueue("tls-strp");
	if (unlikely(!tls_strp_wq))
		return -ENOMEM;

	return 0;
}

void tls_strp_dev_exit(void)
{
	destroy_workqueue(tls_strp_wq);
}
Commit	Line	Data
c618db2a	1	// SPDX-License-Identifier: GPL-2.0-only
84c61fe1	2	/* Copyright (c) 2016 Tom Herbert <tom@herbertland.com> */
c618db2a JK	3
c618db2a JK	4	#include <linux/skbuff.h>
84c61fe1 JK	5	#include <linux/workqueue.h>
	6	#include <net/strparser.h>
	7	#include <net/tcp.h>
	8	#include <net/sock.h>
	9	#include <net/tls.h>
c618db2a JK	10
	11	#include "tls.h"
	12
84c61fe1 JK	13	static struct workqueue_struct *tls_strp_wq;
	14
	15	static void tls_strp_abort_strp(struct tls_strparser *strp, int err)
	16	{
	17	if (strp->stopped)
	18	return;
	19
	20	strp->stopped = 1;
	21
	22	/* Report an error on the lower socket */
	23	strp->sk->sk_err = -err;
	24	sk_error_report(strp->sk);
	25	}
	26
	27	static void tls_strp_anchor_free(struct tls_strparser *strp)
d4e5db64	28	{
84c61fe1 JK	29	struct skb_shared_info *shinfo = skb_shinfo(strp->anchor);
	30
	31	DEBUG_NET_WARN_ON_ONCE(atomic_read(&shinfo->dataref) != 1);
	32	shinfo->frag_list = NULL;
	33	consume_skb(strp->anchor);
	34	strp->anchor = NULL;
	35	}
	36
	37	/* Create a new skb with the contents of input copied to its page frags */
	38	static struct sk_buff tls_strp_msg_make_copy(struct tls_strparser strp)
	39	{
	40	struct strp_msg *rxm;
d4e5db64	41	struct sk_buff *skb;
84c61fe1 JK	42	int i, err, offset;
84c61fe1 JK	43
d800a7b3	44	skb = alloc_skb_with_frags(0, strp->stm.full_len, TLS_PAGE_ORDER,
84c61fe1 JK	45	&err, strp->sk->sk_allocation);
	46	if (!skb)
	47	return NULL;
	48
	49	offset = strp->stm.offset;
	50	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
	51	skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
d4e5db64	52
84c61fe1 JK	53	WARN_ON_ONCE(skb_copy_bits(strp->anchor, offset,
	54	skb_frag_address(frag),
	55	skb_frag_size(frag)));
	56	offset += skb_frag_size(frag);
	57	}
	58
	59	skb_copy_header(skb, strp->anchor);
	60	rxm = strp_msg(skb);
	61	rxm->offset = 0;
d4e5db64 JK	62	return skb;
	63	}
	64
84c61fe1 JK	65	/* Steal the input skb, input msg is invalid after calling this function */
	66	struct sk_buff tls_strp_msg_detach(struct tls_sw_context_rx ctx)
	67	{
	68	struct tls_strparser *strp = &ctx->strp;
	69
	70	#ifdef CONFIG_TLS_DEVICE
	71	DEBUG_NET_WARN_ON_ONCE(!strp->anchor->decrypted);
	72	#else
	73	/* This function turns an input into an output,
	74	* that can only happen if we have offload.
	75	*/
	76	WARN_ON(1);
	77	#endif
	78
	79	if (strp->copy_mode) {
	80	struct sk_buff *skb;
	81
	82	/* Replace anchor with an empty skb, this is a little
	83	* dangerous but __tls_cur_msg() warns on empty skbs
	84	* so hopefully we'll catch abuses.
	85	*/
	86	skb = alloc_skb(0, strp->sk->sk_allocation);
	87	if (!skb)
	88	return NULL;
	89
	90	swap(strp->anchor, skb);
	91	return skb;
	92	}
	93
	94	return tls_strp_msg_make_copy(strp);
	95	}
	96
	97	/* Force the input skb to be in copy mode. The data ownership remains
	98	* with the input skb itself (meaning unpause will wipe it) but it can
	99	* be modified.
	100	*/
8b3c59a7 JK	101	int tls_strp_msg_cow(struct tls_sw_context_rx *ctx)
8b3c59a7 JK	102	{
84c61fe1 JK	103	struct tls_strparser *strp = &ctx->strp;
	104	struct sk_buff *skb;
	105
	106	if (strp->copy_mode)
	107	return 0;
	108
	109	skb = tls_strp_msg_make_copy(strp);
	110	if (!skb)
	111	return -ENOMEM;
	112
	113	tls_strp_anchor_free(strp);
	114	strp->anchor = skb;
	115
	116	tcp_read_done(strp->sk, strp->stm.full_len);
	117	strp->copy_mode = 1;
	118
	119	return 0;
	120	}
	121
	122	/* Make a clone (in the skb sense) of the input msg to keep a reference
	123	* to the underlying data. The reference-holding skbs get placed on
	124	* @dst.
	125	*/
	126	int tls_strp_msg_hold(struct tls_strparser strp, struct sk_buff_head dst)
	127	{
	128	struct skb_shared_info *shinfo = skb_shinfo(strp->anchor);
	129
	130	if (strp->copy_mode) {
	131	struct sk_buff *skb;
	132
	133	WARN_ON_ONCE(!shinfo->nr_frags);
	134
	135	/* We can't skb_clone() the anchor, it gets wiped by unpause */
	136	skb = alloc_skb(0, strp->sk->sk_allocation);
	137	if (!skb)
	138	return -ENOMEM;
	139
	140	__skb_queue_tail(dst, strp->anchor);
	141	strp->anchor = skb;
	142	} else {
	143	struct sk_buff iter, clone;
	144	int chunk, len, offset;
	145
	146	offset = strp->stm.offset;
	147	len = strp->stm.full_len;
	148	iter = shinfo->frag_list;
	149
	150	while (len > 0) {
	151	if (iter->len <= offset) {
	152	offset -= iter->len;
	153	goto next;
	154	}
	155
	156	chunk = iter->len - offset;
	157	offset = 0;
	158
	159	clone = skb_clone(iter, strp->sk->sk_allocation);
	160	if (!clone)
	161	return -ENOMEM;
	162	__skb_queue_tail(dst, clone);
	163
	164	len -= chunk;
	165	next:
	166	iter = iter->next;
167	}
168	}
169
170	return 0;
171	}
172
173	static void tls_strp_flush_anchor_copy(struct tls_strparser *strp)
174	{
175	struct skb_shared_info *shinfo = skb_shinfo(strp->anchor);
176	int i;
177
178	DEBUG_NET_WARN_ON_ONCE(atomic_read(&shinfo->dataref) != 1);
179
180	for (i = 0; i < shinfo->nr_frags; i++)
181	__skb_frag_unref(&shinfo->frags[i], false);
182	shinfo->nr_frags = 0;
183	strp->copy_mode = 0;
184	}
185
186	static int tls_strp_copyin(read_descriptor_t desc, struct sk_buff in_skb,
187	unsigned int offset, size_t in_len)
188	{
189	struct tls_strparser strp = (struct tls_strparser )desc->arg.data;
84c61fe1 JK	190	struct sk_buff *skb;
84c61fe1 JK	191	skb_frag_t *frag;
8fd1e151 YL	192	size_t len, chunk;
8fd1e151 YL	193	int sz;
84c61fe1 JK	194
	195	if (strp->msg_ready)
	196	return 0;
	197
	198	skb = strp->anchor;
	199	frag = &skb_shinfo(skb)->frags[skb->len / PAGE_SIZE];
	200
	201	len = in_len;
	202	/* First make sure we got the header */
	203	if (!strp->stm.full_len) {
	204	/* Assume one page is more than enough for headers */
	205	chunk = min_t(size_t, len, PAGE_SIZE - skb_frag_size(frag));
	206	WARN_ON_ONCE(skb_copy_bits(in_skb, offset,
	207	skb_frag_address(frag) +
	208	skb_frag_size(frag),
	209	chunk));
	210
	211	sz = tls_rx_msg_size(strp, strp->anchor);
	212	if (sz < 0) {
	213	desc->error = sz;
	214	return 0;
	215	}
	216
	217	/* We may have over-read, sz == 0 is guaranteed under-read */
	218	if (sz > 0)
	219	chunk = min_t(size_t, chunk, sz - skb->len);
	220
	221	skb->len += chunk;
	222	skb->data_len += chunk;
	223	skb_frag_size_add(frag, chunk);
	224	frag++;
	225	len -= chunk;
	226	offset += chunk;
	227
	228	strp->stm.full_len = sz;
	229	if (!strp->stm.full_len)
	230	goto read_done;
	231	}
	232
	233	/* Load up more data */
	234	while (len && strp->stm.full_len > skb->len) {
	235	chunk = min_t(size_t, len, strp->stm.full_len - skb->len);
	236	chunk = min_t(size_t, chunk, PAGE_SIZE - skb_frag_size(frag));
	237	WARN_ON_ONCE(skb_copy_bits(in_skb, offset,
	238	skb_frag_address(frag) +
	239	skb_frag_size(frag),
	240	chunk));
	241
	242	skb->len += chunk;
	243	skb->data_len += chunk;
	244	skb_frag_size_add(frag, chunk);
	245	frag++;
	246	len -= chunk;
	247	offset += chunk;
	248	}
	249
	250	if (strp->stm.full_len == skb->len) {
	251	desc->count = 0;
	252
	253	strp->msg_ready = 1;
	254	tls_rx_msg_ready(strp);
	255	}
	256
	257	read_done:
258	return in_len - len;
259	}
260
261	static int tls_strp_read_copyin(struct tls_strparser *strp)
262	{
263	struct socket *sock = strp->sk->sk_socket;
264	read_descriptor_t desc;
265
266	desc.arg.data = strp;
267	desc.error = 0;
268	desc.count = 1; /* give more than one skb per call */
269
270	/* sk should be locked here, so okay to do read_sock */
271	sock->ops->read_sock(strp->sk, &desc, tls_strp_copyin);
272
273	return desc.error;
274	}
275
0d87bbd3	276	static int tls_strp_read_copy(struct tls_strparser *strp, bool qshort)
84c61fe1 JK	277	{
	278	struct skb_shared_info *shinfo;
	279	struct page *page;
	280	int need_spc, len;
	281
	282	/* If the rbuf is small or rcv window has collapsed to 0 we need
	283	* to read the data out. Otherwise the connection will stall.
	284	* Without pressure threshold of INT_MAX will never be ready.
	285	*/
0d87bbd3	286	if (likely(qshort && !tcp_epollin_ready(strp->sk, INT_MAX)))
84c61fe1 JK	287	return 0;
	288
	289	shinfo = skb_shinfo(strp->anchor);
	290	shinfo->frag_list = NULL;
	291
	292	/* If we don't know the length go max plus page for cipher overhead */
	293	need_spc = strp->stm.full_len ?: TLS_MAX_PAYLOAD_SIZE + PAGE_SIZE;
	294
	295	for (len = need_spc; len > 0; len -= PAGE_SIZE) {
	296	page = alloc_page(strp->sk->sk_allocation);
	297	if (!page) {
	298	tls_strp_flush_anchor_copy(strp);
	299	return -ENOMEM;
	300	}
	301
	302	skb_fill_page_desc(strp->anchor, shinfo->nr_frags++,
	303	page, 0, 0);
	304	}
	305
	306	strp->copy_mode = 1;
	307	strp->stm.offset = 0;
	308
	309	strp->anchor->len = 0;
	310	strp->anchor->data_len = 0;
	311	strp->anchor->truesize = round_up(need_spc, PAGE_SIZE);
	312
	313	tls_strp_read_copyin(strp);
	314
	315	return 0;
	316	}
	317
0d87bbd3 JK	318	static bool tls_strp_check_no_dup(struct tls_strparser *strp)
	319	{
	320	unsigned int len = strp->stm.offset + strp->stm.full_len;
	321	struct sk_buff *skb;
	322	u32 seq;
	323
	324	skb = skb_shinfo(strp->anchor)->frag_list;
	325	seq = TCP_SKB_CB(skb)->seq;
	326
	327	while (skb->len < len) {
	328	seq += skb->len;
	329	len -= skb->len;
	330	skb = skb->next;
	331
	332	if (TCP_SKB_CB(skb)->seq != seq)
	333	return false;
	334	}
	335
	336	return true;
	337	}
	338
84c61fe1 JK	339	static void tls_strp_load_anchor_with_queue(struct tls_strparser *strp, int len)
	340	{
	341	struct tcp_sock *tp = tcp_sk(strp->sk);
	342	struct sk_buff *first;
	343	u32 offset;
	344
	345	first = tcp_recv_skb(strp->sk, tp->copied_seq, &offset);
	346	if (WARN_ON_ONCE(!first))
	347	return;
	348
	349	/* Bestow the state onto the anchor */
	350	strp->anchor->len = offset + len;
	351	strp->anchor->data_len = offset + len;
	352	strp->anchor->truesize = offset + len;
	353
	354	skb_shinfo(strp->anchor)->frag_list = first;
	355
	356	skb_copy_header(strp->anchor, first);
	357	strp->anchor->destructor = NULL;
	358
	359	strp->stm.offset = offset;
	360	}
	361
	362	void tls_strp_msg_load(struct tls_strparser *strp, bool force_refresh)
	363	{
	364	struct strp_msg *rxm;
	365	struct tls_msg *tlm;
	366
	367	DEBUG_NET_WARN_ON_ONCE(!strp->msg_ready);
	368	DEBUG_NET_WARN_ON_ONCE(!strp->stm.full_len);
	369
	370	if (!strp->copy_mode && force_refresh) {
	371	if (WARN_ON(tcp_inq(strp->sk) < strp->stm.full_len))
	372	return;
	373
	374	tls_strp_load_anchor_with_queue(strp, strp->stm.full_len);
	375	}
	376
	377	rxm = strp_msg(strp->anchor);
	378	rxm->full_len = strp->stm.full_len;
	379	rxm->offset = strp->stm.offset;
	380	tlm = tls_msg(strp->anchor);
	381	tlm->control = strp->mark;
	382	}
	383
	384	/* Called with lock held on lower socket */
	385	static int tls_strp_read_sock(struct tls_strparser *strp)
	386	{
	387	int sz, inq;
	388
	389	inq = tcp_inq(strp->sk);
	390	if (inq < 1)
	391	return 0;
	392
	393	if (unlikely(strp->copy_mode))
	394	return tls_strp_read_copyin(strp);
	395
	396	if (inq < strp->stm.full_len)
0d87bbd3	397	return tls_strp_read_copy(strp, true);
84c61fe1 JK	398
	399	if (!strp->stm.full_len) {
	400	tls_strp_load_anchor_with_queue(strp, inq);
	401
	402	sz = tls_rx_msg_size(strp, strp->anchor);
	403	if (sz < 0) {
	404	tls_strp_abort_strp(strp, sz);
	405	return sz;
	406	}
	407
	408	strp->stm.full_len = sz;
	409
	410	if (!strp->stm.full_len \|\| inq < strp->stm.full_len)
0d87bbd3	411	return tls_strp_read_copy(strp, true);
84c61fe1 JK	412	}
84c61fe1 JK	413
0d87bbd3 JK	414	if (!tls_strp_check_no_dup(strp))
	415	return tls_strp_read_copy(strp, false);
	416
84c61fe1 JK	417	strp->msg_ready = 1;
	418	tls_rx_msg_ready(strp);
	419
	420	return 0;
	421	}
	422
	423	void tls_strp_check_rcv(struct tls_strparser *strp)
	424	{
	425	if (unlikely(strp->stopped) \|\| strp->msg_ready)
	426	return;
	427
	428	if (tls_strp_read_sock(strp) == -ENOMEM)
	429	queue_work(tls_strp_wq, &strp->work);
	430	}
	431
	432	/* Lower sock lock held */
	433	void tls_strp_data_ready(struct tls_strparser *strp)
	434	{
	435	/* This check is needed to synchronize with do_tls_strp_work.
	436	* do_tls_strp_work acquires a process lock (lock_sock) whereas
	437	* the lock held here is bh_lock_sock. The two locks can be
	438	* held by different threads at the same time, but bh_lock_sock
	439	* allows a thread in BH context to safely check if the process
	440	* lock is held. In this case, if the lock is held, queue work.
	441	*/
	442	if (sock_owned_by_user_nocheck(strp->sk)) {
	443	queue_work(tls_strp_wq, &strp->work);
	444	return;
	445	}
	446
	447	tls_strp_check_rcv(strp);
	448	}
	449
	450	static void tls_strp_work(struct work_struct *w)
	451	{
	452	struct tls_strparser *strp =
	453	container_of(w, struct tls_strparser, work);
	454
	455	lock_sock(strp->sk);
	456	tls_strp_check_rcv(strp);
	457	release_sock(strp->sk);
	458	}
	459
	460	void tls_strp_msg_done(struct tls_strparser *strp)
	461	{
	462	WARN_ON(!strp->stm.full_len);
	463
	464	if (likely(!strp->copy_mode))
	465	tcp_read_done(strp->sk, strp->stm.full_len);
	466	else
	467	tls_strp_flush_anchor_copy(strp);
	468
	469	strp->msg_ready = 0;
	470	memset(&strp->stm, 0, sizeof(strp->stm));
	471
	472	tls_strp_check_rcv(strp);
	473	}
	474
	475	void tls_strp_stop(struct tls_strparser *strp)
	476	{
	477	strp->stopped = 1;
	478	}
	479
	480	int tls_strp_init(struct tls_strparser strp, struct sock sk)
481	{
482	memset(strp, 0, sizeof(*strp));
483
484	strp->sk = sk;
485
486	strp->anchor = alloc_skb(0, GFP_KERNEL);
487	if (!strp->anchor)
488	return -ENOMEM;
489
490	INIT_WORK(&strp->work, tls_strp_work);
8b3c59a7	491
8b3c59a7 JK	492	return 0;
	493	}
	494
84c61fe1 JK	495	/* strp must already be stopped so that tls_strp_recv will no longer be called.
	496	* Note that tls_strp_done is not called with the lower socket held.
	497	*/
	498	void tls_strp_done(struct tls_strparser *strp)
c618db2a	499	{
84c61fe1	500	WARN_ON(!strp->stopped);
c618db2a	501
84c61fe1 JK	502	cancel_work_sync(&strp->work);
	503	tls_strp_anchor_free(strp);
	504	}
	505
	506	int __init tls_strp_dev_init(void)
	507	{
d11ef9cc	508	tls_strp_wq = create_workqueue("tls-strp");
84c61fe1	509	if (unlikely(!tls_strp_wq))
c618db2a	510	return -ENOMEM;
84c61fe1	511
c618db2a JK	512	return 0;
c618db2a JK	513	}
84c61fe1 JK	514
	515	void tls_strp_dev_exit(void)
	516	{
	517	destroy_workqueue(tls_strp_wq);
	518	}