[linux-block.git] / net / mptcp / protocol.c

// SPDX-License-Identifier: GPL-2.0
/* Multipath TCP
 *
 * Copyright (c) 2017 - 2019, Intel Corporation.
 */

#define pr_fmt(fmt) "MPTCP: " fmt

#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/netdevice.h>
#include <net/sock.h>
#include <net/inet_common.h>
#include <net/inet_hashtables.h>
#include <net/protocol.h>
#include <net/tcp.h>
#if IS_ENABLED(CONFIG_MPTCP_IPV6)
#include <net/transp_v6.h>
#endif
#include <net/mptcp.h>
#include "protocol.h"

#define MPTCP_SAME_STATE TCP_MAX_STATES

/* If msk has an initial subflow socket, and the MP_CAPABLE handshake has not
 * completed yet or has failed, return the subflow socket.
 * Otherwise return NULL.
 */
static struct socket *__mptcp_nmpc_socket(const struct mptcp_sock *msk)
{
	if (!msk->subflow || mptcp_subflow_ctx(msk->subflow->sk)->fourth_ack)
		return NULL;

	return msk->subflow;
}

/* if msk has a single subflow, and the mp_capable handshake is failed,
 * return it.
 * Otherwise returns NULL
 */
static struct socket *__mptcp_tcp_fallback(const struct mptcp_sock *msk)
{
	struct socket *ssock = __mptcp_nmpc_socket(msk);

	sock_owned_by_me((const struct sock *)msk);

	if (!ssock || sk_is_mptcp(ssock->sk))
		return NULL;

	return ssock;
}

static bool __mptcp_can_create_subflow(const struct mptcp_sock *msk)
{
	return ((struct sock *)msk)->sk_state == TCP_CLOSE;
}

static struct socket *__mptcp_socket_create(struct mptcp_sock *msk, int state)
{
	struct mptcp_subflow_context *subflow;
	struct sock *sk = (struct sock *)msk;
	struct socket *ssock;
	int err;

	ssock = __mptcp_nmpc_socket(msk);
	if (ssock)
		goto set_state;

	if (!__mptcp_can_create_subflow(msk))
		return ERR_PTR(-EINVAL);

	err = mptcp_subflow_create_socket(sk, &ssock);
	if (err)
		return ERR_PTR(err);

	msk->subflow = ssock;
	subflow = mptcp_subflow_ctx(ssock->sk);
	list_add(&subflow->node, &msk->conn_list);
	subflow->request_mptcp = 1;

set_state:
	if (state != MPTCP_SAME_STATE)
		inet_sk_state_store(sk, state);
	return ssock;
}

static struct sock *mptcp_subflow_get(const struct mptcp_sock *msk)
{
	struct mptcp_subflow_context *subflow;

	sock_owned_by_me((const struct sock *)msk);

	mptcp_for_each_subflow(msk, subflow) {
		return mptcp_subflow_tcp_sock(subflow);
	}

	return NULL;
}

static int mptcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t len)
{
	struct mptcp_sock *msk = mptcp_sk(sk);
	struct socket *ssock;
	struct sock *ssk;
	int ret;

	if (msg->msg_flags & ~(MSG_MORE | MSG_DONTWAIT | MSG_NOSIGNAL))
		return -EOPNOTSUPP;

	lock_sock(sk);
	ssock = __mptcp_tcp_fallback(msk);
	if (ssock) {
		pr_debug("fallback passthrough");
		ret = sock_sendmsg(ssock, msg);
		release_sock(sk);
		return ret;
	}

	ssk = mptcp_subflow_get(msk);
	if (!ssk) {
		release_sock(sk);
		return -ENOTCONN;
	}

	ret = sock_sendmsg(ssk->sk_socket, msg);

	release_sock(sk);
	return ret;
}

static int mptcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len,
			 int nonblock, int flags, int *addr_len)
{
	struct mptcp_sock *msk = mptcp_sk(sk);
	struct socket *ssock;
	struct sock *ssk;
	int copied = 0;

	if (msg->msg_flags & ~(MSG_WAITALL | MSG_DONTWAIT))
		return -EOPNOTSUPP;

	lock_sock(sk);
	ssock = __mptcp_tcp_fallback(msk);
	if (ssock) {
		pr_debug("fallback-read subflow=%p",
			 mptcp_subflow_ctx(ssock->sk));
		copied = sock_recvmsg(ssock, msg, flags);
		release_sock(sk);
		return copied;
	}

	ssk = mptcp_subflow_get(msk);
	if (!ssk) {
		release_sock(sk);
		return -ENOTCONN;
	}

	copied = sock_recvmsg(ssk->sk_socket, msg, flags);

	release_sock(sk);

	return copied;
}

/* subflow sockets can be either outgoing (connect) or incoming
 * (accept).
 *
 * Outgoing subflows use in-kernel sockets.
 * Incoming subflows do not have their own 'struct socket' allocated,
 * so we need to use tcp_close() after detaching them from the mptcp
 * parent socket.
 */
static void __mptcp_close_ssk(struct sock *sk, struct sock *ssk,
			      struct mptcp_subflow_context *subflow,
			      long timeout)
{
	struct socket *sock = READ_ONCE(ssk->sk_socket);

	list_del(&subflow->node);

	if (sock && sock != sk->sk_socket) {
		/* outgoing subflow */
		sock_release(sock);
	} else {
		/* incoming subflow */
		tcp_close(ssk, timeout);
	}
}

static int mptcp_init_sock(struct sock *sk)
{
	struct mptcp_sock *msk = mptcp_sk(sk);

	INIT_LIST_HEAD(&msk->conn_list);

	return 0;
}

static void mptcp_subflow_shutdown(struct sock *ssk, int how)
{
	lock_sock(ssk);

	switch (ssk->sk_state) {
	case TCP_LISTEN:
		if (!(how & RCV_SHUTDOWN))
			break;
		/* fall through */
	case TCP_SYN_SENT:
		tcp_disconnect(ssk, O_NONBLOCK);
		break;
	default:
		ssk->sk_shutdown |= how;
		tcp_shutdown(ssk, how);
		break;
	}

	/* Wake up anyone sleeping in poll. */
	ssk->sk_state_change(ssk);
	release_sock(ssk);
}

static void mptcp_close(struct sock *sk, long timeout)
{
	struct mptcp_subflow_context *subflow, *tmp;
	struct mptcp_sock *msk = mptcp_sk(sk);

	mptcp_token_destroy(msk->token);
	inet_sk_state_store(sk, TCP_CLOSE);

	lock_sock(sk);

	list_for_each_entry_safe(subflow, tmp, &msk->conn_list, node) {
		struct sock *ssk = mptcp_subflow_tcp_sock(subflow);

		__mptcp_close_ssk(sk, ssk, subflow, timeout);
	}

	release_sock(sk);
	sk_common_release(sk);
}

static void mptcp_copy_inaddrs(struct sock *msk, const struct sock *ssk)
{
#if IS_ENABLED(CONFIG_MPTCP_IPV6)
	const struct ipv6_pinfo *ssk6 = inet6_sk(ssk);
	struct ipv6_pinfo *msk6 = inet6_sk(msk);

	msk->sk_v6_daddr = ssk->sk_v6_daddr;
	msk->sk_v6_rcv_saddr = ssk->sk_v6_rcv_saddr;

	if (msk6 && ssk6) {
		msk6->saddr = ssk6->saddr;
		msk6->flow_label = ssk6->flow_label;
	}
#endif

	inet_sk(msk)->inet_num = inet_sk(ssk)->inet_num;
	inet_sk(msk)->inet_dport = inet_sk(ssk)->inet_dport;
	inet_sk(msk)->inet_sport = inet_sk(ssk)->inet_sport;
	inet_sk(msk)->inet_daddr = inet_sk(ssk)->inet_daddr;
	inet_sk(msk)->inet_saddr = inet_sk(ssk)->inet_saddr;
	inet_sk(msk)->inet_rcv_saddr = inet_sk(ssk)->inet_rcv_saddr;
}

static struct sock *mptcp_accept(struct sock *sk, int flags, int *err,
				 bool kern)
{
	struct mptcp_sock *msk = mptcp_sk(sk);
	struct socket *listener;
	struct sock *newsk;

	listener = __mptcp_nmpc_socket(msk);
	if (WARN_ON_ONCE(!listener)) {
		*err = -EINVAL;
		return NULL;
	}

	pr_debug("msk=%p, listener=%p", msk, mptcp_subflow_ctx(listener->sk));
	newsk = inet_csk_accept(listener->sk, flags, err, kern);
	if (!newsk)
		return NULL;

	pr_debug("msk=%p, subflow is mptcp=%d", msk, sk_is_mptcp(newsk));

	if (sk_is_mptcp(newsk)) {
		struct mptcp_subflow_context *subflow;
		struct sock *new_mptcp_sock;
		struct sock *ssk = newsk;

		subflow = mptcp_subflow_ctx(newsk);
		lock_sock(sk);

		local_bh_disable();
		new_mptcp_sock = sk_clone_lock(sk, GFP_ATOMIC);
		if (!new_mptcp_sock) {
			*err = -ENOBUFS;
			local_bh_enable();
			release_sock(sk);
			mptcp_subflow_shutdown(newsk, SHUT_RDWR + 1);
			tcp_close(newsk, 0);
			return NULL;
		}

		mptcp_init_sock(new_mptcp_sock);

		msk = mptcp_sk(new_mptcp_sock);
		msk->remote_key = subflow->remote_key;
		msk->local_key = subflow->local_key;
		msk->token = subflow->token;
		msk->subflow = NULL;

		mptcp_token_update_accept(newsk, new_mptcp_sock);
		newsk = new_mptcp_sock;
		mptcp_copy_inaddrs(newsk, ssk);
		list_add(&subflow->node, &msk->conn_list);

		/* will be fully established at mptcp_stream_accept()
		 * completion.
		 */
		inet_sk_state_store(new_mptcp_sock, TCP_SYN_RECV);
		bh_unlock_sock(new_mptcp_sock);
		local_bh_enable();
		release_sock(sk);
	}

	return newsk;
}

static void mptcp_destroy(struct sock *sk)
{
}

static int mptcp_get_port(struct sock *sk, unsigned short snum)
{
	struct mptcp_sock *msk = mptcp_sk(sk);
	struct socket *ssock;

	ssock = __mptcp_nmpc_socket(msk);
	pr_debug("msk=%p, subflow=%p", msk, ssock);
	if (WARN_ON_ONCE(!ssock))
		return -EINVAL;

	return inet_csk_get_port(ssock->sk, snum);
}

void mptcp_finish_connect(struct sock *ssk)
{
	struct mptcp_subflow_context *subflow;
	struct mptcp_sock *msk;
	struct sock *sk;

	subflow = mptcp_subflow_ctx(ssk);

	if (!subflow->mp_capable)
		return;

	sk = subflow->conn;
	msk = mptcp_sk(sk);

	/* the socket is not connected yet, no msk/subflow ops can access/race
	 * accessing the field below
	 */
	WRITE_ONCE(msk->remote_key, subflow->remote_key);
	WRITE_ONCE(msk->local_key, subflow->local_key);
	WRITE_ONCE(msk->token, subflow->token);
}

static void mptcp_sock_graft(struct sock *sk, struct socket *parent)
{
	write_lock_bh(&sk->sk_callback_lock);
	rcu_assign_pointer(sk->sk_wq, &parent->wq);
	sk_set_socket(sk, parent);
	sk->sk_uid = SOCK_INODE(parent)->i_uid;
	write_unlock_bh(&sk->sk_callback_lock);
}

static struct proto mptcp_prot = {
	.name		= "MPTCP",
	.owner		= THIS_MODULE,
	.init		= mptcp_init_sock,
	.close		= mptcp_close,
	.accept		= mptcp_accept,
	.shutdown	= tcp_shutdown,
	.destroy	= mptcp_destroy,
	.sendmsg	= mptcp_sendmsg,
	.recvmsg	= mptcp_recvmsg,
	.hash		= inet_hash,
	.unhash		= inet_unhash,
	.get_port	= mptcp_get_port,
	.obj_size	= sizeof(struct mptcp_sock),
	.no_autobind	= true,
};

static int mptcp_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len)
{
	struct mptcp_sock *msk = mptcp_sk(sock->sk);
	struct socket *ssock;
	int err;

	lock_sock(sock->sk);
	ssock = __mptcp_socket_create(msk, MPTCP_SAME_STATE);
	if (IS_ERR(ssock)) {
		err = PTR_ERR(ssock);
		goto unlock;
	}

	err = ssock->ops->bind(ssock, uaddr, addr_len);
	if (!err)
		mptcp_copy_inaddrs(sock->sk, ssock->sk);

unlock:
	release_sock(sock->sk);
	return err;
}

static int mptcp_stream_connect(struct socket *sock, struct sockaddr *uaddr,
				int addr_len, int flags)
{
	struct mptcp_sock *msk = mptcp_sk(sock->sk);
	struct socket *ssock;
	int err;

	lock_sock(sock->sk);
	ssock = __mptcp_socket_create(msk, TCP_SYN_SENT);
	if (IS_ERR(ssock)) {
		err = PTR_ERR(ssock);
		goto unlock;
	}

#ifdef CONFIG_TCP_MD5SIG
	/* no MPTCP if MD5SIG is enabled on this socket or we may run out of
	 * TCP option space.
	 */
	if (rcu_access_pointer(tcp_sk(ssock->sk)->md5sig_info))
		mptcp_subflow_ctx(ssock->sk)->request_mptcp = 0;
#endif

	err = ssock->ops->connect(ssock, uaddr, addr_len, flags);
	inet_sk_state_store(sock->sk, inet_sk_state_load(ssock->sk));
	mptcp_copy_inaddrs(sock->sk, ssock->sk);

unlock:
	release_sock(sock->sk);
	return err;
}

static int mptcp_v4_getname(struct socket *sock, struct sockaddr *uaddr,
			    int peer)
{
	if (sock->sk->sk_prot == &tcp_prot) {
		/* we are being invoked from __sys_accept4, after
		 * mptcp_accept() has just accepted a non-mp-capable
		 * flow: sk is a tcp_sk, not an mptcp one.
		 *
		 * Hand the socket over to tcp so all further socket ops
		 * bypass mptcp.
		 */
		sock->ops = &inet_stream_ops;
	}

	return inet_getname(sock, uaddr, peer);
}

#if IS_ENABLED(CONFIG_MPTCP_IPV6)
static int mptcp_v6_getname(struct socket *sock, struct sockaddr *uaddr,
			    int peer)
{
	if (sock->sk->sk_prot == &tcpv6_prot) {
		/* we are being invoked from __sys_accept4 after
		 * mptcp_accept() has accepted a non-mp-capable
		 * subflow: sk is a tcp_sk, not mptcp.
		 *
		 * Hand the socket over to tcp so all further
		 * socket ops bypass mptcp.
		 */
		sock->ops = &inet6_stream_ops;
	}

	return inet6_getname(sock, uaddr, peer);
}
#endif

static int mptcp_listen(struct socket *sock, int backlog)
{
	struct mptcp_sock *msk = mptcp_sk(sock->sk);
	struct socket *ssock;
	int err;

	pr_debug("msk=%p", msk);

	lock_sock(sock->sk);
	ssock = __mptcp_socket_create(msk, TCP_LISTEN);
	if (IS_ERR(ssock)) {
		err = PTR_ERR(ssock);
		goto unlock;
	}

	err = ssock->ops->listen(ssock, backlog);
	inet_sk_state_store(sock->sk, inet_sk_state_load(ssock->sk));
	if (!err)
		mptcp_copy_inaddrs(sock->sk, ssock->sk);

unlock:
	release_sock(sock->sk);
	return err;
}

static bool is_tcp_proto(const struct proto *p)
{
#if IS_ENABLED(CONFIG_MPTCP_IPV6)
	return p == &tcp_prot || p == &tcpv6_prot;
#else
	return p == &tcp_prot;
#endif
}

static int mptcp_stream_accept(struct socket *sock, struct socket *newsock,
			       int flags, bool kern)
{
	struct mptcp_sock *msk = mptcp_sk(sock->sk);
	struct socket *ssock;
	int err;

	pr_debug("msk=%p", msk);

	lock_sock(sock->sk);
	if (sock->sk->sk_state != TCP_LISTEN)
		goto unlock_fail;

	ssock = __mptcp_nmpc_socket(msk);
	if (!ssock)
		goto unlock_fail;

	sock_hold(ssock->sk);
	release_sock(sock->sk);

	err = ssock->ops->accept(sock, newsock, flags, kern);
	if (err == 0 && !is_tcp_proto(newsock->sk->sk_prot)) {
		struct mptcp_sock *msk = mptcp_sk(newsock->sk);
		struct mptcp_subflow_context *subflow;

		/* set ssk->sk_socket of accept()ed flows to mptcp socket.
		 * This is needed so NOSPACE flag can be set from tcp stack.
		 */
		list_for_each_entry(subflow, &msk->conn_list, node) {
			struct sock *ssk = mptcp_subflow_tcp_sock(subflow);

			if (!ssk->sk_socket)
				mptcp_sock_graft(ssk, newsock);
		}

		inet_sk_state_store(newsock->sk, TCP_ESTABLISHED);
	}

	sock_put(ssock->sk);
	return err;

unlock_fail:
	release_sock(sock->sk);
	return -EINVAL;
}

static __poll_t mptcp_poll(struct file *file, struct socket *sock,
			   struct poll_table_struct *wait)
{
	__poll_t mask = 0;

	return mask;
}

static int mptcp_shutdown(struct socket *sock, int how)
{
	struct mptcp_sock *msk = mptcp_sk(sock->sk);
	struct mptcp_subflow_context *subflow;
	int ret = 0;

	pr_debug("sk=%p, how=%d", msk, how);

	lock_sock(sock->sk);

	if (how == SHUT_WR || how == SHUT_RDWR)
		inet_sk_state_store(sock->sk, TCP_FIN_WAIT1);

	how++;

	if ((how & ~SHUTDOWN_MASK) || !how) {
		ret = -EINVAL;
		goto out_unlock;
	}

	if (sock->state == SS_CONNECTING) {
		if ((1 << sock->sk->sk_state) &
		    (TCPF_SYN_SENT | TCPF_SYN_RECV | TCPF_CLOSE))
			sock->state = SS_DISCONNECTING;
		else
			sock->state = SS_CONNECTED;
	}

	mptcp_for_each_subflow(msk, subflow) {
		struct sock *tcp_sk = mptcp_subflow_tcp_sock(subflow);

		mptcp_subflow_shutdown(tcp_sk, how);
	}

out_unlock:
	release_sock(sock->sk);

	return ret;
}

static struct proto_ops mptcp_stream_ops;

static struct inet_protosw mptcp_protosw = {
	.type		= SOCK_STREAM,
	.protocol	= IPPROTO_MPTCP,
	.prot		= &mptcp_prot,
	.ops		= &mptcp_stream_ops,
	.flags		= INET_PROTOSW_ICSK,
};

void __init mptcp_init(void)
{
	mptcp_prot.h.hashinfo = tcp_prot.h.hashinfo;
	mptcp_stream_ops = inet_stream_ops;
	mptcp_stream_ops.bind = mptcp_bind;
	mptcp_stream_ops.connect = mptcp_stream_connect;
	mptcp_stream_ops.poll = mptcp_poll;
	mptcp_stream_ops.accept = mptcp_stream_accept;
	mptcp_stream_ops.getname = mptcp_v4_getname;
	mptcp_stream_ops.listen = mptcp_listen;
	mptcp_stream_ops.shutdown = mptcp_shutdown;

	mptcp_subflow_init();

	if (proto_register(&mptcp_prot, 1) != 0)
		panic("Failed to register MPTCP proto.\n");

	inet_register_protosw(&mptcp_protosw);
}

#if IS_ENABLED(CONFIG_MPTCP_IPV6)
static struct proto_ops mptcp_v6_stream_ops;
static struct proto mptcp_v6_prot;

static void mptcp_v6_destroy(struct sock *sk)
{
	mptcp_destroy(sk);
	inet6_destroy_sock(sk);
}

static struct inet_protosw mptcp_v6_protosw = {
	.type		= SOCK_STREAM,
	.protocol	= IPPROTO_MPTCP,
	.prot		= &mptcp_v6_prot,
	.ops		= &mptcp_v6_stream_ops,
	.flags		= INET_PROTOSW_ICSK,
};

int mptcpv6_init(void)
{
	int err;

	mptcp_v6_prot = mptcp_prot;
	strcpy(mptcp_v6_prot.name, "MPTCPv6");
	mptcp_v6_prot.slab = NULL;
	mptcp_v6_prot.destroy = mptcp_v6_destroy;
	mptcp_v6_prot.obj_size = sizeof(struct mptcp_sock) +
				 sizeof(struct ipv6_pinfo);

	err = proto_register(&mptcp_v6_prot, 1);
	if (err)
		return err;

	mptcp_v6_stream_ops = inet6_stream_ops;
	mptcp_v6_stream_ops.bind = mptcp_bind;
	mptcp_v6_stream_ops.connect = mptcp_stream_connect;
	mptcp_v6_stream_ops.poll = mptcp_poll;
	mptcp_v6_stream_ops.accept = mptcp_stream_accept;
	mptcp_v6_stream_ops.getname = mptcp_v6_getname;
	mptcp_v6_stream_ops.listen = mptcp_listen;
	mptcp_v6_stream_ops.shutdown = mptcp_shutdown;

	err = inet6_register_protosw(&mptcp_v6_protosw);
	if (err)
		proto_unregister(&mptcp_v6_prot);

	return err;
}
#endif
Commit	Line	Data
f870fa0b MM	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Multipath TCP
	3	*
	4	* Copyright (c) 2017 - 2019, Intel Corporation.
	5	*/
	6
	7	#define pr_fmt(fmt) "MPTCP: " fmt
	8
	9	#include <linux/kernel.h>
	10	#include <linux/module.h>
	11	#include <linux/netdevice.h>
	12	#include <net/sock.h>
	13	#include <net/inet_common.h>
	14	#include <net/inet_hashtables.h>
	15	#include <net/protocol.h>
	16	#include <net/tcp.h>
cf7da0d6 PK	17	#if IS_ENABLED(CONFIG_MPTCP_IPV6)
	18	#include <net/transp_v6.h>
	19	#endif
f870fa0b MM	20	#include <net/mptcp.h>
	21	#include "protocol.h"
	22
2303f994 PK	23	#define MPTCP_SAME_STATE TCP_MAX_STATES
	24
	25	/* If msk has an initial subflow socket, and the MP_CAPABLE handshake has not
	26	* completed yet or has failed, return the subflow socket.
	27	* Otherwise return NULL.
	28	*/
	29	static struct socket __mptcp_nmpc_socket(const struct mptcp_sock msk)
	30	{
cec37a6e	31	if (!msk->subflow \|\| mptcp_subflow_ctx(msk->subflow->sk)->fourth_ack)
2303f994 PK	32	return NULL;
	33
	34	return msk->subflow;
	35	}
	36
cec37a6e PK	37	/* if msk has a single subflow, and the mp_capable handshake is failed,
	38	* return it.
	39	* Otherwise returns NULL
	40	*/
	41	static struct socket __mptcp_tcp_fallback(const struct mptcp_sock msk)
	42	{
	43	struct socket *ssock = __mptcp_nmpc_socket(msk);
	44
	45	sock_owned_by_me((const struct sock *)msk);
	46
	47	if (!ssock \|\| sk_is_mptcp(ssock->sk))
	48	return NULL;
	49
	50	return ssock;
	51	}
	52
2303f994 PK	53	static bool __mptcp_can_create_subflow(const struct mptcp_sock *msk)
	54	{
	55	return ((struct sock *)msk)->sk_state == TCP_CLOSE;
	56	}
	57
	58	static struct socket __mptcp_socket_create(struct mptcp_sock msk, int state)
	59	{
	60	struct mptcp_subflow_context *subflow;
	61	struct sock sk = (struct sock )msk;
	62	struct socket *ssock;
	63	int err;
	64
	65	ssock = __mptcp_nmpc_socket(msk);
	66	if (ssock)
	67	goto set_state;
	68
	69	if (!__mptcp_can_create_subflow(msk))
	70	return ERR_PTR(-EINVAL);
	71
	72	err = mptcp_subflow_create_socket(sk, &ssock);
	73	if (err)
	74	return ERR_PTR(err);
	75
	76	msk->subflow = ssock;
	77	subflow = mptcp_subflow_ctx(ssock->sk);
cec37a6e	78	list_add(&subflow->node, &msk->conn_list);
2303f994 PK	79	subflow->request_mptcp = 1;
	80
	81	set_state:
	82	if (state != MPTCP_SAME_STATE)
	83	inet_sk_state_store(sk, state);
	84	return ssock;
	85	}
	86
cec37a6e PK	87	static struct sock mptcp_subflow_get(const struct mptcp_sock msk)
	88	{
	89	struct mptcp_subflow_context *subflow;
	90
	91	sock_owned_by_me((const struct sock *)msk);
	92
	93	mptcp_for_each_subflow(msk, subflow) {
	94	return mptcp_subflow_tcp_sock(subflow);
	95	}
	96
	97	return NULL;
	98	}
	99
f870fa0b MM	100	static int mptcp_sendmsg(struct sock sk, struct msghdr msg, size_t len)
	101	{
	102	struct mptcp_sock *msk = mptcp_sk(sk);
cec37a6e PK	103	struct socket *ssock;
	104	struct sock *ssk;
	105	int ret;
f870fa0b MM	106
	107	if (msg->msg_flags & ~(MSG_MORE \| MSG_DONTWAIT \| MSG_NOSIGNAL))
	108	return -EOPNOTSUPP;
	109
cec37a6e PK	110	lock_sock(sk);
	111	ssock = __mptcp_tcp_fallback(msk);
	112	if (ssock) {
	113	pr_debug("fallback passthrough");
	114	ret = sock_sendmsg(ssock, msg);
	115	release_sock(sk);
	116	return ret;
	117	}
	118
	119	ssk = mptcp_subflow_get(msk);
	120	if (!ssk) {
	121	release_sock(sk);
	122	return -ENOTCONN;
	123	}
	124
	125	ret = sock_sendmsg(ssk->sk_socket, msg);
	126
	127	release_sock(sk);
	128	return ret;
f870fa0b MM	129	}
	130
	131	static int mptcp_recvmsg(struct sock sk, struct msghdr msg, size_t len,
	132	int nonblock, int flags, int *addr_len)
	133	{
	134	struct mptcp_sock *msk = mptcp_sk(sk);
cec37a6e PK	135	struct socket *ssock;
	136	struct sock *ssk;
	137	int copied = 0;
f870fa0b MM	138
	139	if (msg->msg_flags & ~(MSG_WAITALL \| MSG_DONTWAIT))
	140	return -EOPNOTSUPP;
	141
cec37a6e PK	142	lock_sock(sk);
	143	ssock = __mptcp_tcp_fallback(msk);
	144	if (ssock) {
	145	pr_debug("fallback-read subflow=%p",
	146	mptcp_subflow_ctx(ssock->sk));
	147	copied = sock_recvmsg(ssock, msg, flags);
	148	release_sock(sk);
	149	return copied;
	150	}
	151
	152	ssk = mptcp_subflow_get(msk);
	153	if (!ssk) {
	154	release_sock(sk);
	155	return -ENOTCONN;
	156	}
	157
	158	copied = sock_recvmsg(ssk->sk_socket, msg, flags);
	159
	160	release_sock(sk);
	161
	162	return copied;
	163	}
	164
	165	/* subflow sockets can be either outgoing (connect) or incoming
	166	* (accept).
	167	*
	168	* Outgoing subflows use in-kernel sockets.
	169	* Incoming subflows do not have their own 'struct socket' allocated,
	170	* so we need to use tcp_close() after detaching them from the mptcp
	171	* parent socket.
	172	*/
	173	static void __mptcp_close_ssk(struct sock sk, struct sock ssk,
	174	struct mptcp_subflow_context *subflow,
	175	long timeout)
	176	{
	177	struct socket *sock = READ_ONCE(ssk->sk_socket);
	178
	179	list_del(&subflow->node);
	180
	181	if (sock && sock != sk->sk_socket) {
	182	/* outgoing subflow */
	183	sock_release(sock);
	184	} else {
	185	/* incoming subflow */
	186	tcp_close(ssk, timeout);
	187	}
f870fa0b MM	188	}
	189
	190	static int mptcp_init_sock(struct sock *sk)
	191	{
cec37a6e PK	192	struct mptcp_sock *msk = mptcp_sk(sk);
	193
	194	INIT_LIST_HEAD(&msk->conn_list);
	195
f870fa0b MM	196	return 0;
	197	}
	198
21498490 PK	199	static void mptcp_subflow_shutdown(struct sock *ssk, int how)
	200	{
	201	lock_sock(ssk);
	202
	203	switch (ssk->sk_state) {
	204	case TCP_LISTEN:
	205	if (!(how & RCV_SHUTDOWN))
	206	break;
	207	/* fall through */
	208	case TCP_SYN_SENT:
	209	tcp_disconnect(ssk, O_NONBLOCK);
	210	break;
	211	default:
	212	ssk->sk_shutdown \|= how;
	213	tcp_shutdown(ssk, how);
	214	break;
	215	}
	216
	217	/* Wake up anyone sleeping in poll. */
	218	ssk->sk_state_change(ssk);
	219	release_sock(ssk);
	220	}
	221
f870fa0b MM	222	static void mptcp_close(struct sock *sk, long timeout)
f870fa0b MM	223	{
cec37a6e	224	struct mptcp_subflow_context subflow, tmp;
f870fa0b MM	225	struct mptcp_sock *msk = mptcp_sk(sk);
f870fa0b MM	226
79c0949e	227	mptcp_token_destroy(msk->token);
f870fa0b MM	228	inet_sk_state_store(sk, TCP_CLOSE);
f870fa0b MM	229
cec37a6e PK	230	lock_sock(sk);
	231
	232	list_for_each_entry_safe(subflow, tmp, &msk->conn_list, node) {
	233	struct sock *ssk = mptcp_subflow_tcp_sock(subflow);
	234
	235	__mptcp_close_ssk(sk, ssk, subflow, timeout);
f870fa0b MM	236	}
f870fa0b MM	237
cec37a6e PK	238	release_sock(sk);
cec37a6e PK	239	sk_common_release(sk);
f870fa0b MM	240	}
f870fa0b MM	241
cf7da0d6 PK	242	static void mptcp_copy_inaddrs(struct sock msk, const struct sock ssk)
	243	{
	244	#if IS_ENABLED(CONFIG_MPTCP_IPV6)
	245	const struct ipv6_pinfo *ssk6 = inet6_sk(ssk);
	246	struct ipv6_pinfo *msk6 = inet6_sk(msk);
	247
	248	msk->sk_v6_daddr = ssk->sk_v6_daddr;
	249	msk->sk_v6_rcv_saddr = ssk->sk_v6_rcv_saddr;
	250
	251	if (msk6 && ssk6) {
	252	msk6->saddr = ssk6->saddr;
	253	msk6->flow_label = ssk6->flow_label;
	254	}
	255	#endif
	256
	257	inet_sk(msk)->inet_num = inet_sk(ssk)->inet_num;
	258	inet_sk(msk)->inet_dport = inet_sk(ssk)->inet_dport;
	259	inet_sk(msk)->inet_sport = inet_sk(ssk)->inet_sport;
	260	inet_sk(msk)->inet_daddr = inet_sk(ssk)->inet_daddr;
	261	inet_sk(msk)->inet_saddr = inet_sk(ssk)->inet_saddr;
	262	inet_sk(msk)->inet_rcv_saddr = inet_sk(ssk)->inet_rcv_saddr;
	263	}
	264
	265	static struct sock mptcp_accept(struct sock sk, int flags, int *err,
	266	bool kern)
	267	{
	268	struct mptcp_sock *msk = mptcp_sk(sk);
	269	struct socket *listener;
	270	struct sock *newsk;
	271
	272	listener = __mptcp_nmpc_socket(msk);
	273	if (WARN_ON_ONCE(!listener)) {
	274	*err = -EINVAL;
	275	return NULL;
	276	}
	277
	278	pr_debug("msk=%p, listener=%p", msk, mptcp_subflow_ctx(listener->sk));
	279	newsk = inet_csk_accept(listener->sk, flags, err, kern);
	280	if (!newsk)
	281	return NULL;
	282
	283	pr_debug("msk=%p, subflow is mptcp=%d", msk, sk_is_mptcp(newsk));
	284
	285	if (sk_is_mptcp(newsk)) {
	286	struct mptcp_subflow_context *subflow;
	287	struct sock *new_mptcp_sock;
	288	struct sock *ssk = newsk;
	289
	290	subflow = mptcp_subflow_ctx(newsk);
	291	lock_sock(sk);
	292
	293	local_bh_disable();
	294	new_mptcp_sock = sk_clone_lock(sk, GFP_ATOMIC);
	295	if (!new_mptcp_sock) {
	296	*err = -ENOBUFS;
	297	local_bh_enable();
	298	release_sock(sk);
21498490	299	mptcp_subflow_shutdown(newsk, SHUT_RDWR + 1);
cf7da0d6 PK	300	tcp_close(newsk, 0);
	301	return NULL;
	302	}
	303
	304	mptcp_init_sock(new_mptcp_sock);
	305
	306	msk = mptcp_sk(new_mptcp_sock);
	307	msk->remote_key = subflow->remote_key;
	308	msk->local_key = subflow->local_key;
79c0949e	309	msk->token = subflow->token;
cf7da0d6 PK	310	msk->subflow = NULL;
cf7da0d6 PK	311
79c0949e	312	mptcp_token_update_accept(newsk, new_mptcp_sock);
cf7da0d6 PK	313	newsk = new_mptcp_sock;
	314	mptcp_copy_inaddrs(newsk, ssk);
	315	list_add(&subflow->node, &msk->conn_list);
	316
	317	/* will be fully established at mptcp_stream_accept()
	318	* completion.
	319	*/
	320	inet_sk_state_store(new_mptcp_sock, TCP_SYN_RECV);
	321	bh_unlock_sock(new_mptcp_sock);
	322	local_bh_enable();
	323	release_sock(sk);
	324	}
	325
	326	return newsk;
	327	}
	328
79c0949e PK	329	static void mptcp_destroy(struct sock *sk)
	330	{
	331	}
	332
cec37a6e	333	static int mptcp_get_port(struct sock *sk, unsigned short snum)
f870fa0b MM	334	{
f870fa0b MM	335	struct mptcp_sock *msk = mptcp_sk(sk);
cec37a6e	336	struct socket *ssock;
f870fa0b	337
cec37a6e PK	338	ssock = __mptcp_nmpc_socket(msk);
	339	pr_debug("msk=%p, subflow=%p", msk, ssock);
	340	if (WARN_ON_ONCE(!ssock))
	341	return -EINVAL;
f870fa0b	342
cec37a6e PK	343	return inet_csk_get_port(ssock->sk, snum);
cec37a6e PK	344	}
f870fa0b	345
cec37a6e PK	346	void mptcp_finish_connect(struct sock *ssk)
	347	{
	348	struct mptcp_subflow_context *subflow;
	349	struct mptcp_sock *msk;
	350	struct sock *sk;
f870fa0b	351
cec37a6e	352	subflow = mptcp_subflow_ctx(ssk);
f870fa0b	353
cec37a6e PK	354	if (!subflow->mp_capable)
	355	return;
	356
	357	sk = subflow->conn;
	358	msk = mptcp_sk(sk);
	359
	360	/* the socket is not connected yet, no msk/subflow ops can access/race
	361	* accessing the field below
	362	*/
	363	WRITE_ONCE(msk->remote_key, subflow->remote_key);
	364	WRITE_ONCE(msk->local_key, subflow->local_key);
79c0949e	365	WRITE_ONCE(msk->token, subflow->token);
f870fa0b MM	366	}
f870fa0b MM	367
cf7da0d6 PK	368	static void mptcp_sock_graft(struct sock sk, struct socket parent)
	369	{
	370	write_lock_bh(&sk->sk_callback_lock);
	371	rcu_assign_pointer(sk->sk_wq, &parent->wq);
	372	sk_set_socket(sk, parent);
	373	sk->sk_uid = SOCK_INODE(parent)->i_uid;
	374	write_unlock_bh(&sk->sk_callback_lock);
	375	}
	376
f870fa0b MM	377	static struct proto mptcp_prot = {
	378	.name = "MPTCP",
	379	.owner = THIS_MODULE,
	380	.init = mptcp_init_sock,
	381	.close = mptcp_close,
cf7da0d6	382	.accept = mptcp_accept,
f870fa0b	383	.shutdown = tcp_shutdown,
79c0949e	384	.destroy = mptcp_destroy,
f870fa0b MM	385	.sendmsg = mptcp_sendmsg,
	386	.recvmsg = mptcp_recvmsg,
	387	.hash = inet_hash,
	388	.unhash = inet_unhash,
cec37a6e	389	.get_port = mptcp_get_port,
f870fa0b MM	390	.obj_size = sizeof(struct mptcp_sock),
	391	.no_autobind = true,
	392	};
	393
2303f994 PK	394	static int mptcp_bind(struct socket sock, struct sockaddr uaddr, int addr_len)
	395	{
	396	struct mptcp_sock *msk = mptcp_sk(sock->sk);
	397	struct socket *ssock;
cf7da0d6	398	int err;
2303f994 PK	399
	400	lock_sock(sock->sk);
	401	ssock = __mptcp_socket_create(msk, MPTCP_SAME_STATE);
	402	if (IS_ERR(ssock)) {
	403	err = PTR_ERR(ssock);
	404	goto unlock;
	405	}
	406
	407	err = ssock->ops->bind(ssock, uaddr, addr_len);
cf7da0d6 PK	408	if (!err)
cf7da0d6 PK	409	mptcp_copy_inaddrs(sock->sk, ssock->sk);
2303f994 PK	410
	411	unlock:
	412	release_sock(sock->sk);
	413	return err;
	414	}
	415
	416	static int mptcp_stream_connect(struct socket sock, struct sockaddr uaddr,
	417	int addr_len, int flags)
	418	{
	419	struct mptcp_sock *msk = mptcp_sk(sock->sk);
	420	struct socket *ssock;
	421	int err;
	422
	423	lock_sock(sock->sk);
	424	ssock = __mptcp_socket_create(msk, TCP_SYN_SENT);
	425	if (IS_ERR(ssock)) {
	426	err = PTR_ERR(ssock);
	427	goto unlock;
	428	}
	429
cf7da0d6 PK	430	#ifdef CONFIG_TCP_MD5SIG
	431	/* no MPTCP if MD5SIG is enabled on this socket or we may run out of
	432	* TCP option space.
	433	*/
	434	if (rcu_access_pointer(tcp_sk(ssock->sk)->md5sig_info))
	435	mptcp_subflow_ctx(ssock->sk)->request_mptcp = 0;
	436	#endif
	437
2303f994 PK	438	err = ssock->ops->connect(ssock, uaddr, addr_len, flags);
2303f994 PK	439	inet_sk_state_store(sock->sk, inet_sk_state_load(ssock->sk));
cf7da0d6	440	mptcp_copy_inaddrs(sock->sk, ssock->sk);
2303f994 PK	441
	442	unlock:
	443	release_sock(sock->sk);
	444	return err;
	445	}
	446
cf7da0d6 PK	447	static int mptcp_v4_getname(struct socket sock, struct sockaddr uaddr,
	448	int peer)
	449	{
	450	if (sock->sk->sk_prot == &tcp_prot) {
	451	/* we are being invoked from __sys_accept4, after
	452	* mptcp_accept() has just accepted a non-mp-capable
	453	* flow: sk is a tcp_sk, not an mptcp one.
	454	*
	455	* Hand the socket over to tcp so all further socket ops
	456	* bypass mptcp.
	457	*/
	458	sock->ops = &inet_stream_ops;
	459	}
	460
	461	return inet_getname(sock, uaddr, peer);
	462	}
	463
	464	#if IS_ENABLED(CONFIG_MPTCP_IPV6)
	465	static int mptcp_v6_getname(struct socket sock, struct sockaddr uaddr,
	466	int peer)
	467	{
	468	if (sock->sk->sk_prot == &tcpv6_prot) {
	469	/* we are being invoked from __sys_accept4 after
	470	* mptcp_accept() has accepted a non-mp-capable
	471	* subflow: sk is a tcp_sk, not mptcp.
	472	*
	473	* Hand the socket over to tcp so all further
	474	* socket ops bypass mptcp.
	475	*/
	476	sock->ops = &inet6_stream_ops;
	477	}
	478
	479	return inet6_getname(sock, uaddr, peer);
	480	}
	481	#endif
	482
	483	static int mptcp_listen(struct socket *sock, int backlog)
	484	{
	485	struct mptcp_sock *msk = mptcp_sk(sock->sk);
	486	struct socket *ssock;
	487	int err;
	488
	489	pr_debug("msk=%p", msk);
	490
	491	lock_sock(sock->sk);
	492	ssock = __mptcp_socket_create(msk, TCP_LISTEN);
	493	if (IS_ERR(ssock)) {
	494	err = PTR_ERR(ssock);
	495	goto unlock;
	496	}
	497
	498	err = ssock->ops->listen(ssock, backlog);
	499	inet_sk_state_store(sock->sk, inet_sk_state_load(ssock->sk));
	500	if (!err)
	501	mptcp_copy_inaddrs(sock->sk, ssock->sk);
	502
	503	unlock:
	504	release_sock(sock->sk);
	505	return err;
	506	}
	507
	508	static bool is_tcp_proto(const struct proto *p)
	509	{
	510	#if IS_ENABLED(CONFIG_MPTCP_IPV6)
511	return p == &tcp_prot \|\| p == &tcpv6_prot;
512	#else
513	return p == &tcp_prot;
514	#endif
515	}
516
517	static int mptcp_stream_accept(struct socket sock, struct socket newsock,
518	int flags, bool kern)
519	{
520	struct mptcp_sock *msk = mptcp_sk(sock->sk);
521	struct socket *ssock;
522	int err;
523
524	pr_debug("msk=%p", msk);
525
526	lock_sock(sock->sk);
527	if (sock->sk->sk_state != TCP_LISTEN)
528	goto unlock_fail;
529
530	ssock = __mptcp_nmpc_socket(msk);
531	if (!ssock)
532	goto unlock_fail;
533
534	sock_hold(ssock->sk);
535	release_sock(sock->sk);
536
537	err = ssock->ops->accept(sock, newsock, flags, kern);
538	if (err == 0 && !is_tcp_proto(newsock->sk->sk_prot)) {
539	struct mptcp_sock *msk = mptcp_sk(newsock->sk);
540	struct mptcp_subflow_context *subflow;
541
542	/* set ssk->sk_socket of accept()ed flows to mptcp socket.
543	* This is needed so NOSPACE flag can be set from tcp stack.
544	*/
545	list_for_each_entry(subflow, &msk->conn_list, node) {
546	struct sock *ssk = mptcp_subflow_tcp_sock(subflow);
547
548	if (!ssk->sk_socket)
549	mptcp_sock_graft(ssk, newsock);
550	}
551
552	inet_sk_state_store(newsock->sk, TCP_ESTABLISHED);
553	}
554
555	sock_put(ssock->sk);
556	return err;
557
558	unlock_fail:
559	release_sock(sock->sk);
560	return -EINVAL;
561	}
562
2303f994 PK	563	static __poll_t mptcp_poll(struct file file, struct socket sock,
	564	struct poll_table_struct *wait)
	565	{
	566	__poll_t mask = 0;
	567
	568	return mask;
	569	}
	570
21498490 PK	571	static int mptcp_shutdown(struct socket *sock, int how)
	572	{
	573	struct mptcp_sock *msk = mptcp_sk(sock->sk);
	574	struct mptcp_subflow_context *subflow;
	575	int ret = 0;
	576
	577	pr_debug("sk=%p, how=%d", msk, how);
	578
	579	lock_sock(sock->sk);
	580
	581	if (how == SHUT_WR \|\| how == SHUT_RDWR)
	582	inet_sk_state_store(sock->sk, TCP_FIN_WAIT1);
	583
	584	how++;
	585
	586	if ((how & ~SHUTDOWN_MASK) \|\| !how) {
	587	ret = -EINVAL;
	588	goto out_unlock;
	589	}
	590
	591	if (sock->state == SS_CONNECTING) {
	592	if ((1 << sock->sk->sk_state) &
	593	(TCPF_SYN_SENT \| TCPF_SYN_RECV \| TCPF_CLOSE))
	594	sock->state = SS_DISCONNECTING;
	595	else
	596	sock->state = SS_CONNECTED;
	597	}
	598
	599	mptcp_for_each_subflow(msk, subflow) {
	600	struct sock *tcp_sk = mptcp_subflow_tcp_sock(subflow);
	601
	602	mptcp_subflow_shutdown(tcp_sk, how);
	603	}
	604
	605	out_unlock:
	606	release_sock(sock->sk);
	607
	608	return ret;
	609	}
	610
2303f994 PK	611	static struct proto_ops mptcp_stream_ops;
2303f994 PK	612
f870fa0b MM	613	static struct inet_protosw mptcp_protosw = {
	614	.type = SOCK_STREAM,
	615	.protocol = IPPROTO_MPTCP,
	616	.prot = &mptcp_prot,
2303f994 PK	617	.ops = &mptcp_stream_ops,
2303f994 PK	618	.flags = INET_PROTOSW_ICSK,
f870fa0b MM	619	};
	620
	621	void __init mptcp_init(void)
	622	{
2303f994 PK	623	mptcp_prot.h.hashinfo = tcp_prot.h.hashinfo;
	624	mptcp_stream_ops = inet_stream_ops;
	625	mptcp_stream_ops.bind = mptcp_bind;
	626	mptcp_stream_ops.connect = mptcp_stream_connect;
	627	mptcp_stream_ops.poll = mptcp_poll;
cf7da0d6 PK	628	mptcp_stream_ops.accept = mptcp_stream_accept;
	629	mptcp_stream_ops.getname = mptcp_v4_getname;
	630	mptcp_stream_ops.listen = mptcp_listen;
21498490	631	mptcp_stream_ops.shutdown = mptcp_shutdown;
2303f994 PK	632
	633	mptcp_subflow_init();
	634
f870fa0b MM	635	if (proto_register(&mptcp_prot, 1) != 0)
	636	panic("Failed to register MPTCP proto.\n");
	637
	638	inet_register_protosw(&mptcp_protosw);
	639	}
	640
	641	#if IS_ENABLED(CONFIG_MPTCP_IPV6)
2303f994	642	static struct proto_ops mptcp_v6_stream_ops;
f870fa0b MM	643	static struct proto mptcp_v6_prot;
f870fa0b MM	644
79c0949e PK	645	static void mptcp_v6_destroy(struct sock *sk)
	646	{
	647	mptcp_destroy(sk);
	648	inet6_destroy_sock(sk);
	649	}
	650
f870fa0b MM	651	static struct inet_protosw mptcp_v6_protosw = {
	652	.type = SOCK_STREAM,
	653	.protocol = IPPROTO_MPTCP,
	654	.prot = &mptcp_v6_prot,
2303f994	655	.ops = &mptcp_v6_stream_ops,
f870fa0b MM	656	.flags = INET_PROTOSW_ICSK,
	657	};
	658
	659	int mptcpv6_init(void)
	660	{
	661	int err;
	662
	663	mptcp_v6_prot = mptcp_prot;
	664	strcpy(mptcp_v6_prot.name, "MPTCPv6");
	665	mptcp_v6_prot.slab = NULL;
79c0949e	666	mptcp_v6_prot.destroy = mptcp_v6_destroy;
f870fa0b MM	667	mptcp_v6_prot.obj_size = sizeof(struct mptcp_sock) +
	668	sizeof(struct ipv6_pinfo);
	669
	670	err = proto_register(&mptcp_v6_prot, 1);
	671	if (err)
	672	return err;
	673
2303f994 PK	674	mptcp_v6_stream_ops = inet6_stream_ops;
	675	mptcp_v6_stream_ops.bind = mptcp_bind;
	676	mptcp_v6_stream_ops.connect = mptcp_stream_connect;
	677	mptcp_v6_stream_ops.poll = mptcp_poll;
cf7da0d6 PK	678	mptcp_v6_stream_ops.accept = mptcp_stream_accept;
	679	mptcp_v6_stream_ops.getname = mptcp_v6_getname;
	680	mptcp_v6_stream_ops.listen = mptcp_listen;
21498490	681	mptcp_v6_stream_ops.shutdown = mptcp_shutdown;
2303f994	682
f870fa0b MM	683	err = inet6_register_protosw(&mptcp_v6_protosw);
	684	if (err)
	685	proto_unregister(&mptcp_v6_prot);
	686
	687	return err;
	688	}
	689	#endif