[linux-block.git] / net / mptcp / pm.c

// SPDX-License-Identifier: GPL-2.0
/* Multipath TCP
 *
 * Copyright (c) 2019, Intel Corporation.
 */
#define pr_fmt(fmt) "MPTCP: " fmt

#include <linux/kernel.h>
#include <net/mptcp.h>
#include "protocol.h"

#include "mib.h"

/* path manager command handlers */

int mptcp_pm_announce_addr(struct mptcp_sock *msk,
			   const struct mptcp_addr_info *addr,
			   bool echo)
{
	u8 add_addr = READ_ONCE(msk->pm.addr_signal);

	pr_debug("msk=%p, local_id=%d, echo=%d\n", msk, addr->id, echo);

	lockdep_assert_held(&msk->pm.lock);

	if (add_addr &
	    (echo ? BIT(MPTCP_ADD_ADDR_ECHO) : BIT(MPTCP_ADD_ADDR_SIGNAL))) {
		MPTCP_INC_STATS(sock_net((struct sock *)msk),
				echo ? MPTCP_MIB_ECHOADDTXDROP : MPTCP_MIB_ADDADDRTXDROP);
		return -EINVAL;
	}

	if (echo) {
		msk->pm.remote = *addr;
		add_addr |= BIT(MPTCP_ADD_ADDR_ECHO);
	} else {
		msk->pm.local = *addr;
		add_addr |= BIT(MPTCP_ADD_ADDR_SIGNAL);
	}
	WRITE_ONCE(msk->pm.addr_signal, add_addr);
	return 0;
}

int mptcp_pm_remove_addr(struct mptcp_sock *msk, const struct mptcp_rm_list *rm_list)
{
	u8 rm_addr = READ_ONCE(msk->pm.addr_signal);

	pr_debug("msk=%p, rm_list_nr=%d\n", msk, rm_list->nr);

	if (rm_addr) {
		MPTCP_ADD_STATS(sock_net((struct sock *)msk),
				MPTCP_MIB_RMADDRTXDROP, rm_list->nr);
		return -EINVAL;
	}

	msk->pm.rm_list_tx = *rm_list;
	rm_addr |= BIT(MPTCP_RM_ADDR_SIGNAL);
	WRITE_ONCE(msk->pm.addr_signal, rm_addr);
	mptcp_pm_nl_addr_send_ack(msk);
	return 0;
}

/* path manager event handlers */

void mptcp_pm_new_connection(struct mptcp_sock *msk, const struct sock *ssk, int server_side)
{
	struct mptcp_pm_data *pm = &msk->pm;

	pr_debug("msk=%p, token=%u side=%d\n", msk, READ_ONCE(msk->token), server_side);

	WRITE_ONCE(pm->server_side, server_side);
	mptcp_event(MPTCP_EVENT_CREATED, msk, ssk, GFP_ATOMIC);
}

bool mptcp_pm_allow_new_subflow(struct mptcp_sock *msk)
{
	struct mptcp_pm_data *pm = &msk->pm;
	unsigned int subflows_max;
	int ret = 0;

	if (mptcp_pm_is_userspace(msk)) {
		if (mptcp_userspace_pm_active(msk)) {
			spin_lock_bh(&pm->lock);
			pm->subflows++;
			spin_unlock_bh(&pm->lock);
			return true;
		}
		return false;
	}

	subflows_max = mptcp_pm_get_subflows_max(msk);

	pr_debug("msk=%p subflows=%d max=%d allow=%d\n", msk, pm->subflows,
		 subflows_max, READ_ONCE(pm->accept_subflow));

	/* try to avoid acquiring the lock below */
	if (!READ_ONCE(pm->accept_subflow))
		return false;

	spin_lock_bh(&pm->lock);
	if (READ_ONCE(pm->accept_subflow)) {
		ret = pm->subflows < subflows_max;
		if (ret && ++pm->subflows == subflows_max)
			WRITE_ONCE(pm->accept_subflow, false);
	}
	spin_unlock_bh(&pm->lock);

	return ret;
}

/* return true if the new status bit is currently cleared, that is, this event
 * can be server, eventually by an already scheduled work
 */
static bool mptcp_pm_schedule_work(struct mptcp_sock *msk,
				   enum mptcp_pm_status new_status)
{
	pr_debug("msk=%p status=%x new=%lx\n", msk, msk->pm.status,
		 BIT(new_status));
	if (msk->pm.status & BIT(new_status))
		return false;

	msk->pm.status |= BIT(new_status);
	mptcp_schedule_work((struct sock *)msk);
	return true;
}

void mptcp_pm_fully_established(struct mptcp_sock *msk, const struct sock *ssk)
{
	struct mptcp_pm_data *pm = &msk->pm;
	bool announce = false;

	pr_debug("msk=%p\n", msk);

	spin_lock_bh(&pm->lock);

	/* mptcp_pm_fully_established() can be invoked by multiple
	 * racing paths - accept() and check_fully_established()
	 * be sure to serve this event only once.
	 */
	if (READ_ONCE(pm->work_pending) &&
	    !(msk->pm.status & BIT(MPTCP_PM_ALREADY_ESTABLISHED)))
		mptcp_pm_schedule_work(msk, MPTCP_PM_ESTABLISHED);

	if ((msk->pm.status & BIT(MPTCP_PM_ALREADY_ESTABLISHED)) == 0)
		announce = true;

	msk->pm.status |= BIT(MPTCP_PM_ALREADY_ESTABLISHED);
	spin_unlock_bh(&pm->lock);

	if (announce)
		mptcp_event(MPTCP_EVENT_ESTABLISHED, msk, ssk, GFP_ATOMIC);
}

void mptcp_pm_connection_closed(struct mptcp_sock *msk)
{
	pr_debug("msk=%p\n", msk);
}

void mptcp_pm_subflow_established(struct mptcp_sock *msk)
{
	struct mptcp_pm_data *pm = &msk->pm;

	pr_debug("msk=%p\n", msk);

	if (!READ_ONCE(pm->work_pending))
		return;

	spin_lock_bh(&pm->lock);

	if (READ_ONCE(pm->work_pending))
		mptcp_pm_schedule_work(msk, MPTCP_PM_SUBFLOW_ESTABLISHED);

	spin_unlock_bh(&pm->lock);
}

void mptcp_pm_subflow_check_next(struct mptcp_sock *msk,
				 const struct mptcp_subflow_context *subflow)
{
	struct mptcp_pm_data *pm = &msk->pm;
	bool update_subflows;

	update_subflows = subflow->request_join || subflow->mp_join;
	if (mptcp_pm_is_userspace(msk)) {
		if (update_subflows) {
			spin_lock_bh(&pm->lock);
			pm->subflows--;
			spin_unlock_bh(&pm->lock);
		}
		return;
	}

	if (!READ_ONCE(pm->work_pending) && !update_subflows)
		return;

	spin_lock_bh(&pm->lock);
	if (update_subflows)
		__mptcp_pm_close_subflow(msk);

	/* Even if this subflow is not really established, tell the PM to try
	 * to pick the next ones, if possible.
	 */
	if (mptcp_pm_nl_check_work_pending(msk))
		mptcp_pm_schedule_work(msk, MPTCP_PM_SUBFLOW_ESTABLISHED);

	spin_unlock_bh(&pm->lock);
}

void mptcp_pm_add_addr_received(const struct sock *ssk,
				const struct mptcp_addr_info *addr)
{
	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
	struct mptcp_sock *msk = mptcp_sk(subflow->conn);
	struct mptcp_pm_data *pm = &msk->pm;

	pr_debug("msk=%p remote_id=%d accept=%d\n", msk, addr->id,
		 READ_ONCE(pm->accept_addr));

	mptcp_event_addr_announced(ssk, addr);

	spin_lock_bh(&pm->lock);

	if (mptcp_pm_is_userspace(msk)) {
		if (mptcp_userspace_pm_active(msk)) {
			mptcp_pm_announce_addr(msk, addr, true);
			mptcp_pm_add_addr_send_ack(msk);
		} else {
			__MPTCP_INC_STATS(sock_net((struct sock *)msk), MPTCP_MIB_ADDADDRDROP);
		}
	/* id0 should not have a different address */
	} else if ((addr->id == 0 && !mptcp_pm_nl_is_init_remote_addr(msk, addr)) ||
		   (addr->id > 0 && !READ_ONCE(pm->accept_addr))) {
		mptcp_pm_announce_addr(msk, addr, true);
		mptcp_pm_add_addr_send_ack(msk);
	} else if (mptcp_pm_schedule_work(msk, MPTCP_PM_ADD_ADDR_RECEIVED)) {
		pm->remote = *addr;
	} else {
		__MPTCP_INC_STATS(sock_net((struct sock *)msk), MPTCP_MIB_ADDADDRDROP);
	}

	spin_unlock_bh(&pm->lock);
}

void mptcp_pm_add_addr_echoed(struct mptcp_sock *msk,
			      const struct mptcp_addr_info *addr)
{
	struct mptcp_pm_data *pm = &msk->pm;

	pr_debug("msk=%p\n", msk);

	spin_lock_bh(&pm->lock);

	if (mptcp_lookup_anno_list_by_saddr(msk, addr) && READ_ONCE(pm->work_pending))
		mptcp_pm_schedule_work(msk, MPTCP_PM_SUBFLOW_ESTABLISHED);

	spin_unlock_bh(&pm->lock);
}

void mptcp_pm_add_addr_send_ack(struct mptcp_sock *msk)
{
	if (!mptcp_pm_should_add_signal(msk))
		return;

	mptcp_pm_schedule_work(msk, MPTCP_PM_ADD_ADDR_SEND_ACK);
}

void mptcp_pm_rm_addr_received(struct mptcp_sock *msk,
			       const struct mptcp_rm_list *rm_list)
{
	struct mptcp_pm_data *pm = &msk->pm;
	u8 i;

	pr_debug("msk=%p remote_ids_nr=%d\n", msk, rm_list->nr);

	for (i = 0; i < rm_list->nr; i++)
		mptcp_event_addr_removed(msk, rm_list->ids[i]);

	spin_lock_bh(&pm->lock);
	if (mptcp_pm_schedule_work(msk, MPTCP_PM_RM_ADDR_RECEIVED))
		pm->rm_list_rx = *rm_list;
	else
		__MPTCP_INC_STATS(sock_net((struct sock *)msk), MPTCP_MIB_RMADDRDROP);
	spin_unlock_bh(&pm->lock);
}

void mptcp_pm_mp_prio_received(struct sock *ssk, u8 bkup)
{
	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
	struct sock *sk = subflow->conn;
	struct mptcp_sock *msk;

	pr_debug("subflow->backup=%d, bkup=%d\n", subflow->backup, bkup);
	msk = mptcp_sk(sk);
	if (subflow->backup != bkup)
		subflow->backup = bkup;

	mptcp_event(MPTCP_EVENT_SUB_PRIORITY, msk, ssk, GFP_ATOMIC);
}

void mptcp_pm_mp_fail_received(struct sock *sk, u64 fail_seq)
{
	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
	struct mptcp_sock *msk = mptcp_sk(subflow->conn);

	pr_debug("fail_seq=%llu\n", fail_seq);

	if (!READ_ONCE(msk->allow_infinite_fallback))
		return;

	if (!subflow->fail_tout) {
		pr_debug("send MP_FAIL response and infinite map\n");

		subflow->send_mp_fail = 1;
		subflow->send_infinite_map = 1;
		tcp_send_ack(sk);
	} else {
		pr_debug("MP_FAIL response received\n");
		WRITE_ONCE(subflow->fail_tout, 0);
	}
}

/* path manager helpers */

bool mptcp_pm_add_addr_signal(struct mptcp_sock *msk, const struct sk_buff *skb,
			      unsigned int opt_size, unsigned int remaining,
			      struct mptcp_addr_info *addr, bool *echo,
			      bool *drop_other_suboptions)
{
	int ret = false;
	u8 add_addr;
	u8 family;
	bool port;

	spin_lock_bh(&msk->pm.lock);

	/* double check after the lock is acquired */
	if (!mptcp_pm_should_add_signal(msk))
		goto out_unlock;

	/* always drop every other options for pure ack ADD_ADDR; this is a
	 * plain dup-ack from TCP perspective. The other MPTCP-relevant info,
	 * if any, will be carried by the 'original' TCP ack
	 */
	if (skb && skb_is_tcp_pure_ack(skb)) {
		remaining += opt_size;
		*drop_other_suboptions = true;
	}

	*echo = mptcp_pm_should_add_signal_echo(msk);
	port = !!(*echo ? msk->pm.remote.port : msk->pm.local.port);

	family = *echo ? msk->pm.remote.family : msk->pm.local.family;
	if (remaining < mptcp_add_addr_len(family, *echo, port))
		goto out_unlock;

	if (*echo) {
		*addr = msk->pm.remote;
		add_addr = msk->pm.addr_signal & ~BIT(MPTCP_ADD_ADDR_ECHO);
	} else {
		*addr = msk->pm.local;
		add_addr = msk->pm.addr_signal & ~BIT(MPTCP_ADD_ADDR_SIGNAL);
	}
	WRITE_ONCE(msk->pm.addr_signal, add_addr);
	ret = true;

out_unlock:
	spin_unlock_bh(&msk->pm.lock);
	return ret;
}

bool mptcp_pm_rm_addr_signal(struct mptcp_sock *msk, unsigned int remaining,
			     struct mptcp_rm_list *rm_list)
{
	int ret = false, len;
	u8 rm_addr;

	spin_lock_bh(&msk->pm.lock);

	/* double check after the lock is acquired */
	if (!mptcp_pm_should_rm_signal(msk))
		goto out_unlock;

	rm_addr = msk->pm.addr_signal & ~BIT(MPTCP_RM_ADDR_SIGNAL);
	len = mptcp_rm_addr_len(&msk->pm.rm_list_tx);
	if (len < 0) {
		WRITE_ONCE(msk->pm.addr_signal, rm_addr);
		goto out_unlock;
	}
	if (remaining < len)
		goto out_unlock;

	*rm_list = msk->pm.rm_list_tx;
	WRITE_ONCE(msk->pm.addr_signal, rm_addr);
	ret = true;

out_unlock:
	spin_unlock_bh(&msk->pm.lock);
	return ret;
}

int mptcp_pm_get_local_id(struct mptcp_sock *msk, struct sock_common *skc)
{
	struct mptcp_addr_info skc_local;
	struct mptcp_addr_info msk_local;

	if (WARN_ON_ONCE(!msk))
		return -1;

	/* The 0 ID mapping is defined by the first subflow, copied into the msk
	 * addr
	 */
	mptcp_local_address((struct sock_common *)msk, &msk_local);
	mptcp_local_address((struct sock_common *)skc, &skc_local);
	if (mptcp_addresses_equal(&msk_local, &skc_local, false))
		return 0;

	if (mptcp_pm_is_userspace(msk))
		return mptcp_userspace_pm_get_local_id(msk, &skc_local);
	return mptcp_pm_nl_get_local_id(msk, &skc_local);
}

bool mptcp_pm_is_backup(struct mptcp_sock *msk, struct sock_common *skc)
{
	struct mptcp_addr_info skc_local;

	mptcp_local_address((struct sock_common *)skc, &skc_local);

	if (mptcp_pm_is_userspace(msk))
		return mptcp_userspace_pm_is_backup(msk, &skc_local);

	return mptcp_pm_nl_is_backup(msk, &skc_local);
}

int mptcp_pm_get_addr(struct sk_buff *skb, struct genl_info *info)
{
	if (info->attrs[MPTCP_PM_ATTR_TOKEN])
		return mptcp_userspace_pm_get_addr(skb, info);
	return mptcp_pm_nl_get_addr(skb, info);
}

int mptcp_pm_dump_addr(struct sk_buff *msg, struct netlink_callback *cb)
{
	const struct genl_info *info = genl_info_dump(cb);

	if (info->attrs[MPTCP_PM_ATTR_TOKEN])
		return mptcp_userspace_pm_dump_addr(msg, cb);
	return mptcp_pm_nl_dump_addr(msg, cb);
}

int mptcp_pm_set_flags(struct sk_buff *skb, struct genl_info *info)
{
	if (info->attrs[MPTCP_PM_ATTR_TOKEN])
		return mptcp_userspace_pm_set_flags(skb, info);
	return mptcp_pm_nl_set_flags(skb, info);
}

void mptcp_pm_subflow_chk_stale(const struct mptcp_sock *msk, struct sock *ssk)
{
	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
	u32 rcv_tstamp = READ_ONCE(tcp_sk(ssk)->rcv_tstamp);

	/* keep track of rtx periods with no progress */
	if (!subflow->stale_count) {
		subflow->stale_rcv_tstamp = rcv_tstamp;
		subflow->stale_count++;
	} else if (subflow->stale_rcv_tstamp == rcv_tstamp) {
		if (subflow->stale_count < U8_MAX)
			subflow->stale_count++;
		mptcp_pm_nl_subflow_chk_stale(msk, ssk);
	} else {
		subflow->stale_count = 0;
		mptcp_subflow_set_active(subflow);
	}
}

/* if sk is ipv4 or ipv6_only allows only same-family local and remote addresses,
 * otherwise allow any matching local/remote pair
 */
bool mptcp_pm_addr_families_match(const struct sock *sk,
				  const struct mptcp_addr_info *loc,
				  const struct mptcp_addr_info *rem)
{
	bool mptcp_is_v4 = sk->sk_family == AF_INET;

#if IS_ENABLED(CONFIG_MPTCP_IPV6)
	bool loc_is_v4 = loc->family == AF_INET || ipv6_addr_v4mapped(&loc->addr6);
	bool rem_is_v4 = rem->family == AF_INET || ipv6_addr_v4mapped(&rem->addr6);

	if (mptcp_is_v4)
		return loc_is_v4 && rem_is_v4;

	if (ipv6_only_sock(sk))
		return !loc_is_v4 && !rem_is_v4;

	return loc_is_v4 == rem_is_v4;
#else
	return mptcp_is_v4 && loc->family == AF_INET && rem->family == AF_INET;
#endif
}

void mptcp_pm_data_reset(struct mptcp_sock *msk)
{
	u8 pm_type = mptcp_get_pm_type(sock_net((struct sock *)msk));
	struct mptcp_pm_data *pm = &msk->pm;

	pm->add_addr_signaled = 0;
	pm->add_addr_accepted = 0;
	pm->local_addr_used = 0;
	pm->subflows = 0;
	pm->rm_list_tx.nr = 0;
	pm->rm_list_rx.nr = 0;
	WRITE_ONCE(pm->pm_type, pm_type);

	if (pm_type == MPTCP_PM_TYPE_KERNEL) {
		bool subflows_allowed = !!mptcp_pm_get_subflows_max(msk);

		/* pm->work_pending must be only be set to 'true' when
		 * pm->pm_type is set to MPTCP_PM_TYPE_KERNEL
		 */
		WRITE_ONCE(pm->work_pending,
			   (!!mptcp_pm_get_local_addr_max(msk) &&
			    subflows_allowed) ||
			   !!mptcp_pm_get_add_addr_signal_max(msk));
		WRITE_ONCE(pm->accept_addr,
			   !!mptcp_pm_get_add_addr_accept_max(msk) &&
			   subflows_allowed);
		WRITE_ONCE(pm->accept_subflow, subflows_allowed);
	} else {
		WRITE_ONCE(pm->work_pending, 0);
		WRITE_ONCE(pm->accept_addr, 0);
		WRITE_ONCE(pm->accept_subflow, 0);
	}

	WRITE_ONCE(pm->addr_signal, 0);
	WRITE_ONCE(pm->remote_deny_join_id0, false);
	pm->status = 0;
	bitmap_fill(msk->pm.id_avail_bitmap, MPTCP_PM_MAX_ADDR_ID + 1);
}

void mptcp_pm_data_init(struct mptcp_sock *msk)
{
	spin_lock_init(&msk->pm.lock);
	INIT_LIST_HEAD(&msk->pm.anno_list);
	INIT_LIST_HEAD(&msk->pm.userspace_pm_local_addr_list);
	mptcp_pm_data_reset(msk);
}

void __init mptcp_pm_init(void)
{
	mptcp_pm_nl_init();
}
Commit	Line	Data
	1	// SPDX-License-Identifier: GPL-2.0
	2	/* Multipath TCP
	3	*
	4	* Copyright (c) 2019, Intel Corporation.
	5	*/
	6	#define pr_fmt(fmt) "MPTCP: " fmt
	7
	8	#include <linux/kernel.h>
	9	#include <net/mptcp.h>
	10	#include "protocol.h"
	11
	12	#include "mib.h"
	13
	14	/* path manager command handlers */
	15
	16	int mptcp_pm_announce_addr(struct mptcp_sock *msk,
	17	const struct mptcp_addr_info *addr,
	18	bool echo)
	19	{
	20	u8 add_addr = READ_ONCE(msk->pm.addr_signal);
	21
	22	pr_debug("msk=%p, local_id=%d, echo=%d\n", msk, addr->id, echo);
	23
	24	lockdep_assert_held(&msk->pm.lock);
	25
	26	if (add_addr &
	27	(echo ? BIT(MPTCP_ADD_ADDR_ECHO) : BIT(MPTCP_ADD_ADDR_SIGNAL))) {
	28	MPTCP_INC_STATS(sock_net((struct sock *)msk),
	29	echo ? MPTCP_MIB_ECHOADDTXDROP : MPTCP_MIB_ADDADDRTXDROP);
	30	return -EINVAL;
	31	}
	32
	33	if (echo) {
	34	msk->pm.remote = *addr;
	35	add_addr \|= BIT(MPTCP_ADD_ADDR_ECHO);
	36	} else {
	37	msk->pm.local = *addr;
	38	add_addr \|= BIT(MPTCP_ADD_ADDR_SIGNAL);
	39	}
	40	WRITE_ONCE(msk->pm.addr_signal, add_addr);
	41	return 0;
	42	}
	43
	44	int mptcp_pm_remove_addr(struct mptcp_sock msk, const struct mptcp_rm_list rm_list)
	45	{
	46	u8 rm_addr = READ_ONCE(msk->pm.addr_signal);
	47
	48	pr_debug("msk=%p, rm_list_nr=%d\n", msk, rm_list->nr);
	49
	50	if (rm_addr) {
	51	MPTCP_ADD_STATS(sock_net((struct sock *)msk),
	52	MPTCP_MIB_RMADDRTXDROP, rm_list->nr);
	53	return -EINVAL;
	54	}
	55
	56	msk->pm.rm_list_tx = *rm_list;
	57	rm_addr \|= BIT(MPTCP_RM_ADDR_SIGNAL);
	58	WRITE_ONCE(msk->pm.addr_signal, rm_addr);
	59	mptcp_pm_nl_addr_send_ack(msk);
	60	return 0;
	61	}
	62
	63	/* path manager event handlers */
	64
	65	void mptcp_pm_new_connection(struct mptcp_sock msk, const struct sock ssk, int server_side)
	66	{
	67	struct mptcp_pm_data *pm = &msk->pm;
	68
	69	pr_debug("msk=%p, token=%u side=%d\n", msk, READ_ONCE(msk->token), server_side);
	70
	71	WRITE_ONCE(pm->server_side, server_side);
	72	mptcp_event(MPTCP_EVENT_CREATED, msk, ssk, GFP_ATOMIC);
	73	}
	74
	75	bool mptcp_pm_allow_new_subflow(struct mptcp_sock *msk)
	76	{
	77	struct mptcp_pm_data *pm = &msk->pm;
	78	unsigned int subflows_max;
	79	int ret = 0;
	80
	81	if (mptcp_pm_is_userspace(msk)) {
	82	if (mptcp_userspace_pm_active(msk)) {
	83	spin_lock_bh(&pm->lock);
	84	pm->subflows++;
	85	spin_unlock_bh(&pm->lock);
	86	return true;
	87	}
	88	return false;
	89	}
	90
	91	subflows_max = mptcp_pm_get_subflows_max(msk);
	92
	93	pr_debug("msk=%p subflows=%d max=%d allow=%d\n", msk, pm->subflows,
	94	subflows_max, READ_ONCE(pm->accept_subflow));
	95
	96	/* try to avoid acquiring the lock below */
	97	if (!READ_ONCE(pm->accept_subflow))
	98	return false;
	99
	100	spin_lock_bh(&pm->lock);
	101	if (READ_ONCE(pm->accept_subflow)) {
	102	ret = pm->subflows < subflows_max;
	103	if (ret && ++pm->subflows == subflows_max)
	104	WRITE_ONCE(pm->accept_subflow, false);
	105	}
	106	spin_unlock_bh(&pm->lock);
	107
	108	return ret;
	109	}
	110
	111	/* return true if the new status bit is currently cleared, that is, this event
	112	* can be server, eventually by an already scheduled work
	113	*/
	114	static bool mptcp_pm_schedule_work(struct mptcp_sock *msk,
	115	enum mptcp_pm_status new_status)
	116	{
	117	pr_debug("msk=%p status=%x new=%lx\n", msk, msk->pm.status,
	118	BIT(new_status));
	119	if (msk->pm.status & BIT(new_status))
	120	return false;
	121
	122	msk->pm.status \|= BIT(new_status);
	123	mptcp_schedule_work((struct sock *)msk);
	124	return true;
	125	}
	126
	127	void mptcp_pm_fully_established(struct mptcp_sock msk, const struct sock ssk)
	128	{
	129	struct mptcp_pm_data *pm = &msk->pm;
	130	bool announce = false;
	131
	132	pr_debug("msk=%p\n", msk);
	133
	134	spin_lock_bh(&pm->lock);
	135
	136	/* mptcp_pm_fully_established() can be invoked by multiple
	137	* racing paths - accept() and check_fully_established()
	138	* be sure to serve this event only once.
	139	*/
	140	if (READ_ONCE(pm->work_pending) &&
	141	!(msk->pm.status & BIT(MPTCP_PM_ALREADY_ESTABLISHED)))
	142	mptcp_pm_schedule_work(msk, MPTCP_PM_ESTABLISHED);
	143
	144	if ((msk->pm.status & BIT(MPTCP_PM_ALREADY_ESTABLISHED)) == 0)
	145	announce = true;
	146
	147	msk->pm.status \|= BIT(MPTCP_PM_ALREADY_ESTABLISHED);
	148	spin_unlock_bh(&pm->lock);
	149
	150	if (announce)
	151	mptcp_event(MPTCP_EVENT_ESTABLISHED, msk, ssk, GFP_ATOMIC);
	152	}
	153
	154	void mptcp_pm_connection_closed(struct mptcp_sock *msk)
	155	{
	156	pr_debug("msk=%p\n", msk);
	157	}
	158
	159	void mptcp_pm_subflow_established(struct mptcp_sock *msk)
	160	{
	161	struct mptcp_pm_data *pm = &msk->pm;
	162
	163	pr_debug("msk=%p\n", msk);
	164
	165	if (!READ_ONCE(pm->work_pending))
	166	return;
	167
	168	spin_lock_bh(&pm->lock);
	169
	170	if (READ_ONCE(pm->work_pending))
	171	mptcp_pm_schedule_work(msk, MPTCP_PM_SUBFLOW_ESTABLISHED);
	172
	173	spin_unlock_bh(&pm->lock);
	174	}
	175
	176	void mptcp_pm_subflow_check_next(struct mptcp_sock *msk,
	177	const struct mptcp_subflow_context *subflow)
	178	{
	179	struct mptcp_pm_data *pm = &msk->pm;
	180	bool update_subflows;
	181
	182	update_subflows = subflow->request_join \|\| subflow->mp_join;
	183	if (mptcp_pm_is_userspace(msk)) {
	184	if (update_subflows) {
	185	spin_lock_bh(&pm->lock);
	186	pm->subflows--;
	187	spin_unlock_bh(&pm->lock);
	188	}
	189	return;
	190	}
	191
	192	if (!READ_ONCE(pm->work_pending) && !update_subflows)
	193	return;
	194
	195	spin_lock_bh(&pm->lock);
	196	if (update_subflows)
	197	__mptcp_pm_close_subflow(msk);
	198
	199	/* Even if this subflow is not really established, tell the PM to try
	200	* to pick the next ones, if possible.
	201	*/
	202	if (mptcp_pm_nl_check_work_pending(msk))
	203	mptcp_pm_schedule_work(msk, MPTCP_PM_SUBFLOW_ESTABLISHED);
	204
	205	spin_unlock_bh(&pm->lock);
	206	}
	207
	208	void mptcp_pm_add_addr_received(const struct sock *ssk,
	209	const struct mptcp_addr_info *addr)
	210	{
	211	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
	212	struct mptcp_sock *msk = mptcp_sk(subflow->conn);
	213	struct mptcp_pm_data *pm = &msk->pm;
	214
	215	pr_debug("msk=%p remote_id=%d accept=%d\n", msk, addr->id,
	216	READ_ONCE(pm->accept_addr));
	217
	218	mptcp_event_addr_announced(ssk, addr);
	219
	220	spin_lock_bh(&pm->lock);
	221
	222	if (mptcp_pm_is_userspace(msk)) {
	223	if (mptcp_userspace_pm_active(msk)) {
	224	mptcp_pm_announce_addr(msk, addr, true);
	225	mptcp_pm_add_addr_send_ack(msk);
	226	} else {
	227	__MPTCP_INC_STATS(sock_net((struct sock *)msk), MPTCP_MIB_ADDADDRDROP);
	228	}
	229	/* id0 should not have a different address */
	230	} else if ((addr->id == 0 && !mptcp_pm_nl_is_init_remote_addr(msk, addr)) \|\|
	231	(addr->id > 0 && !READ_ONCE(pm->accept_addr))) {
	232	mptcp_pm_announce_addr(msk, addr, true);
	233	mptcp_pm_add_addr_send_ack(msk);
	234	} else if (mptcp_pm_schedule_work(msk, MPTCP_PM_ADD_ADDR_RECEIVED)) {
	235	pm->remote = *addr;
	236	} else {
	237	__MPTCP_INC_STATS(sock_net((struct sock *)msk), MPTCP_MIB_ADDADDRDROP);
	238	}
	239
	240	spin_unlock_bh(&pm->lock);
	241	}
	242
	243	void mptcp_pm_add_addr_echoed(struct mptcp_sock *msk,
	244	const struct mptcp_addr_info *addr)
	245	{
	246	struct mptcp_pm_data *pm = &msk->pm;
	247
	248	pr_debug("msk=%p\n", msk);
	249
	250	spin_lock_bh(&pm->lock);
	251
	252	if (mptcp_lookup_anno_list_by_saddr(msk, addr) && READ_ONCE(pm->work_pending))
	253	mptcp_pm_schedule_work(msk, MPTCP_PM_SUBFLOW_ESTABLISHED);
	254
	255	spin_unlock_bh(&pm->lock);
	256	}
	257
	258	void mptcp_pm_add_addr_send_ack(struct mptcp_sock *msk)
	259	{
	260	if (!mptcp_pm_should_add_signal(msk))
	261	return;
	262
	263	mptcp_pm_schedule_work(msk, MPTCP_PM_ADD_ADDR_SEND_ACK);
	264	}
	265
	266	void mptcp_pm_rm_addr_received(struct mptcp_sock *msk,
	267	const struct mptcp_rm_list *rm_list)
	268	{
	269	struct mptcp_pm_data *pm = &msk->pm;
	270	u8 i;
	271
	272	pr_debug("msk=%p remote_ids_nr=%d\n", msk, rm_list->nr);
	273
	274	for (i = 0; i < rm_list->nr; i++)
	275	mptcp_event_addr_removed(msk, rm_list->ids[i]);
	276
	277	spin_lock_bh(&pm->lock);
	278	if (mptcp_pm_schedule_work(msk, MPTCP_PM_RM_ADDR_RECEIVED))
	279	pm->rm_list_rx = *rm_list;
	280	else
	281	__MPTCP_INC_STATS(sock_net((struct sock *)msk), MPTCP_MIB_RMADDRDROP);
	282	spin_unlock_bh(&pm->lock);
	283	}
	284
	285	void mptcp_pm_mp_prio_received(struct sock *ssk, u8 bkup)
	286	{
	287	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
	288	struct sock *sk = subflow->conn;
	289	struct mptcp_sock *msk;
	290
	291	pr_debug("subflow->backup=%d, bkup=%d\n", subflow->backup, bkup);
	292	msk = mptcp_sk(sk);
	293	if (subflow->backup != bkup)
	294	subflow->backup = bkup;
	295
	296	mptcp_event(MPTCP_EVENT_SUB_PRIORITY, msk, ssk, GFP_ATOMIC);
	297	}
	298
	299	void mptcp_pm_mp_fail_received(struct sock *sk, u64 fail_seq)
	300	{
	301	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
	302	struct mptcp_sock *msk = mptcp_sk(subflow->conn);
	303
	304	pr_debug("fail_seq=%llu\n", fail_seq);
	305
	306	if (!READ_ONCE(msk->allow_infinite_fallback))
	307	return;
	308
	309	if (!subflow->fail_tout) {
	310	pr_debug("send MP_FAIL response and infinite map\n");
	311
	312	subflow->send_mp_fail = 1;
	313	subflow->send_infinite_map = 1;
	314	tcp_send_ack(sk);
	315	} else {
	316	pr_debug("MP_FAIL response received\n");
	317	WRITE_ONCE(subflow->fail_tout, 0);
	318	}
	319	}
	320
	321	/* path manager helpers */
	322
	323	bool mptcp_pm_add_addr_signal(struct mptcp_sock msk, const struct sk_buff skb,
	324	unsigned int opt_size, unsigned int remaining,
	325	struct mptcp_addr_info addr, bool echo,
	326	bool *drop_other_suboptions)
	327	{
	328	int ret = false;
	329	u8 add_addr;
	330	u8 family;
	331	bool port;
	332
	333	spin_lock_bh(&msk->pm.lock);
	334
	335	/* double check after the lock is acquired */
	336	if (!mptcp_pm_should_add_signal(msk))
	337	goto out_unlock;
	338
	339	/* always drop every other options for pure ack ADD_ADDR; this is a
	340	* plain dup-ack from TCP perspective. The other MPTCP-relevant info,
	341	* if any, will be carried by the 'original' TCP ack
	342	*/
	343	if (skb && skb_is_tcp_pure_ack(skb)) {
	344	remaining += opt_size;
	345	*drop_other_suboptions = true;
	346	}
	347
	348	*echo = mptcp_pm_should_add_signal_echo(msk);
	349	port = !!(*echo ? msk->pm.remote.port : msk->pm.local.port);
	350
	351	family = *echo ? msk->pm.remote.family : msk->pm.local.family;
	352	if (remaining < mptcp_add_addr_len(family, *echo, port))
	353	goto out_unlock;
	354
	355	if (*echo) {
	356	*addr = msk->pm.remote;
	357	add_addr = msk->pm.addr_signal & ~BIT(MPTCP_ADD_ADDR_ECHO);
	358	} else {
	359	*addr = msk->pm.local;
	360	add_addr = msk->pm.addr_signal & ~BIT(MPTCP_ADD_ADDR_SIGNAL);
	361	}
	362	WRITE_ONCE(msk->pm.addr_signal, add_addr);
	363	ret = true;
	364
	365	out_unlock:
	366	spin_unlock_bh(&msk->pm.lock);
	367	return ret;
	368	}
	369
	370	bool mptcp_pm_rm_addr_signal(struct mptcp_sock *msk, unsigned int remaining,
	371	struct mptcp_rm_list *rm_list)
	372	{
	373	int ret = false, len;
	374	u8 rm_addr;
	375
	376	spin_lock_bh(&msk->pm.lock);
	377
	378	/* double check after the lock is acquired */
	379	if (!mptcp_pm_should_rm_signal(msk))
	380	goto out_unlock;
	381
	382	rm_addr = msk->pm.addr_signal & ~BIT(MPTCP_RM_ADDR_SIGNAL);
	383	len = mptcp_rm_addr_len(&msk->pm.rm_list_tx);
	384	if (len < 0) {
	385	WRITE_ONCE(msk->pm.addr_signal, rm_addr);
	386	goto out_unlock;
	387	}
	388	if (remaining < len)
	389	goto out_unlock;
	390
	391	*rm_list = msk->pm.rm_list_tx;
	392	WRITE_ONCE(msk->pm.addr_signal, rm_addr);
	393	ret = true;
	394
	395	out_unlock:
	396	spin_unlock_bh(&msk->pm.lock);
	397	return ret;
	398	}
	399
	400	int mptcp_pm_get_local_id(struct mptcp_sock msk, struct sock_common skc)
	401	{
	402	struct mptcp_addr_info skc_local;
	403	struct mptcp_addr_info msk_local;
	404
	405	if (WARN_ON_ONCE(!msk))
	406	return -1;
	407
	408	/* The 0 ID mapping is defined by the first subflow, copied into the msk
	409	* addr
	410	*/
	411	mptcp_local_address((struct sock_common *)msk, &msk_local);
	412	mptcp_local_address((struct sock_common *)skc, &skc_local);
	413	if (mptcp_addresses_equal(&msk_local, &skc_local, false))
	414	return 0;
	415
	416	if (mptcp_pm_is_userspace(msk))
	417	return mptcp_userspace_pm_get_local_id(msk, &skc_local);
	418	return mptcp_pm_nl_get_local_id(msk, &skc_local);
	419	}
	420
	421	bool mptcp_pm_is_backup(struct mptcp_sock msk, struct sock_common skc)
	422	{
	423	struct mptcp_addr_info skc_local;
	424
	425	mptcp_local_address((struct sock_common *)skc, &skc_local);
	426
	427	if (mptcp_pm_is_userspace(msk))
	428	return mptcp_userspace_pm_is_backup(msk, &skc_local);
	429
	430	return mptcp_pm_nl_is_backup(msk, &skc_local);
	431	}
	432
	433	int mptcp_pm_get_addr(struct sk_buff skb, struct genl_info info)
	434	{
	435	if (info->attrs[MPTCP_PM_ATTR_TOKEN])
	436	return mptcp_userspace_pm_get_addr(skb, info);
	437	return mptcp_pm_nl_get_addr(skb, info);
	438	}
	439
	440	int mptcp_pm_dump_addr(struct sk_buff msg, struct netlink_callback cb)
	441	{
	442	const struct genl_info *info = genl_info_dump(cb);
	443
	444	if (info->attrs[MPTCP_PM_ATTR_TOKEN])
	445	return mptcp_userspace_pm_dump_addr(msg, cb);
	446	return mptcp_pm_nl_dump_addr(msg, cb);
	447	}
	448
	449	int mptcp_pm_set_flags(struct sk_buff skb, struct genl_info info)
	450	{
	451	if (info->attrs[MPTCP_PM_ATTR_TOKEN])
	452	return mptcp_userspace_pm_set_flags(skb, info);
	453	return mptcp_pm_nl_set_flags(skb, info);
	454	}
	455
	456	void mptcp_pm_subflow_chk_stale(const struct mptcp_sock msk, struct sock ssk)
	457	{
	458	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(ssk);
	459	u32 rcv_tstamp = READ_ONCE(tcp_sk(ssk)->rcv_tstamp);
	460
	461	/* keep track of rtx periods with no progress */
	462	if (!subflow->stale_count) {
	463	subflow->stale_rcv_tstamp = rcv_tstamp;
	464	subflow->stale_count++;
	465	} else if (subflow->stale_rcv_tstamp == rcv_tstamp) {
	466	if (subflow->stale_count < U8_MAX)
	467	subflow->stale_count++;
	468	mptcp_pm_nl_subflow_chk_stale(msk, ssk);
	469	} else {
	470	subflow->stale_count = 0;
	471	mptcp_subflow_set_active(subflow);
	472	}
	473	}
	474
	475	/* if sk is ipv4 or ipv6_only allows only same-family local and remote addresses,
	476	* otherwise allow any matching local/remote pair
	477	*/
	478	bool mptcp_pm_addr_families_match(const struct sock *sk,
	479	const struct mptcp_addr_info *loc,
	480	const struct mptcp_addr_info *rem)
	481	{
	482	bool mptcp_is_v4 = sk->sk_family == AF_INET;
	483
	484	#if IS_ENABLED(CONFIG_MPTCP_IPV6)
	485	bool loc_is_v4 = loc->family == AF_INET \|\| ipv6_addr_v4mapped(&loc->addr6);
	486	bool rem_is_v4 = rem->family == AF_INET \|\| ipv6_addr_v4mapped(&rem->addr6);
	487
	488	if (mptcp_is_v4)
	489	return loc_is_v4 && rem_is_v4;
	490
	491	if (ipv6_only_sock(sk))
	492	return !loc_is_v4 && !rem_is_v4;
	493
	494	return loc_is_v4 == rem_is_v4;
	495	#else
	496	return mptcp_is_v4 && loc->family == AF_INET && rem->family == AF_INET;
	497	#endif
	498	}
	499
	500	void mptcp_pm_data_reset(struct mptcp_sock *msk)
	501	{
	502	u8 pm_type = mptcp_get_pm_type(sock_net((struct sock *)msk));
	503	struct mptcp_pm_data *pm = &msk->pm;
	504
	505	pm->add_addr_signaled = 0;
	506	pm->add_addr_accepted = 0;
	507	pm->local_addr_used = 0;
	508	pm->subflows = 0;
	509	pm->rm_list_tx.nr = 0;
	510	pm->rm_list_rx.nr = 0;
	511	WRITE_ONCE(pm->pm_type, pm_type);
	512
	513	if (pm_type == MPTCP_PM_TYPE_KERNEL) {
	514	bool subflows_allowed = !!mptcp_pm_get_subflows_max(msk);
	515
	516	/* pm->work_pending must be only be set to 'true' when
	517	* pm->pm_type is set to MPTCP_PM_TYPE_KERNEL
	518	*/
	519	WRITE_ONCE(pm->work_pending,
	520	(!!mptcp_pm_get_local_addr_max(msk) &&
	521	subflows_allowed) \|\|
	522	!!mptcp_pm_get_add_addr_signal_max(msk));
	523	WRITE_ONCE(pm->accept_addr,
	524	!!mptcp_pm_get_add_addr_accept_max(msk) &&
	525	subflows_allowed);
	526	WRITE_ONCE(pm->accept_subflow, subflows_allowed);
	527	} else {
	528	WRITE_ONCE(pm->work_pending, 0);
	529	WRITE_ONCE(pm->accept_addr, 0);
	530	WRITE_ONCE(pm->accept_subflow, 0);
	531	}
	532
	533	WRITE_ONCE(pm->addr_signal, 0);
	534	WRITE_ONCE(pm->remote_deny_join_id0, false);
	535	pm->status = 0;
	536	bitmap_fill(msk->pm.id_avail_bitmap, MPTCP_PM_MAX_ADDR_ID + 1);
	537	}
	538
	539	void mptcp_pm_data_init(struct mptcp_sock *msk)
	540	{
	541	spin_lock_init(&msk->pm.lock);
	542	INIT_LIST_HEAD(&msk->pm.anno_list);
	543	INIT_LIST_HEAD(&msk->pm.userspace_pm_local_addr_list);
	544	mptcp_pm_data_reset(msk);
	545	}
	546
	547	void __init mptcp_pm_init(void)
	548	{
	549	mptcp_pm_nl_init();
	550	}