[linux-block.git] / drivers / net / wireguard / device.c

// SPDX-License-Identifier: GPL-2.0
/*
 * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
 */

#include "queueing.h"
#include "socket.h"
#include "timers.h"
#include "device.h"
#include "ratelimiter.h"
#include "peer.h"
#include "messages.h"

#include <linux/module.h>
#include <linux/rtnetlink.h>
#include <linux/inet.h>
#include <linux/netdevice.h>
#include <linux/inetdevice.h>
#include <linux/if_arp.h>
#include <linux/icmp.h>
#include <linux/suspend.h>
#include <net/icmp.h>
#include <net/rtnetlink.h>
#include <net/ip_tunnels.h>
#include <net/addrconf.h>

static LIST_HEAD(device_list);

static int wg_open(struct net_device *dev)
{
	struct in_device *dev_v4 = __in_dev_get_rtnl(dev);
	struct inet6_dev *dev_v6 = __in6_dev_get(dev);
	struct wg_device *wg = netdev_priv(dev);
	struct wg_peer *peer;
	int ret;

	if (dev_v4) {
		/* At some point we might put this check near the ip_rt_send_
		 * redirect call of ip_forward in net/ipv4/ip_forward.c, similar
		 * to the current secpath check.
		 */
		IN_DEV_CONF_SET(dev_v4, SEND_REDIRECTS, false);
		IPV4_DEVCONF_ALL(dev_net(dev), SEND_REDIRECTS) = false;
	}
	if (dev_v6)
		dev_v6->cnf.addr_gen_mode = IN6_ADDR_GEN_MODE_NONE;

	ret = wg_socket_init(wg, wg->incoming_port);
	if (ret < 0)
		return ret;
	mutex_lock(&wg->device_update_lock);
	list_for_each_entry(peer, &wg->peer_list, peer_list) {
		wg_packet_send_staged_packets(peer);
		if (peer->persistent_keepalive_interval)
			wg_packet_send_keepalive(peer);
	}
	mutex_unlock(&wg->device_update_lock);
	return 0;
}

#ifdef CONFIG_PM_SLEEP
static int wg_pm_notification(struct notifier_block *nb, unsigned long action,
			      void *data)
{
	struct wg_device *wg;
	struct wg_peer *peer;

	/* If the machine is constantly suspending and resuming, as part of
	 * its normal operation rather than as a somewhat rare event, then we
	 * don't actually want to clear keys.
	 */
	if (IS_ENABLED(CONFIG_PM_AUTOSLEEP) || IS_ENABLED(CONFIG_ANDROID))
		return 0;

	if (action != PM_HIBERNATION_PREPARE && action != PM_SUSPEND_PREPARE)
		return 0;

	rtnl_lock();
	list_for_each_entry(wg, &device_list, device_list) {
		mutex_lock(&wg->device_update_lock);
		list_for_each_entry(peer, &wg->peer_list, peer_list) {
			del_timer(&peer->timer_zero_key_material);
			wg_noise_handshake_clear(&peer->handshake);
			wg_noise_keypairs_clear(&peer->keypairs);
		}
		mutex_unlock(&wg->device_update_lock);
	}
	rtnl_unlock();
	rcu_barrier();
	return 0;
}

static struct notifier_block pm_notifier = { .notifier_call = wg_pm_notification };
#endif

static int wg_stop(struct net_device *dev)
{
	struct wg_device *wg = netdev_priv(dev);
	struct wg_peer *peer;

	mutex_lock(&wg->device_update_lock);
	list_for_each_entry(peer, &wg->peer_list, peer_list) {
		wg_packet_purge_staged_packets(peer);
		wg_timers_stop(peer);
		wg_noise_handshake_clear(&peer->handshake);
		wg_noise_keypairs_clear(&peer->keypairs);
		wg_noise_reset_last_sent_handshake(&peer->last_sent_handshake);
	}
	mutex_unlock(&wg->device_update_lock);
	skb_queue_purge(&wg->incoming_handshakes);
	wg_socket_reinit(wg, NULL, NULL);
	return 0;
}

static netdev_tx_t wg_xmit(struct sk_buff *skb, struct net_device *dev)
{
	struct wg_device *wg = netdev_priv(dev);
	struct sk_buff_head packets;
	struct wg_peer *peer;
	struct sk_buff *next;
	sa_family_t family;
	u32 mtu;
	int ret;

	if (unlikely(wg_skb_examine_untrusted_ip_hdr(skb) != skb->protocol)) {
		ret = -EPROTONOSUPPORT;
		net_dbg_ratelimited("%s: Invalid IP packet\n", dev->name);
		goto err;
	}

	peer = wg_allowedips_lookup_dst(&wg->peer_allowedips, skb);
	if (unlikely(!peer)) {
		ret = -ENOKEY;
		if (skb->protocol == htons(ETH_P_IP))
			net_dbg_ratelimited("%s: No peer has allowed IPs matching %pI4\n",
					    dev->name, &ip_hdr(skb)->daddr);
		else if (skb->protocol == htons(ETH_P_IPV6))
			net_dbg_ratelimited("%s: No peer has allowed IPs matching %pI6\n",
					    dev->name, &ipv6_hdr(skb)->daddr);
		goto err;
	}

	family = READ_ONCE(peer->endpoint.addr.sa_family);
	if (unlikely(family != AF_INET && family != AF_INET6)) {
		ret = -EDESTADDRREQ;
		net_dbg_ratelimited("%s: No valid endpoint has been configured or discovered for peer %llu\n",
				    dev->name, peer->internal_id);
		goto err_peer;
	}

	mtu = skb_dst(skb) ? dst_mtu(skb_dst(skb)) : dev->mtu;

	__skb_queue_head_init(&packets);
	if (!skb_is_gso(skb)) {
		skb_mark_not_on_list(skb);
	} else {
		struct sk_buff *segs = skb_gso_segment(skb, 0);

		if (unlikely(IS_ERR(segs))) {
			ret = PTR_ERR(segs);
			goto err_peer;
		}
		dev_kfree_skb(skb);
		skb = segs;
	}

	skb_list_walk_safe(skb, skb, next) {
		skb_mark_not_on_list(skb);

		skb = skb_share_check(skb, GFP_ATOMIC);
		if (unlikely(!skb))
			continue;

		/* We only need to keep the original dst around for icmp,
		 * so at this point we're in a position to drop it.
		 */
		skb_dst_drop(skb);

		PACKET_CB(skb)->mtu = mtu;

		__skb_queue_tail(&packets, skb);
	}

	spin_lock_bh(&peer->staged_packet_queue.lock);
	/* If the queue is getting too big, we start removing the oldest packets
	 * until it's small again. We do this before adding the new packet, so
	 * we don't remove GSO segments that are in excess.
	 */
	while (skb_queue_len(&peer->staged_packet_queue) > MAX_STAGED_PACKETS) {
		dev_kfree_skb(__skb_dequeue(&peer->staged_packet_queue));
		++dev->stats.tx_dropped;
	}
	skb_queue_splice_tail(&packets, &peer->staged_packet_queue);
	spin_unlock_bh(&peer->staged_packet_queue.lock);

	wg_packet_send_staged_packets(peer);

	wg_peer_put(peer);
	return NETDEV_TX_OK;

err_peer:
	wg_peer_put(peer);
err:
	++dev->stats.tx_errors;
	if (skb->protocol == htons(ETH_P_IP))
		icmp_ndo_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_UNREACH, 0);
	else if (skb->protocol == htons(ETH_P_IPV6))
		icmpv6_ndo_send(skb, ICMPV6_DEST_UNREACH, ICMPV6_ADDR_UNREACH, 0);
	kfree_skb(skb);
	return ret;
}

static const struct net_device_ops netdev_ops = {
	.ndo_open		= wg_open,
	.ndo_stop		= wg_stop,
	.ndo_start_xmit		= wg_xmit,
	.ndo_get_stats64	= ip_tunnel_get_stats64
};

static void wg_destruct(struct net_device *dev)
{
	struct wg_device *wg = netdev_priv(dev);

	rtnl_lock();
	list_del(&wg->device_list);
	rtnl_unlock();
	mutex_lock(&wg->device_update_lock);
	wg->incoming_port = 0;
	wg_socket_reinit(wg, NULL, NULL);
	/* The final references are cleared in the below calls to destroy_workqueue. */
	wg_peer_remove_all(wg);
	destroy_workqueue(wg->handshake_receive_wq);
	destroy_workqueue(wg->handshake_send_wq);
	destroy_workqueue(wg->packet_crypt_wq);
	wg_packet_queue_free(&wg->decrypt_queue, true);
	wg_packet_queue_free(&wg->encrypt_queue, true);
	rcu_barrier(); /* Wait for all the peers to be actually freed. */
	wg_ratelimiter_uninit();
	memzero_explicit(&wg->static_identity, sizeof(wg->static_identity));
	skb_queue_purge(&wg->incoming_handshakes);
	free_percpu(dev->tstats);
	free_percpu(wg->incoming_handshakes_worker);
	if (wg->have_creating_net_ref)
		put_net(wg->creating_net);
	kvfree(wg->index_hashtable);
	kvfree(wg->peer_hashtable);
	mutex_unlock(&wg->device_update_lock);

	pr_debug("%s: Interface deleted\n", dev->name);
	free_netdev(dev);
}

static const struct device_type device_type = { .name = KBUILD_MODNAME };

static void wg_setup(struct net_device *dev)
{
	struct wg_device *wg = netdev_priv(dev);
	enum { WG_NETDEV_FEATURES = NETIF_F_HW_CSUM | NETIF_F_RXCSUM |
				    NETIF_F_SG | NETIF_F_GSO |
				    NETIF_F_GSO_SOFTWARE | NETIF_F_HIGHDMA };
	const int overhead = MESSAGE_MINIMUM_LENGTH + sizeof(struct udphdr) +
			     max(sizeof(struct ipv6hdr), sizeof(struct iphdr));

	dev->netdev_ops = &netdev_ops;
	dev->hard_header_len = 0;
	dev->addr_len = 0;
	dev->needed_headroom = DATA_PACKET_HEAD_ROOM;
	dev->needed_tailroom = noise_encrypted_len(MESSAGE_PADDING_MULTIPLE);
	dev->type = ARPHRD_NONE;
	dev->flags = IFF_POINTOPOINT | IFF_NOARP;
	dev->priv_flags |= IFF_NO_QUEUE;
	dev->features |= NETIF_F_LLTX;
	dev->features |= WG_NETDEV_FEATURES;
	dev->hw_features |= WG_NETDEV_FEATURES;
	dev->hw_enc_features |= WG_NETDEV_FEATURES;
	dev->mtu = ETH_DATA_LEN - overhead;
	dev->max_mtu = round_down(INT_MAX, MESSAGE_PADDING_MULTIPLE) - overhead;

	SET_NETDEV_DEVTYPE(dev, &device_type);

	/* We need to keep the dst around in case of icmp replies. */
	netif_keep_dst(dev);

	memset(wg, 0, sizeof(*wg));
	wg->dev = dev;
}

static int wg_newlink(struct net *src_net, struct net_device *dev,
		      struct nlattr *tb[], struct nlattr *data[],
		      struct netlink_ext_ack *extack)
{
	struct wg_device *wg = netdev_priv(dev);
	int ret = -ENOMEM;

	wg->creating_net = src_net;
	init_rwsem(&wg->static_identity.lock);
	mutex_init(&wg->socket_update_lock);
	mutex_init(&wg->device_update_lock);
	skb_queue_head_init(&wg->incoming_handshakes);
	wg_allowedips_init(&wg->peer_allowedips);
	wg_cookie_checker_init(&wg->cookie_checker, wg);
	INIT_LIST_HEAD(&wg->peer_list);
	wg->device_update_gen = 1;

	wg->peer_hashtable = wg_pubkey_hashtable_alloc();
	if (!wg->peer_hashtable)
		return ret;

	wg->index_hashtable = wg_index_hashtable_alloc();
	if (!wg->index_hashtable)
		goto err_free_peer_hashtable;

	dev->tstats = netdev_alloc_pcpu_stats(struct pcpu_sw_netstats);
	if (!dev->tstats)
		goto err_free_index_hashtable;

	wg->incoming_handshakes_worker =
		wg_packet_percpu_multicore_worker_alloc(
				wg_packet_handshake_receive_worker, wg);
	if (!wg->incoming_handshakes_worker)
		goto err_free_tstats;

	wg->handshake_receive_wq = alloc_workqueue("wg-kex-%s",
			WQ_CPU_INTENSIVE | WQ_FREEZABLE, 0, dev->name);
	if (!wg->handshake_receive_wq)
		goto err_free_incoming_handshakes;

	wg->handshake_send_wq = alloc_workqueue("wg-kex-%s",
			WQ_UNBOUND | WQ_FREEZABLE, 0, dev->name);
	if (!wg->handshake_send_wq)
		goto err_destroy_handshake_receive;

	wg->packet_crypt_wq = alloc_workqueue("wg-crypt-%s",
			WQ_CPU_INTENSIVE | WQ_MEM_RECLAIM, 0, dev->name);
	if (!wg->packet_crypt_wq)
		goto err_destroy_handshake_send;

	ret = wg_packet_queue_init(&wg->encrypt_queue, wg_packet_encrypt_worker,
				   true, MAX_QUEUED_PACKETS);
	if (ret < 0)
		goto err_destroy_packet_crypt;

	ret = wg_packet_queue_init(&wg->decrypt_queue, wg_packet_decrypt_worker,
				   true, MAX_QUEUED_PACKETS);
	if (ret < 0)
		goto err_free_encrypt_queue;

	ret = wg_ratelimiter_init();
	if (ret < 0)
		goto err_free_decrypt_queue;

	ret = register_netdevice(dev);
	if (ret < 0)
		goto err_uninit_ratelimiter;

	list_add(&wg->device_list, &device_list);

	/* We wait until the end to assign priv_destructor, so that
	 * register_netdevice doesn't call it for us if it fails.
	 */
	dev->priv_destructor = wg_destruct;

	pr_debug("%s: Interface created\n", dev->name);
	return ret;

err_uninit_ratelimiter:
	wg_ratelimiter_uninit();
err_free_decrypt_queue:
	wg_packet_queue_free(&wg->decrypt_queue, true);
err_free_encrypt_queue:
	wg_packet_queue_free(&wg->encrypt_queue, true);
err_destroy_packet_crypt:
	destroy_workqueue(wg->packet_crypt_wq);
err_destroy_handshake_send:
	destroy_workqueue(wg->handshake_send_wq);
err_destroy_handshake_receive:
	destroy_workqueue(wg->handshake_receive_wq);
err_free_incoming_handshakes:
	free_percpu(wg->incoming_handshakes_worker);
err_free_tstats:
	free_percpu(dev->tstats);
err_free_index_hashtable:
	kvfree(wg->index_hashtable);
err_free_peer_hashtable:
	kvfree(wg->peer_hashtable);
	return ret;
}

static struct rtnl_link_ops link_ops __read_mostly = {
	.kind			= KBUILD_MODNAME,
	.priv_size		= sizeof(struct wg_device),
	.setup			= wg_setup,
	.newlink		= wg_newlink,
};

static int wg_netdevice_notification(struct notifier_block *nb,
				     unsigned long action, void *data)
{
	struct net_device *dev = ((struct netdev_notifier_info *)data)->dev;
	struct wg_device *wg = netdev_priv(dev);

	ASSERT_RTNL();

	if (action != NETDEV_REGISTER || dev->netdev_ops != &netdev_ops)
		return 0;

	if (dev_net(dev) == wg->creating_net && wg->have_creating_net_ref) {
		put_net(wg->creating_net);
		wg->have_creating_net_ref = false;
	} else if (dev_net(dev) != wg->creating_net &&
		   !wg->have_creating_net_ref) {
		wg->have_creating_net_ref = true;
		get_net(wg->creating_net);
	}
	return 0;
}

static struct notifier_block netdevice_notifier = {
	.notifier_call = wg_netdevice_notification
};

int __init wg_device_init(void)
{
	int ret;

#ifdef CONFIG_PM_SLEEP
	ret = register_pm_notifier(&pm_notifier);
	if (ret)
		return ret;
#endif

	ret = register_netdevice_notifier(&netdevice_notifier);
	if (ret)
		goto error_pm;

	ret = rtnl_link_register(&link_ops);
	if (ret)
		goto error_netdevice;

	return 0;

error_netdevice:
	unregister_netdevice_notifier(&netdevice_notifier);
error_pm:
#ifdef CONFIG_PM_SLEEP
	unregister_pm_notifier(&pm_notifier);
#endif
	return ret;
}

void wg_device_uninit(void)
{
	rtnl_link_unregister(&link_ops);
	unregister_netdevice_notifier(&netdevice_notifier);
#ifdef CONFIG_PM_SLEEP
	unregister_pm_notifier(&pm_notifier);
#endif
	rcu_barrier();
}
Commit	Line	Data
e7096c13 JD	1	// SPDX-License-Identifier: GPL-2.0
	2	/*
	3	* Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
	4	*/
	5
	6	#include "queueing.h"
	7	#include "socket.h"
	8	#include "timers.h"
	9	#include "device.h"
	10	#include "ratelimiter.h"
	11	#include "peer.h"
	12	#include "messages.h"
	13
	14	#include <linux/module.h>
	15	#include <linux/rtnetlink.h>
	16	#include <linux/inet.h>
	17	#include <linux/netdevice.h>
	18	#include <linux/inetdevice.h>
	19	#include <linux/if_arp.h>
	20	#include <linux/icmp.h>
	21	#include <linux/suspend.h>
	22	#include <net/icmp.h>
	23	#include <net/rtnetlink.h>
	24	#include <net/ip_tunnels.h>
	25	#include <net/addrconf.h>
	26
	27	static LIST_HEAD(device_list);
	28
	29	static int wg_open(struct net_device *dev)
	30	{
	31	struct in_device *dev_v4 = __in_dev_get_rtnl(dev);
	32	struct inet6_dev *dev_v6 = __in6_dev_get(dev);
	33	struct wg_device *wg = netdev_priv(dev);
	34	struct wg_peer *peer;
	35	int ret;
	36
	37	if (dev_v4) {
	38	/* At some point we might put this check near the ip_rt_send_
	39	* redirect call of ip_forward in net/ipv4/ip_forward.c, similar
	40	* to the current secpath check.
	41	*/
	42	IN_DEV_CONF_SET(dev_v4, SEND_REDIRECTS, false);
	43	IPV4_DEVCONF_ALL(dev_net(dev), SEND_REDIRECTS) = false;
	44	}
	45	if (dev_v6)
	46	dev_v6->cnf.addr_gen_mode = IN6_ADDR_GEN_MODE_NONE;
	47
	48	ret = wg_socket_init(wg, wg->incoming_port);
	49	if (ret < 0)
	50	return ret;
	51	mutex_lock(&wg->device_update_lock);
	52	list_for_each_entry(peer, &wg->peer_list, peer_list) {
	53	wg_packet_send_staged_packets(peer);
	54	if (peer->persistent_keepalive_interval)
	55	wg_packet_send_keepalive(peer);
	56	}
	57	mutex_unlock(&wg->device_update_lock);
	58	return 0;
	59	}
	60
	61	#ifdef CONFIG_PM_SLEEP
	62	static int wg_pm_notification(struct notifier_block *nb, unsigned long action,
	63	void *data)
	64	{
65	struct wg_device *wg;
66	struct wg_peer *peer;
67
68	/* If the machine is constantly suspending and resuming, as part of
69	* its normal operation rather than as a somewhat rare event, then we
70	* don't actually want to clear keys.
71	*/
72	if (IS_ENABLED(CONFIG_PM_AUTOSLEEP) \|\| IS_ENABLED(CONFIG_ANDROID))
73	return 0;
74
75	if (action != PM_HIBERNATION_PREPARE && action != PM_SUSPEND_PREPARE)
76	return 0;
77
78	rtnl_lock();
79	list_for_each_entry(wg, &device_list, device_list) {
80	mutex_lock(&wg->device_update_lock);
81	list_for_each_entry(peer, &wg->peer_list, peer_list) {
82	del_timer(&peer->timer_zero_key_material);
83	wg_noise_handshake_clear(&peer->handshake);
84	wg_noise_keypairs_clear(&peer->keypairs);
85	}
86	mutex_unlock(&wg->device_update_lock);
87	}
88	rtnl_unlock();
89	rcu_barrier();
90	return 0;
91	}
92
93	static struct notifier_block pm_notifier = { .notifier_call = wg_pm_notification };
94	#endif
95
96	static int wg_stop(struct net_device *dev)
97	{
98	struct wg_device *wg = netdev_priv(dev);
99	struct wg_peer *peer;
100
101	mutex_lock(&wg->device_update_lock);
102	list_for_each_entry(peer, &wg->peer_list, peer_list) {
103	wg_packet_purge_staged_packets(peer);
104	wg_timers_stop(peer);
105	wg_noise_handshake_clear(&peer->handshake);
106	wg_noise_keypairs_clear(&peer->keypairs);
107	wg_noise_reset_last_sent_handshake(&peer->last_sent_handshake);
108	}
109	mutex_unlock(&wg->device_update_lock);
110	skb_queue_purge(&wg->incoming_handshakes);
111	wg_socket_reinit(wg, NULL, NULL);
112	return 0;
113	}
114
115	static netdev_tx_t wg_xmit(struct sk_buff skb, struct net_device dev)
116	{
117	struct wg_device *wg = netdev_priv(dev);
118	struct sk_buff_head packets;
119	struct wg_peer *peer;
120	struct sk_buff *next;
121	sa_family_t family;
122	u32 mtu;
123	int ret;
124
125	if (unlikely(wg_skb_examine_untrusted_ip_hdr(skb) != skb->protocol)) {
126	ret = -EPROTONOSUPPORT;
127	net_dbg_ratelimited("%s: Invalid IP packet\n", dev->name);
128	goto err;
129	}
130
131	peer = wg_allowedips_lookup_dst(&wg->peer_allowedips, skb);
132	if (unlikely(!peer)) {
133	ret = -ENOKEY;
134	if (skb->protocol == htons(ETH_P_IP))
135	net_dbg_ratelimited("%s: No peer has allowed IPs matching %pI4\n",
136	dev->name, &ip_hdr(skb)->daddr);
137	else if (skb->protocol == htons(ETH_P_IPV6))
138	net_dbg_ratelimited("%s: No peer has allowed IPs matching %pI6\n",
139	dev->name, &ipv6_hdr(skb)->daddr);
140	goto err;
141	}
142
143	family = READ_ONCE(peer->endpoint.addr.sa_family);
144	if (unlikely(family != AF_INET && family != AF_INET6)) {
145	ret = -EDESTADDRREQ;
146	net_dbg_ratelimited("%s: No valid endpoint has been configured or discovered for peer %llu\n",
147	dev->name, peer->internal_id);
148	goto err_peer;
149	}
150
151	mtu = skb_dst(skb) ? dst_mtu(skb_dst(skb)) : dev->mtu;
152
153	__skb_queue_head_init(&packets);
154	if (!skb_is_gso(skb)) {
155	skb_mark_not_on_list(skb);
156	} else {
157	struct sk_buff *segs = skb_gso_segment(skb, 0);
158
159	if (unlikely(IS_ERR(segs))) {
160	ret = PTR_ERR(segs);
161	goto err_peer;
162	}
163	dev_kfree_skb(skb);
164	skb = segs;
165	}
166
167	skb_list_walk_safe(skb, skb, next) {
168	skb_mark_not_on_list(skb);
169
170	skb = skb_share_check(skb, GFP_ATOMIC);
171	if (unlikely(!skb))
172	continue;
173
174	/* We only need to keep the original dst around for icmp,
175	* so at this point we're in a position to drop it.
176	*/
177	skb_dst_drop(skb);
178
179	PACKET_CB(skb)->mtu = mtu;
180
181	__skb_queue_tail(&packets, skb);
182	}
183
184	spin_lock_bh(&peer->staged_packet_queue.lock);
185	/* If the queue is getting too big, we start removing the oldest packets
186	* until it's small again. We do this before adding the new packet, so
187	* we don't remove GSO segments that are in excess.
188	*/
189	while (skb_queue_len(&peer->staged_packet_queue) > MAX_STAGED_PACKETS) {
190	dev_kfree_skb(__skb_dequeue(&peer->staged_packet_queue));
191	++dev->stats.tx_dropped;
192	}
193	skb_queue_splice_tail(&packets, &peer->staged_packet_queue);
194	spin_unlock_bh(&peer->staged_packet_queue.lock);
195
196	wg_packet_send_staged_packets(peer);
197
198	wg_peer_put(peer);
199	return NETDEV_TX_OK;
200
201	err_peer:
202	wg_peer_put(peer);
203	err:
204	++dev->stats.tx_errors;
205	if (skb->protocol == htons(ETH_P_IP))
a12d7f3c	206	icmp_ndo_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_UNREACH, 0);
e7096c13	207	else if (skb->protocol == htons(ETH_P_IPV6))
a12d7f3c	208	icmpv6_ndo_send(skb, ICMPV6_DEST_UNREACH, ICMPV6_ADDR_UNREACH, 0);
e7096c13 JD	209	kfree_skb(skb);
	210	return ret;
	211	}
	212
	213	static const struct net_device_ops netdev_ops = {
	214	.ndo_open = wg_open,
	215	.ndo_stop = wg_stop,
	216	.ndo_start_xmit = wg_xmit,
	217	.ndo_get_stats64 = ip_tunnel_get_stats64
	218	};
	219
	220	static void wg_destruct(struct net_device *dev)
	221	{
	222	struct wg_device *wg = netdev_priv(dev);
	223
	224	rtnl_lock();
	225	list_del(&wg->device_list);
	226	rtnl_unlock();
	227	mutex_lock(&wg->device_update_lock);
	228	wg->incoming_port = 0;
	229	wg_socket_reinit(wg, NULL, NULL);
	230	/* The final references are cleared in the below calls to destroy_workqueue. */
	231	wg_peer_remove_all(wg);
	232	destroy_workqueue(wg->handshake_receive_wq);
	233	destroy_workqueue(wg->handshake_send_wq);
	234	destroy_workqueue(wg->packet_crypt_wq);
	235	wg_packet_queue_free(&wg->decrypt_queue, true);
	236	wg_packet_queue_free(&wg->encrypt_queue, true);
	237	rcu_barrier(); /* Wait for all the peers to be actually freed. */
	238	wg_ratelimiter_uninit();
	239	memzero_explicit(&wg->static_identity, sizeof(wg->static_identity));
	240	skb_queue_purge(&wg->incoming_handshakes);
	241	free_percpu(dev->tstats);
	242	free_percpu(wg->incoming_handshakes_worker);
	243	if (wg->have_creating_net_ref)
	244	put_net(wg->creating_net);
	245	kvfree(wg->index_hashtable);
	246	kvfree(wg->peer_hashtable);
	247	mutex_unlock(&wg->device_update_lock);
	248
	249	pr_debug("%s: Interface deleted\n", dev->name);
	250	free_netdev(dev);
	251	}
	252
	253	static const struct device_type device_type = { .name = KBUILD_MODNAME };
	254
	255	static void wg_setup(struct net_device *dev)
	256	{
	257	struct wg_device *wg = netdev_priv(dev);
	258	enum { WG_NETDEV_FEATURES = NETIF_F_HW_CSUM \| NETIF_F_RXCSUM \|
	259	NETIF_F_SG \| NETIF_F_GSO \|
	260	NETIF_F_GSO_SOFTWARE \| NETIF_F_HIGHDMA };
175f1ca9 JD	261	const int overhead = MESSAGE_MINIMUM_LENGTH + sizeof(struct udphdr) +
175f1ca9 JD	262	max(sizeof(struct ipv6hdr), sizeof(struct iphdr));
e7096c13 JD	263
	264	dev->netdev_ops = &netdev_ops;
	265	dev->hard_header_len = 0;
	266	dev->addr_len = 0;
	267	dev->needed_headroom = DATA_PACKET_HEAD_ROOM;
	268	dev->needed_tailroom = noise_encrypted_len(MESSAGE_PADDING_MULTIPLE);
	269	dev->type = ARPHRD_NONE;
	270	dev->flags = IFF_POINTOPOINT \| IFF_NOARP;
	271	dev->priv_flags \|= IFF_NO_QUEUE;
	272	dev->features \|= NETIF_F_LLTX;
	273	dev->features \|= WG_NETDEV_FEATURES;
	274	dev->hw_features \|= WG_NETDEV_FEATURES;
	275	dev->hw_enc_features \|= WG_NETDEV_FEATURES;
175f1ca9 JD	276	dev->mtu = ETH_DATA_LEN - overhead;
175f1ca9 JD	277	dev->max_mtu = round_down(INT_MAX, MESSAGE_PADDING_MULTIPLE) - overhead;
e7096c13 JD	278
	279	SET_NETDEV_DEVTYPE(dev, &device_type);
	280
	281	/* We need to keep the dst around in case of icmp replies. */
	282	netif_keep_dst(dev);
	283
	284	memset(wg, 0, sizeof(*wg));
	285	wg->dev = dev;
	286	}
	287
	288	static int wg_newlink(struct net src_net, struct net_device dev,
	289	struct nlattr tb[], struct nlattr data[],
	290	struct netlink_ext_ack *extack)
	291	{
	292	struct wg_device *wg = netdev_priv(dev);
	293	int ret = -ENOMEM;
	294
	295	wg->creating_net = src_net;
	296	init_rwsem(&wg->static_identity.lock);
	297	mutex_init(&wg->socket_update_lock);
	298	mutex_init(&wg->device_update_lock);
	299	skb_queue_head_init(&wg->incoming_handshakes);
	300	wg_allowedips_init(&wg->peer_allowedips);
	301	wg_cookie_checker_init(&wg->cookie_checker, wg);
	302	INIT_LIST_HEAD(&wg->peer_list);
	303	wg->device_update_gen = 1;
	304
	305	wg->peer_hashtable = wg_pubkey_hashtable_alloc();
	306	if (!wg->peer_hashtable)
	307	return ret;
	308
	309	wg->index_hashtable = wg_index_hashtable_alloc();
	310	if (!wg->index_hashtable)
	311	goto err_free_peer_hashtable;
	312
	313	dev->tstats = netdev_alloc_pcpu_stats(struct pcpu_sw_netstats);
	314	if (!dev->tstats)
	315	goto err_free_index_hashtable;
	316
	317	wg->incoming_handshakes_worker =
	318	wg_packet_percpu_multicore_worker_alloc(
	319	wg_packet_handshake_receive_worker, wg);
	320	if (!wg->incoming_handshakes_worker)
	321	goto err_free_tstats;
	322
	323	wg->handshake_receive_wq = alloc_workqueue("wg-kex-%s",
	324	WQ_CPU_INTENSIVE \| WQ_FREEZABLE, 0, dev->name);
	325	if (!wg->handshake_receive_wq)
	326	goto err_free_incoming_handshakes;
	327
	328	wg->handshake_send_wq = alloc_workqueue("wg-kex-%s",
	329	WQ_UNBOUND \| WQ_FREEZABLE, 0, dev->name);
	330	if (!wg->handshake_send_wq)
	331	goto err_destroy_handshake_receive;
	332
	333	wg->packet_crypt_wq = alloc_workqueue("wg-crypt-%s",
	334	WQ_CPU_INTENSIVE \| WQ_MEM_RECLAIM, 0, dev->name);
	335	if (!wg->packet_crypt_wq)
	336	goto err_destroy_handshake_send;
	337
	338	ret = wg_packet_queue_init(&wg->encrypt_queue, wg_packet_encrypt_worker,
	339	true, MAX_QUEUED_PACKETS);
	340	if (ret < 0)
	341	goto err_destroy_packet_crypt;
342
343	ret = wg_packet_queue_init(&wg->decrypt_queue, wg_packet_decrypt_worker,
344	true, MAX_QUEUED_PACKETS);
345	if (ret < 0)
346	goto err_free_encrypt_queue;
347
348	ret = wg_ratelimiter_init();
349	if (ret < 0)
350	goto err_free_decrypt_queue;
351
352	ret = register_netdevice(dev);
353	if (ret < 0)
354	goto err_uninit_ratelimiter;
355
356	list_add(&wg->device_list, &device_list);
357
358	/* We wait until the end to assign priv_destructor, so that
359	* register_netdevice doesn't call it for us if it fails.
360	*/
361	dev->priv_destructor = wg_destruct;
362
363	pr_debug("%s: Interface created\n", dev->name);
364	return ret;
365
366	err_uninit_ratelimiter:
367	wg_ratelimiter_uninit();
368	err_free_decrypt_queue:
369	wg_packet_queue_free(&wg->decrypt_queue, true);
370	err_free_encrypt_queue:
371	wg_packet_queue_free(&wg->encrypt_queue, true);
372	err_destroy_packet_crypt:
373	destroy_workqueue(wg->packet_crypt_wq);
374	err_destroy_handshake_send:
375	destroy_workqueue(wg->handshake_send_wq);
376	err_destroy_handshake_receive:
377	destroy_workqueue(wg->handshake_receive_wq);
378	err_free_incoming_handshakes:
379	free_percpu(wg->incoming_handshakes_worker);
380	err_free_tstats:
381	free_percpu(dev->tstats);
382	err_free_index_hashtable:
383	kvfree(wg->index_hashtable);
384	err_free_peer_hashtable:
385	kvfree(wg->peer_hashtable);
386	return ret;
387	}
388
389	static struct rtnl_link_ops link_ops __read_mostly = {
390	.kind = KBUILD_MODNAME,
391	.priv_size = sizeof(struct wg_device),
392	.setup = wg_setup,
393	.newlink = wg_newlink,
394	};
395
396	static int wg_netdevice_notification(struct notifier_block *nb,
397	unsigned long action, void *data)
398	{
399	struct net_device dev = ((struct netdev_notifier_info )data)->dev;
400	struct wg_device *wg = netdev_priv(dev);
401
402	ASSERT_RTNL();
403
404	if (action != NETDEV_REGISTER \|\| dev->netdev_ops != &netdev_ops)
405	return 0;
406
407	if (dev_net(dev) == wg->creating_net && wg->have_creating_net_ref) {
408	put_net(wg->creating_net);
409	wg->have_creating_net_ref = false;
410	} else if (dev_net(dev) != wg->creating_net &&
411	!wg->have_creating_net_ref) {
412	wg->have_creating_net_ref = true;
413	get_net(wg->creating_net);
414	}
415	return 0;
416	}
417
418	static struct notifier_block netdevice_notifier = {
419	.notifier_call = wg_netdevice_notification
420	};
421
422	int __init wg_device_init(void)
423	{
424	int ret;
425
426	#ifdef CONFIG_PM_SLEEP
427	ret = register_pm_notifier(&pm_notifier);
428	if (ret)
429	return ret;
430	#endif
431
432	ret = register_netdevice_notifier(&netdevice_notifier);
433	if (ret)
434	goto error_pm;
435
436	ret = rtnl_link_register(&link_ops);
437	if (ret)
438	goto error_netdevice;
439
440	return 0;
441
442	error_netdevice:
443	unregister_netdevice_notifier(&netdevice_notifier);
444	error_pm:
445	#ifdef CONFIG_PM_SLEEP
446	unregister_pm_notifier(&pm_notifier);
447	#endif
448	return ret;
449	}
450
451	void wg_device_uninit(void)
452	{
453	rtnl_link_unregister(&link_ops);
454	unregister_netdevice_notifier(&netdevice_notifier);
455	#ifdef CONFIG_PM_SLEEP
456	unregister_pm_notifier(&pm_notifier);
457	#endif
458	rcu_barrier();
459	}