Commit | Line | Data |
---|---|---|
e7096c13 JD |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* | |
3 | * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved. | |
4 | */ | |
5 | ||
6 | #include "peer.h" | |
7 | #include "device.h" | |
8 | #include "queueing.h" | |
9 | #include "timers.h" | |
10 | #include "peerlookup.h" | |
11 | #include "noise.h" | |
12 | ||
13 | #include <linux/kref.h> | |
14 | #include <linux/lockdep.h> | |
15 | #include <linux/rcupdate.h> | |
16 | #include <linux/list.h> | |
17 | ||
18 | static atomic64_t peer_counter = ATOMIC64_INIT(0); | |
19 | ||
20 | struct wg_peer *wg_peer_create(struct wg_device *wg, | |
21 | const u8 public_key[NOISE_PUBLIC_KEY_LEN], | |
22 | const u8 preshared_key[NOISE_SYMMETRIC_KEY_LEN]) | |
23 | { | |
24 | struct wg_peer *peer; | |
25 | int ret = -ENOMEM; | |
26 | ||
27 | lockdep_assert_held(&wg->device_update_lock); | |
28 | ||
29 | if (wg->num_peers >= MAX_PEERS_PER_DEVICE) | |
30 | return ERR_PTR(ret); | |
31 | ||
32 | peer = kzalloc(sizeof(*peer), GFP_KERNEL); | |
33 | if (unlikely(!peer)) | |
34 | return ERR_PTR(ret); | |
35 | peer->device = wg; | |
36 | ||
11a7686a JD |
37 | wg_noise_handshake_init(&peer->handshake, &wg->static_identity, |
38 | public_key, preshared_key, peer); | |
e7096c13 JD |
39 | if (dst_cache_init(&peer->endpoint_cache, GFP_KERNEL)) |
40 | goto err_1; | |
41 | if (wg_packet_queue_init(&peer->tx_queue, wg_packet_tx_worker, false, | |
42 | MAX_QUEUED_PACKETS)) | |
43 | goto err_2; | |
44 | if (wg_packet_queue_init(&peer->rx_queue, NULL, false, | |
45 | MAX_QUEUED_PACKETS)) | |
46 | goto err_3; | |
47 | ||
48 | peer->internal_id = atomic64_inc_return(&peer_counter); | |
49 | peer->serial_work_cpu = nr_cpumask_bits; | |
50 | wg_cookie_init(&peer->latest_cookie); | |
51 | wg_timers_init(peer); | |
52 | wg_cookie_checker_precompute_peer_keys(peer); | |
53 | spin_lock_init(&peer->keypairs.keypair_update_lock); | |
54 | INIT_WORK(&peer->transmit_handshake_work, | |
55 | wg_packet_handshake_send_worker); | |
56 | rwlock_init(&peer->endpoint_lock); | |
57 | kref_init(&peer->refcount); | |
58 | skb_queue_head_init(&peer->staged_packet_queue); | |
59 | wg_noise_reset_last_sent_handshake(&peer->last_sent_handshake); | |
60 | set_bit(NAPI_STATE_NO_BUSY_POLL, &peer->napi.state); | |
61 | netif_napi_add(wg->dev, &peer->napi, wg_packet_rx_poll, | |
62 | NAPI_POLL_WEIGHT); | |
63 | napi_enable(&peer->napi); | |
64 | list_add_tail(&peer->peer_list, &wg->peer_list); | |
65 | INIT_LIST_HEAD(&peer->allowedips_list); | |
66 | wg_pubkey_hashtable_add(wg->peer_hashtable, peer); | |
67 | ++wg->num_peers; | |
68 | pr_debug("%s: Peer %llu created\n", wg->dev->name, peer->internal_id); | |
69 | return peer; | |
70 | ||
71 | err_3: | |
72 | wg_packet_queue_free(&peer->tx_queue, false); | |
73 | err_2: | |
74 | dst_cache_destroy(&peer->endpoint_cache); | |
75 | err_1: | |
76 | kfree(peer); | |
77 | return ERR_PTR(ret); | |
78 | } | |
79 | ||
80 | struct wg_peer *wg_peer_get_maybe_zero(struct wg_peer *peer) | |
81 | { | |
82 | RCU_LOCKDEP_WARN(!rcu_read_lock_bh_held(), | |
83 | "Taking peer reference without holding the RCU read lock"); | |
84 | if (unlikely(!peer || !kref_get_unless_zero(&peer->refcount))) | |
85 | return NULL; | |
86 | return peer; | |
87 | } | |
88 | ||
89 | static void peer_make_dead(struct wg_peer *peer) | |
90 | { | |
91 | /* Remove from configuration-time lookup structures. */ | |
92 | list_del_init(&peer->peer_list); | |
93 | wg_allowedips_remove_by_peer(&peer->device->peer_allowedips, peer, | |
94 | &peer->device->device_update_lock); | |
95 | wg_pubkey_hashtable_remove(peer->device->peer_hashtable, peer); | |
96 | ||
97 | /* Mark as dead, so that we don't allow jumping contexts after. */ | |
98 | WRITE_ONCE(peer->is_dead, true); | |
99 | ||
100 | /* The caller must now synchronize_rcu() for this to take effect. */ | |
101 | } | |
102 | ||
103 | static void peer_remove_after_dead(struct wg_peer *peer) | |
104 | { | |
105 | WARN_ON(!peer->is_dead); | |
106 | ||
107 | /* No more keypairs can be created for this peer, since is_dead protects | |
108 | * add_new_keypair, so we can now destroy existing ones. | |
109 | */ | |
110 | wg_noise_keypairs_clear(&peer->keypairs); | |
111 | ||
112 | /* Destroy all ongoing timers that were in-flight at the beginning of | |
113 | * this function. | |
114 | */ | |
115 | wg_timers_stop(peer); | |
116 | ||
117 | /* The transition between packet encryption/decryption queues isn't | |
118 | * guarded by is_dead, but each reference's life is strictly bounded by | |
119 | * two generations: once for parallel crypto and once for serial | |
120 | * ingestion, so we can simply flush twice, and be sure that we no | |
121 | * longer have references inside these queues. | |
122 | */ | |
123 | ||
124 | /* a) For encrypt/decrypt. */ | |
125 | flush_workqueue(peer->device->packet_crypt_wq); | |
126 | /* b.1) For send (but not receive, since that's napi). */ | |
127 | flush_workqueue(peer->device->packet_crypt_wq); | |
128 | /* b.2.1) For receive (but not send, since that's wq). */ | |
129 | napi_disable(&peer->napi); | |
130 | /* b.2.1) It's now safe to remove the napi struct, which must be done | |
131 | * here from process context. | |
132 | */ | |
133 | netif_napi_del(&peer->napi); | |
134 | ||
135 | /* Ensure any workstructs we own (like transmit_handshake_work or | |
136 | * clear_peer_work) no longer are in use. | |
137 | */ | |
138 | flush_workqueue(peer->device->handshake_send_wq); | |
139 | ||
140 | /* After the above flushes, a peer might still be active in a few | |
141 | * different contexts: 1) from xmit(), before hitting is_dead and | |
142 | * returning, 2) from wg_packet_consume_data(), before hitting is_dead | |
143 | * and returning, 3) from wg_receive_handshake_packet() after a point | |
144 | * where it has processed an incoming handshake packet, but where | |
145 | * all calls to pass it off to timers fails because of is_dead. We won't | |
146 | * have new references in (1) eventually, because we're removed from | |
147 | * allowedips; we won't have new references in (2) eventually, because | |
148 | * wg_index_hashtable_lookup will always return NULL, since we removed | |
149 | * all existing keypairs and no more can be created; we won't have new | |
150 | * references in (3) eventually, because we're removed from the pubkey | |
151 | * hash table, which allows for a maximum of one handshake response, | |
152 | * via the still-uncleared index hashtable entry, but not more than one, | |
153 | * and in wg_cookie_message_consume, the lookup eventually gets a peer | |
154 | * with a refcount of zero, so no new reference is taken. | |
155 | */ | |
156 | ||
157 | --peer->device->num_peers; | |
158 | wg_peer_put(peer); | |
159 | } | |
160 | ||
161 | /* We have a separate "remove" function make sure that all active places where | |
162 | * a peer is currently operating will eventually come to an end and not pass | |
163 | * their reference onto another context. | |
164 | */ | |
165 | void wg_peer_remove(struct wg_peer *peer) | |
166 | { | |
167 | if (unlikely(!peer)) | |
168 | return; | |
169 | lockdep_assert_held(&peer->device->device_update_lock); | |
170 | ||
171 | peer_make_dead(peer); | |
172 | synchronize_rcu(); | |
173 | peer_remove_after_dead(peer); | |
174 | } | |
175 | ||
176 | void wg_peer_remove_all(struct wg_device *wg) | |
177 | { | |
178 | struct wg_peer *peer, *temp; | |
179 | LIST_HEAD(dead_peers); | |
180 | ||
181 | lockdep_assert_held(&wg->device_update_lock); | |
182 | ||
183 | /* Avoid having to traverse individually for each one. */ | |
184 | wg_allowedips_free(&wg->peer_allowedips, &wg->device_update_lock); | |
185 | ||
186 | list_for_each_entry_safe(peer, temp, &wg->peer_list, peer_list) { | |
187 | peer_make_dead(peer); | |
188 | list_add_tail(&peer->peer_list, &dead_peers); | |
189 | } | |
190 | synchronize_rcu(); | |
191 | list_for_each_entry_safe(peer, temp, &dead_peers, peer_list) | |
192 | peer_remove_after_dead(peer); | |
193 | } | |
194 | ||
195 | static void rcu_release(struct rcu_head *rcu) | |
196 | { | |
197 | struct wg_peer *peer = container_of(rcu, struct wg_peer, rcu); | |
198 | ||
199 | dst_cache_destroy(&peer->endpoint_cache); | |
200 | wg_packet_queue_free(&peer->rx_queue, false); | |
201 | wg_packet_queue_free(&peer->tx_queue, false); | |
202 | ||
203 | /* The final zeroing takes care of clearing any remaining handshake key | |
204 | * material and other potentially sensitive information. | |
205 | */ | |
206 | kzfree(peer); | |
207 | } | |
208 | ||
209 | static void kref_release(struct kref *refcount) | |
210 | { | |
211 | struct wg_peer *peer = container_of(refcount, struct wg_peer, refcount); | |
212 | ||
213 | pr_debug("%s: Peer %llu (%pISpfsc) destroyed\n", | |
214 | peer->device->dev->name, peer->internal_id, | |
215 | &peer->endpoint.addr); | |
216 | ||
217 | /* Remove ourself from dynamic runtime lookup structures, now that the | |
218 | * last reference is gone. | |
219 | */ | |
220 | wg_index_hashtable_remove(peer->device->index_hashtable, | |
221 | &peer->handshake.entry); | |
222 | ||
223 | /* Remove any lingering packets that didn't have a chance to be | |
224 | * transmitted. | |
225 | */ | |
226 | wg_packet_purge_staged_packets(peer); | |
227 | ||
228 | /* Free the memory used. */ | |
229 | call_rcu(&peer->rcu, rcu_release); | |
230 | } | |
231 | ||
232 | void wg_peer_put(struct wg_peer *peer) | |
233 | { | |
234 | if (unlikely(!peer)) | |
235 | return; | |
236 | kref_put(&peer->refcount, kref_release); | |
237 | } |