Commit | Line | Data |
---|---|---|
1b262839 ST |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* | |
3 | * Copyright 2019 Google LLC | |
4 | */ | |
5 | ||
6 | /** | |
7 | * DOC: The Keyslot Manager | |
8 | * | |
9 | * Many devices with inline encryption support have a limited number of "slots" | |
10 | * into which encryption contexts may be programmed, and requests can be tagged | |
11 | * with a slot number to specify the key to use for en/decryption. | |
12 | * | |
13 | * As the number of slots is limited, and programming keys is expensive on | |
14 | * many inline encryption hardware, we don't want to program the same key into | |
15 | * multiple slots - if multiple requests are using the same key, we want to | |
16 | * program just one slot with that key and use that slot for all requests. | |
17 | * | |
18 | * The keyslot manager manages these keyslots appropriately, and also acts as | |
19 | * an abstraction between the inline encryption hardware and the upper layers. | |
20 | * | |
21 | * Lower layer devices will set up a keyslot manager in their request queue | |
22 | * and tell it how to perform device specific operations like programming/ | |
23 | * evicting keys from keyslots. | |
24 | * | |
25 | * Upper layers will call blk_ksm_get_slot_for_key() to program a | |
26 | * key into some slot in the inline encryption hardware. | |
27 | */ | |
28 | #include <linux/keyslot-manager.h> | |
29 | #include <linux/atomic.h> | |
30 | #include <linux/mutex.h> | |
31 | #include <linux/pm_runtime.h> | |
32 | #include <linux/wait.h> | |
33 | #include <linux/blkdev.h> | |
34 | ||
35 | struct blk_ksm_keyslot { | |
36 | atomic_t slot_refs; | |
37 | struct list_head idle_slot_node; | |
38 | struct hlist_node hash_node; | |
39 | const struct blk_crypto_key *key; | |
40 | struct blk_keyslot_manager *ksm; | |
41 | }; | |
42 | ||
43 | static inline void blk_ksm_hw_enter(struct blk_keyslot_manager *ksm) | |
44 | { | |
45 | /* | |
46 | * Calling into the driver requires ksm->lock held and the device | |
47 | * resumed. But we must resume the device first, since that can acquire | |
48 | * and release ksm->lock via blk_ksm_reprogram_all_keys(). | |
49 | */ | |
50 | if (ksm->dev) | |
51 | pm_runtime_get_sync(ksm->dev); | |
52 | down_write(&ksm->lock); | |
53 | } | |
54 | ||
55 | static inline void blk_ksm_hw_exit(struct blk_keyslot_manager *ksm) | |
56 | { | |
57 | up_write(&ksm->lock); | |
58 | if (ksm->dev) | |
59 | pm_runtime_put_sync(ksm->dev); | |
60 | } | |
61 | ||
62 | /** | |
63 | * blk_ksm_init() - Initialize a keyslot manager | |
64 | * @ksm: The keyslot_manager to initialize. | |
65 | * @num_slots: The number of key slots to manage. | |
66 | * | |
67 | * Allocate memory for keyslots and initialize a keyslot manager. Called by | |
68 | * e.g. storage drivers to set up a keyslot manager in their request_queue. | |
69 | * | |
70 | * Return: 0 on success, or else a negative error code. | |
71 | */ | |
72 | int blk_ksm_init(struct blk_keyslot_manager *ksm, unsigned int num_slots) | |
73 | { | |
74 | unsigned int slot; | |
75 | unsigned int i; | |
76 | unsigned int slot_hashtable_size; | |
77 | ||
78 | memset(ksm, 0, sizeof(*ksm)); | |
79 | ||
80 | if (num_slots == 0) | |
81 | return -EINVAL; | |
82 | ||
83 | ksm->slots = kvcalloc(num_slots, sizeof(ksm->slots[0]), GFP_KERNEL); | |
84 | if (!ksm->slots) | |
85 | return -ENOMEM; | |
86 | ||
87 | ksm->num_slots = num_slots; | |
88 | ||
89 | init_rwsem(&ksm->lock); | |
90 | ||
91 | init_waitqueue_head(&ksm->idle_slots_wait_queue); | |
92 | INIT_LIST_HEAD(&ksm->idle_slots); | |
93 | ||
94 | for (slot = 0; slot < num_slots; slot++) { | |
95 | ksm->slots[slot].ksm = ksm; | |
96 | list_add_tail(&ksm->slots[slot].idle_slot_node, | |
97 | &ksm->idle_slots); | |
98 | } | |
99 | ||
100 | spin_lock_init(&ksm->idle_slots_lock); | |
101 | ||
102 | slot_hashtable_size = roundup_pow_of_two(num_slots); | |
103 | ksm->log_slot_ht_size = ilog2(slot_hashtable_size); | |
104 | ksm->slot_hashtable = kvmalloc_array(slot_hashtable_size, | |
105 | sizeof(ksm->slot_hashtable[0]), | |
106 | GFP_KERNEL); | |
107 | if (!ksm->slot_hashtable) | |
108 | goto err_destroy_ksm; | |
109 | for (i = 0; i < slot_hashtable_size; i++) | |
110 | INIT_HLIST_HEAD(&ksm->slot_hashtable[i]); | |
111 | ||
112 | return 0; | |
113 | ||
114 | err_destroy_ksm: | |
115 | blk_ksm_destroy(ksm); | |
116 | return -ENOMEM; | |
117 | } | |
118 | EXPORT_SYMBOL_GPL(blk_ksm_init); | |
119 | ||
120 | static inline struct hlist_head * | |
121 | blk_ksm_hash_bucket_for_key(struct blk_keyslot_manager *ksm, | |
122 | const struct blk_crypto_key *key) | |
123 | { | |
124 | return &ksm->slot_hashtable[hash_ptr(key, ksm->log_slot_ht_size)]; | |
125 | } | |
126 | ||
127 | static void blk_ksm_remove_slot_from_lru_list(struct blk_ksm_keyslot *slot) | |
128 | { | |
129 | struct blk_keyslot_manager *ksm = slot->ksm; | |
130 | unsigned long flags; | |
131 | ||
132 | spin_lock_irqsave(&ksm->idle_slots_lock, flags); | |
133 | list_del(&slot->idle_slot_node); | |
134 | spin_unlock_irqrestore(&ksm->idle_slots_lock, flags); | |
135 | } | |
136 | ||
137 | static struct blk_ksm_keyslot *blk_ksm_find_keyslot( | |
138 | struct blk_keyslot_manager *ksm, | |
139 | const struct blk_crypto_key *key) | |
140 | { | |
141 | const struct hlist_head *head = blk_ksm_hash_bucket_for_key(ksm, key); | |
142 | struct blk_ksm_keyslot *slotp; | |
143 | ||
144 | hlist_for_each_entry(slotp, head, hash_node) { | |
145 | if (slotp->key == key) | |
146 | return slotp; | |
147 | } | |
148 | return NULL; | |
149 | } | |
150 | ||
151 | static struct blk_ksm_keyslot *blk_ksm_find_and_grab_keyslot( | |
152 | struct blk_keyslot_manager *ksm, | |
153 | const struct blk_crypto_key *key) | |
154 | { | |
155 | struct blk_ksm_keyslot *slot; | |
156 | ||
157 | slot = blk_ksm_find_keyslot(ksm, key); | |
158 | if (!slot) | |
159 | return NULL; | |
160 | if (atomic_inc_return(&slot->slot_refs) == 1) { | |
161 | /* Took first reference to this slot; remove it from LRU list */ | |
162 | blk_ksm_remove_slot_from_lru_list(slot); | |
163 | } | |
164 | return slot; | |
165 | } | |
166 | ||
167 | unsigned int blk_ksm_get_slot_idx(struct blk_ksm_keyslot *slot) | |
168 | { | |
169 | return slot - slot->ksm->slots; | |
170 | } | |
171 | EXPORT_SYMBOL_GPL(blk_ksm_get_slot_idx); | |
172 | ||
173 | /** | |
174 | * blk_ksm_get_slot_for_key() - Program a key into a keyslot. | |
175 | * @ksm: The keyslot manager to program the key into. | |
176 | * @key: Pointer to the key object to program, including the raw key, crypto | |
177 | * mode, and data unit size. | |
178 | * @slot_ptr: A pointer to return the pointer of the allocated keyslot. | |
179 | * | |
180 | * Get a keyslot that's been programmed with the specified key. If one already | |
181 | * exists, return it with incremented refcount. Otherwise, wait for a keyslot | |
182 | * to become idle and program it. | |
183 | * | |
184 | * Context: Process context. Takes and releases ksm->lock. | |
185 | * Return: BLK_STS_OK on success (and keyslot is set to the pointer of the | |
186 | * allocated keyslot), or some other blk_status_t otherwise (and | |
187 | * keyslot is set to NULL). | |
188 | */ | |
189 | blk_status_t blk_ksm_get_slot_for_key(struct blk_keyslot_manager *ksm, | |
190 | const struct blk_crypto_key *key, | |
191 | struct blk_ksm_keyslot **slot_ptr) | |
192 | { | |
193 | struct blk_ksm_keyslot *slot; | |
194 | int slot_idx; | |
195 | int err; | |
196 | ||
197 | *slot_ptr = NULL; | |
198 | down_read(&ksm->lock); | |
199 | slot = blk_ksm_find_and_grab_keyslot(ksm, key); | |
200 | up_read(&ksm->lock); | |
201 | if (slot) | |
202 | goto success; | |
203 | ||
204 | for (;;) { | |
205 | blk_ksm_hw_enter(ksm); | |
206 | slot = blk_ksm_find_and_grab_keyslot(ksm, key); | |
207 | if (slot) { | |
208 | blk_ksm_hw_exit(ksm); | |
209 | goto success; | |
210 | } | |
211 | ||
212 | /* | |
213 | * If we're here, that means there wasn't a slot that was | |
214 | * already programmed with the key. So try to program it. | |
215 | */ | |
216 | if (!list_empty(&ksm->idle_slots)) | |
217 | break; | |
218 | ||
219 | blk_ksm_hw_exit(ksm); | |
220 | wait_event(ksm->idle_slots_wait_queue, | |
221 | !list_empty(&ksm->idle_slots)); | |
222 | } | |
223 | ||
224 | slot = list_first_entry(&ksm->idle_slots, struct blk_ksm_keyslot, | |
225 | idle_slot_node); | |
226 | slot_idx = blk_ksm_get_slot_idx(slot); | |
227 | ||
228 | err = ksm->ksm_ll_ops.keyslot_program(ksm, key, slot_idx); | |
229 | if (err) { | |
230 | wake_up(&ksm->idle_slots_wait_queue); | |
231 | blk_ksm_hw_exit(ksm); | |
232 | return errno_to_blk_status(err); | |
233 | } | |
234 | ||
235 | /* Move this slot to the hash list for the new key. */ | |
236 | if (slot->key) | |
237 | hlist_del(&slot->hash_node); | |
238 | slot->key = key; | |
239 | hlist_add_head(&slot->hash_node, blk_ksm_hash_bucket_for_key(ksm, key)); | |
240 | ||
241 | atomic_set(&slot->slot_refs, 1); | |
242 | ||
243 | blk_ksm_remove_slot_from_lru_list(slot); | |
244 | ||
245 | blk_ksm_hw_exit(ksm); | |
246 | success: | |
247 | *slot_ptr = slot; | |
248 | return BLK_STS_OK; | |
249 | } | |
250 | ||
251 | /** | |
252 | * blk_ksm_put_slot() - Release a reference to a slot | |
253 | * @slot: The keyslot to release the reference of. | |
254 | * | |
255 | * Context: Any context. | |
256 | */ | |
257 | void blk_ksm_put_slot(struct blk_ksm_keyslot *slot) | |
258 | { | |
259 | struct blk_keyslot_manager *ksm; | |
260 | unsigned long flags; | |
261 | ||
262 | if (!slot) | |
263 | return; | |
264 | ||
265 | ksm = slot->ksm; | |
266 | ||
267 | if (atomic_dec_and_lock_irqsave(&slot->slot_refs, | |
268 | &ksm->idle_slots_lock, flags)) { | |
269 | list_add_tail(&slot->idle_slot_node, &ksm->idle_slots); | |
270 | spin_unlock_irqrestore(&ksm->idle_slots_lock, flags); | |
271 | wake_up(&ksm->idle_slots_wait_queue); | |
272 | } | |
273 | } | |
274 | ||
275 | /** | |
276 | * blk_ksm_crypto_cfg_supported() - Find out if a crypto configuration is | |
277 | * supported by a ksm. | |
278 | * @ksm: The keyslot manager to check | |
279 | * @cfg: The crypto configuration to check for. | |
280 | * | |
281 | * Checks for crypto_mode/data unit size/dun bytes support. | |
282 | * | |
283 | * Return: Whether or not this ksm supports the specified crypto config. | |
284 | */ | |
285 | bool blk_ksm_crypto_cfg_supported(struct blk_keyslot_manager *ksm, | |
286 | const struct blk_crypto_config *cfg) | |
287 | { | |
288 | if (!ksm) | |
289 | return false; | |
290 | if (!(ksm->crypto_modes_supported[cfg->crypto_mode] & | |
291 | cfg->data_unit_size)) | |
292 | return false; | |
293 | if (ksm->max_dun_bytes_supported < cfg->dun_bytes) | |
294 | return false; | |
295 | return true; | |
296 | } | |
297 | ||
298 | /** | |
299 | * blk_ksm_evict_key() - Evict a key from the lower layer device. | |
300 | * @ksm: The keyslot manager to evict from | |
301 | * @key: The key to evict | |
302 | * | |
303 | * Find the keyslot that the specified key was programmed into, and evict that | |
304 | * slot from the lower layer device. The slot must not be in use by any | |
305 | * in-flight IO when this function is called. | |
306 | * | |
307 | * Context: Process context. Takes and releases ksm->lock. | |
308 | * Return: 0 on success or if there's no keyslot with the specified key, -EBUSY | |
309 | * if the keyslot is still in use, or another -errno value on other | |
310 | * error. | |
311 | */ | |
312 | int blk_ksm_evict_key(struct blk_keyslot_manager *ksm, | |
313 | const struct blk_crypto_key *key) | |
314 | { | |
315 | struct blk_ksm_keyslot *slot; | |
316 | int err = 0; | |
317 | ||
318 | blk_ksm_hw_enter(ksm); | |
319 | slot = blk_ksm_find_keyslot(ksm, key); | |
320 | if (!slot) | |
321 | goto out_unlock; | |
322 | ||
323 | if (WARN_ON_ONCE(atomic_read(&slot->slot_refs) != 0)) { | |
324 | err = -EBUSY; | |
325 | goto out_unlock; | |
326 | } | |
327 | err = ksm->ksm_ll_ops.keyslot_evict(ksm, key, | |
328 | blk_ksm_get_slot_idx(slot)); | |
329 | if (err) | |
330 | goto out_unlock; | |
331 | ||
332 | hlist_del(&slot->hash_node); | |
333 | slot->key = NULL; | |
334 | err = 0; | |
335 | out_unlock: | |
336 | blk_ksm_hw_exit(ksm); | |
337 | return err; | |
338 | } | |
339 | ||
340 | /** | |
341 | * blk_ksm_reprogram_all_keys() - Re-program all keyslots. | |
342 | * @ksm: The keyslot manager | |
343 | * | |
344 | * Re-program all keyslots that are supposed to have a key programmed. This is | |
345 | * intended only for use by drivers for hardware that loses its keys on reset. | |
346 | * | |
347 | * Context: Process context. Takes and releases ksm->lock. | |
348 | */ | |
349 | void blk_ksm_reprogram_all_keys(struct blk_keyslot_manager *ksm) | |
350 | { | |
351 | unsigned int slot; | |
352 | ||
353 | /* This is for device initialization, so don't resume the device */ | |
354 | down_write(&ksm->lock); | |
355 | for (slot = 0; slot < ksm->num_slots; slot++) { | |
356 | const struct blk_crypto_key *key = ksm->slots[slot].key; | |
357 | int err; | |
358 | ||
359 | if (!key) | |
360 | continue; | |
361 | ||
362 | err = ksm->ksm_ll_ops.keyslot_program(ksm, key, slot); | |
363 | WARN_ON(err); | |
364 | } | |
365 | up_write(&ksm->lock); | |
366 | } | |
367 | EXPORT_SYMBOL_GPL(blk_ksm_reprogram_all_keys); | |
368 | ||
369 | void blk_ksm_destroy(struct blk_keyslot_manager *ksm) | |
370 | { | |
371 | if (!ksm) | |
372 | return; | |
373 | kvfree(ksm->slot_hashtable); | |
374 | memzero_explicit(ksm->slots, sizeof(ksm->slots[0]) * ksm->num_slots); | |
375 | kvfree(ksm->slots); | |
376 | memzero_explicit(ksm, sizeof(*ksm)); | |
377 | } | |
378 | EXPORT_SYMBOL_GPL(blk_ksm_destroy); |