Commit | Line | Data |
---|---|---|
90e53c5e BL |
1 | // SPDX-License-Identifier: Apache-2.0 OR MIT |
2 | ||
3 | //! API to safely and fallibly initialize pinned `struct`s using in-place constructors. | |
4 | //! | |
5 | //! It also allows in-place initialization of big `struct`s that would otherwise produce a stack | |
6 | //! overflow. | |
7 | //! | |
8 | //! Most `struct`s from the [`sync`] module need to be pinned, because they contain self-referential | |
9 | //! `struct`s from C. [Pinning][pinning] is Rust's way of ensuring data does not move. | |
10 | //! | |
11 | //! # Overview | |
12 | //! | |
13 | //! To initialize a `struct` with an in-place constructor you will need two things: | |
14 | //! - an in-place constructor, | |
15 | //! - a memory location that can hold your `struct`. | |
16 | //! | |
fc6c6baa BL |
17 | //! To get an in-place constructor there are generally three options: |
18 | //! - directly creating an in-place constructor using the [`pin_init!`] macro, | |
90e53c5e BL |
19 | //! - a custom function/macro returning an in-place constructor provided by someone else, |
20 | //! - using the unsafe function [`pin_init_from_closure()`] to manually create an initializer. | |
21 | //! | |
22 | //! Aside from pinned initialization, this API also supports in-place construction without pinning, | |
23 | //! the macros/types/functions are generally named like the pinned variants without the `pin` | |
24 | //! prefix. | |
25 | //! | |
fc6c6baa BL |
26 | //! # Examples |
27 | //! | |
28 | //! ## Using the [`pin_init!`] macro | |
29 | //! | |
30 | //! If you want to use [`PinInit`], then you will have to annotate your `struct` with | |
31 | //! `#[`[`pin_data`]`]`. It is a macro that uses `#[pin]` as a marker for | |
32 | //! [structurally pinned fields]. After doing this, you can then create an in-place constructor via | |
33 | //! [`pin_init!`]. The syntax is almost the same as normal `struct` initializers. The difference is | |
34 | //! that you need to write `<-` instead of `:` for fields that you want to initialize in-place. | |
35 | //! | |
36 | //! ```rust | |
37 | //! # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] | |
38 | //! use kernel::{prelude::*, sync::Mutex, new_mutex}; | |
39 | //! # use core::pin::Pin; | |
40 | //! #[pin_data] | |
41 | //! struct Foo { | |
42 | //! #[pin] | |
43 | //! a: Mutex<usize>, | |
44 | //! b: u32, | |
45 | //! } | |
46 | //! | |
47 | //! let foo = pin_init!(Foo { | |
48 | //! a <- new_mutex!(42, "Foo::a"), | |
49 | //! b: 24, | |
50 | //! }); | |
51 | //! ``` | |
52 | //! | |
53 | //! `foo` now is of the type [`impl PinInit<Foo>`]. We can now use any smart pointer that we like | |
54 | //! (or just the stack) to actually initialize a `Foo`: | |
55 | //! | |
56 | //! ```rust | |
57 | //! # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] | |
58 | //! # use kernel::{prelude::*, sync::Mutex, new_mutex}; | |
59 | //! # use core::pin::Pin; | |
60 | //! # #[pin_data] | |
61 | //! # struct Foo { | |
62 | //! # #[pin] | |
63 | //! # a: Mutex<usize>, | |
64 | //! # b: u32, | |
65 | //! # } | |
66 | //! # let foo = pin_init!(Foo { | |
67 | //! # a <- new_mutex!(42, "Foo::a"), | |
68 | //! # b: 24, | |
69 | //! # }); | |
70 | //! let foo: Result<Pin<Box<Foo>>> = Box::pin_init(foo); | |
71 | //! ``` | |
72 | //! | |
73 | //! For more information see the [`pin_init!`] macro. | |
74 | //! | |
75 | //! ## Using a custom function/macro that returns an initializer | |
76 | //! | |
77 | //! Many types from the kernel supply a function/macro that returns an initializer, because the | |
78 | //! above method only works for types where you can access the fields. | |
79 | //! | |
80 | //! ```rust | |
81 | //! # use kernel::{new_mutex, sync::{Arc, Mutex}}; | |
82 | //! let mtx: Result<Arc<Mutex<usize>>> = Arc::pin_init(new_mutex!(42, "example::mtx")); | |
83 | //! ``` | |
84 | //! | |
85 | //! To declare an init macro/function you just return an [`impl PinInit<T, E>`]: | |
86 | //! | |
87 | //! ```rust | |
88 | //! # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] | |
89 | //! # use kernel::{sync::Mutex, prelude::*, new_mutex, init::PinInit, try_pin_init}; | |
90 | //! #[pin_data] | |
91 | //! struct DriverData { | |
92 | //! #[pin] | |
93 | //! status: Mutex<i32>, | |
94 | //! buffer: Box<[u8; 1_000_000]>, | |
95 | //! } | |
96 | //! | |
97 | //! impl DriverData { | |
98 | //! fn new() -> impl PinInit<Self, Error> { | |
99 | //! try_pin_init!(Self { | |
100 | //! status <- new_mutex!(0, "DriverData::status"), | |
101 | //! buffer: Box::init(kernel::init::zeroed())?, | |
102 | //! }) | |
103 | //! } | |
104 | //! } | |
105 | //! ``` | |
106 | //! | |
d0fdc396 BL |
107 | //! ## Manual creation of an initializer |
108 | //! | |
109 | //! Often when working with primitives the previous approaches are not sufficient. That is where | |
110 | //! [`pin_init_from_closure()`] comes in. This `unsafe` function allows you to create a | |
111 | //! [`impl PinInit<T, E>`] directly from a closure. Of course you have to ensure that the closure | |
112 | //! actually does the initialization in the correct way. Here are the things to look out for | |
113 | //! (we are calling the parameter to the closure `slot`): | |
114 | //! - when the closure returns `Ok(())`, then it has completed the initialization successfully, so | |
115 | //! `slot` now contains a valid bit pattern for the type `T`, | |
116 | //! - when the closure returns `Err(e)`, then the caller may deallocate the memory at `slot`, so | |
117 | //! you need to take care to clean up anything if your initialization fails mid-way, | |
118 | //! - you may assume that `slot` will stay pinned even after the closure returns until `drop` of | |
119 | //! `slot` gets called. | |
120 | //! | |
121 | //! ```rust | |
122 | //! use kernel::{prelude::*, init}; | |
123 | //! use core::{ptr::addr_of_mut, marker::PhantomPinned, pin::Pin}; | |
124 | //! # mod bindings { | |
125 | //! # pub struct foo; | |
126 | //! # pub unsafe fn init_foo(_ptr: *mut foo) {} | |
127 | //! # pub unsafe fn destroy_foo(_ptr: *mut foo) {} | |
128 | //! # pub unsafe fn enable_foo(_ptr: *mut foo, _flags: u32) -> i32 { 0 } | |
129 | //! # } | |
130 | //! /// # Invariants | |
131 | //! /// | |
132 | //! /// `foo` is always initialized | |
133 | //! #[pin_data(PinnedDrop)] | |
134 | //! pub struct RawFoo { | |
135 | //! #[pin] | |
136 | //! foo: Opaque<bindings::foo>, | |
137 | //! #[pin] | |
138 | //! _p: PhantomPinned, | |
139 | //! } | |
140 | //! | |
141 | //! impl RawFoo { | |
142 | //! pub fn new(flags: u32) -> impl PinInit<Self, Error> { | |
143 | //! // SAFETY: | |
144 | //! // - when the closure returns `Ok(())`, then it has successfully initialized and | |
145 | //! // enabled `foo`, | |
146 | //! // - when it returns `Err(e)`, then it has cleaned up before | |
147 | //! unsafe { | |
148 | //! init::pin_init_from_closure(move |slot: *mut Self| { | |
149 | //! // `slot` contains uninit memory, avoid creating a reference. | |
150 | //! let foo = addr_of_mut!((*slot).foo); | |
151 | //! | |
152 | //! // Initialize the `foo` | |
153 | //! bindings::init_foo(Opaque::raw_get(foo)); | |
154 | //! | |
155 | //! // Try to enable it. | |
156 | //! let err = bindings::enable_foo(Opaque::raw_get(foo), flags); | |
157 | //! if err != 0 { | |
158 | //! // Enabling has failed, first clean up the foo and then return the error. | |
159 | //! bindings::destroy_foo(Opaque::raw_get(foo)); | |
160 | //! return Err(Error::from_kernel_errno(err)); | |
161 | //! } | |
162 | //! | |
163 | //! // All fields of `RawFoo` have been initialized, since `_p` is a ZST. | |
164 | //! Ok(()) | |
165 | //! }) | |
166 | //! } | |
167 | //! } | |
168 | //! } | |
169 | //! | |
170 | //! #[pinned_drop] | |
171 | //! impl PinnedDrop for RawFoo { | |
172 | //! fn drop(self: Pin<&mut Self>) { | |
173 | //! // SAFETY: Since `foo` is initialized, destroying is safe. | |
174 | //! unsafe { bindings::destroy_foo(self.foo.get()) }; | |
175 | //! } | |
176 | //! } | |
177 | //! ``` | |
178 | //! | |
90e53c5e BL |
179 | //! [`sync`]: kernel::sync |
180 | //! [pinning]: https://doc.rust-lang.org/std/pin/index.html | |
181 | //! [structurally pinned fields]: | |
182 | //! https://doc.rust-lang.org/std/pin/index.html#pinning-is-structural-for-field | |
183 | //! [`Arc<T>`]: crate::sync::Arc | |
184 | //! [`impl PinInit<Foo>`]: PinInit | |
185 | //! [`impl PinInit<T, E>`]: PinInit | |
186 | //! [`impl Init<T, E>`]: Init | |
187 | //! [`Opaque`]: kernel::types::Opaque | |
188 | //! [`pin_data`]: ::macros::pin_data | |
90e53c5e | 189 | |
92c4a1e7 BL |
190 | use crate::{ |
191 | error::{self, Error}, | |
192 | sync::UniqueArc, | |
193 | }; | |
fc6c6baa | 194 | use alloc::boxed::Box; |
92c4a1e7 BL |
195 | use core::{ |
196 | alloc::AllocError, cell::Cell, convert::Infallible, marker::PhantomData, mem::MaybeUninit, | |
197 | pin::Pin, ptr, | |
198 | }; | |
90e53c5e BL |
199 | |
200 | #[doc(hidden)] | |
201 | pub mod __internal; | |
fc6c6baa BL |
202 | #[doc(hidden)] |
203 | pub mod macros; | |
204 | ||
205 | /// Construct an in-place, pinned initializer for `struct`s. | |
206 | /// | |
207 | /// This macro defaults the error to [`Infallible`]. If you need [`Error`], then use | |
208 | /// [`try_pin_init!`]. | |
209 | /// | |
210 | /// The syntax is almost identical to that of a normal `struct` initializer: | |
211 | /// | |
212 | /// ```rust | |
213 | /// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] | |
214 | /// # use kernel::{init, pin_init, macros::pin_data, init::*}; | |
215 | /// # use core::pin::Pin; | |
216 | /// #[pin_data] | |
217 | /// struct Foo { | |
218 | /// a: usize, | |
219 | /// b: Bar, | |
220 | /// } | |
221 | /// | |
222 | /// #[pin_data] | |
223 | /// struct Bar { | |
224 | /// x: u32, | |
225 | /// } | |
226 | /// | |
227 | /// # fn demo() -> impl PinInit<Foo> { | |
228 | /// let a = 42; | |
229 | /// | |
230 | /// let initializer = pin_init!(Foo { | |
231 | /// a, | |
232 | /// b: Bar { | |
233 | /// x: 64, | |
234 | /// }, | |
235 | /// }); | |
236 | /// # initializer } | |
237 | /// # Box::pin_init(demo()).unwrap(); | |
238 | /// ``` | |
239 | /// | |
240 | /// Arbitrary Rust expressions can be used to set the value of a variable. | |
241 | /// | |
242 | /// The fields are initialized in the order that they appear in the initializer. So it is possible | |
243 | /// to read already initialized fields using raw pointers. | |
244 | /// | |
245 | /// IMPORTANT: You are not allowed to create references to fields of the struct inside of the | |
246 | /// initializer. | |
247 | /// | |
248 | /// # Init-functions | |
249 | /// | |
250 | /// When working with this API it is often desired to let others construct your types without | |
251 | /// giving access to all fields. This is where you would normally write a plain function `new` | |
252 | /// that would return a new instance of your type. With this API that is also possible. | |
253 | /// However, there are a few extra things to keep in mind. | |
254 | /// | |
255 | /// To create an initializer function, simply declare it like this: | |
256 | /// | |
257 | /// ```rust | |
258 | /// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] | |
259 | /// # use kernel::{init, pin_init, prelude::*, init::*}; | |
260 | /// # use core::pin::Pin; | |
261 | /// # #[pin_data] | |
262 | /// # struct Foo { | |
263 | /// # a: usize, | |
264 | /// # b: Bar, | |
265 | /// # } | |
266 | /// # #[pin_data] | |
267 | /// # struct Bar { | |
268 | /// # x: u32, | |
269 | /// # } | |
270 | /// impl Foo { | |
271 | /// fn new() -> impl PinInit<Self> { | |
272 | /// pin_init!(Self { | |
273 | /// a: 42, | |
274 | /// b: Bar { | |
275 | /// x: 64, | |
276 | /// }, | |
277 | /// }) | |
278 | /// } | |
279 | /// } | |
280 | /// ``` | |
281 | /// | |
282 | /// Users of `Foo` can now create it like this: | |
283 | /// | |
284 | /// ```rust | |
285 | /// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] | |
286 | /// # use kernel::{init, pin_init, macros::pin_data, init::*}; | |
287 | /// # use core::pin::Pin; | |
288 | /// # #[pin_data] | |
289 | /// # struct Foo { | |
290 | /// # a: usize, | |
291 | /// # b: Bar, | |
292 | /// # } | |
293 | /// # #[pin_data] | |
294 | /// # struct Bar { | |
295 | /// # x: u32, | |
296 | /// # } | |
297 | /// # impl Foo { | |
298 | /// # fn new() -> impl PinInit<Self> { | |
299 | /// # pin_init!(Self { | |
300 | /// # a: 42, | |
301 | /// # b: Bar { | |
302 | /// # x: 64, | |
303 | /// # }, | |
304 | /// # }) | |
305 | /// # } | |
306 | /// # } | |
307 | /// let foo = Box::pin_init(Foo::new()); | |
308 | /// ``` | |
309 | /// | |
310 | /// They can also easily embed it into their own `struct`s: | |
311 | /// | |
312 | /// ```rust | |
313 | /// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] | |
314 | /// # use kernel::{init, pin_init, macros::pin_data, init::*}; | |
315 | /// # use core::pin::Pin; | |
316 | /// # #[pin_data] | |
317 | /// # struct Foo { | |
318 | /// # a: usize, | |
319 | /// # b: Bar, | |
320 | /// # } | |
321 | /// # #[pin_data] | |
322 | /// # struct Bar { | |
323 | /// # x: u32, | |
324 | /// # } | |
325 | /// # impl Foo { | |
326 | /// # fn new() -> impl PinInit<Self> { | |
327 | /// # pin_init!(Self { | |
328 | /// # a: 42, | |
329 | /// # b: Bar { | |
330 | /// # x: 64, | |
331 | /// # }, | |
332 | /// # }) | |
333 | /// # } | |
334 | /// # } | |
335 | /// #[pin_data] | |
336 | /// struct FooContainer { | |
337 | /// #[pin] | |
338 | /// foo1: Foo, | |
339 | /// #[pin] | |
340 | /// foo2: Foo, | |
341 | /// other: u32, | |
342 | /// } | |
343 | /// | |
344 | /// impl FooContainer { | |
345 | /// fn new(other: u32) -> impl PinInit<Self> { | |
346 | /// pin_init!(Self { | |
347 | /// foo1 <- Foo::new(), | |
348 | /// foo2 <- Foo::new(), | |
349 | /// other, | |
350 | /// }) | |
351 | /// } | |
352 | /// } | |
353 | /// ``` | |
354 | /// | |
355 | /// Here we see that when using `pin_init!` with `PinInit`, one needs to write `<-` instead of `:`. | |
356 | /// This signifies that the given field is initialized in-place. As with `struct` initializers, just | |
357 | /// writing the field (in this case `other`) without `:` or `<-` means `other: other,`. | |
358 | /// | |
359 | /// # Syntax | |
360 | /// | |
361 | /// As already mentioned in the examples above, inside of `pin_init!` a `struct` initializer with | |
362 | /// the following modifications is expected: | |
363 | /// - Fields that you want to initialize in-place have to use `<-` instead of `:`. | |
364 | /// - In front of the initializer you can write `&this in` to have access to a [`NonNull<Self>`] | |
365 | /// pointer named `this` inside of the initializer. | |
366 | /// | |
367 | /// For instance: | |
368 | /// | |
369 | /// ```rust | |
370 | /// # use kernel::pin_init; | |
371 | /// # use macros::pin_data; | |
372 | /// # use core::{ptr::addr_of_mut, marker::PhantomPinned}; | |
373 | /// #[pin_data] | |
374 | /// struct Buf { | |
375 | /// // `ptr` points into `buf`. | |
376 | /// ptr: *mut u8, | |
377 | /// buf: [u8; 64], | |
378 | /// #[pin] | |
379 | /// pin: PhantomPinned, | |
380 | /// } | |
381 | /// pin_init!(&this in Buf { | |
382 | /// buf: [0; 64], | |
383 | /// ptr: unsafe { addr_of_mut!((*this.as_ptr()).buf).cast() }, | |
384 | /// pin: PhantomPinned, | |
385 | /// }); | |
386 | /// ``` | |
387 | /// | |
388 | /// [`try_pin_init!`]: kernel::try_pin_init | |
389 | /// [`NonNull<Self>`]: core::ptr::NonNull | |
fc6c6baa BL |
390 | // For a detailed example of how this macro works, see the module documentation of the hidden |
391 | // module `__internal` inside of `init/__internal.rs`. | |
392 | #[macro_export] | |
393 | macro_rules! pin_init { | |
394 | ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { | |
395 | $($fields:tt)* | |
396 | }) => { | |
397 | $crate::try_pin_init!( | |
398 | @this($($this)?), | |
399 | @typ($t $(::<$($generics),*>)?), | |
400 | @fields($($fields)*), | |
401 | @error(::core::convert::Infallible), | |
402 | ) | |
403 | }; | |
404 | } | |
405 | ||
406 | /// Construct an in-place, fallible pinned initializer for `struct`s. | |
407 | /// | |
408 | /// If the initialization can complete without error (or [`Infallible`]), then use [`pin_init!`]. | |
409 | /// | |
410 | /// You can use the `?` operator or use `return Err(err)` inside the initializer to stop | |
411 | /// initialization and return the error. | |
412 | /// | |
413 | /// IMPORTANT: if you have `unsafe` code inside of the initializer you have to ensure that when | |
414 | /// initialization fails, the memory can be safely deallocated without any further modifications. | |
415 | /// | |
416 | /// This macro defaults the error to [`Error`]. | |
417 | /// | |
418 | /// The syntax is identical to [`pin_init!`] with the following exception: you can append `? $type` | |
419 | /// after the `struct` initializer to specify the error type you want to use. | |
420 | /// | |
421 | /// # Examples | |
422 | /// | |
423 | /// ```rust | |
424 | /// # #![feature(new_uninit)] | |
425 | /// use kernel::{init::{self, PinInit}, error::Error}; | |
426 | /// #[pin_data] | |
427 | /// struct BigBuf { | |
428 | /// big: Box<[u8; 1024 * 1024 * 1024]>, | |
429 | /// small: [u8; 1024 * 1024], | |
430 | /// ptr: *mut u8, | |
431 | /// } | |
432 | /// | |
433 | /// impl BigBuf { | |
434 | /// fn new() -> impl PinInit<Self, Error> { | |
435 | /// try_pin_init!(Self { | |
436 | /// big: Box::init(init::zeroed())?, | |
437 | /// small: [0; 1024 * 1024], | |
438 | /// ptr: core::ptr::null_mut(), | |
439 | /// }? Error) | |
440 | /// } | |
441 | /// } | |
442 | /// ``` | |
fc6c6baa BL |
443 | // For a detailed example of how this macro works, see the module documentation of the hidden |
444 | // module `__internal` inside of `init/__internal.rs`. | |
445 | #[macro_export] | |
446 | macro_rules! try_pin_init { | |
447 | ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { | |
448 | $($fields:tt)* | |
449 | }) => { | |
450 | $crate::try_pin_init!( | |
451 | @this($($this)?), | |
452 | @typ($t $(::<$($generics),*>)? ), | |
453 | @fields($($fields)*), | |
454 | @error($crate::error::Error), | |
455 | ) | |
456 | }; | |
457 | ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { | |
458 | $($fields:tt)* | |
459 | }? $err:ty) => { | |
460 | $crate::try_pin_init!( | |
461 | @this($($this)?), | |
462 | @typ($t $(::<$($generics),*>)? ), | |
463 | @fields($($fields)*), | |
464 | @error($err), | |
465 | ) | |
466 | }; | |
467 | ( | |
468 | @this($($this:ident)?), | |
469 | @typ($t:ident $(::<$($generics:ty),*>)?), | |
470 | @fields($($fields:tt)*), | |
471 | @error($err:ty), | |
472 | ) => {{ | |
473 | // We do not want to allow arbitrary returns, so we declare this type as the `Ok` return | |
474 | // type and shadow it later when we insert the arbitrary user code. That way there will be | |
475 | // no possibility of returning without `unsafe`. | |
476 | struct __InitOk; | |
477 | // Get the pin data from the supplied type. | |
478 | let data = unsafe { | |
479 | use $crate::init::__internal::HasPinData; | |
480 | $t$(::<$($generics),*>)?::__pin_data() | |
481 | }; | |
482 | // Ensure that `data` really is of type `PinData` and help with type inference: | |
483 | let init = $crate::init::__internal::PinData::make_closure::<_, __InitOk, $err>( | |
484 | data, | |
485 | move |slot| { | |
486 | { | |
487 | // Shadow the structure so it cannot be used to return early. | |
488 | struct __InitOk; | |
489 | // Create the `this` so it can be referenced by the user inside of the | |
490 | // expressions creating the individual fields. | |
491 | $(let $this = unsafe { ::core::ptr::NonNull::new_unchecked(slot) };)? | |
492 | // Initialize every field. | |
493 | $crate::try_pin_init!(init_slot: | |
494 | @data(data), | |
495 | @slot(slot), | |
496 | @munch_fields($($fields)*,), | |
497 | ); | |
498 | // We use unreachable code to ensure that all fields have been mentioned exactly | |
499 | // once, this struct initializer will still be type-checked and complain with a | |
500 | // very natural error message if a field is forgotten/mentioned more than once. | |
501 | #[allow(unreachable_code, clippy::diverging_sub_expression)] | |
502 | if false { | |
503 | $crate::try_pin_init!(make_initializer: | |
504 | @slot(slot), | |
505 | @type_name($t), | |
506 | @munch_fields($($fields)*,), | |
507 | @acc(), | |
508 | ); | |
509 | } | |
510 | // Forget all guards, since initialization was a success. | |
511 | $crate::try_pin_init!(forget_guards: | |
512 | @munch_fields($($fields)*,), | |
513 | ); | |
514 | } | |
515 | Ok(__InitOk) | |
516 | } | |
517 | ); | |
518 | let init = move |slot| -> ::core::result::Result<(), $err> { | |
519 | init(slot).map(|__InitOk| ()) | |
520 | }; | |
521 | let init = unsafe { $crate::init::pin_init_from_closure::<_, $err>(init) }; | |
522 | init | |
523 | }}; | |
524 | (init_slot: | |
525 | @data($data:ident), | |
526 | @slot($slot:ident), | |
527 | @munch_fields($(,)?), | |
528 | ) => { | |
529 | // Endpoint of munching, no fields are left. | |
530 | }; | |
531 | (init_slot: | |
532 | @data($data:ident), | |
533 | @slot($slot:ident), | |
534 | // In-place initialization syntax. | |
535 | @munch_fields($field:ident <- $val:expr, $($rest:tt)*), | |
536 | ) => { | |
537 | let $field = $val; | |
538 | // Call the initializer. | |
539 | // | |
540 | // SAFETY: `slot` is valid, because we are inside of an initializer closure, we | |
541 | // return when an error/panic occurs. | |
542 | // We also use the `data` to require the correct trait (`Init` or `PinInit`) for `$field`. | |
543 | unsafe { $data.$field(::core::ptr::addr_of_mut!((*$slot).$field), $field)? }; | |
544 | // Create the drop guard. | |
545 | // | |
546 | // We only give access to `&DropGuard`, so it cannot be forgotten via safe code. | |
547 | // | |
548 | // SAFETY: We forget the guard later when initialization has succeeded. | |
549 | let $field = &unsafe { | |
550 | $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field)) | |
551 | }; | |
552 | ||
553 | $crate::try_pin_init!(init_slot: | |
554 | @data($data), | |
555 | @slot($slot), | |
556 | @munch_fields($($rest)*), | |
557 | ); | |
558 | }; | |
559 | (init_slot: | |
560 | @data($data:ident), | |
561 | @slot($slot:ident), | |
562 | // Direct value init, this is safe for every field. | |
563 | @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), | |
564 | ) => { | |
565 | $(let $field = $val;)? | |
566 | // Initialize the field. | |
567 | // | |
568 | // SAFETY: The memory at `slot` is uninitialized. | |
569 | unsafe { ::core::ptr::write(::core::ptr::addr_of_mut!((*$slot).$field), $field) }; | |
570 | // Create the drop guard: | |
571 | // | |
572 | // We only give access to `&DropGuard`, so it cannot be accidentally forgotten. | |
573 | // | |
574 | // SAFETY: We forget the guard later when initialization has succeeded. | |
575 | let $field = &unsafe { | |
576 | $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field)) | |
577 | }; | |
578 | ||
579 | $crate::try_pin_init!(init_slot: | |
580 | @data($data), | |
581 | @slot($slot), | |
582 | @munch_fields($($rest)*), | |
583 | ); | |
584 | }; | |
585 | (make_initializer: | |
586 | @slot($slot:ident), | |
587 | @type_name($t:ident), | |
588 | @munch_fields($(,)?), | |
589 | @acc($($acc:tt)*), | |
590 | ) => { | |
591 | // Endpoint, nothing more to munch, create the initializer. | |
592 | // Since we are in the `if false` branch, this will never get executed. We abuse `slot` to | |
593 | // get the correct type inference here: | |
594 | unsafe { | |
595 | ::core::ptr::write($slot, $t { | |
596 | $($acc)* | |
597 | }); | |
598 | } | |
599 | }; | |
600 | (make_initializer: | |
601 | @slot($slot:ident), | |
602 | @type_name($t:ident), | |
603 | @munch_fields($field:ident <- $val:expr, $($rest:tt)*), | |
604 | @acc($($acc:tt)*), | |
605 | ) => { | |
606 | $crate::try_pin_init!(make_initializer: | |
607 | @slot($slot), | |
608 | @type_name($t), | |
609 | @munch_fields($($rest)*), | |
610 | @acc($($acc)* $field: ::core::panic!(),), | |
611 | ); | |
612 | }; | |
613 | (make_initializer: | |
614 | @slot($slot:ident), | |
615 | @type_name($t:ident), | |
616 | @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), | |
617 | @acc($($acc:tt)*), | |
618 | ) => { | |
619 | $crate::try_pin_init!(make_initializer: | |
620 | @slot($slot), | |
621 | @type_name($t), | |
622 | @munch_fields($($rest)*), | |
623 | @acc($($acc)* $field: ::core::panic!(),), | |
624 | ); | |
625 | }; | |
626 | (forget_guards: | |
627 | @munch_fields($(,)?), | |
628 | ) => { | |
629 | // Munching finished. | |
630 | }; | |
631 | (forget_guards: | |
632 | @munch_fields($field:ident <- $val:expr, $($rest:tt)*), | |
633 | ) => { | |
634 | unsafe { $crate::init::__internal::DropGuard::forget($field) }; | |
635 | ||
636 | $crate::try_pin_init!(forget_guards: | |
637 | @munch_fields($($rest)*), | |
638 | ); | |
639 | }; | |
640 | (forget_guards: | |
641 | @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), | |
642 | ) => { | |
643 | unsafe { $crate::init::__internal::DropGuard::forget($field) }; | |
644 | ||
645 | $crate::try_pin_init!(forget_guards: | |
646 | @munch_fields($($rest)*), | |
647 | ); | |
648 | }; | |
649 | } | |
650 | ||
651 | /// Construct an in-place initializer for `struct`s. | |
652 | /// | |
653 | /// This macro defaults the error to [`Infallible`]. If you need [`Error`], then use | |
654 | /// [`try_init!`]. | |
655 | /// | |
656 | /// The syntax is identical to [`pin_init!`] and its safety caveats also apply: | |
657 | /// - `unsafe` code must guarantee either full initialization or return an error and allow | |
658 | /// deallocation of the memory. | |
659 | /// - the fields are initialized in the order given in the initializer. | |
660 | /// - no references to fields are allowed to be created inside of the initializer. | |
661 | /// | |
662 | /// This initializer is for initializing data in-place that might later be moved. If you want to | |
663 | /// pin-initialize, use [`pin_init!`]. | |
fc6c6baa BL |
664 | // For a detailed example of how this macro works, see the module documentation of the hidden |
665 | // module `__internal` inside of `init/__internal.rs`. | |
666 | #[macro_export] | |
667 | macro_rules! init { | |
668 | ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { | |
669 | $($fields:tt)* | |
670 | }) => { | |
671 | $crate::try_init!( | |
672 | @this($($this)?), | |
673 | @typ($t $(::<$($generics),*>)?), | |
674 | @fields($($fields)*), | |
675 | @error(::core::convert::Infallible), | |
676 | ) | |
677 | } | |
678 | } | |
679 | ||
680 | /// Construct an in-place fallible initializer for `struct`s. | |
681 | /// | |
682 | /// This macro defaults the error to [`Error`]. If you need [`Infallible`], then use | |
683 | /// [`init!`]. | |
684 | /// | |
685 | /// The syntax is identical to [`try_pin_init!`]. If you want to specify a custom error, | |
686 | /// append `? $type` after the `struct` initializer. | |
687 | /// The safety caveats from [`try_pin_init!`] also apply: | |
688 | /// - `unsafe` code must guarantee either full initialization or return an error and allow | |
689 | /// deallocation of the memory. | |
690 | /// - the fields are initialized in the order given in the initializer. | |
691 | /// - no references to fields are allowed to be created inside of the initializer. | |
692 | /// | |
693 | /// # Examples | |
694 | /// | |
695 | /// ```rust | |
696 | /// use kernel::{init::PinInit, error::Error, InPlaceInit}; | |
697 | /// struct BigBuf { | |
698 | /// big: Box<[u8; 1024 * 1024 * 1024]>, | |
699 | /// small: [u8; 1024 * 1024], | |
700 | /// } | |
701 | /// | |
702 | /// impl BigBuf { | |
703 | /// fn new() -> impl Init<Self, Error> { | |
704 | /// try_init!(Self { | |
705 | /// big: Box::init(zeroed())?, | |
706 | /// small: [0; 1024 * 1024], | |
707 | /// }? Error) | |
708 | /// } | |
709 | /// } | |
710 | /// ``` | |
fc6c6baa BL |
711 | // For a detailed example of how this macro works, see the module documentation of the hidden |
712 | // module `__internal` inside of `init/__internal.rs`. | |
713 | #[macro_export] | |
714 | macro_rules! try_init { | |
715 | ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { | |
716 | $($fields:tt)* | |
717 | }) => { | |
718 | $crate::try_init!( | |
719 | @this($($this)?), | |
720 | @typ($t $(::<$($generics),*>)?), | |
721 | @fields($($fields)*), | |
722 | @error($crate::error::Error), | |
723 | ) | |
724 | }; | |
725 | ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { | |
726 | $($fields:tt)* | |
727 | }? $err:ty) => { | |
728 | $crate::try_init!( | |
729 | @this($($this)?), | |
730 | @typ($t $(::<$($generics),*>)?), | |
731 | @fields($($fields)*), | |
732 | @error($err), | |
733 | ) | |
734 | }; | |
735 | ( | |
736 | @this($($this:ident)?), | |
737 | @typ($t:ident $(::<$($generics:ty),*>)?), | |
738 | @fields($($fields:tt)*), | |
739 | @error($err:ty), | |
740 | ) => {{ | |
741 | // We do not want to allow arbitrary returns, so we declare this type as the `Ok` return | |
742 | // type and shadow it later when we insert the arbitrary user code. That way there will be | |
743 | // no possibility of returning without `unsafe`. | |
744 | struct __InitOk; | |
745 | // Get the init data from the supplied type. | |
746 | let data = unsafe { | |
747 | use $crate::init::__internal::HasInitData; | |
748 | $t$(::<$($generics),*>)?::__init_data() | |
749 | }; | |
750 | // Ensure that `data` really is of type `InitData` and help with type inference: | |
751 | let init = $crate::init::__internal::InitData::make_closure::<_, __InitOk, $err>( | |
752 | data, | |
753 | move |slot| { | |
754 | { | |
755 | // Shadow the structure so it cannot be used to return early. | |
756 | struct __InitOk; | |
757 | // Create the `this` so it can be referenced by the user inside of the | |
758 | // expressions creating the individual fields. | |
759 | $(let $this = unsafe { ::core::ptr::NonNull::new_unchecked(slot) };)? | |
760 | // Initialize every field. | |
761 | $crate::try_init!(init_slot: | |
762 | @slot(slot), | |
763 | @munch_fields($($fields)*,), | |
764 | ); | |
765 | // We use unreachable code to ensure that all fields have been mentioned exactly | |
766 | // once, this struct initializer will still be type-checked and complain with a | |
767 | // very natural error message if a field is forgotten/mentioned more than once. | |
768 | #[allow(unreachable_code, clippy::diverging_sub_expression)] | |
769 | if false { | |
770 | $crate::try_init!(make_initializer: | |
771 | @slot(slot), | |
772 | @type_name($t), | |
773 | @munch_fields($($fields)*,), | |
774 | @acc(), | |
775 | ); | |
776 | } | |
777 | // Forget all guards, since initialization was a success. | |
778 | $crate::try_init!(forget_guards: | |
779 | @munch_fields($($fields)*,), | |
780 | ); | |
781 | } | |
782 | Ok(__InitOk) | |
783 | } | |
784 | ); | |
785 | let init = move |slot| -> ::core::result::Result<(), $err> { | |
786 | init(slot).map(|__InitOk| ()) | |
787 | }; | |
788 | let init = unsafe { $crate::init::init_from_closure::<_, $err>(init) }; | |
789 | init | |
790 | }}; | |
791 | (init_slot: | |
792 | @slot($slot:ident), | |
793 | @munch_fields( $(,)?), | |
794 | ) => { | |
795 | // Endpoint of munching, no fields are left. | |
796 | }; | |
797 | (init_slot: | |
798 | @slot($slot:ident), | |
799 | @munch_fields($field:ident <- $val:expr, $($rest:tt)*), | |
800 | ) => { | |
801 | let $field = $val; | |
802 | // Call the initializer. | |
803 | // | |
804 | // SAFETY: `slot` is valid, because we are inside of an initializer closure, we | |
805 | // return when an error/panic occurs. | |
806 | unsafe { | |
807 | $crate::init::Init::__init($field, ::core::ptr::addr_of_mut!((*$slot).$field))?; | |
808 | } | |
809 | // Create the drop guard. | |
810 | // | |
811 | // We only give access to `&DropGuard`, so it cannot be accidentally forgotten. | |
812 | // | |
813 | // SAFETY: We forget the guard later when initialization has succeeded. | |
814 | let $field = &unsafe { | |
815 | $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field)) | |
816 | }; | |
817 | ||
818 | $crate::try_init!(init_slot: | |
819 | @slot($slot), | |
820 | @munch_fields($($rest)*), | |
821 | ); | |
822 | }; | |
823 | (init_slot: | |
824 | @slot($slot:ident), | |
825 | // Direct value init. | |
826 | @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), | |
827 | ) => { | |
828 | $(let $field = $val;)? | |
829 | // Call the initializer. | |
830 | // | |
831 | // SAFETY: The memory at `slot` is uninitialized. | |
832 | unsafe { ::core::ptr::write(::core::ptr::addr_of_mut!((*$slot).$field), $field) }; | |
833 | // Create the drop guard. | |
834 | // | |
835 | // We only give access to `&DropGuard`, so it cannot be accidentally forgotten. | |
836 | // | |
837 | // SAFETY: We forget the guard later when initialization has succeeded. | |
838 | let $field = &unsafe { | |
839 | $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field)) | |
840 | }; | |
841 | ||
842 | $crate::try_init!(init_slot: | |
843 | @slot($slot), | |
844 | @munch_fields($($rest)*), | |
845 | ); | |
846 | }; | |
847 | (make_initializer: | |
848 | @slot($slot:ident), | |
849 | @type_name($t:ident), | |
850 | @munch_fields( $(,)?), | |
851 | @acc($($acc:tt)*), | |
852 | ) => { | |
853 | // Endpoint, nothing more to munch, create the initializer. | |
854 | // Since we are in the `if false` branch, this will never get executed. We abuse `slot` to | |
855 | // get the correct type inference here: | |
856 | unsafe { | |
857 | ::core::ptr::write($slot, $t { | |
858 | $($acc)* | |
859 | }); | |
860 | } | |
861 | }; | |
862 | (make_initializer: | |
863 | @slot($slot:ident), | |
864 | @type_name($t:ident), | |
865 | @munch_fields($field:ident <- $val:expr, $($rest:tt)*), | |
866 | @acc($($acc:tt)*), | |
867 | ) => { | |
868 | $crate::try_init!(make_initializer: | |
869 | @slot($slot), | |
870 | @type_name($t), | |
871 | @munch_fields($($rest)*), | |
872 | @acc($($acc)*$field: ::core::panic!(),), | |
873 | ); | |
874 | }; | |
875 | (make_initializer: | |
876 | @slot($slot:ident), | |
877 | @type_name($t:ident), | |
878 | @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), | |
879 | @acc($($acc:tt)*), | |
880 | ) => { | |
881 | $crate::try_init!(make_initializer: | |
882 | @slot($slot), | |
883 | @type_name($t), | |
884 | @munch_fields($($rest)*), | |
885 | @acc($($acc)*$field: ::core::panic!(),), | |
886 | ); | |
887 | }; | |
888 | (forget_guards: | |
889 | @munch_fields($(,)?), | |
890 | ) => { | |
891 | // Munching finished. | |
892 | }; | |
893 | (forget_guards: | |
894 | @munch_fields($field:ident <- $val:expr, $($rest:tt)*), | |
895 | ) => { | |
896 | unsafe { $crate::init::__internal::DropGuard::forget($field) }; | |
897 | ||
898 | $crate::try_init!(forget_guards: | |
899 | @munch_fields($($rest)*), | |
900 | ); | |
901 | }; | |
902 | (forget_guards: | |
903 | @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), | |
904 | ) => { | |
905 | unsafe { $crate::init::__internal::DropGuard::forget($field) }; | |
906 | ||
907 | $crate::try_init!(forget_guards: | |
908 | @munch_fields($($rest)*), | |
909 | ); | |
910 | }; | |
911 | } | |
90e53c5e BL |
912 | |
913 | /// A pin-initializer for the type `T`. | |
914 | /// | |
915 | /// To use this initializer, you will need a suitable memory location that can hold a `T`. This can | |
92c4a1e7 BL |
916 | /// be [`Box<T>`], [`Arc<T>`], [`UniqueArc<T>`]. Use the [`InPlaceInit::pin_init`] function of a |
917 | /// smart pointer like [`Arc<T>`] on this. | |
90e53c5e BL |
918 | /// |
919 | /// Also see the [module description](self). | |
920 | /// | |
921 | /// # Safety | |
922 | /// | |
923 | /// When implementing this type you will need to take great care. Also there are probably very few | |
924 | /// cases where a manual implementation is necessary. Use [`pin_init_from_closure`] where possible. | |
925 | /// | |
926 | /// The [`PinInit::__pinned_init`] function | |
927 | /// - returns `Ok(())` if it initialized every field of `slot`, | |
928 | /// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means: | |
929 | /// - `slot` can be deallocated without UB occurring, | |
930 | /// - `slot` does not need to be dropped, | |
931 | /// - `slot` is not partially initialized. | |
932 | /// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`. | |
933 | /// | |
934 | /// [`Arc<T>`]: crate::sync::Arc | |
935 | /// [`Arc::pin_init`]: crate::sync::Arc::pin_init | |
90e53c5e BL |
936 | #[must_use = "An initializer must be used in order to create its value."] |
937 | pub unsafe trait PinInit<T: ?Sized, E = Infallible>: Sized { | |
938 | /// Initializes `slot`. | |
939 | /// | |
940 | /// # Safety | |
941 | /// | |
942 | /// - `slot` is a valid pointer to uninitialized memory. | |
943 | /// - the caller does not touch `slot` when `Err` is returned, they are only permitted to | |
944 | /// deallocate. | |
945 | /// - `slot` will not move until it is dropped, i.e. it will be pinned. | |
946 | unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E>; | |
947 | } | |
948 | ||
949 | /// An initializer for `T`. | |
950 | /// | |
951 | /// To use this initializer, you will need a suitable memory location that can hold a `T`. This can | |
92c4a1e7 BL |
952 | /// be [`Box<T>`], [`Arc<T>`], [`UniqueArc<T>`]. Use the [`InPlaceInit::init`] function of a smart |
953 | /// pointer like [`Arc<T>`] on this. Because [`PinInit<T, E>`] is a super trait, you can | |
90e53c5e BL |
954 | /// use every function that takes it as well. |
955 | /// | |
956 | /// Also see the [module description](self). | |
957 | /// | |
958 | /// # Safety | |
959 | /// | |
960 | /// When implementing this type you will need to take great care. Also there are probably very few | |
961 | /// cases where a manual implementation is necessary. Use [`init_from_closure`] where possible. | |
962 | /// | |
963 | /// The [`Init::__init`] function | |
964 | /// - returns `Ok(())` if it initialized every field of `slot`, | |
965 | /// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means: | |
966 | /// - `slot` can be deallocated without UB occurring, | |
967 | /// - `slot` does not need to be dropped, | |
968 | /// - `slot` is not partially initialized. | |
969 | /// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`. | |
970 | /// | |
971 | /// The `__pinned_init` function from the supertrait [`PinInit`] needs to execute the exact same | |
972 | /// code as `__init`. | |
973 | /// | |
974 | /// Contrary to its supertype [`PinInit<T, E>`] the caller is allowed to | |
975 | /// move the pointee after initialization. | |
976 | /// | |
977 | /// [`Arc<T>`]: crate::sync::Arc | |
90e53c5e BL |
978 | #[must_use = "An initializer must be used in order to create its value."] |
979 | pub unsafe trait Init<T: ?Sized, E = Infallible>: Sized { | |
980 | /// Initializes `slot`. | |
981 | /// | |
982 | /// # Safety | |
983 | /// | |
984 | /// - `slot` is a valid pointer to uninitialized memory. | |
985 | /// - the caller does not touch `slot` when `Err` is returned, they are only permitted to | |
986 | /// deallocate. | |
987 | unsafe fn __init(self, slot: *mut T) -> Result<(), E>; | |
988 | } | |
989 | ||
990 | // SAFETY: Every in-place initializer can also be used as a pin-initializer. | |
991 | unsafe impl<T: ?Sized, E, I> PinInit<T, E> for I | |
992 | where | |
993 | I: Init<T, E>, | |
994 | { | |
995 | unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> { | |
996 | // SAFETY: `__init` meets the same requirements as `__pinned_init`, except that it does not | |
997 | // require `slot` to not move after init. | |
998 | unsafe { self.__init(slot) } | |
999 | } | |
1000 | } | |
1001 | ||
1002 | /// Creates a new [`PinInit<T, E>`] from the given closure. | |
1003 | /// | |
1004 | /// # Safety | |
1005 | /// | |
1006 | /// The closure: | |
1007 | /// - returns `Ok(())` if it initialized every field of `slot`, | |
1008 | /// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means: | |
1009 | /// - `slot` can be deallocated without UB occurring, | |
1010 | /// - `slot` does not need to be dropped, | |
1011 | /// - `slot` is not partially initialized. | |
1012 | /// - may assume that the `slot` does not move if `T: !Unpin`, | |
1013 | /// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`. | |
1014 | #[inline] | |
1015 | pub const unsafe fn pin_init_from_closure<T: ?Sized, E>( | |
1016 | f: impl FnOnce(*mut T) -> Result<(), E>, | |
1017 | ) -> impl PinInit<T, E> { | |
1018 | __internal::InitClosure(f, PhantomData) | |
1019 | } | |
1020 | ||
1021 | /// Creates a new [`Init<T, E>`] from the given closure. | |
1022 | /// | |
1023 | /// # Safety | |
1024 | /// | |
1025 | /// The closure: | |
1026 | /// - returns `Ok(())` if it initialized every field of `slot`, | |
1027 | /// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means: | |
1028 | /// - `slot` can be deallocated without UB occurring, | |
1029 | /// - `slot` does not need to be dropped, | |
1030 | /// - `slot` is not partially initialized. | |
1031 | /// - the `slot` may move after initialization. | |
1032 | /// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`. | |
1033 | #[inline] | |
1034 | pub const unsafe fn init_from_closure<T: ?Sized, E>( | |
1035 | f: impl FnOnce(*mut T) -> Result<(), E>, | |
1036 | ) -> impl Init<T, E> { | |
1037 | __internal::InitClosure(f, PhantomData) | |
1038 | } | |
1039 | ||
1040 | /// An initializer that leaves the memory uninitialized. | |
1041 | /// | |
1042 | /// The initializer is a no-op. The `slot` memory is not changed. | |
1043 | #[inline] | |
1044 | pub fn uninit<T, E>() -> impl Init<MaybeUninit<T>, E> { | |
1045 | // SAFETY: The memory is allowed to be uninitialized. | |
1046 | unsafe { init_from_closure(|_| Ok(())) } | |
1047 | } | |
1048 | ||
1049 | // SAFETY: Every type can be initialized by-value. | |
1050 | unsafe impl<T> Init<T> for T { | |
1051 | unsafe fn __init(self, slot: *mut T) -> Result<(), Infallible> { | |
1052 | unsafe { slot.write(self) }; | |
1053 | Ok(()) | |
1054 | } | |
1055 | } | |
92c4a1e7 BL |
1056 | |
1057 | /// Smart pointer that can initialize memory in-place. | |
1058 | pub trait InPlaceInit<T>: Sized { | |
1059 | /// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this | |
1060 | /// type. | |
1061 | /// | |
1062 | /// If `T: !Unpin` it will not be able to move afterwards. | |
1063 | fn try_pin_init<E>(init: impl PinInit<T, E>) -> Result<Pin<Self>, E> | |
1064 | where | |
1065 | E: From<AllocError>; | |
1066 | ||
1067 | /// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this | |
1068 | /// type. | |
1069 | /// | |
1070 | /// If `T: !Unpin` it will not be able to move afterwards. | |
1071 | fn pin_init<E>(init: impl PinInit<T, E>) -> error::Result<Pin<Self>> | |
1072 | where | |
1073 | Error: From<E>, | |
1074 | { | |
1075 | // SAFETY: We delegate to `init` and only change the error type. | |
1076 | let init = unsafe { | |
1077 | pin_init_from_closure(|slot| init.__pinned_init(slot).map_err(|e| Error::from(e))) | |
1078 | }; | |
1079 | Self::try_pin_init(init) | |
1080 | } | |
1081 | ||
1082 | /// Use the given initializer to in-place initialize a `T`. | |
1083 | fn try_init<E>(init: impl Init<T, E>) -> Result<Self, E> | |
1084 | where | |
1085 | E: From<AllocError>; | |
1086 | ||
1087 | /// Use the given initializer to in-place initialize a `T`. | |
1088 | fn init<E>(init: impl Init<T, E>) -> error::Result<Self> | |
1089 | where | |
1090 | Error: From<E>, | |
1091 | { | |
1092 | // SAFETY: We delegate to `init` and only change the error type. | |
1093 | let init = unsafe { | |
1094 | init_from_closure(|slot| init.__pinned_init(slot).map_err(|e| Error::from(e))) | |
1095 | }; | |
1096 | Self::try_init(init) | |
1097 | } | |
1098 | } | |
1099 | ||
1100 | impl<T> InPlaceInit<T> for Box<T> { | |
1101 | #[inline] | |
1102 | fn try_pin_init<E>(init: impl PinInit<T, E>) -> Result<Pin<Self>, E> | |
1103 | where | |
1104 | E: From<AllocError>, | |
1105 | { | |
1106 | let mut this = Box::try_new_uninit()?; | |
1107 | let slot = this.as_mut_ptr(); | |
1108 | // SAFETY: When init errors/panics, slot will get deallocated but not dropped, | |
1109 | // slot is valid and will not be moved, because we pin it later. | |
1110 | unsafe { init.__pinned_init(slot)? }; | |
1111 | // SAFETY: All fields have been initialized. | |
1112 | Ok(unsafe { this.assume_init() }.into()) | |
1113 | } | |
1114 | ||
1115 | #[inline] | |
1116 | fn try_init<E>(init: impl Init<T, E>) -> Result<Self, E> | |
1117 | where | |
1118 | E: From<AllocError>, | |
1119 | { | |
1120 | let mut this = Box::try_new_uninit()?; | |
1121 | let slot = this.as_mut_ptr(); | |
1122 | // SAFETY: When init errors/panics, slot will get deallocated but not dropped, | |
1123 | // slot is valid. | |
1124 | unsafe { init.__init(slot)? }; | |
1125 | // SAFETY: All fields have been initialized. | |
1126 | Ok(unsafe { this.assume_init() }) | |
1127 | } | |
1128 | } | |
1129 | ||
1130 | impl<T> InPlaceInit<T> for UniqueArc<T> { | |
1131 | #[inline] | |
1132 | fn try_pin_init<E>(init: impl PinInit<T, E>) -> Result<Pin<Self>, E> | |
1133 | where | |
1134 | E: From<AllocError>, | |
1135 | { | |
1136 | let mut this = UniqueArc::try_new_uninit()?; | |
1137 | let slot = this.as_mut_ptr(); | |
1138 | // SAFETY: When init errors/panics, slot will get deallocated but not dropped, | |
1139 | // slot is valid and will not be moved, because we pin it later. | |
1140 | unsafe { init.__pinned_init(slot)? }; | |
1141 | // SAFETY: All fields have been initialized. | |
1142 | Ok(unsafe { this.assume_init() }.into()) | |
1143 | } | |
1144 | ||
1145 | #[inline] | |
1146 | fn try_init<E>(init: impl Init<T, E>) -> Result<Self, E> | |
1147 | where | |
1148 | E: From<AllocError>, | |
1149 | { | |
1150 | let mut this = UniqueArc::try_new_uninit()?; | |
1151 | let slot = this.as_mut_ptr(); | |
1152 | // SAFETY: When init errors/panics, slot will get deallocated but not dropped, | |
1153 | // slot is valid. | |
1154 | unsafe { init.__init(slot)? }; | |
1155 | // SAFETY: All fields have been initialized. | |
1156 | Ok(unsafe { this.assume_init() }) | |
1157 | } | |
1158 | } | |
d0fdc396 BL |
1159 | |
1160 | /// Trait facilitating pinned destruction. | |
1161 | /// | |
1162 | /// Use [`pinned_drop`] to implement this trait safely: | |
1163 | /// | |
1164 | /// ```rust | |
1165 | /// # use kernel::sync::Mutex; | |
1166 | /// use kernel::macros::pinned_drop; | |
1167 | /// use core::pin::Pin; | |
1168 | /// #[pin_data(PinnedDrop)] | |
1169 | /// struct Foo { | |
1170 | /// #[pin] | |
1171 | /// mtx: Mutex<usize>, | |
1172 | /// } | |
1173 | /// | |
1174 | /// #[pinned_drop] | |
1175 | /// impl PinnedDrop for Foo { | |
1176 | /// fn drop(self: Pin<&mut Self>) { | |
1177 | /// pr_info!("Foo is being dropped!"); | |
1178 | /// } | |
1179 | /// } | |
1180 | /// ``` | |
1181 | /// | |
1182 | /// # Safety | |
1183 | /// | |
1184 | /// This trait must be implemented via the [`pinned_drop`] proc-macro attribute on the impl. | |
1185 | /// | |
1186 | /// [`pinned_drop`]: kernel::macros::pinned_drop | |
1187 | pub unsafe trait PinnedDrop: __internal::HasPinData { | |
1188 | /// Executes the pinned destructor of this type. | |
1189 | /// | |
1190 | /// While this function is marked safe, it is actually unsafe to call it manually. For this | |
1191 | /// reason it takes an additional parameter. This type can only be constructed by `unsafe` code | |
1192 | /// and thus prevents this function from being called where it should not. | |
1193 | /// | |
1194 | /// This extra parameter will be generated by the `#[pinned_drop]` proc-macro attribute | |
1195 | /// automatically. | |
1196 | fn drop(self: Pin<&mut Self>, only_call_from_drop: __internal::OnlyCallFromDrop); | |
1197 | } |