Commit | Line | Data |
---|---|---|
90e53c5e BL |
1 | // SPDX-License-Identifier: Apache-2.0 OR MIT |
2 | ||
3 | //! API to safely and fallibly initialize pinned `struct`s using in-place constructors. | |
4 | //! | |
5 | //! It also allows in-place initialization of big `struct`s that would otherwise produce a stack | |
6 | //! overflow. | |
7 | //! | |
8 | //! Most `struct`s from the [`sync`] module need to be pinned, because they contain self-referential | |
9 | //! `struct`s from C. [Pinning][pinning] is Rust's way of ensuring data does not move. | |
10 | //! | |
11 | //! # Overview | |
12 | //! | |
13 | //! To initialize a `struct` with an in-place constructor you will need two things: | |
14 | //! - an in-place constructor, | |
6841d45a BL |
15 | //! - a memory location that can hold your `struct` (this can be the [stack], an [`Arc<T>`], |
16 | //! [`UniqueArc<T>`], [`Box<T>`] or any other smart pointer that implements [`InPlaceInit`]). | |
90e53c5e | 17 | //! |
fc6c6baa BL |
18 | //! To get an in-place constructor there are generally three options: |
19 | //! - directly creating an in-place constructor using the [`pin_init!`] macro, | |
90e53c5e BL |
20 | //! - a custom function/macro returning an in-place constructor provided by someone else, |
21 | //! - using the unsafe function [`pin_init_from_closure()`] to manually create an initializer. | |
22 | //! | |
23 | //! Aside from pinned initialization, this API also supports in-place construction without pinning, | |
24 | //! the macros/types/functions are generally named like the pinned variants without the `pin` | |
25 | //! prefix. | |
26 | //! | |
fc6c6baa BL |
27 | //! # Examples |
28 | //! | |
29 | //! ## Using the [`pin_init!`] macro | |
30 | //! | |
31 | //! If you want to use [`PinInit`], then you will have to annotate your `struct` with | |
32 | //! `#[`[`pin_data`]`]`. It is a macro that uses `#[pin]` as a marker for | |
33 | //! [structurally pinned fields]. After doing this, you can then create an in-place constructor via | |
34 | //! [`pin_init!`]. The syntax is almost the same as normal `struct` initializers. The difference is | |
35 | //! that you need to write `<-` instead of `:` for fields that you want to initialize in-place. | |
36 | //! | |
37 | //! ```rust | |
b2516f7a | 38 | //! # #![allow(clippy::disallowed_names)] |
fc6c6baa BL |
39 | //! use kernel::{prelude::*, sync::Mutex, new_mutex}; |
40 | //! # use core::pin::Pin; | |
41 | //! #[pin_data] | |
42 | //! struct Foo { | |
43 | //! #[pin] | |
44 | //! a: Mutex<usize>, | |
45 | //! b: u32, | |
46 | //! } | |
47 | //! | |
48 | //! let foo = pin_init!(Foo { | |
49 | //! a <- new_mutex!(42, "Foo::a"), | |
50 | //! b: 24, | |
51 | //! }); | |
52 | //! ``` | |
53 | //! | |
54 | //! `foo` now is of the type [`impl PinInit<Foo>`]. We can now use any smart pointer that we like | |
55 | //! (or just the stack) to actually initialize a `Foo`: | |
56 | //! | |
57 | //! ```rust | |
b2516f7a | 58 | //! # #![allow(clippy::disallowed_names)] |
fc6c6baa BL |
59 | //! # use kernel::{prelude::*, sync::Mutex, new_mutex}; |
60 | //! # use core::pin::Pin; | |
61 | //! # #[pin_data] | |
62 | //! # struct Foo { | |
63 | //! # #[pin] | |
64 | //! # a: Mutex<usize>, | |
65 | //! # b: u32, | |
66 | //! # } | |
67 | //! # let foo = pin_init!(Foo { | |
68 | //! # a <- new_mutex!(42, "Foo::a"), | |
69 | //! # b: 24, | |
70 | //! # }); | |
71 | //! let foo: Result<Pin<Box<Foo>>> = Box::pin_init(foo); | |
72 | //! ``` | |
73 | //! | |
74 | //! For more information see the [`pin_init!`] macro. | |
75 | //! | |
76 | //! ## Using a custom function/macro that returns an initializer | |
77 | //! | |
78 | //! Many types from the kernel supply a function/macro that returns an initializer, because the | |
79 | //! above method only works for types where you can access the fields. | |
80 | //! | |
81 | //! ```rust | |
82 | //! # use kernel::{new_mutex, sync::{Arc, Mutex}}; | |
83 | //! let mtx: Result<Arc<Mutex<usize>>> = Arc::pin_init(new_mutex!(42, "example::mtx")); | |
84 | //! ``` | |
85 | //! | |
86 | //! To declare an init macro/function you just return an [`impl PinInit<T, E>`]: | |
87 | //! | |
88 | //! ```rust | |
b2516f7a | 89 | //! # #![allow(clippy::disallowed_names)] |
fc6c6baa BL |
90 | //! # use kernel::{sync::Mutex, prelude::*, new_mutex, init::PinInit, try_pin_init}; |
91 | //! #[pin_data] | |
92 | //! struct DriverData { | |
93 | //! #[pin] | |
94 | //! status: Mutex<i32>, | |
95 | //! buffer: Box<[u8; 1_000_000]>, | |
96 | //! } | |
97 | //! | |
98 | //! impl DriverData { | |
99 | //! fn new() -> impl PinInit<Self, Error> { | |
100 | //! try_pin_init!(Self { | |
101 | //! status <- new_mutex!(0, "DriverData::status"), | |
102 | //! buffer: Box::init(kernel::init::zeroed())?, | |
103 | //! }) | |
104 | //! } | |
105 | //! } | |
106 | //! ``` | |
107 | //! | |
d0fdc396 BL |
108 | //! ## Manual creation of an initializer |
109 | //! | |
110 | //! Often when working with primitives the previous approaches are not sufficient. That is where | |
111 | //! [`pin_init_from_closure()`] comes in. This `unsafe` function allows you to create a | |
112 | //! [`impl PinInit<T, E>`] directly from a closure. Of course you have to ensure that the closure | |
113 | //! actually does the initialization in the correct way. Here are the things to look out for | |
114 | //! (we are calling the parameter to the closure `slot`): | |
115 | //! - when the closure returns `Ok(())`, then it has completed the initialization successfully, so | |
116 | //! `slot` now contains a valid bit pattern for the type `T`, | |
117 | //! - when the closure returns `Err(e)`, then the caller may deallocate the memory at `slot`, so | |
118 | //! you need to take care to clean up anything if your initialization fails mid-way, | |
119 | //! - you may assume that `slot` will stay pinned even after the closure returns until `drop` of | |
120 | //! `slot` gets called. | |
121 | //! | |
122 | //! ```rust | |
a30e94c2 MO |
123 | //! # #![allow(unreachable_pub, clippy::disallowed_names)] |
124 | //! use kernel::{prelude::*, init, types::Opaque}; | |
d0fdc396 BL |
125 | //! use core::{ptr::addr_of_mut, marker::PhantomPinned, pin::Pin}; |
126 | //! # mod bindings { | |
a30e94c2 | 127 | //! # #![allow(non_camel_case_types)] |
d0fdc396 BL |
128 | //! # pub struct foo; |
129 | //! # pub unsafe fn init_foo(_ptr: *mut foo) {} | |
130 | //! # pub unsafe fn destroy_foo(_ptr: *mut foo) {} | |
131 | //! # pub unsafe fn enable_foo(_ptr: *mut foo, _flags: u32) -> i32 { 0 } | |
132 | //! # } | |
a30e94c2 MO |
133 | //! # // `Error::from_errno` is `pub(crate)` in the `kernel` crate, thus provide a workaround. |
134 | //! # trait FromErrno { | |
135 | //! # fn from_errno(errno: core::ffi::c_int) -> Error { | |
136 | //! # // Dummy error that can be constructed outside the `kernel` crate. | |
137 | //! # Error::from(core::fmt::Error) | |
138 | //! # } | |
139 | //! # } | |
140 | //! # impl FromErrno for Error {} | |
d0fdc396 BL |
141 | //! /// # Invariants |
142 | //! /// | |
143 | //! /// `foo` is always initialized | |
144 | //! #[pin_data(PinnedDrop)] | |
145 | //! pub struct RawFoo { | |
146 | //! #[pin] | |
147 | //! foo: Opaque<bindings::foo>, | |
148 | //! #[pin] | |
149 | //! _p: PhantomPinned, | |
150 | //! } | |
151 | //! | |
152 | //! impl RawFoo { | |
153 | //! pub fn new(flags: u32) -> impl PinInit<Self, Error> { | |
154 | //! // SAFETY: | |
155 | //! // - when the closure returns `Ok(())`, then it has successfully initialized and | |
156 | //! // enabled `foo`, | |
157 | //! // - when it returns `Err(e)`, then it has cleaned up before | |
158 | //! unsafe { | |
159 | //! init::pin_init_from_closure(move |slot: *mut Self| { | |
160 | //! // `slot` contains uninit memory, avoid creating a reference. | |
161 | //! let foo = addr_of_mut!((*slot).foo); | |
162 | //! | |
163 | //! // Initialize the `foo` | |
164 | //! bindings::init_foo(Opaque::raw_get(foo)); | |
165 | //! | |
166 | //! // Try to enable it. | |
167 | //! let err = bindings::enable_foo(Opaque::raw_get(foo), flags); | |
168 | //! if err != 0 { | |
169 | //! // Enabling has failed, first clean up the foo and then return the error. | |
170 | //! bindings::destroy_foo(Opaque::raw_get(foo)); | |
a30e94c2 | 171 | //! return Err(Error::from_errno(err)); |
d0fdc396 BL |
172 | //! } |
173 | //! | |
174 | //! // All fields of `RawFoo` have been initialized, since `_p` is a ZST. | |
175 | //! Ok(()) | |
176 | //! }) | |
177 | //! } | |
178 | //! } | |
179 | //! } | |
180 | //! | |
181 | //! #[pinned_drop] | |
182 | //! impl PinnedDrop for RawFoo { | |
183 | //! fn drop(self: Pin<&mut Self>) { | |
184 | //! // SAFETY: Since `foo` is initialized, destroying is safe. | |
185 | //! unsafe { bindings::destroy_foo(self.foo.get()) }; | |
186 | //! } | |
187 | //! } | |
188 | //! ``` | |
189 | //! | |
692e8935 BL |
190 | //! For the special case where initializing a field is a single FFI-function call that cannot fail, |
191 | //! there exist the helper function [`Opaque::ffi_init`]. This function initialize a single | |
192 | //! [`Opaque`] field by just delegating to the supplied closure. You can use these in combination | |
193 | //! with [`pin_init!`]. | |
194 | //! | |
195 | //! For more information on how to use [`pin_init_from_closure()`], take a look at the uses inside | |
196 | //! the `kernel` crate. The [`sync`] module is a good starting point. | |
197 | //! | |
90e53c5e BL |
198 | //! [`sync`]: kernel::sync |
199 | //! [pinning]: https://doc.rust-lang.org/std/pin/index.html | |
200 | //! [structurally pinned fields]: | |
201 | //! https://doc.rust-lang.org/std/pin/index.html#pinning-is-structural-for-field | |
6841d45a | 202 | //! [stack]: crate::stack_pin_init |
90e53c5e BL |
203 | //! [`Arc<T>`]: crate::sync::Arc |
204 | //! [`impl PinInit<Foo>`]: PinInit | |
205 | //! [`impl PinInit<T, E>`]: PinInit | |
206 | //! [`impl Init<T, E>`]: Init | |
207 | //! [`Opaque`]: kernel::types::Opaque | |
692e8935 | 208 | //! [`Opaque::ffi_init`]: kernel::types::Opaque::ffi_init |
90e53c5e | 209 | //! [`pin_data`]: ::macros::pin_data |
3ed03f4d | 210 | //! [`pin_init!`]: crate::pin_init! |
90e53c5e | 211 | |
92c4a1e7 BL |
212 | use crate::{ |
213 | error::{self, Error}, | |
214 | sync::UniqueArc, | |
2e704f18 | 215 | types::{Opaque, ScopeGuard}, |
92c4a1e7 | 216 | }; |
fc6c6baa | 217 | use alloc::boxed::Box; |
92c4a1e7 | 218 | use core::{ |
38cde0bd | 219 | alloc::AllocError, |
2e704f18 | 220 | cell::UnsafeCell, |
38cde0bd BL |
221 | convert::Infallible, |
222 | marker::PhantomData, | |
223 | mem::MaybeUninit, | |
224 | num::*, | |
225 | pin::Pin, | |
226 | ptr::{self, NonNull}, | |
92c4a1e7 | 227 | }; |
90e53c5e BL |
228 | |
229 | #[doc(hidden)] | |
230 | pub mod __internal; | |
fc6c6baa BL |
231 | #[doc(hidden)] |
232 | pub mod macros; | |
233 | ||
6841d45a BL |
234 | /// Initialize and pin a type directly on the stack. |
235 | /// | |
236 | /// # Examples | |
237 | /// | |
238 | /// ```rust | |
b2516f7a | 239 | /// # #![allow(clippy::disallowed_names)] |
a30e94c2 | 240 | /// # use kernel::{init, macros::pin_data, pin_init, stack_pin_init, init::*, sync::Mutex, new_mutex}; |
6841d45a BL |
241 | /// # use core::pin::Pin; |
242 | /// #[pin_data] | |
243 | /// struct Foo { | |
244 | /// #[pin] | |
245 | /// a: Mutex<usize>, | |
246 | /// b: Bar, | |
247 | /// } | |
248 | /// | |
249 | /// #[pin_data] | |
250 | /// struct Bar { | |
251 | /// x: u32, | |
252 | /// } | |
253 | /// | |
254 | /// stack_pin_init!(let foo = pin_init!(Foo { | |
255 | /// a <- new_mutex!(42), | |
256 | /// b: Bar { | |
257 | /// x: 64, | |
258 | /// }, | |
259 | /// })); | |
260 | /// let foo: Pin<&mut Foo> = foo; | |
261 | /// pr_info!("a: {}", &*foo.a.lock()); | |
262 | /// ``` | |
263 | /// | |
264 | /// # Syntax | |
265 | /// | |
266 | /// A normal `let` binding with optional type annotation. The expression is expected to implement | |
267 | /// [`PinInit`]/[`Init`] with the error type [`Infallible`]. If you want to use a different error | |
268 | /// type, then use [`stack_try_pin_init!`]. | |
3ed03f4d MO |
269 | /// |
270 | /// [`stack_try_pin_init!`]: crate::stack_try_pin_init! | |
6841d45a BL |
271 | #[macro_export] |
272 | macro_rules! stack_pin_init { | |
273 | (let $var:ident $(: $t:ty)? = $val:expr) => { | |
274 | let val = $val; | |
275 | let mut $var = ::core::pin::pin!($crate::init::__internal::StackInit$(::<$t>)?::uninit()); | |
276 | let mut $var = match $crate::init::__internal::StackInit::init($var, val) { | |
277 | Ok(res) => res, | |
278 | Err(x) => { | |
279 | let x: ::core::convert::Infallible = x; | |
280 | match x {} | |
281 | } | |
282 | }; | |
283 | }; | |
284 | } | |
285 | ||
286 | /// Initialize and pin a type directly on the stack. | |
287 | /// | |
288 | /// # Examples | |
289 | /// | |
a30e94c2 | 290 | /// ```rust,ignore |
b2516f7a | 291 | /// # #![allow(clippy::disallowed_names)] |
6841d45a BL |
292 | /// # use kernel::{init, pin_init, stack_try_pin_init, init::*, sync::Mutex, new_mutex}; |
293 | /// # use macros::pin_data; | |
294 | /// # use core::{alloc::AllocError, pin::Pin}; | |
295 | /// #[pin_data] | |
296 | /// struct Foo { | |
297 | /// #[pin] | |
298 | /// a: Mutex<usize>, | |
299 | /// b: Box<Bar>, | |
300 | /// } | |
301 | /// | |
302 | /// struct Bar { | |
303 | /// x: u32, | |
304 | /// } | |
305 | /// | |
306 | /// stack_try_pin_init!(let foo: Result<Pin<&mut Foo>, AllocError> = pin_init!(Foo { | |
307 | /// a <- new_mutex!(42), | |
308 | /// b: Box::try_new(Bar { | |
309 | /// x: 64, | |
310 | /// })?, | |
311 | /// })); | |
312 | /// let foo = foo.unwrap(); | |
313 | /// pr_info!("a: {}", &*foo.a.lock()); | |
314 | /// ``` | |
315 | /// | |
a30e94c2 | 316 | /// ```rust,ignore |
b2516f7a | 317 | /// # #![allow(clippy::disallowed_names)] |
6841d45a BL |
318 | /// # use kernel::{init, pin_init, stack_try_pin_init, init::*, sync::Mutex, new_mutex}; |
319 | /// # use macros::pin_data; | |
320 | /// # use core::{alloc::AllocError, pin::Pin}; | |
321 | /// #[pin_data] | |
322 | /// struct Foo { | |
323 | /// #[pin] | |
324 | /// a: Mutex<usize>, | |
325 | /// b: Box<Bar>, | |
326 | /// } | |
327 | /// | |
328 | /// struct Bar { | |
329 | /// x: u32, | |
330 | /// } | |
331 | /// | |
332 | /// stack_try_pin_init!(let foo: Pin<&mut Foo> =? pin_init!(Foo { | |
333 | /// a <- new_mutex!(42), | |
334 | /// b: Box::try_new(Bar { | |
335 | /// x: 64, | |
336 | /// })?, | |
337 | /// })); | |
338 | /// pr_info!("a: {}", &*foo.a.lock()); | |
339 | /// # Ok::<_, AllocError>(()) | |
340 | /// ``` | |
341 | /// | |
342 | /// # Syntax | |
343 | /// | |
344 | /// A normal `let` binding with optional type annotation. The expression is expected to implement | |
345 | /// [`PinInit`]/[`Init`]. This macro assigns a result to the given variable, adding a `?` after the | |
346 | /// `=` will propagate this error. | |
347 | #[macro_export] | |
348 | macro_rules! stack_try_pin_init { | |
349 | (let $var:ident $(: $t:ty)? = $val:expr) => { | |
350 | let val = $val; | |
351 | let mut $var = ::core::pin::pin!($crate::init::__internal::StackInit$(::<$t>)?::uninit()); | |
352 | let mut $var = $crate::init::__internal::StackInit::init($var, val); | |
353 | }; | |
354 | (let $var:ident $(: $t:ty)? =? $val:expr) => { | |
355 | let val = $val; | |
356 | let mut $var = ::core::pin::pin!($crate::init::__internal::StackInit$(::<$t>)?::uninit()); | |
357 | let mut $var = $crate::init::__internal::StackInit::init($var, val)?; | |
358 | }; | |
359 | } | |
360 | ||
fc6c6baa BL |
361 | /// Construct an in-place, pinned initializer for `struct`s. |
362 | /// | |
363 | /// This macro defaults the error to [`Infallible`]. If you need [`Error`], then use | |
364 | /// [`try_pin_init!`]. | |
365 | /// | |
366 | /// The syntax is almost identical to that of a normal `struct` initializer: | |
367 | /// | |
368 | /// ```rust | |
b2516f7a | 369 | /// # #![allow(clippy::disallowed_names)] |
fc6c6baa BL |
370 | /// # use kernel::{init, pin_init, macros::pin_data, init::*}; |
371 | /// # use core::pin::Pin; | |
372 | /// #[pin_data] | |
373 | /// struct Foo { | |
374 | /// a: usize, | |
375 | /// b: Bar, | |
376 | /// } | |
377 | /// | |
378 | /// #[pin_data] | |
379 | /// struct Bar { | |
380 | /// x: u32, | |
381 | /// } | |
382 | /// | |
383 | /// # fn demo() -> impl PinInit<Foo> { | |
384 | /// let a = 42; | |
385 | /// | |
386 | /// let initializer = pin_init!(Foo { | |
387 | /// a, | |
388 | /// b: Bar { | |
389 | /// x: 64, | |
390 | /// }, | |
391 | /// }); | |
392 | /// # initializer } | |
393 | /// # Box::pin_init(demo()).unwrap(); | |
394 | /// ``` | |
395 | /// | |
396 | /// Arbitrary Rust expressions can be used to set the value of a variable. | |
397 | /// | |
398 | /// The fields are initialized in the order that they appear in the initializer. So it is possible | |
399 | /// to read already initialized fields using raw pointers. | |
400 | /// | |
401 | /// IMPORTANT: You are not allowed to create references to fields of the struct inside of the | |
402 | /// initializer. | |
403 | /// | |
404 | /// # Init-functions | |
405 | /// | |
406 | /// When working with this API it is often desired to let others construct your types without | |
407 | /// giving access to all fields. This is where you would normally write a plain function `new` | |
408 | /// that would return a new instance of your type. With this API that is also possible. | |
409 | /// However, there are a few extra things to keep in mind. | |
410 | /// | |
411 | /// To create an initializer function, simply declare it like this: | |
412 | /// | |
413 | /// ```rust | |
b2516f7a | 414 | /// # #![allow(clippy::disallowed_names)] |
fc6c6baa BL |
415 | /// # use kernel::{init, pin_init, prelude::*, init::*}; |
416 | /// # use core::pin::Pin; | |
417 | /// # #[pin_data] | |
418 | /// # struct Foo { | |
419 | /// # a: usize, | |
420 | /// # b: Bar, | |
421 | /// # } | |
422 | /// # #[pin_data] | |
423 | /// # struct Bar { | |
424 | /// # x: u32, | |
425 | /// # } | |
426 | /// impl Foo { | |
427 | /// fn new() -> impl PinInit<Self> { | |
428 | /// pin_init!(Self { | |
429 | /// a: 42, | |
430 | /// b: Bar { | |
431 | /// x: 64, | |
432 | /// }, | |
433 | /// }) | |
434 | /// } | |
435 | /// } | |
436 | /// ``` | |
437 | /// | |
438 | /// Users of `Foo` can now create it like this: | |
439 | /// | |
440 | /// ```rust | |
b2516f7a | 441 | /// # #![allow(clippy::disallowed_names)] |
fc6c6baa BL |
442 | /// # use kernel::{init, pin_init, macros::pin_data, init::*}; |
443 | /// # use core::pin::Pin; | |
444 | /// # #[pin_data] | |
445 | /// # struct Foo { | |
446 | /// # a: usize, | |
447 | /// # b: Bar, | |
448 | /// # } | |
449 | /// # #[pin_data] | |
450 | /// # struct Bar { | |
451 | /// # x: u32, | |
452 | /// # } | |
453 | /// # impl Foo { | |
454 | /// # fn new() -> impl PinInit<Self> { | |
455 | /// # pin_init!(Self { | |
456 | /// # a: 42, | |
457 | /// # b: Bar { | |
458 | /// # x: 64, | |
459 | /// # }, | |
460 | /// # }) | |
461 | /// # } | |
462 | /// # } | |
463 | /// let foo = Box::pin_init(Foo::new()); | |
464 | /// ``` | |
465 | /// | |
466 | /// They can also easily embed it into their own `struct`s: | |
467 | /// | |
468 | /// ```rust | |
b2516f7a | 469 | /// # #![allow(clippy::disallowed_names)] |
fc6c6baa BL |
470 | /// # use kernel::{init, pin_init, macros::pin_data, init::*}; |
471 | /// # use core::pin::Pin; | |
472 | /// # #[pin_data] | |
473 | /// # struct Foo { | |
474 | /// # a: usize, | |
475 | /// # b: Bar, | |
476 | /// # } | |
477 | /// # #[pin_data] | |
478 | /// # struct Bar { | |
479 | /// # x: u32, | |
480 | /// # } | |
481 | /// # impl Foo { | |
482 | /// # fn new() -> impl PinInit<Self> { | |
483 | /// # pin_init!(Self { | |
484 | /// # a: 42, | |
485 | /// # b: Bar { | |
486 | /// # x: 64, | |
487 | /// # }, | |
488 | /// # }) | |
489 | /// # } | |
490 | /// # } | |
491 | /// #[pin_data] | |
492 | /// struct FooContainer { | |
493 | /// #[pin] | |
494 | /// foo1: Foo, | |
495 | /// #[pin] | |
496 | /// foo2: Foo, | |
497 | /// other: u32, | |
498 | /// } | |
499 | /// | |
500 | /// impl FooContainer { | |
501 | /// fn new(other: u32) -> impl PinInit<Self> { | |
502 | /// pin_init!(Self { | |
503 | /// foo1 <- Foo::new(), | |
504 | /// foo2 <- Foo::new(), | |
505 | /// other, | |
506 | /// }) | |
507 | /// } | |
508 | /// } | |
509 | /// ``` | |
510 | /// | |
511 | /// Here we see that when using `pin_init!` with `PinInit`, one needs to write `<-` instead of `:`. | |
512 | /// This signifies that the given field is initialized in-place. As with `struct` initializers, just | |
513 | /// writing the field (in this case `other`) without `:` or `<-` means `other: other,`. | |
514 | /// | |
515 | /// # Syntax | |
516 | /// | |
517 | /// As already mentioned in the examples above, inside of `pin_init!` a `struct` initializer with | |
518 | /// the following modifications is expected: | |
519 | /// - Fields that you want to initialize in-place have to use `<-` instead of `:`. | |
520 | /// - In front of the initializer you can write `&this in` to have access to a [`NonNull<Self>`] | |
521 | /// pointer named `this` inside of the initializer. | |
35e7fca2 BL |
522 | /// - Using struct update syntax one can place `..Zeroable::zeroed()` at the very end of the |
523 | /// struct, this initializes every field with 0 and then runs all initializers specified in the | |
524 | /// body. This can only be done if [`Zeroable`] is implemented for the struct. | |
fc6c6baa BL |
525 | /// |
526 | /// For instance: | |
527 | /// | |
528 | /// ```rust | |
a031fe8d | 529 | /// # use kernel::{macros::{Zeroable, pin_data}, pin_init}; |
fc6c6baa BL |
530 | /// # use core::{ptr::addr_of_mut, marker::PhantomPinned}; |
531 | /// #[pin_data] | |
35e7fca2 | 532 | /// #[derive(Zeroable)] |
fc6c6baa BL |
533 | /// struct Buf { |
534 | /// // `ptr` points into `buf`. | |
535 | /// ptr: *mut u8, | |
536 | /// buf: [u8; 64], | |
537 | /// #[pin] | |
538 | /// pin: PhantomPinned, | |
539 | /// } | |
540 | /// pin_init!(&this in Buf { | |
541 | /// buf: [0; 64], | |
542 | /// ptr: unsafe { addr_of_mut!((*this.as_ptr()).buf).cast() }, | |
543 | /// pin: PhantomPinned, | |
544 | /// }); | |
35e7fca2 BL |
545 | /// pin_init!(Buf { |
546 | /// buf: [1; 64], | |
547 | /// ..Zeroable::zeroed() | |
548 | /// }); | |
fc6c6baa BL |
549 | /// ``` |
550 | /// | |
551 | /// [`try_pin_init!`]: kernel::try_pin_init | |
552 | /// [`NonNull<Self>`]: core::ptr::NonNull | |
fc6c6baa BL |
553 | // For a detailed example of how this macro works, see the module documentation of the hidden |
554 | // module `__internal` inside of `init/__internal.rs`. | |
555 | #[macro_export] | |
556 | macro_rules! pin_init { | |
557 | ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { | |
558 | $($fields:tt)* | |
559 | }) => { | |
b3068ac3 | 560 | $crate::__init_internal!( |
fc6c6baa BL |
561 | @this($($this)?), |
562 | @typ($t $(::<$($generics),*>)?), | |
563 | @fields($($fields)*), | |
564 | @error(::core::convert::Infallible), | |
b3068ac3 BL |
565 | @data(PinData, use_data), |
566 | @has_data(HasPinData, __pin_data), | |
567 | @construct_closure(pin_init_from_closure), | |
35e7fca2 | 568 | @munch_fields($($fields)*), |
fc6c6baa BL |
569 | ) |
570 | }; | |
571 | } | |
572 | ||
573 | /// Construct an in-place, fallible pinned initializer for `struct`s. | |
574 | /// | |
575 | /// If the initialization can complete without error (or [`Infallible`]), then use [`pin_init!`]. | |
576 | /// | |
577 | /// You can use the `?` operator or use `return Err(err)` inside the initializer to stop | |
578 | /// initialization and return the error. | |
579 | /// | |
580 | /// IMPORTANT: if you have `unsafe` code inside of the initializer you have to ensure that when | |
581 | /// initialization fails, the memory can be safely deallocated without any further modifications. | |
582 | /// | |
583 | /// This macro defaults the error to [`Error`]. | |
584 | /// | |
585 | /// The syntax is identical to [`pin_init!`] with the following exception: you can append `? $type` | |
586 | /// after the `struct` initializer to specify the error type you want to use. | |
587 | /// | |
588 | /// # Examples | |
589 | /// | |
590 | /// ```rust | |
591 | /// # #![feature(new_uninit)] | |
592 | /// use kernel::{init::{self, PinInit}, error::Error}; | |
593 | /// #[pin_data] | |
594 | /// struct BigBuf { | |
595 | /// big: Box<[u8; 1024 * 1024 * 1024]>, | |
596 | /// small: [u8; 1024 * 1024], | |
597 | /// ptr: *mut u8, | |
598 | /// } | |
599 | /// | |
600 | /// impl BigBuf { | |
601 | /// fn new() -> impl PinInit<Self, Error> { | |
602 | /// try_pin_init!(Self { | |
603 | /// big: Box::init(init::zeroed())?, | |
604 | /// small: [0; 1024 * 1024], | |
605 | /// ptr: core::ptr::null_mut(), | |
606 | /// }? Error) | |
607 | /// } | |
608 | /// } | |
609 | /// ``` | |
fc6c6baa BL |
610 | // For a detailed example of how this macro works, see the module documentation of the hidden |
611 | // module `__internal` inside of `init/__internal.rs`. | |
612 | #[macro_export] | |
613 | macro_rules! try_pin_init { | |
614 | ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { | |
615 | $($fields:tt)* | |
616 | }) => { | |
b3068ac3 | 617 | $crate::__init_internal!( |
fc6c6baa BL |
618 | @this($($this)?), |
619 | @typ($t $(::<$($generics),*>)? ), | |
620 | @fields($($fields)*), | |
621 | @error($crate::error::Error), | |
b3068ac3 BL |
622 | @data(PinData, use_data), |
623 | @has_data(HasPinData, __pin_data), | |
624 | @construct_closure(pin_init_from_closure), | |
35e7fca2 | 625 | @munch_fields($($fields)*), |
fc6c6baa BL |
626 | ) |
627 | }; | |
628 | ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { | |
629 | $($fields:tt)* | |
630 | }? $err:ty) => { | |
b3068ac3 | 631 | $crate::__init_internal!( |
fc6c6baa BL |
632 | @this($($this)?), |
633 | @typ($t $(::<$($generics),*>)? ), | |
634 | @fields($($fields)*), | |
635 | @error($err), | |
b3068ac3 BL |
636 | @data(PinData, use_data), |
637 | @has_data(HasPinData, __pin_data), | |
638 | @construct_closure(pin_init_from_closure), | |
35e7fca2 | 639 | @munch_fields($($fields)*), |
fc6c6baa BL |
640 | ) |
641 | }; | |
fc6c6baa BL |
642 | } |
643 | ||
644 | /// Construct an in-place initializer for `struct`s. | |
645 | /// | |
646 | /// This macro defaults the error to [`Infallible`]. If you need [`Error`], then use | |
647 | /// [`try_init!`]. | |
648 | /// | |
649 | /// The syntax is identical to [`pin_init!`] and its safety caveats also apply: | |
650 | /// - `unsafe` code must guarantee either full initialization or return an error and allow | |
651 | /// deallocation of the memory. | |
652 | /// - the fields are initialized in the order given in the initializer. | |
653 | /// - no references to fields are allowed to be created inside of the initializer. | |
654 | /// | |
655 | /// This initializer is for initializing data in-place that might later be moved. If you want to | |
656 | /// pin-initialize, use [`pin_init!`]. | |
3ed03f4d MO |
657 | /// |
658 | /// [`try_init!`]: crate::try_init! | |
fc6c6baa BL |
659 | // For a detailed example of how this macro works, see the module documentation of the hidden |
660 | // module `__internal` inside of `init/__internal.rs`. | |
661 | #[macro_export] | |
662 | macro_rules! init { | |
663 | ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { | |
664 | $($fields:tt)* | |
665 | }) => { | |
b3068ac3 | 666 | $crate::__init_internal!( |
fc6c6baa BL |
667 | @this($($this)?), |
668 | @typ($t $(::<$($generics),*>)?), | |
669 | @fields($($fields)*), | |
670 | @error(::core::convert::Infallible), | |
b3068ac3 BL |
671 | @data(InitData, /*no use_data*/), |
672 | @has_data(HasInitData, __init_data), | |
673 | @construct_closure(init_from_closure), | |
35e7fca2 | 674 | @munch_fields($($fields)*), |
fc6c6baa BL |
675 | ) |
676 | } | |
677 | } | |
678 | ||
679 | /// Construct an in-place fallible initializer for `struct`s. | |
680 | /// | |
681 | /// This macro defaults the error to [`Error`]. If you need [`Infallible`], then use | |
682 | /// [`init!`]. | |
683 | /// | |
684 | /// The syntax is identical to [`try_pin_init!`]. If you want to specify a custom error, | |
685 | /// append `? $type` after the `struct` initializer. | |
686 | /// The safety caveats from [`try_pin_init!`] also apply: | |
687 | /// - `unsafe` code must guarantee either full initialization or return an error and allow | |
688 | /// deallocation of the memory. | |
689 | /// - the fields are initialized in the order given in the initializer. | |
690 | /// - no references to fields are allowed to be created inside of the initializer. | |
691 | /// | |
692 | /// # Examples | |
693 | /// | |
694 | /// ```rust | |
a30e94c2 | 695 | /// use kernel::{init::{PinInit, zeroed}, error::Error}; |
fc6c6baa BL |
696 | /// struct BigBuf { |
697 | /// big: Box<[u8; 1024 * 1024 * 1024]>, | |
698 | /// small: [u8; 1024 * 1024], | |
699 | /// } | |
700 | /// | |
701 | /// impl BigBuf { | |
702 | /// fn new() -> impl Init<Self, Error> { | |
703 | /// try_init!(Self { | |
704 | /// big: Box::init(zeroed())?, | |
705 | /// small: [0; 1024 * 1024], | |
706 | /// }? Error) | |
707 | /// } | |
708 | /// } | |
709 | /// ``` | |
fc6c6baa BL |
710 | // For a detailed example of how this macro works, see the module documentation of the hidden |
711 | // module `__internal` inside of `init/__internal.rs`. | |
712 | #[macro_export] | |
713 | macro_rules! try_init { | |
714 | ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { | |
715 | $($fields:tt)* | |
716 | }) => { | |
b3068ac3 | 717 | $crate::__init_internal!( |
fc6c6baa BL |
718 | @this($($this)?), |
719 | @typ($t $(::<$($generics),*>)?), | |
720 | @fields($($fields)*), | |
721 | @error($crate::error::Error), | |
b3068ac3 BL |
722 | @data(InitData, /*no use_data*/), |
723 | @has_data(HasInitData, __init_data), | |
724 | @construct_closure(init_from_closure), | |
35e7fca2 | 725 | @munch_fields($($fields)*), |
fc6c6baa BL |
726 | ) |
727 | }; | |
728 | ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { | |
729 | $($fields:tt)* | |
730 | }? $err:ty) => { | |
b3068ac3 | 731 | $crate::__init_internal!( |
fc6c6baa BL |
732 | @this($($this)?), |
733 | @typ($t $(::<$($generics),*>)?), | |
734 | @fields($($fields)*), | |
735 | @error($err), | |
b3068ac3 BL |
736 | @data(InitData, /*no use_data*/), |
737 | @has_data(HasInitData, __init_data), | |
738 | @construct_closure(init_from_closure), | |
35e7fca2 | 739 | @munch_fields($($fields)*), |
fc6c6baa BL |
740 | ) |
741 | }; | |
fc6c6baa | 742 | } |
90e53c5e BL |
743 | |
744 | /// A pin-initializer for the type `T`. | |
745 | /// | |
746 | /// To use this initializer, you will need a suitable memory location that can hold a `T`. This can | |
6841d45a BL |
747 | /// be [`Box<T>`], [`Arc<T>`], [`UniqueArc<T>`] or even the stack (see [`stack_pin_init!`]). Use the |
748 | /// [`InPlaceInit::pin_init`] function of a smart pointer like [`Arc<T>`] on this. | |
90e53c5e BL |
749 | /// |
750 | /// Also see the [module description](self). | |
751 | /// | |
752 | /// # Safety | |
753 | /// | |
754 | /// When implementing this type you will need to take great care. Also there are probably very few | |
755 | /// cases where a manual implementation is necessary. Use [`pin_init_from_closure`] where possible. | |
756 | /// | |
757 | /// The [`PinInit::__pinned_init`] function | |
758 | /// - returns `Ok(())` if it initialized every field of `slot`, | |
759 | /// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means: | |
760 | /// - `slot` can be deallocated without UB occurring, | |
761 | /// - `slot` does not need to be dropped, | |
762 | /// - `slot` is not partially initialized. | |
763 | /// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`. | |
764 | /// | |
765 | /// [`Arc<T>`]: crate::sync::Arc | |
766 | /// [`Arc::pin_init`]: crate::sync::Arc::pin_init | |
90e53c5e BL |
767 | #[must_use = "An initializer must be used in order to create its value."] |
768 | pub unsafe trait PinInit<T: ?Sized, E = Infallible>: Sized { | |
769 | /// Initializes `slot`. | |
770 | /// | |
771 | /// # Safety | |
772 | /// | |
773 | /// - `slot` is a valid pointer to uninitialized memory. | |
774 | /// - the caller does not touch `slot` when `Err` is returned, they are only permitted to | |
775 | /// deallocate. | |
776 | /// - `slot` will not move until it is dropped, i.e. it will be pinned. | |
777 | unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E>; | |
7f8977a7 BL |
778 | |
779 | /// First initializes the value using `self` then calls the function `f` with the initialized | |
780 | /// value. | |
781 | /// | |
782 | /// If `f` returns an error the value is dropped and the initializer will forward the error. | |
783 | /// | |
784 | /// # Examples | |
785 | /// | |
786 | /// ```rust | |
787 | /// # #![allow(clippy::disallowed_names)] | |
788 | /// use kernel::{types::Opaque, init::pin_init_from_closure}; | |
789 | /// #[repr(C)] | |
790 | /// struct RawFoo([u8; 16]); | |
791 | /// extern { | |
792 | /// fn init_foo(_: *mut RawFoo); | |
793 | /// } | |
794 | /// | |
795 | /// #[pin_data] | |
796 | /// struct Foo { | |
797 | /// #[pin] | |
798 | /// raw: Opaque<RawFoo>, | |
799 | /// } | |
800 | /// | |
801 | /// impl Foo { | |
802 | /// fn setup(self: Pin<&mut Self>) { | |
803 | /// pr_info!("Setting up foo"); | |
804 | /// } | |
805 | /// } | |
806 | /// | |
807 | /// let foo = pin_init!(Foo { | |
808 | /// raw <- unsafe { | |
809 | /// Opaque::ffi_init(|s| { | |
810 | /// init_foo(s); | |
811 | /// }) | |
812 | /// }, | |
813 | /// }).pin_chain(|foo| { | |
814 | /// foo.setup(); | |
815 | /// Ok(()) | |
816 | /// }); | |
817 | /// ``` | |
818 | fn pin_chain<F>(self, f: F) -> ChainPinInit<Self, F, T, E> | |
819 | where | |
820 | F: FnOnce(Pin<&mut T>) -> Result<(), E>, | |
821 | { | |
822 | ChainPinInit(self, f, PhantomData) | |
823 | } | |
824 | } | |
825 | ||
826 | /// An initializer returned by [`PinInit::pin_chain`]. | |
827 | pub struct ChainPinInit<I, F, T: ?Sized, E>(I, F, __internal::Invariant<(E, Box<T>)>); | |
828 | ||
829 | // SAFETY: The `__pinned_init` function is implemented such that it | |
830 | // - returns `Ok(())` on successful initialization, | |
831 | // - returns `Err(err)` on error and in this case `slot` will be dropped. | |
832 | // - considers `slot` pinned. | |
833 | unsafe impl<T: ?Sized, E, I, F> PinInit<T, E> for ChainPinInit<I, F, T, E> | |
834 | where | |
835 | I: PinInit<T, E>, | |
836 | F: FnOnce(Pin<&mut T>) -> Result<(), E>, | |
837 | { | |
838 | unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> { | |
839 | // SAFETY: All requirements fulfilled since this function is `__pinned_init`. | |
840 | unsafe { self.0.__pinned_init(slot)? }; | |
841 | // SAFETY: The above call initialized `slot` and we still have unique access. | |
842 | let val = unsafe { &mut *slot }; | |
843 | // SAFETY: `slot` is considered pinned. | |
844 | let val = unsafe { Pin::new_unchecked(val) }; | |
845 | (self.1)(val).map_err(|e| { | |
846 | // SAFETY: `slot` was initialized above. | |
847 | unsafe { core::ptr::drop_in_place(slot) }; | |
848 | e | |
849 | }) | |
850 | } | |
90e53c5e BL |
851 | } |
852 | ||
853 | /// An initializer for `T`. | |
854 | /// | |
855 | /// To use this initializer, you will need a suitable memory location that can hold a `T`. This can | |
6841d45a BL |
856 | /// be [`Box<T>`], [`Arc<T>`], [`UniqueArc<T>`] or even the stack (see [`stack_pin_init!`]). Use the |
857 | /// [`InPlaceInit::init`] function of a smart pointer like [`Arc<T>`] on this. Because | |
858 | /// [`PinInit<T, E>`] is a super trait, you can use every function that takes it as well. | |
90e53c5e BL |
859 | /// |
860 | /// Also see the [module description](self). | |
861 | /// | |
862 | /// # Safety | |
863 | /// | |
864 | /// When implementing this type you will need to take great care. Also there are probably very few | |
865 | /// cases where a manual implementation is necessary. Use [`init_from_closure`] where possible. | |
866 | /// | |
867 | /// The [`Init::__init`] function | |
868 | /// - returns `Ok(())` if it initialized every field of `slot`, | |
869 | /// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means: | |
870 | /// - `slot` can be deallocated without UB occurring, | |
871 | /// - `slot` does not need to be dropped, | |
872 | /// - `slot` is not partially initialized. | |
873 | /// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`. | |
874 | /// | |
875 | /// The `__pinned_init` function from the supertrait [`PinInit`] needs to execute the exact same | |
876 | /// code as `__init`. | |
877 | /// | |
878 | /// Contrary to its supertype [`PinInit<T, E>`] the caller is allowed to | |
879 | /// move the pointee after initialization. | |
880 | /// | |
881 | /// [`Arc<T>`]: crate::sync::Arc | |
90e53c5e | 882 | #[must_use = "An initializer must be used in order to create its value."] |
1a8076ac | 883 | pub unsafe trait Init<T: ?Sized, E = Infallible>: PinInit<T, E> { |
90e53c5e BL |
884 | /// Initializes `slot`. |
885 | /// | |
886 | /// # Safety | |
887 | /// | |
888 | /// - `slot` is a valid pointer to uninitialized memory. | |
889 | /// - the caller does not touch `slot` when `Err` is returned, they are only permitted to | |
890 | /// deallocate. | |
891 | unsafe fn __init(self, slot: *mut T) -> Result<(), E>; | |
7f8977a7 BL |
892 | |
893 | /// First initializes the value using `self` then calls the function `f` with the initialized | |
894 | /// value. | |
895 | /// | |
896 | /// If `f` returns an error the value is dropped and the initializer will forward the error. | |
897 | /// | |
898 | /// # Examples | |
899 | /// | |
900 | /// ```rust | |
901 | /// # #![allow(clippy::disallowed_names)] | |
902 | /// use kernel::{types::Opaque, init::{self, init_from_closure}}; | |
903 | /// struct Foo { | |
904 | /// buf: [u8; 1_000_000], | |
905 | /// } | |
906 | /// | |
907 | /// impl Foo { | |
908 | /// fn setup(&mut self) { | |
909 | /// pr_info!("Setting up foo"); | |
910 | /// } | |
911 | /// } | |
912 | /// | |
913 | /// let foo = init!(Foo { | |
914 | /// buf <- init::zeroed() | |
915 | /// }).chain(|foo| { | |
916 | /// foo.setup(); | |
917 | /// Ok(()) | |
918 | /// }); | |
919 | /// ``` | |
920 | fn chain<F>(self, f: F) -> ChainInit<Self, F, T, E> | |
921 | where | |
922 | F: FnOnce(&mut T) -> Result<(), E>, | |
923 | { | |
924 | ChainInit(self, f, PhantomData) | |
925 | } | |
90e53c5e BL |
926 | } |
927 | ||
7f8977a7 BL |
928 | /// An initializer returned by [`Init::chain`]. |
929 | pub struct ChainInit<I, F, T: ?Sized, E>(I, F, __internal::Invariant<(E, Box<T>)>); | |
930 | ||
931 | // SAFETY: The `__init` function is implemented such that it | |
932 | // - returns `Ok(())` on successful initialization, | |
933 | // - returns `Err(err)` on error and in this case `slot` will be dropped. | |
934 | unsafe impl<T: ?Sized, E, I, F> Init<T, E> for ChainInit<I, F, T, E> | |
90e53c5e BL |
935 | where |
936 | I: Init<T, E>, | |
7f8977a7 BL |
937 | F: FnOnce(&mut T) -> Result<(), E>, |
938 | { | |
939 | unsafe fn __init(self, slot: *mut T) -> Result<(), E> { | |
940 | // SAFETY: All requirements fulfilled since this function is `__init`. | |
941 | unsafe { self.0.__pinned_init(slot)? }; | |
942 | // SAFETY: The above call initialized `slot` and we still have unique access. | |
943 | (self.1)(unsafe { &mut *slot }).map_err(|e| { | |
944 | // SAFETY: `slot` was initialized above. | |
945 | unsafe { core::ptr::drop_in_place(slot) }; | |
946 | e | |
947 | }) | |
948 | } | |
949 | } | |
950 | ||
951 | // SAFETY: `__pinned_init` behaves exactly the same as `__init`. | |
952 | unsafe impl<T: ?Sized, E, I, F> PinInit<T, E> for ChainInit<I, F, T, E> | |
953 | where | |
954 | I: Init<T, E>, | |
955 | F: FnOnce(&mut T) -> Result<(), E>, | |
90e53c5e BL |
956 | { |
957 | unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> { | |
7f8977a7 | 958 | // SAFETY: `__init` has less strict requirements compared to `__pinned_init`. |
90e53c5e BL |
959 | unsafe { self.__init(slot) } |
960 | } | |
961 | } | |
962 | ||
963 | /// Creates a new [`PinInit<T, E>`] from the given closure. | |
964 | /// | |
965 | /// # Safety | |
966 | /// | |
967 | /// The closure: | |
968 | /// - returns `Ok(())` if it initialized every field of `slot`, | |
969 | /// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means: | |
970 | /// - `slot` can be deallocated without UB occurring, | |
971 | /// - `slot` does not need to be dropped, | |
972 | /// - `slot` is not partially initialized. | |
973 | /// - may assume that the `slot` does not move if `T: !Unpin`, | |
974 | /// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`. | |
975 | #[inline] | |
976 | pub const unsafe fn pin_init_from_closure<T: ?Sized, E>( | |
977 | f: impl FnOnce(*mut T) -> Result<(), E>, | |
978 | ) -> impl PinInit<T, E> { | |
979 | __internal::InitClosure(f, PhantomData) | |
980 | } | |
981 | ||
982 | /// Creates a new [`Init<T, E>`] from the given closure. | |
983 | /// | |
984 | /// # Safety | |
985 | /// | |
986 | /// The closure: | |
987 | /// - returns `Ok(())` if it initialized every field of `slot`, | |
988 | /// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means: | |
989 | /// - `slot` can be deallocated without UB occurring, | |
990 | /// - `slot` does not need to be dropped, | |
991 | /// - `slot` is not partially initialized. | |
992 | /// - the `slot` may move after initialization. | |
993 | /// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`. | |
994 | #[inline] | |
995 | pub const unsafe fn init_from_closure<T: ?Sized, E>( | |
996 | f: impl FnOnce(*mut T) -> Result<(), E>, | |
997 | ) -> impl Init<T, E> { | |
998 | __internal::InitClosure(f, PhantomData) | |
999 | } | |
1000 | ||
1001 | /// An initializer that leaves the memory uninitialized. | |
1002 | /// | |
1003 | /// The initializer is a no-op. The `slot` memory is not changed. | |
1004 | #[inline] | |
1005 | pub fn uninit<T, E>() -> impl Init<MaybeUninit<T>, E> { | |
1006 | // SAFETY: The memory is allowed to be uninitialized. | |
1007 | unsafe { init_from_closure(|_| Ok(())) } | |
1008 | } | |
1009 | ||
9e494390 BL |
1010 | /// Initializes an array by initializing each element via the provided initializer. |
1011 | /// | |
1012 | /// # Examples | |
1013 | /// | |
1014 | /// ```rust | |
1015 | /// use kernel::{error::Error, init::init_array_from_fn}; | |
1016 | /// let array: Box<[usize; 1_000]>= Box::init::<Error>(init_array_from_fn(|i| i)).unwrap(); | |
1017 | /// assert_eq!(array.len(), 1_000); | |
1018 | /// ``` | |
1019 | pub fn init_array_from_fn<I, const N: usize, T, E>( | |
1020 | mut make_init: impl FnMut(usize) -> I, | |
1021 | ) -> impl Init<[T; N], E> | |
1022 | where | |
1023 | I: Init<T, E>, | |
1024 | { | |
1025 | let init = move |slot: *mut [T; N]| { | |
1026 | let slot = slot.cast::<T>(); | |
1027 | // Counts the number of initialized elements and when dropped drops that many elements from | |
1028 | // `slot`. | |
1029 | let mut init_count = ScopeGuard::new_with_data(0, |i| { | |
1030 | // We now free every element that has been initialized before: | |
1031 | // SAFETY: The loop initialized exactly the values from 0..i and since we | |
1032 | // return `Err` below, the caller will consider the memory at `slot` as | |
1033 | // uninitialized. | |
1034 | unsafe { ptr::drop_in_place(ptr::slice_from_raw_parts_mut(slot, i)) }; | |
1035 | }); | |
1036 | for i in 0..N { | |
1037 | let init = make_init(i); | |
1038 | // SAFETY: Since 0 <= `i` < N, it is still in bounds of `[T; N]`. | |
1039 | let ptr = unsafe { slot.add(i) }; | |
1040 | // SAFETY: The pointer is derived from `slot` and thus satisfies the `__init` | |
1041 | // requirements. | |
1042 | unsafe { init.__init(ptr) }?; | |
1043 | *init_count += 1; | |
1044 | } | |
1045 | init_count.dismiss(); | |
1046 | Ok(()) | |
1047 | }; | |
1048 | // SAFETY: The initializer above initializes every element of the array. On failure it drops | |
1049 | // any initialized elements and returns `Err`. | |
1050 | unsafe { init_from_closure(init) } | |
1051 | } | |
1052 | ||
1053 | /// Initializes an array by initializing each element via the provided initializer. | |
1054 | /// | |
1055 | /// # Examples | |
1056 | /// | |
1057 | /// ```rust | |
1058 | /// use kernel::{sync::{Arc, Mutex}, init::pin_init_array_from_fn, new_mutex}; | |
1059 | /// let array: Arc<[Mutex<usize>; 1_000]>= | |
1060 | /// Arc::pin_init(pin_init_array_from_fn(|i| new_mutex!(i))).unwrap(); | |
1061 | /// assert_eq!(array.len(), 1_000); | |
1062 | /// ``` | |
1063 | pub fn pin_init_array_from_fn<I, const N: usize, T, E>( | |
1064 | mut make_init: impl FnMut(usize) -> I, | |
1065 | ) -> impl PinInit<[T; N], E> | |
1066 | where | |
1067 | I: PinInit<T, E>, | |
1068 | { | |
1069 | let init = move |slot: *mut [T; N]| { | |
1070 | let slot = slot.cast::<T>(); | |
1071 | // Counts the number of initialized elements and when dropped drops that many elements from | |
1072 | // `slot`. | |
1073 | let mut init_count = ScopeGuard::new_with_data(0, |i| { | |
1074 | // We now free every element that has been initialized before: | |
1075 | // SAFETY: The loop initialized exactly the values from 0..i and since we | |
1076 | // return `Err` below, the caller will consider the memory at `slot` as | |
1077 | // uninitialized. | |
1078 | unsafe { ptr::drop_in_place(ptr::slice_from_raw_parts_mut(slot, i)) }; | |
1079 | }); | |
1080 | for i in 0..N { | |
1081 | let init = make_init(i); | |
1082 | // SAFETY: Since 0 <= `i` < N, it is still in bounds of `[T; N]`. | |
1083 | let ptr = unsafe { slot.add(i) }; | |
1084 | // SAFETY: The pointer is derived from `slot` and thus satisfies the `__init` | |
1085 | // requirements. | |
1086 | unsafe { init.__pinned_init(ptr) }?; | |
1087 | *init_count += 1; | |
1088 | } | |
1089 | init_count.dismiss(); | |
1090 | Ok(()) | |
1091 | }; | |
1092 | // SAFETY: The initializer above initializes every element of the array. On failure it drops | |
1093 | // any initialized elements and returns `Err`. | |
1094 | unsafe { pin_init_from_closure(init) } | |
1095 | } | |
1096 | ||
90e53c5e | 1097 | // SAFETY: Every type can be initialized by-value. |
52a7f2de BL |
1098 | unsafe impl<T, E> Init<T, E> for T { |
1099 | unsafe fn __init(self, slot: *mut T) -> Result<(), E> { | |
90e53c5e BL |
1100 | unsafe { slot.write(self) }; |
1101 | Ok(()) | |
1102 | } | |
1103 | } | |
92c4a1e7 | 1104 | |
1a8076ac BL |
1105 | // SAFETY: Every type can be initialized by-value. `__pinned_init` calls `__init`. |
1106 | unsafe impl<T, E> PinInit<T, E> for T { | |
1107 | unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> { | |
1108 | unsafe { self.__init(slot) } | |
1109 | } | |
1110 | } | |
1111 | ||
92c4a1e7 BL |
1112 | /// Smart pointer that can initialize memory in-place. |
1113 | pub trait InPlaceInit<T>: Sized { | |
1114 | /// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this | |
1115 | /// type. | |
1116 | /// | |
1117 | /// If `T: !Unpin` it will not be able to move afterwards. | |
1118 | fn try_pin_init<E>(init: impl PinInit<T, E>) -> Result<Pin<Self>, E> | |
1119 | where | |
1120 | E: From<AllocError>; | |
1121 | ||
1122 | /// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this | |
1123 | /// type. | |
1124 | /// | |
1125 | /// If `T: !Unpin` it will not be able to move afterwards. | |
1126 | fn pin_init<E>(init: impl PinInit<T, E>) -> error::Result<Pin<Self>> | |
1127 | where | |
1128 | Error: From<E>, | |
1129 | { | |
1130 | // SAFETY: We delegate to `init` and only change the error type. | |
1131 | let init = unsafe { | |
1132 | pin_init_from_closure(|slot| init.__pinned_init(slot).map_err(|e| Error::from(e))) | |
1133 | }; | |
1134 | Self::try_pin_init(init) | |
1135 | } | |
1136 | ||
1137 | /// Use the given initializer to in-place initialize a `T`. | |
1138 | fn try_init<E>(init: impl Init<T, E>) -> Result<Self, E> | |
1139 | where | |
1140 | E: From<AllocError>; | |
1141 | ||
1142 | /// Use the given initializer to in-place initialize a `T`. | |
1143 | fn init<E>(init: impl Init<T, E>) -> error::Result<Self> | |
1144 | where | |
1145 | Error: From<E>, | |
1146 | { | |
1147 | // SAFETY: We delegate to `init` and only change the error type. | |
1148 | let init = unsafe { | |
1149 | init_from_closure(|slot| init.__pinned_init(slot).map_err(|e| Error::from(e))) | |
1150 | }; | |
1151 | Self::try_init(init) | |
1152 | } | |
1153 | } | |
1154 | ||
1155 | impl<T> InPlaceInit<T> for Box<T> { | |
1156 | #[inline] | |
1157 | fn try_pin_init<E>(init: impl PinInit<T, E>) -> Result<Pin<Self>, E> | |
1158 | where | |
1159 | E: From<AllocError>, | |
1160 | { | |
1161 | let mut this = Box::try_new_uninit()?; | |
1162 | let slot = this.as_mut_ptr(); | |
1163 | // SAFETY: When init errors/panics, slot will get deallocated but not dropped, | |
1164 | // slot is valid and will not be moved, because we pin it later. | |
1165 | unsafe { init.__pinned_init(slot)? }; | |
1166 | // SAFETY: All fields have been initialized. | |
1167 | Ok(unsafe { this.assume_init() }.into()) | |
1168 | } | |
1169 | ||
1170 | #[inline] | |
1171 | fn try_init<E>(init: impl Init<T, E>) -> Result<Self, E> | |
1172 | where | |
1173 | E: From<AllocError>, | |
1174 | { | |
1175 | let mut this = Box::try_new_uninit()?; | |
1176 | let slot = this.as_mut_ptr(); | |
1177 | // SAFETY: When init errors/panics, slot will get deallocated but not dropped, | |
1178 | // slot is valid. | |
1179 | unsafe { init.__init(slot)? }; | |
1180 | // SAFETY: All fields have been initialized. | |
1181 | Ok(unsafe { this.assume_init() }) | |
1182 | } | |
1183 | } | |
1184 | ||
1185 | impl<T> InPlaceInit<T> for UniqueArc<T> { | |
1186 | #[inline] | |
1187 | fn try_pin_init<E>(init: impl PinInit<T, E>) -> Result<Pin<Self>, E> | |
1188 | where | |
1189 | E: From<AllocError>, | |
1190 | { | |
1191 | let mut this = UniqueArc::try_new_uninit()?; | |
1192 | let slot = this.as_mut_ptr(); | |
1193 | // SAFETY: When init errors/panics, slot will get deallocated but not dropped, | |
1194 | // slot is valid and will not be moved, because we pin it later. | |
1195 | unsafe { init.__pinned_init(slot)? }; | |
1196 | // SAFETY: All fields have been initialized. | |
1197 | Ok(unsafe { this.assume_init() }.into()) | |
1198 | } | |
1199 | ||
1200 | #[inline] | |
1201 | fn try_init<E>(init: impl Init<T, E>) -> Result<Self, E> | |
1202 | where | |
1203 | E: From<AllocError>, | |
1204 | { | |
1205 | let mut this = UniqueArc::try_new_uninit()?; | |
1206 | let slot = this.as_mut_ptr(); | |
1207 | // SAFETY: When init errors/panics, slot will get deallocated but not dropped, | |
1208 | // slot is valid. | |
1209 | unsafe { init.__init(slot)? }; | |
1210 | // SAFETY: All fields have been initialized. | |
1211 | Ok(unsafe { this.assume_init() }) | |
1212 | } | |
1213 | } | |
d0fdc396 BL |
1214 | |
1215 | /// Trait facilitating pinned destruction. | |
1216 | /// | |
1217 | /// Use [`pinned_drop`] to implement this trait safely: | |
1218 | /// | |
1219 | /// ```rust | |
1220 | /// # use kernel::sync::Mutex; | |
1221 | /// use kernel::macros::pinned_drop; | |
1222 | /// use core::pin::Pin; | |
1223 | /// #[pin_data(PinnedDrop)] | |
1224 | /// struct Foo { | |
1225 | /// #[pin] | |
1226 | /// mtx: Mutex<usize>, | |
1227 | /// } | |
1228 | /// | |
1229 | /// #[pinned_drop] | |
1230 | /// impl PinnedDrop for Foo { | |
1231 | /// fn drop(self: Pin<&mut Self>) { | |
1232 | /// pr_info!("Foo is being dropped!"); | |
1233 | /// } | |
1234 | /// } | |
1235 | /// ``` | |
1236 | /// | |
1237 | /// # Safety | |
1238 | /// | |
1239 | /// This trait must be implemented via the [`pinned_drop`] proc-macro attribute on the impl. | |
1240 | /// | |
1241 | /// [`pinned_drop`]: kernel::macros::pinned_drop | |
1242 | pub unsafe trait PinnedDrop: __internal::HasPinData { | |
1243 | /// Executes the pinned destructor of this type. | |
1244 | /// | |
1245 | /// While this function is marked safe, it is actually unsafe to call it manually. For this | |
1246 | /// reason it takes an additional parameter. This type can only be constructed by `unsafe` code | |
1247 | /// and thus prevents this function from being called where it should not. | |
1248 | /// | |
1249 | /// This extra parameter will be generated by the `#[pinned_drop]` proc-macro attribute | |
1250 | /// automatically. | |
1251 | fn drop(self: Pin<&mut Self>, only_call_from_drop: __internal::OnlyCallFromDrop); | |
1252 | } | |
38cde0bd BL |
1253 | |
1254 | /// Marker trait for types that can be initialized by writing just zeroes. | |
1255 | /// | |
1256 | /// # Safety | |
1257 | /// | |
1258 | /// The bit pattern consisting of only zeroes is a valid bit pattern for this type. In other words, | |
1259 | /// this is not UB: | |
1260 | /// | |
1261 | /// ```rust,ignore | |
1262 | /// let val: Self = unsafe { core::mem::zeroed() }; | |
1263 | /// ``` | |
1264 | pub unsafe trait Zeroable {} | |
1265 | ||
1266 | /// Create a new zeroed T. | |
1267 | /// | |
1268 | /// The returned initializer will write `0x00` to every byte of the given `slot`. | |
1269 | #[inline] | |
1270 | pub fn zeroed<T: Zeroable>() -> impl Init<T> { | |
1271 | // SAFETY: Because `T: Zeroable`, all bytes zero is a valid bit pattern for `T` | |
1272 | // and because we write all zeroes, the memory is initialized. | |
1273 | unsafe { | |
1274 | init_from_closure(|slot: *mut T| { | |
1275 | slot.write_bytes(0, 1); | |
1276 | Ok(()) | |
1277 | }) | |
1278 | } | |
1279 | } | |
1280 | ||
1281 | macro_rules! impl_zeroable { | |
1282 | ($($({$($generics:tt)*})? $t:ty, )*) => { | |
1283 | $(unsafe impl$($($generics)*)? Zeroable for $t {})* | |
1284 | }; | |
1285 | } | |
1286 | ||
1287 | impl_zeroable! { | |
1288 | // SAFETY: All primitives that are allowed to be zero. | |
1289 | bool, | |
1290 | char, | |
1291 | u8, u16, u32, u64, u128, usize, | |
1292 | i8, i16, i32, i64, i128, isize, | |
1293 | f32, f64, | |
1294 | ||
1295 | // SAFETY: These are ZSTs, there is nothing to zero. | |
1296 | {<T: ?Sized>} PhantomData<T>, core::marker::PhantomPinned, Infallible, (), | |
1297 | ||
1298 | // SAFETY: Type is allowed to take any value, including all zeros. | |
1299 | {<T>} MaybeUninit<T>, | |
2e704f18 BL |
1300 | // SAFETY: Type is allowed to take any value, including all zeros. |
1301 | {<T>} Opaque<T>, | |
1302 | ||
1303 | // SAFETY: `T: Zeroable` and `UnsafeCell` is `repr(transparent)`. | |
1304 | {<T: ?Sized + Zeroable>} UnsafeCell<T>, | |
38cde0bd BL |
1305 | |
1306 | // SAFETY: All zeros is equivalent to `None` (option layout optimization guarantee). | |
1307 | Option<NonZeroU8>, Option<NonZeroU16>, Option<NonZeroU32>, Option<NonZeroU64>, | |
1308 | Option<NonZeroU128>, Option<NonZeroUsize>, | |
1309 | Option<NonZeroI8>, Option<NonZeroI16>, Option<NonZeroI32>, Option<NonZeroI64>, | |
1310 | Option<NonZeroI128>, Option<NonZeroIsize>, | |
1311 | ||
1312 | // SAFETY: All zeros is equivalent to `None` (option layout optimization guarantee). | |
1313 | // | |
1314 | // In this case we are allowed to use `T: ?Sized`, since all zeros is the `None` variant. | |
1315 | {<T: ?Sized>} Option<NonNull<T>>, | |
1316 | {<T: ?Sized>} Option<Box<T>>, | |
1317 | ||
1318 | // SAFETY: `null` pointer is valid. | |
1319 | // | |
1320 | // We cannot use `T: ?Sized`, since the VTABLE pointer part of fat pointers is not allowed to be | |
1321 | // null. | |
1322 | // | |
1323 | // When `Pointee` gets stabilized, we could use | |
1324 | // `T: ?Sized where <T as Pointee>::Metadata: Zeroable` | |
1325 | {<T>} *mut T, {<T>} *const T, | |
1326 | ||
1327 | // SAFETY: `null` pointer is valid and the metadata part of these fat pointers is allowed to be | |
1328 | // zero. | |
1329 | {<T>} *mut [T], {<T>} *const [T], *mut str, *const str, | |
1330 | ||
1331 | // SAFETY: `T` is `Zeroable`. | |
1332 | {<const N: usize, T: Zeroable>} [T; N], {<T: Zeroable>} Wrapping<T>, | |
1333 | } | |
1334 | ||
1335 | macro_rules! impl_tuple_zeroable { | |
1336 | ($(,)?) => {}; | |
1337 | ($first:ident, $($t:ident),* $(,)?) => { | |
1338 | // SAFETY: All elements are zeroable and padding can be zero. | |
1339 | unsafe impl<$first: Zeroable, $($t: Zeroable),*> Zeroable for ($first, $($t),*) {} | |
1340 | impl_tuple_zeroable!($($t),* ,); | |
1341 | } | |
1342 | } | |
1343 | ||
1344 | impl_tuple_zeroable!(A, B, C, D, E, F, G, H, I, J); |