Commit | Line | Data |
---|---|---|
90e53c5e BL |
1 | // SPDX-License-Identifier: Apache-2.0 OR MIT |
2 | ||
3 | //! API to safely and fallibly initialize pinned `struct`s using in-place constructors. | |
4 | //! | |
5 | //! It also allows in-place initialization of big `struct`s that would otherwise produce a stack | |
6 | //! overflow. | |
7 | //! | |
8 | //! Most `struct`s from the [`sync`] module need to be pinned, because they contain self-referential | |
9 | //! `struct`s from C. [Pinning][pinning] is Rust's way of ensuring data does not move. | |
10 | //! | |
11 | //! # Overview | |
12 | //! | |
13 | //! To initialize a `struct` with an in-place constructor you will need two things: | |
14 | //! - an in-place constructor, | |
6841d45a BL |
15 | //! - a memory location that can hold your `struct` (this can be the [stack], an [`Arc<T>`], |
16 | //! [`UniqueArc<T>`], [`Box<T>`] or any other smart pointer that implements [`InPlaceInit`]). | |
90e53c5e | 17 | //! |
fc6c6baa BL |
18 | //! To get an in-place constructor there are generally three options: |
19 | //! - directly creating an in-place constructor using the [`pin_init!`] macro, | |
90e53c5e BL |
20 | //! - a custom function/macro returning an in-place constructor provided by someone else, |
21 | //! - using the unsafe function [`pin_init_from_closure()`] to manually create an initializer. | |
22 | //! | |
23 | //! Aside from pinned initialization, this API also supports in-place construction without pinning, | |
24 | //! the macros/types/functions are generally named like the pinned variants without the `pin` | |
25 | //! prefix. | |
26 | //! | |
fc6c6baa BL |
27 | //! # Examples |
28 | //! | |
29 | //! ## Using the [`pin_init!`] macro | |
30 | //! | |
31 | //! If you want to use [`PinInit`], then you will have to annotate your `struct` with | |
32 | //! `#[`[`pin_data`]`]`. It is a macro that uses `#[pin]` as a marker for | |
33 | //! [structurally pinned fields]. After doing this, you can then create an in-place constructor via | |
34 | //! [`pin_init!`]. The syntax is almost the same as normal `struct` initializers. The difference is | |
35 | //! that you need to write `<-` instead of `:` for fields that you want to initialize in-place. | |
36 | //! | |
37 | //! ```rust | |
38 | //! # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] | |
39 | //! use kernel::{prelude::*, sync::Mutex, new_mutex}; | |
40 | //! # use core::pin::Pin; | |
41 | //! #[pin_data] | |
42 | //! struct Foo { | |
43 | //! #[pin] | |
44 | //! a: Mutex<usize>, | |
45 | //! b: u32, | |
46 | //! } | |
47 | //! | |
48 | //! let foo = pin_init!(Foo { | |
49 | //! a <- new_mutex!(42, "Foo::a"), | |
50 | //! b: 24, | |
51 | //! }); | |
52 | //! ``` | |
53 | //! | |
54 | //! `foo` now is of the type [`impl PinInit<Foo>`]. We can now use any smart pointer that we like | |
55 | //! (or just the stack) to actually initialize a `Foo`: | |
56 | //! | |
57 | //! ```rust | |
58 | //! # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] | |
59 | //! # use kernel::{prelude::*, sync::Mutex, new_mutex}; | |
60 | //! # use core::pin::Pin; | |
61 | //! # #[pin_data] | |
62 | //! # struct Foo { | |
63 | //! # #[pin] | |
64 | //! # a: Mutex<usize>, | |
65 | //! # b: u32, | |
66 | //! # } | |
67 | //! # let foo = pin_init!(Foo { | |
68 | //! # a <- new_mutex!(42, "Foo::a"), | |
69 | //! # b: 24, | |
70 | //! # }); | |
71 | //! let foo: Result<Pin<Box<Foo>>> = Box::pin_init(foo); | |
72 | //! ``` | |
73 | //! | |
74 | //! For more information see the [`pin_init!`] macro. | |
75 | //! | |
76 | //! ## Using a custom function/macro that returns an initializer | |
77 | //! | |
78 | //! Many types from the kernel supply a function/macro that returns an initializer, because the | |
79 | //! above method only works for types where you can access the fields. | |
80 | //! | |
81 | //! ```rust | |
82 | //! # use kernel::{new_mutex, sync::{Arc, Mutex}}; | |
83 | //! let mtx: Result<Arc<Mutex<usize>>> = Arc::pin_init(new_mutex!(42, "example::mtx")); | |
84 | //! ``` | |
85 | //! | |
86 | //! To declare an init macro/function you just return an [`impl PinInit<T, E>`]: | |
87 | //! | |
88 | //! ```rust | |
89 | //! # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] | |
90 | //! # use kernel::{sync::Mutex, prelude::*, new_mutex, init::PinInit, try_pin_init}; | |
91 | //! #[pin_data] | |
92 | //! struct DriverData { | |
93 | //! #[pin] | |
94 | //! status: Mutex<i32>, | |
95 | //! buffer: Box<[u8; 1_000_000]>, | |
96 | //! } | |
97 | //! | |
98 | //! impl DriverData { | |
99 | //! fn new() -> impl PinInit<Self, Error> { | |
100 | //! try_pin_init!(Self { | |
101 | //! status <- new_mutex!(0, "DriverData::status"), | |
102 | //! buffer: Box::init(kernel::init::zeroed())?, | |
103 | //! }) | |
104 | //! } | |
105 | //! } | |
106 | //! ``` | |
107 | //! | |
d0fdc396 BL |
108 | //! ## Manual creation of an initializer |
109 | //! | |
110 | //! Often when working with primitives the previous approaches are not sufficient. That is where | |
111 | //! [`pin_init_from_closure()`] comes in. This `unsafe` function allows you to create a | |
112 | //! [`impl PinInit<T, E>`] directly from a closure. Of course you have to ensure that the closure | |
113 | //! actually does the initialization in the correct way. Here are the things to look out for | |
114 | //! (we are calling the parameter to the closure `slot`): | |
115 | //! - when the closure returns `Ok(())`, then it has completed the initialization successfully, so | |
116 | //! `slot` now contains a valid bit pattern for the type `T`, | |
117 | //! - when the closure returns `Err(e)`, then the caller may deallocate the memory at `slot`, so | |
118 | //! you need to take care to clean up anything if your initialization fails mid-way, | |
119 | //! - you may assume that `slot` will stay pinned even after the closure returns until `drop` of | |
120 | //! `slot` gets called. | |
121 | //! | |
122 | //! ```rust | |
123 | //! use kernel::{prelude::*, init}; | |
124 | //! use core::{ptr::addr_of_mut, marker::PhantomPinned, pin::Pin}; | |
125 | //! # mod bindings { | |
126 | //! # pub struct foo; | |
127 | //! # pub unsafe fn init_foo(_ptr: *mut foo) {} | |
128 | //! # pub unsafe fn destroy_foo(_ptr: *mut foo) {} | |
129 | //! # pub unsafe fn enable_foo(_ptr: *mut foo, _flags: u32) -> i32 { 0 } | |
130 | //! # } | |
131 | //! /// # Invariants | |
132 | //! /// | |
133 | //! /// `foo` is always initialized | |
134 | //! #[pin_data(PinnedDrop)] | |
135 | //! pub struct RawFoo { | |
136 | //! #[pin] | |
137 | //! foo: Opaque<bindings::foo>, | |
138 | //! #[pin] | |
139 | //! _p: PhantomPinned, | |
140 | //! } | |
141 | //! | |
142 | //! impl RawFoo { | |
143 | //! pub fn new(flags: u32) -> impl PinInit<Self, Error> { | |
144 | //! // SAFETY: | |
145 | //! // - when the closure returns `Ok(())`, then it has successfully initialized and | |
146 | //! // enabled `foo`, | |
147 | //! // - when it returns `Err(e)`, then it has cleaned up before | |
148 | //! unsafe { | |
149 | //! init::pin_init_from_closure(move |slot: *mut Self| { | |
150 | //! // `slot` contains uninit memory, avoid creating a reference. | |
151 | //! let foo = addr_of_mut!((*slot).foo); | |
152 | //! | |
153 | //! // Initialize the `foo` | |
154 | //! bindings::init_foo(Opaque::raw_get(foo)); | |
155 | //! | |
156 | //! // Try to enable it. | |
157 | //! let err = bindings::enable_foo(Opaque::raw_get(foo), flags); | |
158 | //! if err != 0 { | |
159 | //! // Enabling has failed, first clean up the foo and then return the error. | |
160 | //! bindings::destroy_foo(Opaque::raw_get(foo)); | |
161 | //! return Err(Error::from_kernel_errno(err)); | |
162 | //! } | |
163 | //! | |
164 | //! // All fields of `RawFoo` have been initialized, since `_p` is a ZST. | |
165 | //! Ok(()) | |
166 | //! }) | |
167 | //! } | |
168 | //! } | |
169 | //! } | |
170 | //! | |
171 | //! #[pinned_drop] | |
172 | //! impl PinnedDrop for RawFoo { | |
173 | //! fn drop(self: Pin<&mut Self>) { | |
174 | //! // SAFETY: Since `foo` is initialized, destroying is safe. | |
175 | //! unsafe { bindings::destroy_foo(self.foo.get()) }; | |
176 | //! } | |
177 | //! } | |
178 | //! ``` | |
179 | //! | |
692e8935 BL |
180 | //! For the special case where initializing a field is a single FFI-function call that cannot fail, |
181 | //! there exist the helper function [`Opaque::ffi_init`]. This function initialize a single | |
182 | //! [`Opaque`] field by just delegating to the supplied closure. You can use these in combination | |
183 | //! with [`pin_init!`]. | |
184 | //! | |
185 | //! For more information on how to use [`pin_init_from_closure()`], take a look at the uses inside | |
186 | //! the `kernel` crate. The [`sync`] module is a good starting point. | |
187 | //! | |
90e53c5e BL |
188 | //! [`sync`]: kernel::sync |
189 | //! [pinning]: https://doc.rust-lang.org/std/pin/index.html | |
190 | //! [structurally pinned fields]: | |
191 | //! https://doc.rust-lang.org/std/pin/index.html#pinning-is-structural-for-field | |
6841d45a | 192 | //! [stack]: crate::stack_pin_init |
90e53c5e BL |
193 | //! [`Arc<T>`]: crate::sync::Arc |
194 | //! [`impl PinInit<Foo>`]: PinInit | |
195 | //! [`impl PinInit<T, E>`]: PinInit | |
196 | //! [`impl Init<T, E>`]: Init | |
197 | //! [`Opaque`]: kernel::types::Opaque | |
692e8935 | 198 | //! [`Opaque::ffi_init`]: kernel::types::Opaque::ffi_init |
90e53c5e | 199 | //! [`pin_data`]: ::macros::pin_data |
3ed03f4d | 200 | //! [`pin_init!`]: crate::pin_init! |
90e53c5e | 201 | |
92c4a1e7 BL |
202 | use crate::{ |
203 | error::{self, Error}, | |
204 | sync::UniqueArc, | |
205 | }; | |
fc6c6baa | 206 | use alloc::boxed::Box; |
92c4a1e7 | 207 | use core::{ |
38cde0bd BL |
208 | alloc::AllocError, |
209 | cell::Cell, | |
210 | convert::Infallible, | |
211 | marker::PhantomData, | |
212 | mem::MaybeUninit, | |
213 | num::*, | |
214 | pin::Pin, | |
215 | ptr::{self, NonNull}, | |
92c4a1e7 | 216 | }; |
90e53c5e BL |
217 | |
218 | #[doc(hidden)] | |
219 | pub mod __internal; | |
fc6c6baa BL |
220 | #[doc(hidden)] |
221 | pub mod macros; | |
222 | ||
6841d45a BL |
223 | /// Initialize and pin a type directly on the stack. |
224 | /// | |
225 | /// # Examples | |
226 | /// | |
227 | /// ```rust | |
228 | /// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] | |
229 | /// # use kernel::{init, pin_init, stack_pin_init, init::*, sync::Mutex, new_mutex}; | |
230 | /// # use macros::pin_data; | |
231 | /// # use core::pin::Pin; | |
232 | /// #[pin_data] | |
233 | /// struct Foo { | |
234 | /// #[pin] | |
235 | /// a: Mutex<usize>, | |
236 | /// b: Bar, | |
237 | /// } | |
238 | /// | |
239 | /// #[pin_data] | |
240 | /// struct Bar { | |
241 | /// x: u32, | |
242 | /// } | |
243 | /// | |
244 | /// stack_pin_init!(let foo = pin_init!(Foo { | |
245 | /// a <- new_mutex!(42), | |
246 | /// b: Bar { | |
247 | /// x: 64, | |
248 | /// }, | |
249 | /// })); | |
250 | /// let foo: Pin<&mut Foo> = foo; | |
251 | /// pr_info!("a: {}", &*foo.a.lock()); | |
252 | /// ``` | |
253 | /// | |
254 | /// # Syntax | |
255 | /// | |
256 | /// A normal `let` binding with optional type annotation. The expression is expected to implement | |
257 | /// [`PinInit`]/[`Init`] with the error type [`Infallible`]. If you want to use a different error | |
258 | /// type, then use [`stack_try_pin_init!`]. | |
3ed03f4d MO |
259 | /// |
260 | /// [`stack_try_pin_init!`]: crate::stack_try_pin_init! | |
6841d45a BL |
261 | #[macro_export] |
262 | macro_rules! stack_pin_init { | |
263 | (let $var:ident $(: $t:ty)? = $val:expr) => { | |
264 | let val = $val; | |
265 | let mut $var = ::core::pin::pin!($crate::init::__internal::StackInit$(::<$t>)?::uninit()); | |
266 | let mut $var = match $crate::init::__internal::StackInit::init($var, val) { | |
267 | Ok(res) => res, | |
268 | Err(x) => { | |
269 | let x: ::core::convert::Infallible = x; | |
270 | match x {} | |
271 | } | |
272 | }; | |
273 | }; | |
274 | } | |
275 | ||
276 | /// Initialize and pin a type directly on the stack. | |
277 | /// | |
278 | /// # Examples | |
279 | /// | |
280 | /// ```rust | |
281 | /// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] | |
282 | /// # use kernel::{init, pin_init, stack_try_pin_init, init::*, sync::Mutex, new_mutex}; | |
283 | /// # use macros::pin_data; | |
284 | /// # use core::{alloc::AllocError, pin::Pin}; | |
285 | /// #[pin_data] | |
286 | /// struct Foo { | |
287 | /// #[pin] | |
288 | /// a: Mutex<usize>, | |
289 | /// b: Box<Bar>, | |
290 | /// } | |
291 | /// | |
292 | /// struct Bar { | |
293 | /// x: u32, | |
294 | /// } | |
295 | /// | |
296 | /// stack_try_pin_init!(let foo: Result<Pin<&mut Foo>, AllocError> = pin_init!(Foo { | |
297 | /// a <- new_mutex!(42), | |
298 | /// b: Box::try_new(Bar { | |
299 | /// x: 64, | |
300 | /// })?, | |
301 | /// })); | |
302 | /// let foo = foo.unwrap(); | |
303 | /// pr_info!("a: {}", &*foo.a.lock()); | |
304 | /// ``` | |
305 | /// | |
306 | /// ```rust | |
307 | /// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] | |
308 | /// # use kernel::{init, pin_init, stack_try_pin_init, init::*, sync::Mutex, new_mutex}; | |
309 | /// # use macros::pin_data; | |
310 | /// # use core::{alloc::AllocError, pin::Pin}; | |
311 | /// #[pin_data] | |
312 | /// struct Foo { | |
313 | /// #[pin] | |
314 | /// a: Mutex<usize>, | |
315 | /// b: Box<Bar>, | |
316 | /// } | |
317 | /// | |
318 | /// struct Bar { | |
319 | /// x: u32, | |
320 | /// } | |
321 | /// | |
322 | /// stack_try_pin_init!(let foo: Pin<&mut Foo> =? pin_init!(Foo { | |
323 | /// a <- new_mutex!(42), | |
324 | /// b: Box::try_new(Bar { | |
325 | /// x: 64, | |
326 | /// })?, | |
327 | /// })); | |
328 | /// pr_info!("a: {}", &*foo.a.lock()); | |
329 | /// # Ok::<_, AllocError>(()) | |
330 | /// ``` | |
331 | /// | |
332 | /// # Syntax | |
333 | /// | |
334 | /// A normal `let` binding with optional type annotation. The expression is expected to implement | |
335 | /// [`PinInit`]/[`Init`]. This macro assigns a result to the given variable, adding a `?` after the | |
336 | /// `=` will propagate this error. | |
337 | #[macro_export] | |
338 | macro_rules! stack_try_pin_init { | |
339 | (let $var:ident $(: $t:ty)? = $val:expr) => { | |
340 | let val = $val; | |
341 | let mut $var = ::core::pin::pin!($crate::init::__internal::StackInit$(::<$t>)?::uninit()); | |
342 | let mut $var = $crate::init::__internal::StackInit::init($var, val); | |
343 | }; | |
344 | (let $var:ident $(: $t:ty)? =? $val:expr) => { | |
345 | let val = $val; | |
346 | let mut $var = ::core::pin::pin!($crate::init::__internal::StackInit$(::<$t>)?::uninit()); | |
347 | let mut $var = $crate::init::__internal::StackInit::init($var, val)?; | |
348 | }; | |
349 | } | |
350 | ||
fc6c6baa BL |
351 | /// Construct an in-place, pinned initializer for `struct`s. |
352 | /// | |
353 | /// This macro defaults the error to [`Infallible`]. If you need [`Error`], then use | |
354 | /// [`try_pin_init!`]. | |
355 | /// | |
356 | /// The syntax is almost identical to that of a normal `struct` initializer: | |
357 | /// | |
358 | /// ```rust | |
359 | /// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] | |
360 | /// # use kernel::{init, pin_init, macros::pin_data, init::*}; | |
361 | /// # use core::pin::Pin; | |
362 | /// #[pin_data] | |
363 | /// struct Foo { | |
364 | /// a: usize, | |
365 | /// b: Bar, | |
366 | /// } | |
367 | /// | |
368 | /// #[pin_data] | |
369 | /// struct Bar { | |
370 | /// x: u32, | |
371 | /// } | |
372 | /// | |
373 | /// # fn demo() -> impl PinInit<Foo> { | |
374 | /// let a = 42; | |
375 | /// | |
376 | /// let initializer = pin_init!(Foo { | |
377 | /// a, | |
378 | /// b: Bar { | |
379 | /// x: 64, | |
380 | /// }, | |
381 | /// }); | |
382 | /// # initializer } | |
383 | /// # Box::pin_init(demo()).unwrap(); | |
384 | /// ``` | |
385 | /// | |
386 | /// Arbitrary Rust expressions can be used to set the value of a variable. | |
387 | /// | |
388 | /// The fields are initialized in the order that they appear in the initializer. So it is possible | |
389 | /// to read already initialized fields using raw pointers. | |
390 | /// | |
391 | /// IMPORTANT: You are not allowed to create references to fields of the struct inside of the | |
392 | /// initializer. | |
393 | /// | |
394 | /// # Init-functions | |
395 | /// | |
396 | /// When working with this API it is often desired to let others construct your types without | |
397 | /// giving access to all fields. This is where you would normally write a plain function `new` | |
398 | /// that would return a new instance of your type. With this API that is also possible. | |
399 | /// However, there are a few extra things to keep in mind. | |
400 | /// | |
401 | /// To create an initializer function, simply declare it like this: | |
402 | /// | |
403 | /// ```rust | |
404 | /// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] | |
405 | /// # use kernel::{init, pin_init, prelude::*, init::*}; | |
406 | /// # use core::pin::Pin; | |
407 | /// # #[pin_data] | |
408 | /// # struct Foo { | |
409 | /// # a: usize, | |
410 | /// # b: Bar, | |
411 | /// # } | |
412 | /// # #[pin_data] | |
413 | /// # struct Bar { | |
414 | /// # x: u32, | |
415 | /// # } | |
416 | /// impl Foo { | |
417 | /// fn new() -> impl PinInit<Self> { | |
418 | /// pin_init!(Self { | |
419 | /// a: 42, | |
420 | /// b: Bar { | |
421 | /// x: 64, | |
422 | /// }, | |
423 | /// }) | |
424 | /// } | |
425 | /// } | |
426 | /// ``` | |
427 | /// | |
428 | /// Users of `Foo` can now create it like this: | |
429 | /// | |
430 | /// ```rust | |
431 | /// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] | |
432 | /// # use kernel::{init, pin_init, macros::pin_data, init::*}; | |
433 | /// # use core::pin::Pin; | |
434 | /// # #[pin_data] | |
435 | /// # struct Foo { | |
436 | /// # a: usize, | |
437 | /// # b: Bar, | |
438 | /// # } | |
439 | /// # #[pin_data] | |
440 | /// # struct Bar { | |
441 | /// # x: u32, | |
442 | /// # } | |
443 | /// # impl Foo { | |
444 | /// # fn new() -> impl PinInit<Self> { | |
445 | /// # pin_init!(Self { | |
446 | /// # a: 42, | |
447 | /// # b: Bar { | |
448 | /// # x: 64, | |
449 | /// # }, | |
450 | /// # }) | |
451 | /// # } | |
452 | /// # } | |
453 | /// let foo = Box::pin_init(Foo::new()); | |
454 | /// ``` | |
455 | /// | |
456 | /// They can also easily embed it into their own `struct`s: | |
457 | /// | |
458 | /// ```rust | |
459 | /// # #![allow(clippy::disallowed_names, clippy::new_ret_no_self)] | |
460 | /// # use kernel::{init, pin_init, macros::pin_data, init::*}; | |
461 | /// # use core::pin::Pin; | |
462 | /// # #[pin_data] | |
463 | /// # struct Foo { | |
464 | /// # a: usize, | |
465 | /// # b: Bar, | |
466 | /// # } | |
467 | /// # #[pin_data] | |
468 | /// # struct Bar { | |
469 | /// # x: u32, | |
470 | /// # } | |
471 | /// # impl Foo { | |
472 | /// # fn new() -> impl PinInit<Self> { | |
473 | /// # pin_init!(Self { | |
474 | /// # a: 42, | |
475 | /// # b: Bar { | |
476 | /// # x: 64, | |
477 | /// # }, | |
478 | /// # }) | |
479 | /// # } | |
480 | /// # } | |
481 | /// #[pin_data] | |
482 | /// struct FooContainer { | |
483 | /// #[pin] | |
484 | /// foo1: Foo, | |
485 | /// #[pin] | |
486 | /// foo2: Foo, | |
487 | /// other: u32, | |
488 | /// } | |
489 | /// | |
490 | /// impl FooContainer { | |
491 | /// fn new(other: u32) -> impl PinInit<Self> { | |
492 | /// pin_init!(Self { | |
493 | /// foo1 <- Foo::new(), | |
494 | /// foo2 <- Foo::new(), | |
495 | /// other, | |
496 | /// }) | |
497 | /// } | |
498 | /// } | |
499 | /// ``` | |
500 | /// | |
501 | /// Here we see that when using `pin_init!` with `PinInit`, one needs to write `<-` instead of `:`. | |
502 | /// This signifies that the given field is initialized in-place. As with `struct` initializers, just | |
503 | /// writing the field (in this case `other`) without `:` or `<-` means `other: other,`. | |
504 | /// | |
505 | /// # Syntax | |
506 | /// | |
507 | /// As already mentioned in the examples above, inside of `pin_init!` a `struct` initializer with | |
508 | /// the following modifications is expected: | |
509 | /// - Fields that you want to initialize in-place have to use `<-` instead of `:`. | |
510 | /// - In front of the initializer you can write `&this in` to have access to a [`NonNull<Self>`] | |
511 | /// pointer named `this` inside of the initializer. | |
512 | /// | |
513 | /// For instance: | |
514 | /// | |
515 | /// ```rust | |
516 | /// # use kernel::pin_init; | |
517 | /// # use macros::pin_data; | |
518 | /// # use core::{ptr::addr_of_mut, marker::PhantomPinned}; | |
519 | /// #[pin_data] | |
520 | /// struct Buf { | |
521 | /// // `ptr` points into `buf`. | |
522 | /// ptr: *mut u8, | |
523 | /// buf: [u8; 64], | |
524 | /// #[pin] | |
525 | /// pin: PhantomPinned, | |
526 | /// } | |
527 | /// pin_init!(&this in Buf { | |
528 | /// buf: [0; 64], | |
529 | /// ptr: unsafe { addr_of_mut!((*this.as_ptr()).buf).cast() }, | |
530 | /// pin: PhantomPinned, | |
531 | /// }); | |
532 | /// ``` | |
533 | /// | |
534 | /// [`try_pin_init!`]: kernel::try_pin_init | |
535 | /// [`NonNull<Self>`]: core::ptr::NonNull | |
fc6c6baa BL |
536 | // For a detailed example of how this macro works, see the module documentation of the hidden |
537 | // module `__internal` inside of `init/__internal.rs`. | |
538 | #[macro_export] | |
539 | macro_rules! pin_init { | |
540 | ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { | |
541 | $($fields:tt)* | |
542 | }) => { | |
543 | $crate::try_pin_init!( | |
544 | @this($($this)?), | |
545 | @typ($t $(::<$($generics),*>)?), | |
546 | @fields($($fields)*), | |
547 | @error(::core::convert::Infallible), | |
548 | ) | |
549 | }; | |
550 | } | |
551 | ||
552 | /// Construct an in-place, fallible pinned initializer for `struct`s. | |
553 | /// | |
554 | /// If the initialization can complete without error (or [`Infallible`]), then use [`pin_init!`]. | |
555 | /// | |
556 | /// You can use the `?` operator or use `return Err(err)` inside the initializer to stop | |
557 | /// initialization and return the error. | |
558 | /// | |
559 | /// IMPORTANT: if you have `unsafe` code inside of the initializer you have to ensure that when | |
560 | /// initialization fails, the memory can be safely deallocated without any further modifications. | |
561 | /// | |
562 | /// This macro defaults the error to [`Error`]. | |
563 | /// | |
564 | /// The syntax is identical to [`pin_init!`] with the following exception: you can append `? $type` | |
565 | /// after the `struct` initializer to specify the error type you want to use. | |
566 | /// | |
567 | /// # Examples | |
568 | /// | |
569 | /// ```rust | |
570 | /// # #![feature(new_uninit)] | |
571 | /// use kernel::{init::{self, PinInit}, error::Error}; | |
572 | /// #[pin_data] | |
573 | /// struct BigBuf { | |
574 | /// big: Box<[u8; 1024 * 1024 * 1024]>, | |
575 | /// small: [u8; 1024 * 1024], | |
576 | /// ptr: *mut u8, | |
577 | /// } | |
578 | /// | |
579 | /// impl BigBuf { | |
580 | /// fn new() -> impl PinInit<Self, Error> { | |
581 | /// try_pin_init!(Self { | |
582 | /// big: Box::init(init::zeroed())?, | |
583 | /// small: [0; 1024 * 1024], | |
584 | /// ptr: core::ptr::null_mut(), | |
585 | /// }? Error) | |
586 | /// } | |
587 | /// } | |
588 | /// ``` | |
fc6c6baa BL |
589 | // For a detailed example of how this macro works, see the module documentation of the hidden |
590 | // module `__internal` inside of `init/__internal.rs`. | |
591 | #[macro_export] | |
592 | macro_rules! try_pin_init { | |
593 | ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { | |
594 | $($fields:tt)* | |
595 | }) => { | |
596 | $crate::try_pin_init!( | |
597 | @this($($this)?), | |
598 | @typ($t $(::<$($generics),*>)? ), | |
599 | @fields($($fields)*), | |
600 | @error($crate::error::Error), | |
601 | ) | |
602 | }; | |
603 | ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { | |
604 | $($fields:tt)* | |
605 | }? $err:ty) => { | |
606 | $crate::try_pin_init!( | |
607 | @this($($this)?), | |
608 | @typ($t $(::<$($generics),*>)? ), | |
609 | @fields($($fields)*), | |
610 | @error($err), | |
611 | ) | |
612 | }; | |
613 | ( | |
614 | @this($($this:ident)?), | |
615 | @typ($t:ident $(::<$($generics:ty),*>)?), | |
616 | @fields($($fields:tt)*), | |
617 | @error($err:ty), | |
618 | ) => {{ | |
619 | // We do not want to allow arbitrary returns, so we declare this type as the `Ok` return | |
620 | // type and shadow it later when we insert the arbitrary user code. That way there will be | |
621 | // no possibility of returning without `unsafe`. | |
622 | struct __InitOk; | |
623 | // Get the pin data from the supplied type. | |
624 | let data = unsafe { | |
625 | use $crate::init::__internal::HasPinData; | |
626 | $t$(::<$($generics),*>)?::__pin_data() | |
627 | }; | |
628 | // Ensure that `data` really is of type `PinData` and help with type inference: | |
629 | let init = $crate::init::__internal::PinData::make_closure::<_, __InitOk, $err>( | |
630 | data, | |
631 | move |slot| { | |
632 | { | |
633 | // Shadow the structure so it cannot be used to return early. | |
634 | struct __InitOk; | |
635 | // Create the `this` so it can be referenced by the user inside of the | |
636 | // expressions creating the individual fields. | |
637 | $(let $this = unsafe { ::core::ptr::NonNull::new_unchecked(slot) };)? | |
638 | // Initialize every field. | |
639 | $crate::try_pin_init!(init_slot: | |
640 | @data(data), | |
641 | @slot(slot), | |
642 | @munch_fields($($fields)*,), | |
643 | ); | |
644 | // We use unreachable code to ensure that all fields have been mentioned exactly | |
645 | // once, this struct initializer will still be type-checked and complain with a | |
646 | // very natural error message if a field is forgotten/mentioned more than once. | |
647 | #[allow(unreachable_code, clippy::diverging_sub_expression)] | |
648 | if false { | |
649 | $crate::try_pin_init!(make_initializer: | |
650 | @slot(slot), | |
651 | @type_name($t), | |
652 | @munch_fields($($fields)*,), | |
653 | @acc(), | |
654 | ); | |
655 | } | |
656 | // Forget all guards, since initialization was a success. | |
657 | $crate::try_pin_init!(forget_guards: | |
658 | @munch_fields($($fields)*,), | |
659 | ); | |
660 | } | |
661 | Ok(__InitOk) | |
662 | } | |
663 | ); | |
664 | let init = move |slot| -> ::core::result::Result<(), $err> { | |
665 | init(slot).map(|__InitOk| ()) | |
666 | }; | |
667 | let init = unsafe { $crate::init::pin_init_from_closure::<_, $err>(init) }; | |
668 | init | |
669 | }}; | |
670 | (init_slot: | |
671 | @data($data:ident), | |
672 | @slot($slot:ident), | |
673 | @munch_fields($(,)?), | |
674 | ) => { | |
675 | // Endpoint of munching, no fields are left. | |
676 | }; | |
677 | (init_slot: | |
678 | @data($data:ident), | |
679 | @slot($slot:ident), | |
680 | // In-place initialization syntax. | |
681 | @munch_fields($field:ident <- $val:expr, $($rest:tt)*), | |
682 | ) => { | |
683 | let $field = $val; | |
684 | // Call the initializer. | |
685 | // | |
686 | // SAFETY: `slot` is valid, because we are inside of an initializer closure, we | |
687 | // return when an error/panic occurs. | |
688 | // We also use the `data` to require the correct trait (`Init` or `PinInit`) for `$field`. | |
689 | unsafe { $data.$field(::core::ptr::addr_of_mut!((*$slot).$field), $field)? }; | |
690 | // Create the drop guard. | |
691 | // | |
692 | // We only give access to `&DropGuard`, so it cannot be forgotten via safe code. | |
693 | // | |
694 | // SAFETY: We forget the guard later when initialization has succeeded. | |
695 | let $field = &unsafe { | |
696 | $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field)) | |
697 | }; | |
698 | ||
699 | $crate::try_pin_init!(init_slot: | |
700 | @data($data), | |
701 | @slot($slot), | |
702 | @munch_fields($($rest)*), | |
703 | ); | |
704 | }; | |
705 | (init_slot: | |
706 | @data($data:ident), | |
707 | @slot($slot:ident), | |
708 | // Direct value init, this is safe for every field. | |
709 | @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), | |
710 | ) => { | |
711 | $(let $field = $val;)? | |
712 | // Initialize the field. | |
713 | // | |
714 | // SAFETY: The memory at `slot` is uninitialized. | |
715 | unsafe { ::core::ptr::write(::core::ptr::addr_of_mut!((*$slot).$field), $field) }; | |
716 | // Create the drop guard: | |
717 | // | |
718 | // We only give access to `&DropGuard`, so it cannot be accidentally forgotten. | |
719 | // | |
720 | // SAFETY: We forget the guard later when initialization has succeeded. | |
721 | let $field = &unsafe { | |
722 | $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field)) | |
723 | }; | |
724 | ||
725 | $crate::try_pin_init!(init_slot: | |
726 | @data($data), | |
727 | @slot($slot), | |
728 | @munch_fields($($rest)*), | |
729 | ); | |
730 | }; | |
731 | (make_initializer: | |
732 | @slot($slot:ident), | |
733 | @type_name($t:ident), | |
734 | @munch_fields($(,)?), | |
735 | @acc($($acc:tt)*), | |
736 | ) => { | |
737 | // Endpoint, nothing more to munch, create the initializer. | |
738 | // Since we are in the `if false` branch, this will never get executed. We abuse `slot` to | |
739 | // get the correct type inference here: | |
740 | unsafe { | |
741 | ::core::ptr::write($slot, $t { | |
742 | $($acc)* | |
743 | }); | |
744 | } | |
745 | }; | |
746 | (make_initializer: | |
747 | @slot($slot:ident), | |
748 | @type_name($t:ident), | |
749 | @munch_fields($field:ident <- $val:expr, $($rest:tt)*), | |
750 | @acc($($acc:tt)*), | |
751 | ) => { | |
752 | $crate::try_pin_init!(make_initializer: | |
753 | @slot($slot), | |
754 | @type_name($t), | |
755 | @munch_fields($($rest)*), | |
756 | @acc($($acc)* $field: ::core::panic!(),), | |
757 | ); | |
758 | }; | |
759 | (make_initializer: | |
760 | @slot($slot:ident), | |
761 | @type_name($t:ident), | |
762 | @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), | |
763 | @acc($($acc:tt)*), | |
764 | ) => { | |
765 | $crate::try_pin_init!(make_initializer: | |
766 | @slot($slot), | |
767 | @type_name($t), | |
768 | @munch_fields($($rest)*), | |
769 | @acc($($acc)* $field: ::core::panic!(),), | |
770 | ); | |
771 | }; | |
772 | (forget_guards: | |
773 | @munch_fields($(,)?), | |
774 | ) => { | |
775 | // Munching finished. | |
776 | }; | |
777 | (forget_guards: | |
778 | @munch_fields($field:ident <- $val:expr, $($rest:tt)*), | |
779 | ) => { | |
780 | unsafe { $crate::init::__internal::DropGuard::forget($field) }; | |
781 | ||
782 | $crate::try_pin_init!(forget_guards: | |
783 | @munch_fields($($rest)*), | |
784 | ); | |
785 | }; | |
786 | (forget_guards: | |
787 | @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), | |
788 | ) => { | |
789 | unsafe { $crate::init::__internal::DropGuard::forget($field) }; | |
790 | ||
791 | $crate::try_pin_init!(forget_guards: | |
792 | @munch_fields($($rest)*), | |
793 | ); | |
794 | }; | |
795 | } | |
796 | ||
797 | /// Construct an in-place initializer for `struct`s. | |
798 | /// | |
799 | /// This macro defaults the error to [`Infallible`]. If you need [`Error`], then use | |
800 | /// [`try_init!`]. | |
801 | /// | |
802 | /// The syntax is identical to [`pin_init!`] and its safety caveats also apply: | |
803 | /// - `unsafe` code must guarantee either full initialization or return an error and allow | |
804 | /// deallocation of the memory. | |
805 | /// - the fields are initialized in the order given in the initializer. | |
806 | /// - no references to fields are allowed to be created inside of the initializer. | |
807 | /// | |
808 | /// This initializer is for initializing data in-place that might later be moved. If you want to | |
809 | /// pin-initialize, use [`pin_init!`]. | |
3ed03f4d MO |
810 | /// |
811 | /// [`try_init!`]: crate::try_init! | |
fc6c6baa BL |
812 | // For a detailed example of how this macro works, see the module documentation of the hidden |
813 | // module `__internal` inside of `init/__internal.rs`. | |
814 | #[macro_export] | |
815 | macro_rules! init { | |
816 | ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { | |
817 | $($fields:tt)* | |
818 | }) => { | |
819 | $crate::try_init!( | |
820 | @this($($this)?), | |
821 | @typ($t $(::<$($generics),*>)?), | |
822 | @fields($($fields)*), | |
823 | @error(::core::convert::Infallible), | |
824 | ) | |
825 | } | |
826 | } | |
827 | ||
828 | /// Construct an in-place fallible initializer for `struct`s. | |
829 | /// | |
830 | /// This macro defaults the error to [`Error`]. If you need [`Infallible`], then use | |
831 | /// [`init!`]. | |
832 | /// | |
833 | /// The syntax is identical to [`try_pin_init!`]. If you want to specify a custom error, | |
834 | /// append `? $type` after the `struct` initializer. | |
835 | /// The safety caveats from [`try_pin_init!`] also apply: | |
836 | /// - `unsafe` code must guarantee either full initialization or return an error and allow | |
837 | /// deallocation of the memory. | |
838 | /// - the fields are initialized in the order given in the initializer. | |
839 | /// - no references to fields are allowed to be created inside of the initializer. | |
840 | /// | |
841 | /// # Examples | |
842 | /// | |
843 | /// ```rust | |
844 | /// use kernel::{init::PinInit, error::Error, InPlaceInit}; | |
845 | /// struct BigBuf { | |
846 | /// big: Box<[u8; 1024 * 1024 * 1024]>, | |
847 | /// small: [u8; 1024 * 1024], | |
848 | /// } | |
849 | /// | |
850 | /// impl BigBuf { | |
851 | /// fn new() -> impl Init<Self, Error> { | |
852 | /// try_init!(Self { | |
853 | /// big: Box::init(zeroed())?, | |
854 | /// small: [0; 1024 * 1024], | |
855 | /// }? Error) | |
856 | /// } | |
857 | /// } | |
858 | /// ``` | |
fc6c6baa BL |
859 | // For a detailed example of how this macro works, see the module documentation of the hidden |
860 | // module `__internal` inside of `init/__internal.rs`. | |
861 | #[macro_export] | |
862 | macro_rules! try_init { | |
863 | ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { | |
864 | $($fields:tt)* | |
865 | }) => { | |
866 | $crate::try_init!( | |
867 | @this($($this)?), | |
868 | @typ($t $(::<$($generics),*>)?), | |
869 | @fields($($fields)*), | |
870 | @error($crate::error::Error), | |
871 | ) | |
872 | }; | |
873 | ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { | |
874 | $($fields:tt)* | |
875 | }? $err:ty) => { | |
876 | $crate::try_init!( | |
877 | @this($($this)?), | |
878 | @typ($t $(::<$($generics),*>)?), | |
879 | @fields($($fields)*), | |
880 | @error($err), | |
881 | ) | |
882 | }; | |
883 | ( | |
884 | @this($($this:ident)?), | |
885 | @typ($t:ident $(::<$($generics:ty),*>)?), | |
886 | @fields($($fields:tt)*), | |
887 | @error($err:ty), | |
888 | ) => {{ | |
889 | // We do not want to allow arbitrary returns, so we declare this type as the `Ok` return | |
890 | // type and shadow it later when we insert the arbitrary user code. That way there will be | |
891 | // no possibility of returning without `unsafe`. | |
892 | struct __InitOk; | |
893 | // Get the init data from the supplied type. | |
894 | let data = unsafe { | |
895 | use $crate::init::__internal::HasInitData; | |
896 | $t$(::<$($generics),*>)?::__init_data() | |
897 | }; | |
898 | // Ensure that `data` really is of type `InitData` and help with type inference: | |
899 | let init = $crate::init::__internal::InitData::make_closure::<_, __InitOk, $err>( | |
900 | data, | |
901 | move |slot| { | |
902 | { | |
903 | // Shadow the structure so it cannot be used to return early. | |
904 | struct __InitOk; | |
905 | // Create the `this` so it can be referenced by the user inside of the | |
906 | // expressions creating the individual fields. | |
907 | $(let $this = unsafe { ::core::ptr::NonNull::new_unchecked(slot) };)? | |
908 | // Initialize every field. | |
909 | $crate::try_init!(init_slot: | |
910 | @slot(slot), | |
911 | @munch_fields($($fields)*,), | |
912 | ); | |
913 | // We use unreachable code to ensure that all fields have been mentioned exactly | |
914 | // once, this struct initializer will still be type-checked and complain with a | |
915 | // very natural error message if a field is forgotten/mentioned more than once. | |
916 | #[allow(unreachable_code, clippy::diverging_sub_expression)] | |
917 | if false { | |
918 | $crate::try_init!(make_initializer: | |
919 | @slot(slot), | |
920 | @type_name($t), | |
921 | @munch_fields($($fields)*,), | |
922 | @acc(), | |
923 | ); | |
924 | } | |
925 | // Forget all guards, since initialization was a success. | |
926 | $crate::try_init!(forget_guards: | |
927 | @munch_fields($($fields)*,), | |
928 | ); | |
929 | } | |
930 | Ok(__InitOk) | |
931 | } | |
932 | ); | |
933 | let init = move |slot| -> ::core::result::Result<(), $err> { | |
934 | init(slot).map(|__InitOk| ()) | |
935 | }; | |
936 | let init = unsafe { $crate::init::init_from_closure::<_, $err>(init) }; | |
937 | init | |
938 | }}; | |
939 | (init_slot: | |
940 | @slot($slot:ident), | |
941 | @munch_fields( $(,)?), | |
942 | ) => { | |
943 | // Endpoint of munching, no fields are left. | |
944 | }; | |
945 | (init_slot: | |
946 | @slot($slot:ident), | |
947 | @munch_fields($field:ident <- $val:expr, $($rest:tt)*), | |
948 | ) => { | |
949 | let $field = $val; | |
950 | // Call the initializer. | |
951 | // | |
952 | // SAFETY: `slot` is valid, because we are inside of an initializer closure, we | |
953 | // return when an error/panic occurs. | |
954 | unsafe { | |
955 | $crate::init::Init::__init($field, ::core::ptr::addr_of_mut!((*$slot).$field))?; | |
956 | } | |
957 | // Create the drop guard. | |
958 | // | |
959 | // We only give access to `&DropGuard`, so it cannot be accidentally forgotten. | |
960 | // | |
961 | // SAFETY: We forget the guard later when initialization has succeeded. | |
962 | let $field = &unsafe { | |
963 | $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field)) | |
964 | }; | |
965 | ||
966 | $crate::try_init!(init_slot: | |
967 | @slot($slot), | |
968 | @munch_fields($($rest)*), | |
969 | ); | |
970 | }; | |
971 | (init_slot: | |
972 | @slot($slot:ident), | |
973 | // Direct value init. | |
974 | @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), | |
975 | ) => { | |
976 | $(let $field = $val;)? | |
977 | // Call the initializer. | |
978 | // | |
979 | // SAFETY: The memory at `slot` is uninitialized. | |
980 | unsafe { ::core::ptr::write(::core::ptr::addr_of_mut!((*$slot).$field), $field) }; | |
981 | // Create the drop guard. | |
982 | // | |
983 | // We only give access to `&DropGuard`, so it cannot be accidentally forgotten. | |
984 | // | |
985 | // SAFETY: We forget the guard later when initialization has succeeded. | |
986 | let $field = &unsafe { | |
987 | $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field)) | |
988 | }; | |
989 | ||
990 | $crate::try_init!(init_slot: | |
991 | @slot($slot), | |
992 | @munch_fields($($rest)*), | |
993 | ); | |
994 | }; | |
995 | (make_initializer: | |
996 | @slot($slot:ident), | |
997 | @type_name($t:ident), | |
998 | @munch_fields( $(,)?), | |
999 | @acc($($acc:tt)*), | |
1000 | ) => { | |
1001 | // Endpoint, nothing more to munch, create the initializer. | |
1002 | // Since we are in the `if false` branch, this will never get executed. We abuse `slot` to | |
1003 | // get the correct type inference here: | |
1004 | unsafe { | |
1005 | ::core::ptr::write($slot, $t { | |
1006 | $($acc)* | |
1007 | }); | |
1008 | } | |
1009 | }; | |
1010 | (make_initializer: | |
1011 | @slot($slot:ident), | |
1012 | @type_name($t:ident), | |
1013 | @munch_fields($field:ident <- $val:expr, $($rest:tt)*), | |
1014 | @acc($($acc:tt)*), | |
1015 | ) => { | |
1016 | $crate::try_init!(make_initializer: | |
1017 | @slot($slot), | |
1018 | @type_name($t), | |
1019 | @munch_fields($($rest)*), | |
1020 | @acc($($acc)*$field: ::core::panic!(),), | |
1021 | ); | |
1022 | }; | |
1023 | (make_initializer: | |
1024 | @slot($slot:ident), | |
1025 | @type_name($t:ident), | |
1026 | @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), | |
1027 | @acc($($acc:tt)*), | |
1028 | ) => { | |
1029 | $crate::try_init!(make_initializer: | |
1030 | @slot($slot), | |
1031 | @type_name($t), | |
1032 | @munch_fields($($rest)*), | |
1033 | @acc($($acc)*$field: ::core::panic!(),), | |
1034 | ); | |
1035 | }; | |
1036 | (forget_guards: | |
1037 | @munch_fields($(,)?), | |
1038 | ) => { | |
1039 | // Munching finished. | |
1040 | }; | |
1041 | (forget_guards: | |
1042 | @munch_fields($field:ident <- $val:expr, $($rest:tt)*), | |
1043 | ) => { | |
1044 | unsafe { $crate::init::__internal::DropGuard::forget($field) }; | |
1045 | ||
1046 | $crate::try_init!(forget_guards: | |
1047 | @munch_fields($($rest)*), | |
1048 | ); | |
1049 | }; | |
1050 | (forget_guards: | |
1051 | @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*), | |
1052 | ) => { | |
1053 | unsafe { $crate::init::__internal::DropGuard::forget($field) }; | |
1054 | ||
1055 | $crate::try_init!(forget_guards: | |
1056 | @munch_fields($($rest)*), | |
1057 | ); | |
1058 | }; | |
1059 | } | |
90e53c5e BL |
1060 | |
1061 | /// A pin-initializer for the type `T`. | |
1062 | /// | |
1063 | /// To use this initializer, you will need a suitable memory location that can hold a `T`. This can | |
6841d45a BL |
1064 | /// be [`Box<T>`], [`Arc<T>`], [`UniqueArc<T>`] or even the stack (see [`stack_pin_init!`]). Use the |
1065 | /// [`InPlaceInit::pin_init`] function of a smart pointer like [`Arc<T>`] on this. | |
90e53c5e BL |
1066 | /// |
1067 | /// Also see the [module description](self). | |
1068 | /// | |
1069 | /// # Safety | |
1070 | /// | |
1071 | /// When implementing this type you will need to take great care. Also there are probably very few | |
1072 | /// cases where a manual implementation is necessary. Use [`pin_init_from_closure`] where possible. | |
1073 | /// | |
1074 | /// The [`PinInit::__pinned_init`] function | |
1075 | /// - returns `Ok(())` if it initialized every field of `slot`, | |
1076 | /// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means: | |
1077 | /// - `slot` can be deallocated without UB occurring, | |
1078 | /// - `slot` does not need to be dropped, | |
1079 | /// - `slot` is not partially initialized. | |
1080 | /// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`. | |
1081 | /// | |
1082 | /// [`Arc<T>`]: crate::sync::Arc | |
1083 | /// [`Arc::pin_init`]: crate::sync::Arc::pin_init | |
90e53c5e BL |
1084 | #[must_use = "An initializer must be used in order to create its value."] |
1085 | pub unsafe trait PinInit<T: ?Sized, E = Infallible>: Sized { | |
1086 | /// Initializes `slot`. | |
1087 | /// | |
1088 | /// # Safety | |
1089 | /// | |
1090 | /// - `slot` is a valid pointer to uninitialized memory. | |
1091 | /// - the caller does not touch `slot` when `Err` is returned, they are only permitted to | |
1092 | /// deallocate. | |
1093 | /// - `slot` will not move until it is dropped, i.e. it will be pinned. | |
1094 | unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E>; | |
1095 | } | |
1096 | ||
1097 | /// An initializer for `T`. | |
1098 | /// | |
1099 | /// To use this initializer, you will need a suitable memory location that can hold a `T`. This can | |
6841d45a BL |
1100 | /// be [`Box<T>`], [`Arc<T>`], [`UniqueArc<T>`] or even the stack (see [`stack_pin_init!`]). Use the |
1101 | /// [`InPlaceInit::init`] function of a smart pointer like [`Arc<T>`] on this. Because | |
1102 | /// [`PinInit<T, E>`] is a super trait, you can use every function that takes it as well. | |
90e53c5e BL |
1103 | /// |
1104 | /// Also see the [module description](self). | |
1105 | /// | |
1106 | /// # Safety | |
1107 | /// | |
1108 | /// When implementing this type you will need to take great care. Also there are probably very few | |
1109 | /// cases where a manual implementation is necessary. Use [`init_from_closure`] where possible. | |
1110 | /// | |
1111 | /// The [`Init::__init`] function | |
1112 | /// - returns `Ok(())` if it initialized every field of `slot`, | |
1113 | /// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means: | |
1114 | /// - `slot` can be deallocated without UB occurring, | |
1115 | /// - `slot` does not need to be dropped, | |
1116 | /// - `slot` is not partially initialized. | |
1117 | /// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`. | |
1118 | /// | |
1119 | /// The `__pinned_init` function from the supertrait [`PinInit`] needs to execute the exact same | |
1120 | /// code as `__init`. | |
1121 | /// | |
1122 | /// Contrary to its supertype [`PinInit<T, E>`] the caller is allowed to | |
1123 | /// move the pointee after initialization. | |
1124 | /// | |
1125 | /// [`Arc<T>`]: crate::sync::Arc | |
90e53c5e BL |
1126 | #[must_use = "An initializer must be used in order to create its value."] |
1127 | pub unsafe trait Init<T: ?Sized, E = Infallible>: Sized { | |
1128 | /// Initializes `slot`. | |
1129 | /// | |
1130 | /// # Safety | |
1131 | /// | |
1132 | /// - `slot` is a valid pointer to uninitialized memory. | |
1133 | /// - the caller does not touch `slot` when `Err` is returned, they are only permitted to | |
1134 | /// deallocate. | |
1135 | unsafe fn __init(self, slot: *mut T) -> Result<(), E>; | |
1136 | } | |
1137 | ||
1138 | // SAFETY: Every in-place initializer can also be used as a pin-initializer. | |
1139 | unsafe impl<T: ?Sized, E, I> PinInit<T, E> for I | |
1140 | where | |
1141 | I: Init<T, E>, | |
1142 | { | |
1143 | unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> { | |
1144 | // SAFETY: `__init` meets the same requirements as `__pinned_init`, except that it does not | |
1145 | // require `slot` to not move after init. | |
1146 | unsafe { self.__init(slot) } | |
1147 | } | |
1148 | } | |
1149 | ||
1150 | /// Creates a new [`PinInit<T, E>`] from the given closure. | |
1151 | /// | |
1152 | /// # Safety | |
1153 | /// | |
1154 | /// The closure: | |
1155 | /// - returns `Ok(())` if it initialized every field of `slot`, | |
1156 | /// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means: | |
1157 | /// - `slot` can be deallocated without UB occurring, | |
1158 | /// - `slot` does not need to be dropped, | |
1159 | /// - `slot` is not partially initialized. | |
1160 | /// - may assume that the `slot` does not move if `T: !Unpin`, | |
1161 | /// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`. | |
1162 | #[inline] | |
1163 | pub const unsafe fn pin_init_from_closure<T: ?Sized, E>( | |
1164 | f: impl FnOnce(*mut T) -> Result<(), E>, | |
1165 | ) -> impl PinInit<T, E> { | |
1166 | __internal::InitClosure(f, PhantomData) | |
1167 | } | |
1168 | ||
1169 | /// Creates a new [`Init<T, E>`] from the given closure. | |
1170 | /// | |
1171 | /// # Safety | |
1172 | /// | |
1173 | /// The closure: | |
1174 | /// - returns `Ok(())` if it initialized every field of `slot`, | |
1175 | /// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means: | |
1176 | /// - `slot` can be deallocated without UB occurring, | |
1177 | /// - `slot` does not need to be dropped, | |
1178 | /// - `slot` is not partially initialized. | |
1179 | /// - the `slot` may move after initialization. | |
1180 | /// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`. | |
1181 | #[inline] | |
1182 | pub const unsafe fn init_from_closure<T: ?Sized, E>( | |
1183 | f: impl FnOnce(*mut T) -> Result<(), E>, | |
1184 | ) -> impl Init<T, E> { | |
1185 | __internal::InitClosure(f, PhantomData) | |
1186 | } | |
1187 | ||
1188 | /// An initializer that leaves the memory uninitialized. | |
1189 | /// | |
1190 | /// The initializer is a no-op. The `slot` memory is not changed. | |
1191 | #[inline] | |
1192 | pub fn uninit<T, E>() -> impl Init<MaybeUninit<T>, E> { | |
1193 | // SAFETY: The memory is allowed to be uninitialized. | |
1194 | unsafe { init_from_closure(|_| Ok(())) } | |
1195 | } | |
1196 | ||
1197 | // SAFETY: Every type can be initialized by-value. | |
52a7f2de BL |
1198 | unsafe impl<T, E> Init<T, E> for T { |
1199 | unsafe fn __init(self, slot: *mut T) -> Result<(), E> { | |
90e53c5e BL |
1200 | unsafe { slot.write(self) }; |
1201 | Ok(()) | |
1202 | } | |
1203 | } | |
92c4a1e7 BL |
1204 | |
1205 | /// Smart pointer that can initialize memory in-place. | |
1206 | pub trait InPlaceInit<T>: Sized { | |
1207 | /// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this | |
1208 | /// type. | |
1209 | /// | |
1210 | /// If `T: !Unpin` it will not be able to move afterwards. | |
1211 | fn try_pin_init<E>(init: impl PinInit<T, E>) -> Result<Pin<Self>, E> | |
1212 | where | |
1213 | E: From<AllocError>; | |
1214 | ||
1215 | /// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this | |
1216 | /// type. | |
1217 | /// | |
1218 | /// If `T: !Unpin` it will not be able to move afterwards. | |
1219 | fn pin_init<E>(init: impl PinInit<T, E>) -> error::Result<Pin<Self>> | |
1220 | where | |
1221 | Error: From<E>, | |
1222 | { | |
1223 | // SAFETY: We delegate to `init` and only change the error type. | |
1224 | let init = unsafe { | |
1225 | pin_init_from_closure(|slot| init.__pinned_init(slot).map_err(|e| Error::from(e))) | |
1226 | }; | |
1227 | Self::try_pin_init(init) | |
1228 | } | |
1229 | ||
1230 | /// Use the given initializer to in-place initialize a `T`. | |
1231 | fn try_init<E>(init: impl Init<T, E>) -> Result<Self, E> | |
1232 | where | |
1233 | E: From<AllocError>; | |
1234 | ||
1235 | /// Use the given initializer to in-place initialize a `T`. | |
1236 | fn init<E>(init: impl Init<T, E>) -> error::Result<Self> | |
1237 | where | |
1238 | Error: From<E>, | |
1239 | { | |
1240 | // SAFETY: We delegate to `init` and only change the error type. | |
1241 | let init = unsafe { | |
1242 | init_from_closure(|slot| init.__pinned_init(slot).map_err(|e| Error::from(e))) | |
1243 | }; | |
1244 | Self::try_init(init) | |
1245 | } | |
1246 | } | |
1247 | ||
1248 | impl<T> InPlaceInit<T> for Box<T> { | |
1249 | #[inline] | |
1250 | fn try_pin_init<E>(init: impl PinInit<T, E>) -> Result<Pin<Self>, E> | |
1251 | where | |
1252 | E: From<AllocError>, | |
1253 | { | |
1254 | let mut this = Box::try_new_uninit()?; | |
1255 | let slot = this.as_mut_ptr(); | |
1256 | // SAFETY: When init errors/panics, slot will get deallocated but not dropped, | |
1257 | // slot is valid and will not be moved, because we pin it later. | |
1258 | unsafe { init.__pinned_init(slot)? }; | |
1259 | // SAFETY: All fields have been initialized. | |
1260 | Ok(unsafe { this.assume_init() }.into()) | |
1261 | } | |
1262 | ||
1263 | #[inline] | |
1264 | fn try_init<E>(init: impl Init<T, E>) -> Result<Self, E> | |
1265 | where | |
1266 | E: From<AllocError>, | |
1267 | { | |
1268 | let mut this = Box::try_new_uninit()?; | |
1269 | let slot = this.as_mut_ptr(); | |
1270 | // SAFETY: When init errors/panics, slot will get deallocated but not dropped, | |
1271 | // slot is valid. | |
1272 | unsafe { init.__init(slot)? }; | |
1273 | // SAFETY: All fields have been initialized. | |
1274 | Ok(unsafe { this.assume_init() }) | |
1275 | } | |
1276 | } | |
1277 | ||
1278 | impl<T> InPlaceInit<T> for UniqueArc<T> { | |
1279 | #[inline] | |
1280 | fn try_pin_init<E>(init: impl PinInit<T, E>) -> Result<Pin<Self>, E> | |
1281 | where | |
1282 | E: From<AllocError>, | |
1283 | { | |
1284 | let mut this = UniqueArc::try_new_uninit()?; | |
1285 | let slot = this.as_mut_ptr(); | |
1286 | // SAFETY: When init errors/panics, slot will get deallocated but not dropped, | |
1287 | // slot is valid and will not be moved, because we pin it later. | |
1288 | unsafe { init.__pinned_init(slot)? }; | |
1289 | // SAFETY: All fields have been initialized. | |
1290 | Ok(unsafe { this.assume_init() }.into()) | |
1291 | } | |
1292 | ||
1293 | #[inline] | |
1294 | fn try_init<E>(init: impl Init<T, E>) -> Result<Self, E> | |
1295 | where | |
1296 | E: From<AllocError>, | |
1297 | { | |
1298 | let mut this = UniqueArc::try_new_uninit()?; | |
1299 | let slot = this.as_mut_ptr(); | |
1300 | // SAFETY: When init errors/panics, slot will get deallocated but not dropped, | |
1301 | // slot is valid. | |
1302 | unsafe { init.__init(slot)? }; | |
1303 | // SAFETY: All fields have been initialized. | |
1304 | Ok(unsafe { this.assume_init() }) | |
1305 | } | |
1306 | } | |
d0fdc396 BL |
1307 | |
1308 | /// Trait facilitating pinned destruction. | |
1309 | /// | |
1310 | /// Use [`pinned_drop`] to implement this trait safely: | |
1311 | /// | |
1312 | /// ```rust | |
1313 | /// # use kernel::sync::Mutex; | |
1314 | /// use kernel::macros::pinned_drop; | |
1315 | /// use core::pin::Pin; | |
1316 | /// #[pin_data(PinnedDrop)] | |
1317 | /// struct Foo { | |
1318 | /// #[pin] | |
1319 | /// mtx: Mutex<usize>, | |
1320 | /// } | |
1321 | /// | |
1322 | /// #[pinned_drop] | |
1323 | /// impl PinnedDrop for Foo { | |
1324 | /// fn drop(self: Pin<&mut Self>) { | |
1325 | /// pr_info!("Foo is being dropped!"); | |
1326 | /// } | |
1327 | /// } | |
1328 | /// ``` | |
1329 | /// | |
1330 | /// # Safety | |
1331 | /// | |
1332 | /// This trait must be implemented via the [`pinned_drop`] proc-macro attribute on the impl. | |
1333 | /// | |
1334 | /// [`pinned_drop`]: kernel::macros::pinned_drop | |
1335 | pub unsafe trait PinnedDrop: __internal::HasPinData { | |
1336 | /// Executes the pinned destructor of this type. | |
1337 | /// | |
1338 | /// While this function is marked safe, it is actually unsafe to call it manually. For this | |
1339 | /// reason it takes an additional parameter. This type can only be constructed by `unsafe` code | |
1340 | /// and thus prevents this function from being called where it should not. | |
1341 | /// | |
1342 | /// This extra parameter will be generated by the `#[pinned_drop]` proc-macro attribute | |
1343 | /// automatically. | |
1344 | fn drop(self: Pin<&mut Self>, only_call_from_drop: __internal::OnlyCallFromDrop); | |
1345 | } | |
38cde0bd BL |
1346 | |
1347 | /// Marker trait for types that can be initialized by writing just zeroes. | |
1348 | /// | |
1349 | /// # Safety | |
1350 | /// | |
1351 | /// The bit pattern consisting of only zeroes is a valid bit pattern for this type. In other words, | |
1352 | /// this is not UB: | |
1353 | /// | |
1354 | /// ```rust,ignore | |
1355 | /// let val: Self = unsafe { core::mem::zeroed() }; | |
1356 | /// ``` | |
1357 | pub unsafe trait Zeroable {} | |
1358 | ||
1359 | /// Create a new zeroed T. | |
1360 | /// | |
1361 | /// The returned initializer will write `0x00` to every byte of the given `slot`. | |
1362 | #[inline] | |
1363 | pub fn zeroed<T: Zeroable>() -> impl Init<T> { | |
1364 | // SAFETY: Because `T: Zeroable`, all bytes zero is a valid bit pattern for `T` | |
1365 | // and because we write all zeroes, the memory is initialized. | |
1366 | unsafe { | |
1367 | init_from_closure(|slot: *mut T| { | |
1368 | slot.write_bytes(0, 1); | |
1369 | Ok(()) | |
1370 | }) | |
1371 | } | |
1372 | } | |
1373 | ||
1374 | macro_rules! impl_zeroable { | |
1375 | ($($({$($generics:tt)*})? $t:ty, )*) => { | |
1376 | $(unsafe impl$($($generics)*)? Zeroable for $t {})* | |
1377 | }; | |
1378 | } | |
1379 | ||
1380 | impl_zeroable! { | |
1381 | // SAFETY: All primitives that are allowed to be zero. | |
1382 | bool, | |
1383 | char, | |
1384 | u8, u16, u32, u64, u128, usize, | |
1385 | i8, i16, i32, i64, i128, isize, | |
1386 | f32, f64, | |
1387 | ||
1388 | // SAFETY: These are ZSTs, there is nothing to zero. | |
1389 | {<T: ?Sized>} PhantomData<T>, core::marker::PhantomPinned, Infallible, (), | |
1390 | ||
1391 | // SAFETY: Type is allowed to take any value, including all zeros. | |
1392 | {<T>} MaybeUninit<T>, | |
1393 | ||
1394 | // SAFETY: All zeros is equivalent to `None` (option layout optimization guarantee). | |
1395 | Option<NonZeroU8>, Option<NonZeroU16>, Option<NonZeroU32>, Option<NonZeroU64>, | |
1396 | Option<NonZeroU128>, Option<NonZeroUsize>, | |
1397 | Option<NonZeroI8>, Option<NonZeroI16>, Option<NonZeroI32>, Option<NonZeroI64>, | |
1398 | Option<NonZeroI128>, Option<NonZeroIsize>, | |
1399 | ||
1400 | // SAFETY: All zeros is equivalent to `None` (option layout optimization guarantee). | |
1401 | // | |
1402 | // In this case we are allowed to use `T: ?Sized`, since all zeros is the `None` variant. | |
1403 | {<T: ?Sized>} Option<NonNull<T>>, | |
1404 | {<T: ?Sized>} Option<Box<T>>, | |
1405 | ||
1406 | // SAFETY: `null` pointer is valid. | |
1407 | // | |
1408 | // We cannot use `T: ?Sized`, since the VTABLE pointer part of fat pointers is not allowed to be | |
1409 | // null. | |
1410 | // | |
1411 | // When `Pointee` gets stabilized, we could use | |
1412 | // `T: ?Sized where <T as Pointee>::Metadata: Zeroable` | |
1413 | {<T>} *mut T, {<T>} *const T, | |
1414 | ||
1415 | // SAFETY: `null` pointer is valid and the metadata part of these fat pointers is allowed to be | |
1416 | // zero. | |
1417 | {<T>} *mut [T], {<T>} *const [T], *mut str, *const str, | |
1418 | ||
1419 | // SAFETY: `T` is `Zeroable`. | |
1420 | {<const N: usize, T: Zeroable>} [T; N], {<T: Zeroable>} Wrapping<T>, | |
1421 | } | |
1422 | ||
1423 | macro_rules! impl_tuple_zeroable { | |
1424 | ($(,)?) => {}; | |
1425 | ($first:ident, $($t:ident),* $(,)?) => { | |
1426 | // SAFETY: All elements are zeroable and padding can be zero. | |
1427 | unsafe impl<$first: Zeroable, $($t: Zeroable),*> Zeroable for ($first, $($t),*) {} | |
1428 | impl_tuple_zeroable!($($t),* ,); | |
1429 | } | |
1430 | } | |
1431 | ||
1432 | impl_tuple_zeroable!(A, B, C, D, E, F, G, H, I, J); |