Commit | Line | Data |
---|---|---|
90e53c5e BL |
1 | // SPDX-License-Identifier: Apache-2.0 OR MIT |
2 | ||
3 | //! API to safely and fallibly initialize pinned `struct`s using in-place constructors. | |
4 | //! | |
5 | //! It also allows in-place initialization of big `struct`s that would otherwise produce a stack | |
6 | //! overflow. | |
7 | //! | |
8 | //! Most `struct`s from the [`sync`] module need to be pinned, because they contain self-referential | |
9 | //! `struct`s from C. [Pinning][pinning] is Rust's way of ensuring data does not move. | |
10 | //! | |
11 | //! # Overview | |
12 | //! | |
13 | //! To initialize a `struct` with an in-place constructor you will need two things: | |
14 | //! - an in-place constructor, | |
6841d45a BL |
15 | //! - a memory location that can hold your `struct` (this can be the [stack], an [`Arc<T>`], |
16 | //! [`UniqueArc<T>`], [`Box<T>`] or any other smart pointer that implements [`InPlaceInit`]). | |
90e53c5e | 17 | //! |
fc6c6baa BL |
18 | //! To get an in-place constructor there are generally three options: |
19 | //! - directly creating an in-place constructor using the [`pin_init!`] macro, | |
90e53c5e BL |
20 | //! - a custom function/macro returning an in-place constructor provided by someone else, |
21 | //! - using the unsafe function [`pin_init_from_closure()`] to manually create an initializer. | |
22 | //! | |
23 | //! Aside from pinned initialization, this API also supports in-place construction without pinning, | |
24 | //! the macros/types/functions are generally named like the pinned variants without the `pin` | |
25 | //! prefix. | |
26 | //! | |
fc6c6baa BL |
27 | //! # Examples |
28 | //! | |
29 | //! ## Using the [`pin_init!`] macro | |
30 | //! | |
31 | //! If you want to use [`PinInit`], then you will have to annotate your `struct` with | |
32 | //! `#[`[`pin_data`]`]`. It is a macro that uses `#[pin]` as a marker for | |
33 | //! [structurally pinned fields]. After doing this, you can then create an in-place constructor via | |
34 | //! [`pin_init!`]. The syntax is almost the same as normal `struct` initializers. The difference is | |
35 | //! that you need to write `<-` instead of `:` for fields that you want to initialize in-place. | |
36 | //! | |
37 | //! ```rust | |
b2516f7a | 38 | //! # #![allow(clippy::disallowed_names)] |
e283ee23 | 39 | //! use kernel::sync::{new_mutex, Mutex}; |
fc6c6baa BL |
40 | //! # use core::pin::Pin; |
41 | //! #[pin_data] | |
42 | //! struct Foo { | |
43 | //! #[pin] | |
44 | //! a: Mutex<usize>, | |
45 | //! b: u32, | |
46 | //! } | |
47 | //! | |
48 | //! let foo = pin_init!(Foo { | |
49 | //! a <- new_mutex!(42, "Foo::a"), | |
50 | //! b: 24, | |
51 | //! }); | |
52 | //! ``` | |
53 | //! | |
54 | //! `foo` now is of the type [`impl PinInit<Foo>`]. We can now use any smart pointer that we like | |
55 | //! (or just the stack) to actually initialize a `Foo`: | |
56 | //! | |
57 | //! ```rust | |
b2516f7a | 58 | //! # #![allow(clippy::disallowed_names)] |
e283ee23 | 59 | //! # use kernel::sync::{new_mutex, Mutex}; |
fc6c6baa BL |
60 | //! # use core::pin::Pin; |
61 | //! # #[pin_data] | |
62 | //! # struct Foo { | |
63 | //! # #[pin] | |
64 | //! # a: Mutex<usize>, | |
65 | //! # b: u32, | |
66 | //! # } | |
67 | //! # let foo = pin_init!(Foo { | |
68 | //! # a <- new_mutex!(42, "Foo::a"), | |
69 | //! # b: 24, | |
70 | //! # }); | |
c34aa00d | 71 | //! let foo: Result<Pin<Box<Foo>>> = Box::pin_init(foo, GFP_KERNEL); |
fc6c6baa BL |
72 | //! ``` |
73 | //! | |
74 | //! For more information see the [`pin_init!`] macro. | |
75 | //! | |
76 | //! ## Using a custom function/macro that returns an initializer | |
77 | //! | |
78 | //! Many types from the kernel supply a function/macro that returns an initializer, because the | |
79 | //! above method only works for types where you can access the fields. | |
80 | //! | |
81 | //! ```rust | |
e283ee23 | 82 | //! # use kernel::sync::{new_mutex, Arc, Mutex}; |
c34aa00d WAF |
83 | //! let mtx: Result<Arc<Mutex<usize>>> = |
84 | //! Arc::pin_init(new_mutex!(42, "example::mtx"), GFP_KERNEL); | |
fc6c6baa BL |
85 | //! ``` |
86 | //! | |
87 | //! To declare an init macro/function you just return an [`impl PinInit<T, E>`]: | |
88 | //! | |
89 | //! ```rust | |
b2516f7a | 90 | //! # #![allow(clippy::disallowed_names)] |
4a2ae880 | 91 | //! # use kernel::{sync::Mutex, new_mutex, init::PinInit, try_pin_init}; |
fc6c6baa BL |
92 | //! #[pin_data] |
93 | //! struct DriverData { | |
94 | //! #[pin] | |
95 | //! status: Mutex<i32>, | |
96 | //! buffer: Box<[u8; 1_000_000]>, | |
97 | //! } | |
98 | //! | |
99 | //! impl DriverData { | |
100 | //! fn new() -> impl PinInit<Self, Error> { | |
101 | //! try_pin_init!(Self { | |
102 | //! status <- new_mutex!(0, "DriverData::status"), | |
c34aa00d | 103 | //! buffer: Box::init(kernel::init::zeroed(), GFP_KERNEL)?, |
fc6c6baa BL |
104 | //! }) |
105 | //! } | |
106 | //! } | |
107 | //! ``` | |
108 | //! | |
d0fdc396 BL |
109 | //! ## Manual creation of an initializer |
110 | //! | |
111 | //! Often when working with primitives the previous approaches are not sufficient. That is where | |
112 | //! [`pin_init_from_closure()`] comes in. This `unsafe` function allows you to create a | |
113 | //! [`impl PinInit<T, E>`] directly from a closure. Of course you have to ensure that the closure | |
114 | //! actually does the initialization in the correct way. Here are the things to look out for | |
115 | //! (we are calling the parameter to the closure `slot`): | |
116 | //! - when the closure returns `Ok(())`, then it has completed the initialization successfully, so | |
117 | //! `slot` now contains a valid bit pattern for the type `T`, | |
118 | //! - when the closure returns `Err(e)`, then the caller may deallocate the memory at `slot`, so | |
119 | //! you need to take care to clean up anything if your initialization fails mid-way, | |
120 | //! - you may assume that `slot` will stay pinned even after the closure returns until `drop` of | |
121 | //! `slot` gets called. | |
122 | //! | |
123 | //! ```rust | |
a30e94c2 | 124 | //! # #![allow(unreachable_pub, clippy::disallowed_names)] |
4a2ae880 | 125 | //! use kernel::{init, types::Opaque}; |
d0fdc396 BL |
126 | //! use core::{ptr::addr_of_mut, marker::PhantomPinned, pin::Pin}; |
127 | //! # mod bindings { | |
a30e94c2 | 128 | //! # #![allow(non_camel_case_types)] |
d0fdc396 BL |
129 | //! # pub struct foo; |
130 | //! # pub unsafe fn init_foo(_ptr: *mut foo) {} | |
131 | //! # pub unsafe fn destroy_foo(_ptr: *mut foo) {} | |
132 | //! # pub unsafe fn enable_foo(_ptr: *mut foo, _flags: u32) -> i32 { 0 } | |
133 | //! # } | |
a30e94c2 MO |
134 | //! # // `Error::from_errno` is `pub(crate)` in the `kernel` crate, thus provide a workaround. |
135 | //! # trait FromErrno { | |
136 | //! # fn from_errno(errno: core::ffi::c_int) -> Error { | |
137 | //! # // Dummy error that can be constructed outside the `kernel` crate. | |
138 | //! # Error::from(core::fmt::Error) | |
139 | //! # } | |
140 | //! # } | |
141 | //! # impl FromErrno for Error {} | |
d0fdc396 BL |
142 | //! /// # Invariants |
143 | //! /// | |
144 | //! /// `foo` is always initialized | |
145 | //! #[pin_data(PinnedDrop)] | |
146 | //! pub struct RawFoo { | |
147 | //! #[pin] | |
148 | //! foo: Opaque<bindings::foo>, | |
149 | //! #[pin] | |
150 | //! _p: PhantomPinned, | |
151 | //! } | |
152 | //! | |
153 | //! impl RawFoo { | |
154 | //! pub fn new(flags: u32) -> impl PinInit<Self, Error> { | |
155 | //! // SAFETY: | |
156 | //! // - when the closure returns `Ok(())`, then it has successfully initialized and | |
157 | //! // enabled `foo`, | |
158 | //! // - when it returns `Err(e)`, then it has cleaned up before | |
159 | //! unsafe { | |
160 | //! init::pin_init_from_closure(move |slot: *mut Self| { | |
161 | //! // `slot` contains uninit memory, avoid creating a reference. | |
162 | //! let foo = addr_of_mut!((*slot).foo); | |
163 | //! | |
164 | //! // Initialize the `foo` | |
165 | //! bindings::init_foo(Opaque::raw_get(foo)); | |
166 | //! | |
167 | //! // Try to enable it. | |
168 | //! let err = bindings::enable_foo(Opaque::raw_get(foo), flags); | |
169 | //! if err != 0 { | |
170 | //! // Enabling has failed, first clean up the foo and then return the error. | |
171 | //! bindings::destroy_foo(Opaque::raw_get(foo)); | |
a30e94c2 | 172 | //! return Err(Error::from_errno(err)); |
d0fdc396 BL |
173 | //! } |
174 | //! | |
175 | //! // All fields of `RawFoo` have been initialized, since `_p` is a ZST. | |
176 | //! Ok(()) | |
177 | //! }) | |
178 | //! } | |
179 | //! } | |
180 | //! } | |
181 | //! | |
182 | //! #[pinned_drop] | |
183 | //! impl PinnedDrop for RawFoo { | |
184 | //! fn drop(self: Pin<&mut Self>) { | |
185 | //! // SAFETY: Since `foo` is initialized, destroying is safe. | |
186 | //! unsafe { bindings::destroy_foo(self.foo.get()) }; | |
187 | //! } | |
188 | //! } | |
189 | //! ``` | |
190 | //! | |
692e8935 BL |
191 | //! For the special case where initializing a field is a single FFI-function call that cannot fail, |
192 | //! there exist the helper function [`Opaque::ffi_init`]. This function initialize a single | |
193 | //! [`Opaque`] field by just delegating to the supplied closure. You can use these in combination | |
194 | //! with [`pin_init!`]. | |
195 | //! | |
196 | //! For more information on how to use [`pin_init_from_closure()`], take a look at the uses inside | |
197 | //! the `kernel` crate. The [`sync`] module is a good starting point. | |
198 | //! | |
90e53c5e BL |
199 | //! [`sync`]: kernel::sync |
200 | //! [pinning]: https://doc.rust-lang.org/std/pin/index.html | |
201 | //! [structurally pinned fields]: | |
202 | //! https://doc.rust-lang.org/std/pin/index.html#pinning-is-structural-for-field | |
6841d45a | 203 | //! [stack]: crate::stack_pin_init |
90e53c5e BL |
204 | //! [`Arc<T>`]: crate::sync::Arc |
205 | //! [`impl PinInit<Foo>`]: PinInit | |
206 | //! [`impl PinInit<T, E>`]: PinInit | |
207 | //! [`impl Init<T, E>`]: Init | |
208 | //! [`Opaque`]: kernel::types::Opaque | |
692e8935 | 209 | //! [`Opaque::ffi_init`]: kernel::types::Opaque::ffi_init |
90e53c5e | 210 | //! [`pin_data`]: ::macros::pin_data |
3ed03f4d | 211 | //! [`pin_init!`]: crate::pin_init! |
90e53c5e | 212 | |
92c4a1e7 | 213 | use crate::{ |
2c109285 | 214 | alloc::{box_ext::BoxExt, AllocError, Flags}, |
92c4a1e7 BL |
215 | error::{self, Error}, |
216 | sync::UniqueArc, | |
2e704f18 | 217 | types::{Opaque, ScopeGuard}, |
92c4a1e7 | 218 | }; |
fc6c6baa | 219 | use alloc::boxed::Box; |
92c4a1e7 | 220 | use core::{ |
2e704f18 | 221 | cell::UnsafeCell, |
38cde0bd BL |
222 | convert::Infallible, |
223 | marker::PhantomData, | |
224 | mem::MaybeUninit, | |
225 | num::*, | |
226 | pin::Pin, | |
227 | ptr::{self, NonNull}, | |
92c4a1e7 | 228 | }; |
90e53c5e BL |
229 | |
230 | #[doc(hidden)] | |
231 | pub mod __internal; | |
fc6c6baa BL |
232 | #[doc(hidden)] |
233 | pub mod macros; | |
234 | ||
6841d45a BL |
235 | /// Initialize and pin a type directly on the stack. |
236 | /// | |
237 | /// # Examples | |
238 | /// | |
239 | /// ```rust | |
b2516f7a | 240 | /// # #![allow(clippy::disallowed_names)] |
a30e94c2 | 241 | /// # use kernel::{init, macros::pin_data, pin_init, stack_pin_init, init::*, sync::Mutex, new_mutex}; |
6841d45a BL |
242 | /// # use core::pin::Pin; |
243 | /// #[pin_data] | |
244 | /// struct Foo { | |
245 | /// #[pin] | |
246 | /// a: Mutex<usize>, | |
247 | /// b: Bar, | |
248 | /// } | |
249 | /// | |
250 | /// #[pin_data] | |
251 | /// struct Bar { | |
252 | /// x: u32, | |
253 | /// } | |
254 | /// | |
255 | /// stack_pin_init!(let foo = pin_init!(Foo { | |
256 | /// a <- new_mutex!(42), | |
257 | /// b: Bar { | |
258 | /// x: 64, | |
259 | /// }, | |
260 | /// })); | |
261 | /// let foo: Pin<&mut Foo> = foo; | |
262 | /// pr_info!("a: {}", &*foo.a.lock()); | |
263 | /// ``` | |
264 | /// | |
265 | /// # Syntax | |
266 | /// | |
267 | /// A normal `let` binding with optional type annotation. The expression is expected to implement | |
268 | /// [`PinInit`]/[`Init`] with the error type [`Infallible`]. If you want to use a different error | |
269 | /// type, then use [`stack_try_pin_init!`]. | |
3ed03f4d MO |
270 | /// |
271 | /// [`stack_try_pin_init!`]: crate::stack_try_pin_init! | |
6841d45a BL |
272 | #[macro_export] |
273 | macro_rules! stack_pin_init { | |
274 | (let $var:ident $(: $t:ty)? = $val:expr) => { | |
275 | let val = $val; | |
276 | let mut $var = ::core::pin::pin!($crate::init::__internal::StackInit$(::<$t>)?::uninit()); | |
277 | let mut $var = match $crate::init::__internal::StackInit::init($var, val) { | |
278 | Ok(res) => res, | |
279 | Err(x) => { | |
280 | let x: ::core::convert::Infallible = x; | |
281 | match x {} | |
282 | } | |
283 | }; | |
284 | }; | |
285 | } | |
286 | ||
287 | /// Initialize and pin a type directly on the stack. | |
288 | /// | |
289 | /// # Examples | |
290 | /// | |
a30e94c2 | 291 | /// ```rust,ignore |
b2516f7a | 292 | /// # #![allow(clippy::disallowed_names)] |
6841d45a BL |
293 | /// # use kernel::{init, pin_init, stack_try_pin_init, init::*, sync::Mutex, new_mutex}; |
294 | /// # use macros::pin_data; | |
295 | /// # use core::{alloc::AllocError, pin::Pin}; | |
296 | /// #[pin_data] | |
297 | /// struct Foo { | |
298 | /// #[pin] | |
299 | /// a: Mutex<usize>, | |
300 | /// b: Box<Bar>, | |
301 | /// } | |
302 | /// | |
303 | /// struct Bar { | |
304 | /// x: u32, | |
305 | /// } | |
306 | /// | |
307 | /// stack_try_pin_init!(let foo: Result<Pin<&mut Foo>, AllocError> = pin_init!(Foo { | |
308 | /// a <- new_mutex!(42), | |
08d3f549 | 309 | /// b: Box::new(Bar { |
6841d45a | 310 | /// x: 64, |
08d3f549 | 311 | /// }, GFP_KERNEL)?, |
6841d45a BL |
312 | /// })); |
313 | /// let foo = foo.unwrap(); | |
314 | /// pr_info!("a: {}", &*foo.a.lock()); | |
315 | /// ``` | |
316 | /// | |
a30e94c2 | 317 | /// ```rust,ignore |
b2516f7a | 318 | /// # #![allow(clippy::disallowed_names)] |
6841d45a BL |
319 | /// # use kernel::{init, pin_init, stack_try_pin_init, init::*, sync::Mutex, new_mutex}; |
320 | /// # use macros::pin_data; | |
321 | /// # use core::{alloc::AllocError, pin::Pin}; | |
322 | /// #[pin_data] | |
323 | /// struct Foo { | |
324 | /// #[pin] | |
325 | /// a: Mutex<usize>, | |
326 | /// b: Box<Bar>, | |
327 | /// } | |
328 | /// | |
329 | /// struct Bar { | |
330 | /// x: u32, | |
331 | /// } | |
332 | /// | |
333 | /// stack_try_pin_init!(let foo: Pin<&mut Foo> =? pin_init!(Foo { | |
334 | /// a <- new_mutex!(42), | |
08d3f549 | 335 | /// b: Box::new(Bar { |
6841d45a | 336 | /// x: 64, |
08d3f549 | 337 | /// }, GFP_KERNEL)?, |
6841d45a BL |
338 | /// })); |
339 | /// pr_info!("a: {}", &*foo.a.lock()); | |
340 | /// # Ok::<_, AllocError>(()) | |
341 | /// ``` | |
342 | /// | |
343 | /// # Syntax | |
344 | /// | |
345 | /// A normal `let` binding with optional type annotation. The expression is expected to implement | |
346 | /// [`PinInit`]/[`Init`]. This macro assigns a result to the given variable, adding a `?` after the | |
347 | /// `=` will propagate this error. | |
348 | #[macro_export] | |
349 | macro_rules! stack_try_pin_init { | |
350 | (let $var:ident $(: $t:ty)? = $val:expr) => { | |
351 | let val = $val; | |
352 | let mut $var = ::core::pin::pin!($crate::init::__internal::StackInit$(::<$t>)?::uninit()); | |
353 | let mut $var = $crate::init::__internal::StackInit::init($var, val); | |
354 | }; | |
355 | (let $var:ident $(: $t:ty)? =? $val:expr) => { | |
356 | let val = $val; | |
357 | let mut $var = ::core::pin::pin!($crate::init::__internal::StackInit$(::<$t>)?::uninit()); | |
358 | let mut $var = $crate::init::__internal::StackInit::init($var, val)?; | |
359 | }; | |
360 | } | |
361 | ||
fc6c6baa BL |
362 | /// Construct an in-place, pinned initializer for `struct`s. |
363 | /// | |
364 | /// This macro defaults the error to [`Infallible`]. If you need [`Error`], then use | |
365 | /// [`try_pin_init!`]. | |
366 | /// | |
367 | /// The syntax is almost identical to that of a normal `struct` initializer: | |
368 | /// | |
369 | /// ```rust | |
b2516f7a | 370 | /// # #![allow(clippy::disallowed_names)] |
fc6c6baa BL |
371 | /// # use kernel::{init, pin_init, macros::pin_data, init::*}; |
372 | /// # use core::pin::Pin; | |
373 | /// #[pin_data] | |
374 | /// struct Foo { | |
375 | /// a: usize, | |
376 | /// b: Bar, | |
377 | /// } | |
378 | /// | |
379 | /// #[pin_data] | |
380 | /// struct Bar { | |
381 | /// x: u32, | |
382 | /// } | |
383 | /// | |
384 | /// # fn demo() -> impl PinInit<Foo> { | |
385 | /// let a = 42; | |
386 | /// | |
387 | /// let initializer = pin_init!(Foo { | |
388 | /// a, | |
389 | /// b: Bar { | |
390 | /// x: 64, | |
391 | /// }, | |
392 | /// }); | |
393 | /// # initializer } | |
c34aa00d | 394 | /// # Box::pin_init(demo(), GFP_KERNEL).unwrap(); |
fc6c6baa BL |
395 | /// ``` |
396 | /// | |
397 | /// Arbitrary Rust expressions can be used to set the value of a variable. | |
398 | /// | |
399 | /// The fields are initialized in the order that they appear in the initializer. So it is possible | |
400 | /// to read already initialized fields using raw pointers. | |
401 | /// | |
402 | /// IMPORTANT: You are not allowed to create references to fields of the struct inside of the | |
403 | /// initializer. | |
404 | /// | |
405 | /// # Init-functions | |
406 | /// | |
407 | /// When working with this API it is often desired to let others construct your types without | |
408 | /// giving access to all fields. This is where you would normally write a plain function `new` | |
409 | /// that would return a new instance of your type. With this API that is also possible. | |
410 | /// However, there are a few extra things to keep in mind. | |
411 | /// | |
412 | /// To create an initializer function, simply declare it like this: | |
413 | /// | |
414 | /// ```rust | |
b2516f7a | 415 | /// # #![allow(clippy::disallowed_names)] |
4a2ae880 | 416 | /// # use kernel::{init, pin_init, init::*}; |
fc6c6baa BL |
417 | /// # use core::pin::Pin; |
418 | /// # #[pin_data] | |
419 | /// # struct Foo { | |
420 | /// # a: usize, | |
421 | /// # b: Bar, | |
422 | /// # } | |
423 | /// # #[pin_data] | |
424 | /// # struct Bar { | |
425 | /// # x: u32, | |
426 | /// # } | |
427 | /// impl Foo { | |
428 | /// fn new() -> impl PinInit<Self> { | |
429 | /// pin_init!(Self { | |
430 | /// a: 42, | |
431 | /// b: Bar { | |
432 | /// x: 64, | |
433 | /// }, | |
434 | /// }) | |
435 | /// } | |
436 | /// } | |
437 | /// ``` | |
438 | /// | |
439 | /// Users of `Foo` can now create it like this: | |
440 | /// | |
441 | /// ```rust | |
b2516f7a | 442 | /// # #![allow(clippy::disallowed_names)] |
fc6c6baa BL |
443 | /// # use kernel::{init, pin_init, macros::pin_data, init::*}; |
444 | /// # use core::pin::Pin; | |
445 | /// # #[pin_data] | |
446 | /// # struct Foo { | |
447 | /// # a: usize, | |
448 | /// # b: Bar, | |
449 | /// # } | |
450 | /// # #[pin_data] | |
451 | /// # struct Bar { | |
452 | /// # x: u32, | |
453 | /// # } | |
454 | /// # impl Foo { | |
455 | /// # fn new() -> impl PinInit<Self> { | |
456 | /// # pin_init!(Self { | |
457 | /// # a: 42, | |
458 | /// # b: Bar { | |
459 | /// # x: 64, | |
460 | /// # }, | |
461 | /// # }) | |
462 | /// # } | |
463 | /// # } | |
c34aa00d | 464 | /// let foo = Box::pin_init(Foo::new(), GFP_KERNEL); |
fc6c6baa BL |
465 | /// ``` |
466 | /// | |
467 | /// They can also easily embed it into their own `struct`s: | |
468 | /// | |
469 | /// ```rust | |
b2516f7a | 470 | /// # #![allow(clippy::disallowed_names)] |
fc6c6baa BL |
471 | /// # use kernel::{init, pin_init, macros::pin_data, init::*}; |
472 | /// # use core::pin::Pin; | |
473 | /// # #[pin_data] | |
474 | /// # struct Foo { | |
475 | /// # a: usize, | |
476 | /// # b: Bar, | |
477 | /// # } | |
478 | /// # #[pin_data] | |
479 | /// # struct Bar { | |
480 | /// # x: u32, | |
481 | /// # } | |
482 | /// # impl Foo { | |
483 | /// # fn new() -> impl PinInit<Self> { | |
484 | /// # pin_init!(Self { | |
485 | /// # a: 42, | |
486 | /// # b: Bar { | |
487 | /// # x: 64, | |
488 | /// # }, | |
489 | /// # }) | |
490 | /// # } | |
491 | /// # } | |
492 | /// #[pin_data] | |
493 | /// struct FooContainer { | |
494 | /// #[pin] | |
495 | /// foo1: Foo, | |
496 | /// #[pin] | |
497 | /// foo2: Foo, | |
498 | /// other: u32, | |
499 | /// } | |
500 | /// | |
501 | /// impl FooContainer { | |
502 | /// fn new(other: u32) -> impl PinInit<Self> { | |
503 | /// pin_init!(Self { | |
504 | /// foo1 <- Foo::new(), | |
505 | /// foo2 <- Foo::new(), | |
506 | /// other, | |
507 | /// }) | |
508 | /// } | |
509 | /// } | |
510 | /// ``` | |
511 | /// | |
512 | /// Here we see that when using `pin_init!` with `PinInit`, one needs to write `<-` instead of `:`. | |
513 | /// This signifies that the given field is initialized in-place. As with `struct` initializers, just | |
514 | /// writing the field (in this case `other`) without `:` or `<-` means `other: other,`. | |
515 | /// | |
516 | /// # Syntax | |
517 | /// | |
518 | /// As already mentioned in the examples above, inside of `pin_init!` a `struct` initializer with | |
519 | /// the following modifications is expected: | |
520 | /// - Fields that you want to initialize in-place have to use `<-` instead of `:`. | |
521 | /// - In front of the initializer you can write `&this in` to have access to a [`NonNull<Self>`] | |
522 | /// pointer named `this` inside of the initializer. | |
35e7fca2 BL |
523 | /// - Using struct update syntax one can place `..Zeroable::zeroed()` at the very end of the |
524 | /// struct, this initializes every field with 0 and then runs all initializers specified in the | |
525 | /// body. This can only be done if [`Zeroable`] is implemented for the struct. | |
fc6c6baa BL |
526 | /// |
527 | /// For instance: | |
528 | /// | |
529 | /// ```rust | |
a031fe8d | 530 | /// # use kernel::{macros::{Zeroable, pin_data}, pin_init}; |
fc6c6baa BL |
531 | /// # use core::{ptr::addr_of_mut, marker::PhantomPinned}; |
532 | /// #[pin_data] | |
35e7fca2 | 533 | /// #[derive(Zeroable)] |
fc6c6baa BL |
534 | /// struct Buf { |
535 | /// // `ptr` points into `buf`. | |
536 | /// ptr: *mut u8, | |
537 | /// buf: [u8; 64], | |
538 | /// #[pin] | |
539 | /// pin: PhantomPinned, | |
540 | /// } | |
541 | /// pin_init!(&this in Buf { | |
542 | /// buf: [0; 64], | |
543 | /// ptr: unsafe { addr_of_mut!((*this.as_ptr()).buf).cast() }, | |
544 | /// pin: PhantomPinned, | |
545 | /// }); | |
35e7fca2 BL |
546 | /// pin_init!(Buf { |
547 | /// buf: [1; 64], | |
548 | /// ..Zeroable::zeroed() | |
549 | /// }); | |
fc6c6baa BL |
550 | /// ``` |
551 | /// | |
552 | /// [`try_pin_init!`]: kernel::try_pin_init | |
553 | /// [`NonNull<Self>`]: core::ptr::NonNull | |
fc6c6baa BL |
554 | // For a detailed example of how this macro works, see the module documentation of the hidden |
555 | // module `__internal` inside of `init/__internal.rs`. | |
556 | #[macro_export] | |
557 | macro_rules! pin_init { | |
558 | ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { | |
559 | $($fields:tt)* | |
560 | }) => { | |
b3068ac3 | 561 | $crate::__init_internal!( |
fc6c6baa BL |
562 | @this($($this)?), |
563 | @typ($t $(::<$($generics),*>)?), | |
564 | @fields($($fields)*), | |
565 | @error(::core::convert::Infallible), | |
b3068ac3 BL |
566 | @data(PinData, use_data), |
567 | @has_data(HasPinData, __pin_data), | |
568 | @construct_closure(pin_init_from_closure), | |
35e7fca2 | 569 | @munch_fields($($fields)*), |
fc6c6baa BL |
570 | ) |
571 | }; | |
572 | } | |
573 | ||
574 | /// Construct an in-place, fallible pinned initializer for `struct`s. | |
575 | /// | |
576 | /// If the initialization can complete without error (or [`Infallible`]), then use [`pin_init!`]. | |
577 | /// | |
578 | /// You can use the `?` operator or use `return Err(err)` inside the initializer to stop | |
579 | /// initialization and return the error. | |
580 | /// | |
581 | /// IMPORTANT: if you have `unsafe` code inside of the initializer you have to ensure that when | |
582 | /// initialization fails, the memory can be safely deallocated without any further modifications. | |
583 | /// | |
584 | /// This macro defaults the error to [`Error`]. | |
585 | /// | |
586 | /// The syntax is identical to [`pin_init!`] with the following exception: you can append `? $type` | |
587 | /// after the `struct` initializer to specify the error type you want to use. | |
588 | /// | |
589 | /// # Examples | |
590 | /// | |
591 | /// ```rust | |
592 | /// # #![feature(new_uninit)] | |
593 | /// use kernel::{init::{self, PinInit}, error::Error}; | |
594 | /// #[pin_data] | |
595 | /// struct BigBuf { | |
596 | /// big: Box<[u8; 1024 * 1024 * 1024]>, | |
597 | /// small: [u8; 1024 * 1024], | |
598 | /// ptr: *mut u8, | |
599 | /// } | |
600 | /// | |
601 | /// impl BigBuf { | |
602 | /// fn new() -> impl PinInit<Self, Error> { | |
603 | /// try_pin_init!(Self { | |
c34aa00d | 604 | /// big: Box::init(init::zeroed(), GFP_KERNEL)?, |
fc6c6baa BL |
605 | /// small: [0; 1024 * 1024], |
606 | /// ptr: core::ptr::null_mut(), | |
607 | /// }? Error) | |
608 | /// } | |
609 | /// } | |
610 | /// ``` | |
fc6c6baa BL |
611 | // For a detailed example of how this macro works, see the module documentation of the hidden |
612 | // module `__internal` inside of `init/__internal.rs`. | |
613 | #[macro_export] | |
614 | macro_rules! try_pin_init { | |
615 | ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { | |
616 | $($fields:tt)* | |
617 | }) => { | |
b3068ac3 | 618 | $crate::__init_internal!( |
fc6c6baa BL |
619 | @this($($this)?), |
620 | @typ($t $(::<$($generics),*>)? ), | |
621 | @fields($($fields)*), | |
622 | @error($crate::error::Error), | |
b3068ac3 BL |
623 | @data(PinData, use_data), |
624 | @has_data(HasPinData, __pin_data), | |
625 | @construct_closure(pin_init_from_closure), | |
35e7fca2 | 626 | @munch_fields($($fields)*), |
fc6c6baa BL |
627 | ) |
628 | }; | |
629 | ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { | |
630 | $($fields:tt)* | |
631 | }? $err:ty) => { | |
b3068ac3 | 632 | $crate::__init_internal!( |
fc6c6baa BL |
633 | @this($($this)?), |
634 | @typ($t $(::<$($generics),*>)? ), | |
635 | @fields($($fields)*), | |
636 | @error($err), | |
b3068ac3 BL |
637 | @data(PinData, use_data), |
638 | @has_data(HasPinData, __pin_data), | |
639 | @construct_closure(pin_init_from_closure), | |
35e7fca2 | 640 | @munch_fields($($fields)*), |
fc6c6baa BL |
641 | ) |
642 | }; | |
fc6c6baa BL |
643 | } |
644 | ||
645 | /// Construct an in-place initializer for `struct`s. | |
646 | /// | |
647 | /// This macro defaults the error to [`Infallible`]. If you need [`Error`], then use | |
648 | /// [`try_init!`]. | |
649 | /// | |
650 | /// The syntax is identical to [`pin_init!`] and its safety caveats also apply: | |
651 | /// - `unsafe` code must guarantee either full initialization or return an error and allow | |
652 | /// deallocation of the memory. | |
653 | /// - the fields are initialized in the order given in the initializer. | |
654 | /// - no references to fields are allowed to be created inside of the initializer. | |
655 | /// | |
656 | /// This initializer is for initializing data in-place that might later be moved. If you want to | |
657 | /// pin-initialize, use [`pin_init!`]. | |
3ed03f4d MO |
658 | /// |
659 | /// [`try_init!`]: crate::try_init! | |
fc6c6baa BL |
660 | // For a detailed example of how this macro works, see the module documentation of the hidden |
661 | // module `__internal` inside of `init/__internal.rs`. | |
662 | #[macro_export] | |
663 | macro_rules! init { | |
664 | ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { | |
665 | $($fields:tt)* | |
666 | }) => { | |
b3068ac3 | 667 | $crate::__init_internal!( |
fc6c6baa BL |
668 | @this($($this)?), |
669 | @typ($t $(::<$($generics),*>)?), | |
670 | @fields($($fields)*), | |
671 | @error(::core::convert::Infallible), | |
b3068ac3 BL |
672 | @data(InitData, /*no use_data*/), |
673 | @has_data(HasInitData, __init_data), | |
674 | @construct_closure(init_from_closure), | |
35e7fca2 | 675 | @munch_fields($($fields)*), |
fc6c6baa BL |
676 | ) |
677 | } | |
678 | } | |
679 | ||
680 | /// Construct an in-place fallible initializer for `struct`s. | |
681 | /// | |
682 | /// This macro defaults the error to [`Error`]. If you need [`Infallible`], then use | |
683 | /// [`init!`]. | |
684 | /// | |
685 | /// The syntax is identical to [`try_pin_init!`]. If you want to specify a custom error, | |
686 | /// append `? $type` after the `struct` initializer. | |
687 | /// The safety caveats from [`try_pin_init!`] also apply: | |
688 | /// - `unsafe` code must guarantee either full initialization or return an error and allow | |
689 | /// deallocation of the memory. | |
690 | /// - the fields are initialized in the order given in the initializer. | |
691 | /// - no references to fields are allowed to be created inside of the initializer. | |
692 | /// | |
693 | /// # Examples | |
694 | /// | |
695 | /// ```rust | |
a30e94c2 | 696 | /// use kernel::{init::{PinInit, zeroed}, error::Error}; |
fc6c6baa BL |
697 | /// struct BigBuf { |
698 | /// big: Box<[u8; 1024 * 1024 * 1024]>, | |
699 | /// small: [u8; 1024 * 1024], | |
700 | /// } | |
701 | /// | |
702 | /// impl BigBuf { | |
703 | /// fn new() -> impl Init<Self, Error> { | |
704 | /// try_init!(Self { | |
c34aa00d | 705 | /// big: Box::init(zeroed(), GFP_KERNEL)?, |
fc6c6baa BL |
706 | /// small: [0; 1024 * 1024], |
707 | /// }? Error) | |
708 | /// } | |
709 | /// } | |
710 | /// ``` | |
fc6c6baa BL |
711 | // For a detailed example of how this macro works, see the module documentation of the hidden |
712 | // module `__internal` inside of `init/__internal.rs`. | |
713 | #[macro_export] | |
714 | macro_rules! try_init { | |
715 | ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { | |
716 | $($fields:tt)* | |
717 | }) => { | |
b3068ac3 | 718 | $crate::__init_internal!( |
fc6c6baa BL |
719 | @this($($this)?), |
720 | @typ($t $(::<$($generics),*>)?), | |
721 | @fields($($fields)*), | |
722 | @error($crate::error::Error), | |
b3068ac3 BL |
723 | @data(InitData, /*no use_data*/), |
724 | @has_data(HasInitData, __init_data), | |
725 | @construct_closure(init_from_closure), | |
35e7fca2 | 726 | @munch_fields($($fields)*), |
fc6c6baa BL |
727 | ) |
728 | }; | |
729 | ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? { | |
730 | $($fields:tt)* | |
731 | }? $err:ty) => { | |
b3068ac3 | 732 | $crate::__init_internal!( |
fc6c6baa BL |
733 | @this($($this)?), |
734 | @typ($t $(::<$($generics),*>)?), | |
735 | @fields($($fields)*), | |
736 | @error($err), | |
b3068ac3 BL |
737 | @data(InitData, /*no use_data*/), |
738 | @has_data(HasInitData, __init_data), | |
739 | @construct_closure(init_from_closure), | |
35e7fca2 | 740 | @munch_fields($($fields)*), |
fc6c6baa BL |
741 | ) |
742 | }; | |
fc6c6baa | 743 | } |
90e53c5e BL |
744 | |
745 | /// A pin-initializer for the type `T`. | |
746 | /// | |
747 | /// To use this initializer, you will need a suitable memory location that can hold a `T`. This can | |
6841d45a BL |
748 | /// be [`Box<T>`], [`Arc<T>`], [`UniqueArc<T>`] or even the stack (see [`stack_pin_init!`]). Use the |
749 | /// [`InPlaceInit::pin_init`] function of a smart pointer like [`Arc<T>`] on this. | |
90e53c5e BL |
750 | /// |
751 | /// Also see the [module description](self). | |
752 | /// | |
753 | /// # Safety | |
754 | /// | |
b6cda913 | 755 | /// When implementing this trait you will need to take great care. Also there are probably very few |
90e53c5e BL |
756 | /// cases where a manual implementation is necessary. Use [`pin_init_from_closure`] where possible. |
757 | /// | |
b6cda913 | 758 | /// The [`PinInit::__pinned_init`] function: |
90e53c5e BL |
759 | /// - returns `Ok(())` if it initialized every field of `slot`, |
760 | /// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means: | |
761 | /// - `slot` can be deallocated without UB occurring, | |
762 | /// - `slot` does not need to be dropped, | |
763 | /// - `slot` is not partially initialized. | |
764 | /// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`. | |
765 | /// | |
766 | /// [`Arc<T>`]: crate::sync::Arc | |
767 | /// [`Arc::pin_init`]: crate::sync::Arc::pin_init | |
90e53c5e BL |
768 | #[must_use = "An initializer must be used in order to create its value."] |
769 | pub unsafe trait PinInit<T: ?Sized, E = Infallible>: Sized { | |
770 | /// Initializes `slot`. | |
771 | /// | |
772 | /// # Safety | |
773 | /// | |
774 | /// - `slot` is a valid pointer to uninitialized memory. | |
775 | /// - the caller does not touch `slot` when `Err` is returned, they are only permitted to | |
776 | /// deallocate. | |
777 | /// - `slot` will not move until it is dropped, i.e. it will be pinned. | |
778 | unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E>; | |
7f8977a7 BL |
779 | |
780 | /// First initializes the value using `self` then calls the function `f` with the initialized | |
781 | /// value. | |
782 | /// | |
783 | /// If `f` returns an error the value is dropped and the initializer will forward the error. | |
784 | /// | |
785 | /// # Examples | |
786 | /// | |
787 | /// ```rust | |
788 | /// # #![allow(clippy::disallowed_names)] | |
789 | /// use kernel::{types::Opaque, init::pin_init_from_closure}; | |
790 | /// #[repr(C)] | |
791 | /// struct RawFoo([u8; 16]); | |
792 | /// extern { | |
793 | /// fn init_foo(_: *mut RawFoo); | |
794 | /// } | |
795 | /// | |
796 | /// #[pin_data] | |
797 | /// struct Foo { | |
798 | /// #[pin] | |
799 | /// raw: Opaque<RawFoo>, | |
800 | /// } | |
801 | /// | |
802 | /// impl Foo { | |
803 | /// fn setup(self: Pin<&mut Self>) { | |
804 | /// pr_info!("Setting up foo"); | |
805 | /// } | |
806 | /// } | |
807 | /// | |
808 | /// let foo = pin_init!(Foo { | |
809 | /// raw <- unsafe { | |
810 | /// Opaque::ffi_init(|s| { | |
811 | /// init_foo(s); | |
812 | /// }) | |
813 | /// }, | |
814 | /// }).pin_chain(|foo| { | |
815 | /// foo.setup(); | |
816 | /// Ok(()) | |
817 | /// }); | |
818 | /// ``` | |
819 | fn pin_chain<F>(self, f: F) -> ChainPinInit<Self, F, T, E> | |
820 | where | |
821 | F: FnOnce(Pin<&mut T>) -> Result<(), E>, | |
822 | { | |
823 | ChainPinInit(self, f, PhantomData) | |
824 | } | |
825 | } | |
826 | ||
827 | /// An initializer returned by [`PinInit::pin_chain`]. | |
828 | pub struct ChainPinInit<I, F, T: ?Sized, E>(I, F, __internal::Invariant<(E, Box<T>)>); | |
829 | ||
830 | // SAFETY: The `__pinned_init` function is implemented such that it | |
831 | // - returns `Ok(())` on successful initialization, | |
832 | // - returns `Err(err)` on error and in this case `slot` will be dropped. | |
833 | // - considers `slot` pinned. | |
834 | unsafe impl<T: ?Sized, E, I, F> PinInit<T, E> for ChainPinInit<I, F, T, E> | |
835 | where | |
836 | I: PinInit<T, E>, | |
837 | F: FnOnce(Pin<&mut T>) -> Result<(), E>, | |
838 | { | |
839 | unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> { | |
840 | // SAFETY: All requirements fulfilled since this function is `__pinned_init`. | |
841 | unsafe { self.0.__pinned_init(slot)? }; | |
842 | // SAFETY: The above call initialized `slot` and we still have unique access. | |
843 | let val = unsafe { &mut *slot }; | |
844 | // SAFETY: `slot` is considered pinned. | |
845 | let val = unsafe { Pin::new_unchecked(val) }; | |
846 | (self.1)(val).map_err(|e| { | |
847 | // SAFETY: `slot` was initialized above. | |
848 | unsafe { core::ptr::drop_in_place(slot) }; | |
849 | e | |
850 | }) | |
851 | } | |
90e53c5e BL |
852 | } |
853 | ||
854 | /// An initializer for `T`. | |
855 | /// | |
856 | /// To use this initializer, you will need a suitable memory location that can hold a `T`. This can | |
6841d45a BL |
857 | /// be [`Box<T>`], [`Arc<T>`], [`UniqueArc<T>`] or even the stack (see [`stack_pin_init!`]). Use the |
858 | /// [`InPlaceInit::init`] function of a smart pointer like [`Arc<T>`] on this. Because | |
859 | /// [`PinInit<T, E>`] is a super trait, you can use every function that takes it as well. | |
90e53c5e BL |
860 | /// |
861 | /// Also see the [module description](self). | |
862 | /// | |
863 | /// # Safety | |
864 | /// | |
b6cda913 | 865 | /// When implementing this trait you will need to take great care. Also there are probably very few |
90e53c5e BL |
866 | /// cases where a manual implementation is necessary. Use [`init_from_closure`] where possible. |
867 | /// | |
b6cda913 | 868 | /// The [`Init::__init`] function: |
90e53c5e BL |
869 | /// - returns `Ok(())` if it initialized every field of `slot`, |
870 | /// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means: | |
871 | /// - `slot` can be deallocated without UB occurring, | |
872 | /// - `slot` does not need to be dropped, | |
873 | /// - `slot` is not partially initialized. | |
874 | /// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`. | |
875 | /// | |
876 | /// The `__pinned_init` function from the supertrait [`PinInit`] needs to execute the exact same | |
877 | /// code as `__init`. | |
878 | /// | |
879 | /// Contrary to its supertype [`PinInit<T, E>`] the caller is allowed to | |
880 | /// move the pointee after initialization. | |
881 | /// | |
882 | /// [`Arc<T>`]: crate::sync::Arc | |
90e53c5e | 883 | #[must_use = "An initializer must be used in order to create its value."] |
1a8076ac | 884 | pub unsafe trait Init<T: ?Sized, E = Infallible>: PinInit<T, E> { |
90e53c5e BL |
885 | /// Initializes `slot`. |
886 | /// | |
887 | /// # Safety | |
888 | /// | |
889 | /// - `slot` is a valid pointer to uninitialized memory. | |
890 | /// - the caller does not touch `slot` when `Err` is returned, they are only permitted to | |
891 | /// deallocate. | |
892 | unsafe fn __init(self, slot: *mut T) -> Result<(), E>; | |
7f8977a7 BL |
893 | |
894 | /// First initializes the value using `self` then calls the function `f` with the initialized | |
895 | /// value. | |
896 | /// | |
897 | /// If `f` returns an error the value is dropped and the initializer will forward the error. | |
898 | /// | |
899 | /// # Examples | |
900 | /// | |
901 | /// ```rust | |
902 | /// # #![allow(clippy::disallowed_names)] | |
903 | /// use kernel::{types::Opaque, init::{self, init_from_closure}}; | |
904 | /// struct Foo { | |
905 | /// buf: [u8; 1_000_000], | |
906 | /// } | |
907 | /// | |
908 | /// impl Foo { | |
909 | /// fn setup(&mut self) { | |
910 | /// pr_info!("Setting up foo"); | |
911 | /// } | |
912 | /// } | |
913 | /// | |
914 | /// let foo = init!(Foo { | |
915 | /// buf <- init::zeroed() | |
916 | /// }).chain(|foo| { | |
917 | /// foo.setup(); | |
918 | /// Ok(()) | |
919 | /// }); | |
920 | /// ``` | |
921 | fn chain<F>(self, f: F) -> ChainInit<Self, F, T, E> | |
922 | where | |
923 | F: FnOnce(&mut T) -> Result<(), E>, | |
924 | { | |
925 | ChainInit(self, f, PhantomData) | |
926 | } | |
90e53c5e BL |
927 | } |
928 | ||
7f8977a7 BL |
929 | /// An initializer returned by [`Init::chain`]. |
930 | pub struct ChainInit<I, F, T: ?Sized, E>(I, F, __internal::Invariant<(E, Box<T>)>); | |
931 | ||
932 | // SAFETY: The `__init` function is implemented such that it | |
933 | // - returns `Ok(())` on successful initialization, | |
934 | // - returns `Err(err)` on error and in this case `slot` will be dropped. | |
935 | unsafe impl<T: ?Sized, E, I, F> Init<T, E> for ChainInit<I, F, T, E> | |
90e53c5e BL |
936 | where |
937 | I: Init<T, E>, | |
7f8977a7 BL |
938 | F: FnOnce(&mut T) -> Result<(), E>, |
939 | { | |
940 | unsafe fn __init(self, slot: *mut T) -> Result<(), E> { | |
941 | // SAFETY: All requirements fulfilled since this function is `__init`. | |
942 | unsafe { self.0.__pinned_init(slot)? }; | |
943 | // SAFETY: The above call initialized `slot` and we still have unique access. | |
944 | (self.1)(unsafe { &mut *slot }).map_err(|e| { | |
945 | // SAFETY: `slot` was initialized above. | |
946 | unsafe { core::ptr::drop_in_place(slot) }; | |
947 | e | |
948 | }) | |
949 | } | |
950 | } | |
951 | ||
952 | // SAFETY: `__pinned_init` behaves exactly the same as `__init`. | |
953 | unsafe impl<T: ?Sized, E, I, F> PinInit<T, E> for ChainInit<I, F, T, E> | |
954 | where | |
955 | I: Init<T, E>, | |
956 | F: FnOnce(&mut T) -> Result<(), E>, | |
90e53c5e BL |
957 | { |
958 | unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> { | |
7f8977a7 | 959 | // SAFETY: `__init` has less strict requirements compared to `__pinned_init`. |
90e53c5e BL |
960 | unsafe { self.__init(slot) } |
961 | } | |
962 | } | |
963 | ||
964 | /// Creates a new [`PinInit<T, E>`] from the given closure. | |
965 | /// | |
966 | /// # Safety | |
967 | /// | |
968 | /// The closure: | |
969 | /// - returns `Ok(())` if it initialized every field of `slot`, | |
970 | /// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means: | |
971 | /// - `slot` can be deallocated without UB occurring, | |
972 | /// - `slot` does not need to be dropped, | |
973 | /// - `slot` is not partially initialized. | |
974 | /// - may assume that the `slot` does not move if `T: !Unpin`, | |
975 | /// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`. | |
976 | #[inline] | |
977 | pub const unsafe fn pin_init_from_closure<T: ?Sized, E>( | |
978 | f: impl FnOnce(*mut T) -> Result<(), E>, | |
979 | ) -> impl PinInit<T, E> { | |
980 | __internal::InitClosure(f, PhantomData) | |
981 | } | |
982 | ||
983 | /// Creates a new [`Init<T, E>`] from the given closure. | |
984 | /// | |
985 | /// # Safety | |
986 | /// | |
987 | /// The closure: | |
988 | /// - returns `Ok(())` if it initialized every field of `slot`, | |
989 | /// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means: | |
990 | /// - `slot` can be deallocated without UB occurring, | |
991 | /// - `slot` does not need to be dropped, | |
992 | /// - `slot` is not partially initialized. | |
993 | /// - the `slot` may move after initialization. | |
994 | /// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`. | |
995 | #[inline] | |
996 | pub const unsafe fn init_from_closure<T: ?Sized, E>( | |
997 | f: impl FnOnce(*mut T) -> Result<(), E>, | |
998 | ) -> impl Init<T, E> { | |
999 | __internal::InitClosure(f, PhantomData) | |
1000 | } | |
1001 | ||
1002 | /// An initializer that leaves the memory uninitialized. | |
1003 | /// | |
1004 | /// The initializer is a no-op. The `slot` memory is not changed. | |
1005 | #[inline] | |
1006 | pub fn uninit<T, E>() -> impl Init<MaybeUninit<T>, E> { | |
1007 | // SAFETY: The memory is allowed to be uninitialized. | |
1008 | unsafe { init_from_closure(|_| Ok(())) } | |
1009 | } | |
1010 | ||
9e494390 BL |
1011 | /// Initializes an array by initializing each element via the provided initializer. |
1012 | /// | |
1013 | /// # Examples | |
1014 | /// | |
1015 | /// ```rust | |
1016 | /// use kernel::{error::Error, init::init_array_from_fn}; | |
c34aa00d | 1017 | /// let array: Box<[usize; 1_000]> = Box::init::<Error>(init_array_from_fn(|i| i), GFP_KERNEL).unwrap(); |
9e494390 BL |
1018 | /// assert_eq!(array.len(), 1_000); |
1019 | /// ``` | |
1020 | pub fn init_array_from_fn<I, const N: usize, T, E>( | |
1021 | mut make_init: impl FnMut(usize) -> I, | |
1022 | ) -> impl Init<[T; N], E> | |
1023 | where | |
1024 | I: Init<T, E>, | |
1025 | { | |
1026 | let init = move |slot: *mut [T; N]| { | |
1027 | let slot = slot.cast::<T>(); | |
1028 | // Counts the number of initialized elements and when dropped drops that many elements from | |
1029 | // `slot`. | |
1030 | let mut init_count = ScopeGuard::new_with_data(0, |i| { | |
b6cda913 | 1031 | // We now free every element that has been initialized before. |
9e494390 BL |
1032 | // SAFETY: The loop initialized exactly the values from 0..i and since we |
1033 | // return `Err` below, the caller will consider the memory at `slot` as | |
1034 | // uninitialized. | |
1035 | unsafe { ptr::drop_in_place(ptr::slice_from_raw_parts_mut(slot, i)) }; | |
1036 | }); | |
1037 | for i in 0..N { | |
1038 | let init = make_init(i); | |
1039 | // SAFETY: Since 0 <= `i` < N, it is still in bounds of `[T; N]`. | |
1040 | let ptr = unsafe { slot.add(i) }; | |
1041 | // SAFETY: The pointer is derived from `slot` and thus satisfies the `__init` | |
1042 | // requirements. | |
1043 | unsafe { init.__init(ptr) }?; | |
1044 | *init_count += 1; | |
1045 | } | |
1046 | init_count.dismiss(); | |
1047 | Ok(()) | |
1048 | }; | |
1049 | // SAFETY: The initializer above initializes every element of the array. On failure it drops | |
1050 | // any initialized elements and returns `Err`. | |
1051 | unsafe { init_from_closure(init) } | |
1052 | } | |
1053 | ||
1054 | /// Initializes an array by initializing each element via the provided initializer. | |
1055 | /// | |
1056 | /// # Examples | |
1057 | /// | |
1058 | /// ```rust | |
1059 | /// use kernel::{sync::{Arc, Mutex}, init::pin_init_array_from_fn, new_mutex}; | |
b6cda913 | 1060 | /// let array: Arc<[Mutex<usize>; 1_000]> = |
c34aa00d | 1061 | /// Arc::pin_init(pin_init_array_from_fn(|i| new_mutex!(i)), GFP_KERNEL).unwrap(); |
9e494390 BL |
1062 | /// assert_eq!(array.len(), 1_000); |
1063 | /// ``` | |
1064 | pub fn pin_init_array_from_fn<I, const N: usize, T, E>( | |
1065 | mut make_init: impl FnMut(usize) -> I, | |
1066 | ) -> impl PinInit<[T; N], E> | |
1067 | where | |
1068 | I: PinInit<T, E>, | |
1069 | { | |
1070 | let init = move |slot: *mut [T; N]| { | |
1071 | let slot = slot.cast::<T>(); | |
1072 | // Counts the number of initialized elements and when dropped drops that many elements from | |
1073 | // `slot`. | |
1074 | let mut init_count = ScopeGuard::new_with_data(0, |i| { | |
b6cda913 | 1075 | // We now free every element that has been initialized before. |
9e494390 BL |
1076 | // SAFETY: The loop initialized exactly the values from 0..i and since we |
1077 | // return `Err` below, the caller will consider the memory at `slot` as | |
1078 | // uninitialized. | |
1079 | unsafe { ptr::drop_in_place(ptr::slice_from_raw_parts_mut(slot, i)) }; | |
1080 | }); | |
1081 | for i in 0..N { | |
1082 | let init = make_init(i); | |
1083 | // SAFETY: Since 0 <= `i` < N, it is still in bounds of `[T; N]`. | |
1084 | let ptr = unsafe { slot.add(i) }; | |
1085 | // SAFETY: The pointer is derived from `slot` and thus satisfies the `__init` | |
1086 | // requirements. | |
1087 | unsafe { init.__pinned_init(ptr) }?; | |
1088 | *init_count += 1; | |
1089 | } | |
1090 | init_count.dismiss(); | |
1091 | Ok(()) | |
1092 | }; | |
1093 | // SAFETY: The initializer above initializes every element of the array. On failure it drops | |
1094 | // any initialized elements and returns `Err`. | |
1095 | unsafe { pin_init_from_closure(init) } | |
1096 | } | |
1097 | ||
90e53c5e | 1098 | // SAFETY: Every type can be initialized by-value. |
52a7f2de BL |
1099 | unsafe impl<T, E> Init<T, E> for T { |
1100 | unsafe fn __init(self, slot: *mut T) -> Result<(), E> { | |
90e53c5e BL |
1101 | unsafe { slot.write(self) }; |
1102 | Ok(()) | |
1103 | } | |
1104 | } | |
92c4a1e7 | 1105 | |
1a8076ac BL |
1106 | // SAFETY: Every type can be initialized by-value. `__pinned_init` calls `__init`. |
1107 | unsafe impl<T, E> PinInit<T, E> for T { | |
1108 | unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> { | |
1109 | unsafe { self.__init(slot) } | |
1110 | } | |
1111 | } | |
1112 | ||
92c4a1e7 BL |
1113 | /// Smart pointer that can initialize memory in-place. |
1114 | pub trait InPlaceInit<T>: Sized { | |
1115 | /// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this | |
1116 | /// type. | |
1117 | /// | |
1118 | /// If `T: !Unpin` it will not be able to move afterwards. | |
c34aa00d | 1119 | fn try_pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> Result<Pin<Self>, E> |
92c4a1e7 BL |
1120 | where |
1121 | E: From<AllocError>; | |
1122 | ||
1123 | /// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this | |
1124 | /// type. | |
1125 | /// | |
1126 | /// If `T: !Unpin` it will not be able to move afterwards. | |
c34aa00d | 1127 | fn pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> error::Result<Pin<Self>> |
92c4a1e7 BL |
1128 | where |
1129 | Error: From<E>, | |
1130 | { | |
1131 | // SAFETY: We delegate to `init` and only change the error type. | |
1132 | let init = unsafe { | |
1133 | pin_init_from_closure(|slot| init.__pinned_init(slot).map_err(|e| Error::from(e))) | |
1134 | }; | |
c34aa00d | 1135 | Self::try_pin_init(init, flags) |
92c4a1e7 BL |
1136 | } |
1137 | ||
1138 | /// Use the given initializer to in-place initialize a `T`. | |
c34aa00d | 1139 | fn try_init<E>(init: impl Init<T, E>, flags: Flags) -> Result<Self, E> |
92c4a1e7 BL |
1140 | where |
1141 | E: From<AllocError>; | |
1142 | ||
1143 | /// Use the given initializer to in-place initialize a `T`. | |
c34aa00d | 1144 | fn init<E>(init: impl Init<T, E>, flags: Flags) -> error::Result<Self> |
92c4a1e7 BL |
1145 | where |
1146 | Error: From<E>, | |
1147 | { | |
1148 | // SAFETY: We delegate to `init` and only change the error type. | |
1149 | let init = unsafe { | |
1150 | init_from_closure(|slot| init.__pinned_init(slot).map_err(|e| Error::from(e))) | |
1151 | }; | |
c34aa00d | 1152 | Self::try_init(init, flags) |
92c4a1e7 BL |
1153 | } |
1154 | } | |
1155 | ||
1156 | impl<T> InPlaceInit<T> for Box<T> { | |
1157 | #[inline] | |
c34aa00d | 1158 | fn try_pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> Result<Pin<Self>, E> |
92c4a1e7 BL |
1159 | where |
1160 | E: From<AllocError>, | |
1161 | { | |
c34aa00d | 1162 | let mut this = <Box<_> as BoxExt<_>>::new_uninit(flags)?; |
92c4a1e7 BL |
1163 | let slot = this.as_mut_ptr(); |
1164 | // SAFETY: When init errors/panics, slot will get deallocated but not dropped, | |
1165 | // slot is valid and will not be moved, because we pin it later. | |
1166 | unsafe { init.__pinned_init(slot)? }; | |
1167 | // SAFETY: All fields have been initialized. | |
1168 | Ok(unsafe { this.assume_init() }.into()) | |
1169 | } | |
1170 | ||
1171 | #[inline] | |
c34aa00d | 1172 | fn try_init<E>(init: impl Init<T, E>, flags: Flags) -> Result<Self, E> |
92c4a1e7 BL |
1173 | where |
1174 | E: From<AllocError>, | |
1175 | { | |
c34aa00d | 1176 | let mut this = <Box<_> as BoxExt<_>>::new_uninit(flags)?; |
92c4a1e7 BL |
1177 | let slot = this.as_mut_ptr(); |
1178 | // SAFETY: When init errors/panics, slot will get deallocated but not dropped, | |
1179 | // slot is valid. | |
1180 | unsafe { init.__init(slot)? }; | |
1181 | // SAFETY: All fields have been initialized. | |
1182 | Ok(unsafe { this.assume_init() }) | |
1183 | } | |
1184 | } | |
1185 | ||
1186 | impl<T> InPlaceInit<T> for UniqueArc<T> { | |
1187 | #[inline] | |
c34aa00d | 1188 | fn try_pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> Result<Pin<Self>, E> |
92c4a1e7 BL |
1189 | where |
1190 | E: From<AllocError>, | |
1191 | { | |
c34aa00d | 1192 | let mut this = UniqueArc::new_uninit(flags)?; |
92c4a1e7 BL |
1193 | let slot = this.as_mut_ptr(); |
1194 | // SAFETY: When init errors/panics, slot will get deallocated but not dropped, | |
1195 | // slot is valid and will not be moved, because we pin it later. | |
1196 | unsafe { init.__pinned_init(slot)? }; | |
1197 | // SAFETY: All fields have been initialized. | |
1198 | Ok(unsafe { this.assume_init() }.into()) | |
1199 | } | |
1200 | ||
1201 | #[inline] | |
c34aa00d | 1202 | fn try_init<E>(init: impl Init<T, E>, flags: Flags) -> Result<Self, E> |
92c4a1e7 BL |
1203 | where |
1204 | E: From<AllocError>, | |
1205 | { | |
c34aa00d | 1206 | let mut this = UniqueArc::new_uninit(flags)?; |
92c4a1e7 BL |
1207 | let slot = this.as_mut_ptr(); |
1208 | // SAFETY: When init errors/panics, slot will get deallocated but not dropped, | |
1209 | // slot is valid. | |
1210 | unsafe { init.__init(slot)? }; | |
1211 | // SAFETY: All fields have been initialized. | |
1212 | Ok(unsafe { this.assume_init() }) | |
1213 | } | |
1214 | } | |
d0fdc396 BL |
1215 | |
1216 | /// Trait facilitating pinned destruction. | |
1217 | /// | |
1218 | /// Use [`pinned_drop`] to implement this trait safely: | |
1219 | /// | |
1220 | /// ```rust | |
1221 | /// # use kernel::sync::Mutex; | |
1222 | /// use kernel::macros::pinned_drop; | |
1223 | /// use core::pin::Pin; | |
1224 | /// #[pin_data(PinnedDrop)] | |
1225 | /// struct Foo { | |
1226 | /// #[pin] | |
1227 | /// mtx: Mutex<usize>, | |
1228 | /// } | |
1229 | /// | |
1230 | /// #[pinned_drop] | |
1231 | /// impl PinnedDrop for Foo { | |
1232 | /// fn drop(self: Pin<&mut Self>) { | |
1233 | /// pr_info!("Foo is being dropped!"); | |
1234 | /// } | |
1235 | /// } | |
1236 | /// ``` | |
1237 | /// | |
1238 | /// # Safety | |
1239 | /// | |
1240 | /// This trait must be implemented via the [`pinned_drop`] proc-macro attribute on the impl. | |
1241 | /// | |
1242 | /// [`pinned_drop`]: kernel::macros::pinned_drop | |
1243 | pub unsafe trait PinnedDrop: __internal::HasPinData { | |
1244 | /// Executes the pinned destructor of this type. | |
1245 | /// | |
1246 | /// While this function is marked safe, it is actually unsafe to call it manually. For this | |
1247 | /// reason it takes an additional parameter. This type can only be constructed by `unsafe` code | |
1248 | /// and thus prevents this function from being called where it should not. | |
1249 | /// | |
1250 | /// This extra parameter will be generated by the `#[pinned_drop]` proc-macro attribute | |
1251 | /// automatically. | |
1252 | fn drop(self: Pin<&mut Self>, only_call_from_drop: __internal::OnlyCallFromDrop); | |
1253 | } | |
38cde0bd BL |
1254 | |
1255 | /// Marker trait for types that can be initialized by writing just zeroes. | |
1256 | /// | |
1257 | /// # Safety | |
1258 | /// | |
1259 | /// The bit pattern consisting of only zeroes is a valid bit pattern for this type. In other words, | |
1260 | /// this is not UB: | |
1261 | /// | |
1262 | /// ```rust,ignore | |
1263 | /// let val: Self = unsafe { core::mem::zeroed() }; | |
1264 | /// ``` | |
1265 | pub unsafe trait Zeroable {} | |
1266 | ||
1267 | /// Create a new zeroed T. | |
1268 | /// | |
1269 | /// The returned initializer will write `0x00` to every byte of the given `slot`. | |
1270 | #[inline] | |
1271 | pub fn zeroed<T: Zeroable>() -> impl Init<T> { | |
1272 | // SAFETY: Because `T: Zeroable`, all bytes zero is a valid bit pattern for `T` | |
1273 | // and because we write all zeroes, the memory is initialized. | |
1274 | unsafe { | |
1275 | init_from_closure(|slot: *mut T| { | |
1276 | slot.write_bytes(0, 1); | |
1277 | Ok(()) | |
1278 | }) | |
1279 | } | |
1280 | } | |
1281 | ||
1282 | macro_rules! impl_zeroable { | |
1283 | ($($({$($generics:tt)*})? $t:ty, )*) => { | |
1284 | $(unsafe impl$($($generics)*)? Zeroable for $t {})* | |
1285 | }; | |
1286 | } | |
1287 | ||
1288 | impl_zeroable! { | |
1289 | // SAFETY: All primitives that are allowed to be zero. | |
1290 | bool, | |
1291 | char, | |
1292 | u8, u16, u32, u64, u128, usize, | |
1293 | i8, i16, i32, i64, i128, isize, | |
1294 | f32, f64, | |
1295 | ||
1296 | // SAFETY: These are ZSTs, there is nothing to zero. | |
1297 | {<T: ?Sized>} PhantomData<T>, core::marker::PhantomPinned, Infallible, (), | |
1298 | ||
1299 | // SAFETY: Type is allowed to take any value, including all zeros. | |
1300 | {<T>} MaybeUninit<T>, | |
2e704f18 BL |
1301 | // SAFETY: Type is allowed to take any value, including all zeros. |
1302 | {<T>} Opaque<T>, | |
1303 | ||
1304 | // SAFETY: `T: Zeroable` and `UnsafeCell` is `repr(transparent)`. | |
1305 | {<T: ?Sized + Zeroable>} UnsafeCell<T>, | |
38cde0bd BL |
1306 | |
1307 | // SAFETY: All zeros is equivalent to `None` (option layout optimization guarantee). | |
1308 | Option<NonZeroU8>, Option<NonZeroU16>, Option<NonZeroU32>, Option<NonZeroU64>, | |
1309 | Option<NonZeroU128>, Option<NonZeroUsize>, | |
1310 | Option<NonZeroI8>, Option<NonZeroI16>, Option<NonZeroI32>, Option<NonZeroI64>, | |
1311 | Option<NonZeroI128>, Option<NonZeroIsize>, | |
1312 | ||
1313 | // SAFETY: All zeros is equivalent to `None` (option layout optimization guarantee). | |
1314 | // | |
1315 | // In this case we are allowed to use `T: ?Sized`, since all zeros is the `None` variant. | |
1316 | {<T: ?Sized>} Option<NonNull<T>>, | |
1317 | {<T: ?Sized>} Option<Box<T>>, | |
1318 | ||
1319 | // SAFETY: `null` pointer is valid. | |
1320 | // | |
1321 | // We cannot use `T: ?Sized`, since the VTABLE pointer part of fat pointers is not allowed to be | |
1322 | // null. | |
1323 | // | |
1324 | // When `Pointee` gets stabilized, we could use | |
1325 | // `T: ?Sized where <T as Pointee>::Metadata: Zeroable` | |
1326 | {<T>} *mut T, {<T>} *const T, | |
1327 | ||
1328 | // SAFETY: `null` pointer is valid and the metadata part of these fat pointers is allowed to be | |
1329 | // zero. | |
1330 | {<T>} *mut [T], {<T>} *const [T], *mut str, *const str, | |
1331 | ||
1332 | // SAFETY: `T` is `Zeroable`. | |
1333 | {<const N: usize, T: Zeroable>} [T; N], {<T: Zeroable>} Wrapping<T>, | |
1334 | } | |
1335 | ||
1336 | macro_rules! impl_tuple_zeroable { | |
1337 | ($(,)?) => {}; | |
1338 | ($first:ident, $($t:ident),* $(,)?) => { | |
1339 | // SAFETY: All elements are zeroable and padding can be zero. | |
1340 | unsafe impl<$first: Zeroable, $($t: Zeroable),*> Zeroable for ($first, $($t),*) {} | |
1341 | impl_tuple_zeroable!($($t),* ,); | |
1342 | } | |
1343 | } | |
1344 | ||
1345 | impl_tuple_zeroable!(A, B, C, D, E, F, G, H, I, J); |