+++ /dev/null
-// SPDX-License-Identifier: Apache-2.0 OR MIT
-
-//! API to safely and fallibly initialize pinned `struct`s using in-place constructors.
-//!
-//! It also allows in-place initialization of big `struct`s that would otherwise produce a stack
-//! overflow.
-//!
-//! Most `struct`s from the [`sync`] module need to be pinned, because they contain self-referential
-//! `struct`s from C. [Pinning][pinning] is Rust's way of ensuring data does not move.
-//!
-//! # Overview
-//!
-//! To initialize a `struct` with an in-place constructor you will need two things:
-//! - an in-place constructor,
-//! - a memory location that can hold your `struct` (this can be the [stack], an [`Arc<T>`],
-//! [`UniqueArc<T>`], [`KBox<T>`] or any other smart pointer that implements [`InPlaceInit`]).
-//!
-//! To get an in-place constructor there are generally three options:
-//! - directly creating an in-place constructor using the [`pin_init!`] macro,
-//! - a custom function/macro returning an in-place constructor provided by someone else,
-//! - using the unsafe function [`pin_init_from_closure()`] to manually create an initializer.
-//!
-//! Aside from pinned initialization, this API also supports in-place construction without pinning,
-//! the macros/types/functions are generally named like the pinned variants without the `pin`
-//! prefix.
-//!
-//! # Examples
-//!
-//! ## Using the [`pin_init!`] macro
-//!
-//! If you want to use [`PinInit`], then you will have to annotate your `struct` with
-//! `#[`[`pin_data`]`]`. It is a macro that uses `#[pin]` as a marker for
-//! [structurally pinned fields]. After doing this, you can then create an in-place constructor via
-//! [`pin_init!`]. The syntax is almost the same as normal `struct` initializers. The difference is
-//! that you need to write `<-` instead of `:` for fields that you want to initialize in-place.
-//!
-//! ```rust,ignore
-//! # #![expect(clippy::disallowed_names)]
-//! use kernel::sync::{new_mutex, Mutex};
-//! # use core::pin::Pin;
-//! #[pin_data]
-//! struct Foo {
-//! #[pin]
-//! a: Mutex<usize>,
-//! b: u32,
-//! }
-//!
-//! let foo = pin_init!(Foo {
-//! a <- new_mutex!(42, "Foo::a"),
-//! b: 24,
-//! });
-//! ```
-//!
-//! `foo` now is of the type [`impl PinInit<Foo>`]. We can now use any smart pointer that we like
-//! (or just the stack) to actually initialize a `Foo`:
-//!
-//! ```rust,ignore
-//! # #![expect(clippy::disallowed_names)]
-//! # use kernel::sync::{new_mutex, Mutex};
-//! # use core::pin::Pin;
-//! # #[pin_data]
-//! # struct Foo {
-//! # #[pin]
-//! # a: Mutex<usize>,
-//! # b: u32,
-//! # }
-//! # let foo = pin_init!(Foo {
-//! # a <- new_mutex!(42, "Foo::a"),
-//! # b: 24,
-//! # });
-//! let foo: Result<Pin<KBox<Foo>>> = KBox::pin_init(foo, GFP_KERNEL);
-//! ```
-//!
-//! For more information see the [`pin_init!`] macro.
-//!
-//! ## Using a custom function/macro that returns an initializer
-//!
-//! Many types from the kernel supply a function/macro that returns an initializer, because the
-//! above method only works for types where you can access the fields.
-//!
-//! ```rust,ignore
-//! # use kernel::sync::{new_mutex, Arc, Mutex};
-//! let mtx: Result<Arc<Mutex<usize>>> =
-//! Arc::pin_init(new_mutex!(42, "example::mtx"), GFP_KERNEL);
-//! ```
-//!
-//! To declare an init macro/function you just return an [`impl PinInit<T, E>`]:
-//!
-//! ```rust,ignore
-//! # use kernel::{sync::Mutex, new_mutex, init::PinInit, try_pin_init};
-//! #[pin_data]
-//! struct DriverData {
-//! #[pin]
-//! status: Mutex<i32>,
-//! buffer: KBox<[u8; 1_000_000]>,
-//! }
-//!
-//! impl DriverData {
-//! fn new() -> impl PinInit<Self, Error> {
-//! try_pin_init!(Self {
-//! status <- new_mutex!(0, "DriverData::status"),
-//! buffer: KBox::init(kernel::init::zeroed(), GFP_KERNEL)?,
-//! })
-//! }
-//! }
-//! ```
-//!
-//! ## Manual creation of an initializer
-//!
-//! Often when working with primitives the previous approaches are not sufficient. That is where
-//! [`pin_init_from_closure()`] comes in. This `unsafe` function allows you to create a
-//! [`impl PinInit<T, E>`] directly from a closure. Of course you have to ensure that the closure
-//! actually does the initialization in the correct way. Here are the things to look out for
-//! (we are calling the parameter to the closure `slot`):
-//! - when the closure returns `Ok(())`, then it has completed the initialization successfully, so
-//! `slot` now contains a valid bit pattern for the type `T`,
-//! - when the closure returns `Err(e)`, then the caller may deallocate the memory at `slot`, so
-//! you need to take care to clean up anything if your initialization fails mid-way,
-//! - you may assume that `slot` will stay pinned even after the closure returns until `drop` of
-//! `slot` gets called.
-//!
-//! ```rust,ignore
-//! # #![expect(unreachable_pub, clippy::disallowed_names)]
-//! use kernel::{init, types::Opaque};
-//! use core::{ptr::addr_of_mut, marker::PhantomPinned, pin::Pin};
-//! # mod bindings {
-//! # #![expect(non_camel_case_types)]
-//! # #![expect(clippy::missing_safety_doc)]
-//! # pub struct foo;
-//! # pub unsafe fn init_foo(_ptr: *mut foo) {}
-//! # pub unsafe fn destroy_foo(_ptr: *mut foo) {}
-//! # pub unsafe fn enable_foo(_ptr: *mut foo, _flags: u32) -> i32 { 0 }
-//! # }
-//! # // `Error::from_errno` is `pub(crate)` in the `kernel` crate, thus provide a workaround.
-//! # trait FromErrno {
-//! # fn from_errno(errno: kernel::ffi::c_int) -> Error {
-//! # // Dummy error that can be constructed outside the `kernel` crate.
-//! # Error::from(core::fmt::Error)
-//! # }
-//! # }
-//! # impl FromErrno for Error {}
-//! /// # Invariants
-//! ///
-//! /// `foo` is always initialized
-//! #[pin_data(PinnedDrop)]
-//! pub struct RawFoo {
-//! #[pin]
-//! foo: Opaque<bindings::foo>,
-//! #[pin]
-//! _p: PhantomPinned,
-//! }
-//!
-//! impl RawFoo {
-//! pub fn new(flags: u32) -> impl PinInit<Self, Error> {
-//! // SAFETY:
-//! // - when the closure returns `Ok(())`, then it has successfully initialized and
-//! // enabled `foo`,
-//! // - when it returns `Err(e)`, then it has cleaned up before
-//! unsafe {
-//! init::pin_init_from_closure(move |slot: *mut Self| {
-//! // `slot` contains uninit memory, avoid creating a reference.
-//! let foo = addr_of_mut!((*slot).foo);
-//!
-//! // Initialize the `foo`
-//! bindings::init_foo(Opaque::raw_get(foo));
-//!
-//! // Try to enable it.
-//! let err = bindings::enable_foo(Opaque::raw_get(foo), flags);
-//! if err != 0 {
-//! // Enabling has failed, first clean up the foo and then return the error.
-//! bindings::destroy_foo(Opaque::raw_get(foo));
-//! return Err(Error::from_errno(err));
-//! }
-//!
-//! // All fields of `RawFoo` have been initialized, since `_p` is a ZST.
-//! Ok(())
-//! })
-//! }
-//! }
-//! }
-//!
-//! #[pinned_drop]
-//! impl PinnedDrop for RawFoo {
-//! fn drop(self: Pin<&mut Self>) {
-//! // SAFETY: Since `foo` is initialized, destroying is safe.
-//! unsafe { bindings::destroy_foo(self.foo.get()) };
-//! }
-//! }
-//! ```
-//!
-//! For the special case where initializing a field is a single FFI-function call that cannot fail,
-//! there exist the helper function [`Opaque::ffi_init`]. This function initialize a single
-//! [`Opaque`] field by just delegating to the supplied closure. You can use these in combination
-//! with [`pin_init!`].
-//!
-//! For more information on how to use [`pin_init_from_closure()`], take a look at the uses inside
-//! the `kernel` crate. The [`sync`] module is a good starting point.
-//!
-//! [`sync`]: kernel::sync
-//! [pinning]: https://doc.rust-lang.org/std/pin/index.html
-//! [structurally pinned fields]:
-//! https://doc.rust-lang.org/std/pin/index.html#pinning-is-structural-for-field
-//! [stack]: crate::stack_pin_init
-//! [`Arc<T>`]: crate::sync::Arc
-//! [`impl PinInit<Foo>`]: PinInit
-//! [`impl PinInit<T, E>`]: PinInit
-//! [`impl Init<T, E>`]: Init
-//! [`Opaque`]: kernel::types::Opaque
-//! [`Opaque::ffi_init`]: kernel::types::Opaque::ffi_init
-//! [`pin_data`]: ::macros::pin_data
-//! [`pin_init!`]: crate::pin_init!
-
-use crate::{
- alloc::{AllocError, Flags, KBox},
- error::{self, Error},
- sync::Arc,
- sync::UniqueArc,
- types::{Opaque, ScopeGuard},
-};
-use core::{
- cell::UnsafeCell,
- convert::Infallible,
- marker::PhantomData,
- mem::MaybeUninit,
- num::*,
- pin::Pin,
- ptr::{self, NonNull},
-};
-
-#[doc(hidden)]
-pub mod __internal;
-#[doc(hidden)]
-pub mod macros;
-
-/// Initialize and pin a type directly on the stack.
-///
-/// # Examples
-///
-/// ```rust,ignore
-/// # #![expect(clippy::disallowed_names)]
-/// # use kernel::{init, macros::pin_data, pin_init, stack_pin_init, init::*, sync::Mutex, new_mutex};
-/// # use core::pin::Pin;
-/// #[pin_data]
-/// struct Foo {
-/// #[pin]
-/// a: Mutex<usize>,
-/// b: Bar,
-/// }
-///
-/// #[pin_data]
-/// struct Bar {
-/// x: u32,
-/// }
-///
-/// stack_pin_init!(let foo = pin_init!(Foo {
-/// a <- new_mutex!(42),
-/// b: Bar {
-/// x: 64,
-/// },
-/// }));
-/// let foo: Pin<&mut Foo> = foo;
-/// pr_info!("a: {}", &*foo.a.lock());
-/// ```
-///
-/// # Syntax
-///
-/// A normal `let` binding with optional type annotation. The expression is expected to implement
-/// [`PinInit`]/[`Init`] with the error type [`Infallible`]. If you want to use a different error
-/// type, then use [`stack_try_pin_init!`].
-///
-/// [`stack_try_pin_init!`]: crate::stack_try_pin_init!
-#[macro_export]
-macro_rules! stack_pin_init {
- (let $var:ident $(: $t:ty)? = $val:expr) => {
- let val = $val;
- let mut $var = ::core::pin::pin!($crate::init::__internal::StackInit$(::<$t>)?::uninit());
- let mut $var = match $crate::init::__internal::StackInit::init($var, val) {
- Ok(res) => res,
- Err(x) => {
- let x: ::core::convert::Infallible = x;
- match x {}
- }
- };
- };
-}
-
-/// Initialize and pin a type directly on the stack.
-///
-/// # Examples
-///
-/// ```rust,ignore
-/// # #![expect(clippy::disallowed_names)]
-/// # use kernel::{
-/// # init,
-/// # pin_init,
-/// # stack_try_pin_init,
-/// # init::*,
-/// # sync::Mutex,
-/// # new_mutex,
-/// # alloc::AllocError,
-/// # };
-/// # use macros::pin_data;
-/// # use core::pin::Pin;
-/// #[pin_data]
-/// struct Foo {
-/// #[pin]
-/// a: Mutex<usize>,
-/// b: KBox<Bar>,
-/// }
-///
-/// struct Bar {
-/// x: u32,
-/// }
-///
-/// stack_try_pin_init!(let foo: Result<Pin<&mut Foo>, AllocError> = pin_init!(Foo {
-/// a <- new_mutex!(42),
-/// b: KBox::new(Bar {
-/// x: 64,
-/// }, GFP_KERNEL)?,
-/// }));
-/// let foo = foo.unwrap();
-/// pr_info!("a: {}", &*foo.a.lock());
-/// ```
-///
-/// ```rust,ignore
-/// # #![expect(clippy::disallowed_names)]
-/// # use kernel::{
-/// # init,
-/// # pin_init,
-/// # stack_try_pin_init,
-/// # init::*,
-/// # sync::Mutex,
-/// # new_mutex,
-/// # alloc::AllocError,
-/// # };
-/// # use macros::pin_data;
-/// # use core::pin::Pin;
-/// #[pin_data]
-/// struct Foo {
-/// #[pin]
-/// a: Mutex<usize>,
-/// b: KBox<Bar>,
-/// }
-///
-/// struct Bar {
-/// x: u32,
-/// }
-///
-/// stack_try_pin_init!(let foo: Pin<&mut Foo> =? pin_init!(Foo {
-/// a <- new_mutex!(42),
-/// b: KBox::new(Bar {
-/// x: 64,
-/// }, GFP_KERNEL)?,
-/// }));
-/// pr_info!("a: {}", &*foo.a.lock());
-/// # Ok::<_, AllocError>(())
-/// ```
-///
-/// # Syntax
-///
-/// A normal `let` binding with optional type annotation. The expression is expected to implement
-/// [`PinInit`]/[`Init`]. This macro assigns a result to the given variable, adding a `?` after the
-/// `=` will propagate this error.
-#[macro_export]
-macro_rules! stack_try_pin_init {
- (let $var:ident $(: $t:ty)? = $val:expr) => {
- let val = $val;
- let mut $var = ::core::pin::pin!($crate::init::__internal::StackInit$(::<$t>)?::uninit());
- let mut $var = $crate::init::__internal::StackInit::init($var, val);
- };
- (let $var:ident $(: $t:ty)? =? $val:expr) => {
- let val = $val;
- let mut $var = ::core::pin::pin!($crate::init::__internal::StackInit$(::<$t>)?::uninit());
- let mut $var = $crate::init::__internal::StackInit::init($var, val)?;
- };
-}
-
-/// Construct an in-place, pinned initializer for `struct`s.
-///
-/// This macro defaults the error to [`Infallible`]. If you need [`Error`], then use
-/// [`try_pin_init!`].
-///
-/// The syntax is almost identical to that of a normal `struct` initializer:
-///
-/// ```rust,ignore
-/// # use kernel::{init, pin_init, macros::pin_data, init::*};
-/// # use core::pin::Pin;
-/// #[pin_data]
-/// struct Foo {
-/// a: usize,
-/// b: Bar,
-/// }
-///
-/// #[pin_data]
-/// struct Bar {
-/// x: u32,
-/// }
-///
-/// # fn demo() -> impl PinInit<Foo> {
-/// let a = 42;
-///
-/// let initializer = pin_init!(Foo {
-/// a,
-/// b: Bar {
-/// x: 64,
-/// },
-/// });
-/// # initializer }
-/// # KBox::pin_init(demo(), GFP_KERNEL).unwrap();
-/// ```
-///
-/// Arbitrary Rust expressions can be used to set the value of a variable.
-///
-/// The fields are initialized in the order that they appear in the initializer. So it is possible
-/// to read already initialized fields using raw pointers.
-///
-/// IMPORTANT: You are not allowed to create references to fields of the struct inside of the
-/// initializer.
-///
-/// # Init-functions
-///
-/// When working with this API it is often desired to let others construct your types without
-/// giving access to all fields. This is where you would normally write a plain function `new`
-/// that would return a new instance of your type. With this API that is also possible.
-/// However, there are a few extra things to keep in mind.
-///
-/// To create an initializer function, simply declare it like this:
-///
-/// ```rust,ignore
-/// # use kernel::{init, pin_init, init::*};
-/// # use core::pin::Pin;
-/// # #[pin_data]
-/// # struct Foo {
-/// # a: usize,
-/// # b: Bar,
-/// # }
-/// # #[pin_data]
-/// # struct Bar {
-/// # x: u32,
-/// # }
-/// impl Foo {
-/// fn new() -> impl PinInit<Self> {
-/// pin_init!(Self {
-/// a: 42,
-/// b: Bar {
-/// x: 64,
-/// },
-/// })
-/// }
-/// }
-/// ```
-///
-/// Users of `Foo` can now create it like this:
-///
-/// ```rust,ignore
-/// # #![expect(clippy::disallowed_names)]
-/// # use kernel::{init, pin_init, macros::pin_data, init::*};
-/// # use core::pin::Pin;
-/// # #[pin_data]
-/// # struct Foo {
-/// # a: usize,
-/// # b: Bar,
-/// # }
-/// # #[pin_data]
-/// # struct Bar {
-/// # x: u32,
-/// # }
-/// # impl Foo {
-/// # fn new() -> impl PinInit<Self> {
-/// # pin_init!(Self {
-/// # a: 42,
-/// # b: Bar {
-/// # x: 64,
-/// # },
-/// # })
-/// # }
-/// # }
-/// let foo = KBox::pin_init(Foo::new(), GFP_KERNEL);
-/// ```
-///
-/// They can also easily embed it into their own `struct`s:
-///
-/// ```rust,ignore
-/// # use kernel::{init, pin_init, macros::pin_data, init::*};
-/// # use core::pin::Pin;
-/// # #[pin_data]
-/// # struct Foo {
-/// # a: usize,
-/// # b: Bar,
-/// # }
-/// # #[pin_data]
-/// # struct Bar {
-/// # x: u32,
-/// # }
-/// # impl Foo {
-/// # fn new() -> impl PinInit<Self> {
-/// # pin_init!(Self {
-/// # a: 42,
-/// # b: Bar {
-/// # x: 64,
-/// # },
-/// # })
-/// # }
-/// # }
-/// #[pin_data]
-/// struct FooContainer {
-/// #[pin]
-/// foo1: Foo,
-/// #[pin]
-/// foo2: Foo,
-/// other: u32,
-/// }
-///
-/// impl FooContainer {
-/// fn new(other: u32) -> impl PinInit<Self> {
-/// pin_init!(Self {
-/// foo1 <- Foo::new(),
-/// foo2 <- Foo::new(),
-/// other,
-/// })
-/// }
-/// }
-/// ```
-///
-/// Here we see that when using `pin_init!` with `PinInit`, one needs to write `<-` instead of `:`.
-/// This signifies that the given field is initialized in-place. As with `struct` initializers, just
-/// writing the field (in this case `other`) without `:` or `<-` means `other: other,`.
-///
-/// # Syntax
-///
-/// As already mentioned in the examples above, inside of `pin_init!` a `struct` initializer with
-/// the following modifications is expected:
-/// - Fields that you want to initialize in-place have to use `<-` instead of `:`.
-/// - In front of the initializer you can write `&this in` to have access to a [`NonNull<Self>`]
-/// pointer named `this` inside of the initializer.
-/// - Using struct update syntax one can place `..Zeroable::zeroed()` at the very end of the
-/// struct, this initializes every field with 0 and then runs all initializers specified in the
-/// body. This can only be done if [`Zeroable`] is implemented for the struct.
-///
-/// For instance:
-///
-/// ```rust,ignore
-/// # use kernel::{macros::{Zeroable, pin_data}, pin_init};
-/// # use core::{ptr::addr_of_mut, marker::PhantomPinned};
-/// #[pin_data]
-/// #[derive(Zeroable)]
-/// struct Buf {
-/// // `ptr` points into `buf`.
-/// ptr: *mut u8,
-/// buf: [u8; 64],
-/// #[pin]
-/// pin: PhantomPinned,
-/// }
-/// pin_init!(&this in Buf {
-/// buf: [0; 64],
-/// // SAFETY: TODO.
-/// ptr: unsafe { addr_of_mut!((*this.as_ptr()).buf).cast() },
-/// pin: PhantomPinned,
-/// });
-/// pin_init!(Buf {
-/// buf: [1; 64],
-/// ..Zeroable::zeroed()
-/// });
-/// ```
-///
-/// [`try_pin_init!`]: kernel::try_pin_init
-/// [`NonNull<Self>`]: core::ptr::NonNull
-// For a detailed example of how this macro works, see the module documentation of the hidden
-// module `__internal` inside of `init/__internal.rs`.
-#[macro_export]
-macro_rules! pin_init {
- ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
- $($fields:tt)*
- }) => {
- $crate::__init_internal!(
- @this($($this)?),
- @typ($t $(::<$($generics),*>)?),
- @fields($($fields)*),
- @error(::core::convert::Infallible),
- @data(PinData, use_data),
- @has_data(HasPinData, __pin_data),
- @construct_closure(pin_init_from_closure),
- @munch_fields($($fields)*),
- )
- };
-}
-
-/// Construct an in-place, fallible pinned initializer for `struct`s.
-///
-/// If the initialization can complete without error (or [`Infallible`]), then use [`pin_init!`].
-///
-/// You can use the `?` operator or use `return Err(err)` inside the initializer to stop
-/// initialization and return the error.
-///
-/// IMPORTANT: if you have `unsafe` code inside of the initializer you have to ensure that when
-/// initialization fails, the memory can be safely deallocated without any further modifications.
-///
-/// This macro defaults the error to [`Error`].
-///
-/// The syntax is identical to [`pin_init!`] with the following exception: you can append `? $type`
-/// after the `struct` initializer to specify the error type you want to use.
-///
-/// # Examples
-///
-/// ```rust,ignore
-/// use kernel::{init::{self, PinInit}, error::Error};
-/// #[pin_data]
-/// struct BigBuf {
-/// big: KBox<[u8; 1024 * 1024 * 1024]>,
-/// small: [u8; 1024 * 1024],
-/// ptr: *mut u8,
-/// }
-///
-/// impl BigBuf {
-/// fn new() -> impl PinInit<Self, Error> {
-/// try_pin_init!(Self {
-/// big: KBox::init(init::zeroed(), GFP_KERNEL)?,
-/// small: [0; 1024 * 1024],
-/// ptr: core::ptr::null_mut(),
-/// }? Error)
-/// }
-/// }
-/// ```
-// For a detailed example of how this macro works, see the module documentation of the hidden
-// module `__internal` inside of `init/__internal.rs`.
-#[macro_export]
-macro_rules! try_pin_init {
- ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
- $($fields:tt)*
- }) => {
- $crate::__init_internal!(
- @this($($this)?),
- @typ($t $(::<$($generics),*>)? ),
- @fields($($fields)*),
- @error($crate::error::Error),
- @data(PinData, use_data),
- @has_data(HasPinData, __pin_data),
- @construct_closure(pin_init_from_closure),
- @munch_fields($($fields)*),
- )
- };
- ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
- $($fields:tt)*
- }? $err:ty) => {
- $crate::__init_internal!(
- @this($($this)?),
- @typ($t $(::<$($generics),*>)? ),
- @fields($($fields)*),
- @error($err),
- @data(PinData, use_data),
- @has_data(HasPinData, __pin_data),
- @construct_closure(pin_init_from_closure),
- @munch_fields($($fields)*),
- )
- };
-}
-
-/// Construct an in-place initializer for `struct`s.
-///
-/// This macro defaults the error to [`Infallible`]. If you need [`Error`], then use
-/// [`try_init!`].
-///
-/// The syntax is identical to [`pin_init!`] and its safety caveats also apply:
-/// - `unsafe` code must guarantee either full initialization or return an error and allow
-/// deallocation of the memory.
-/// - the fields are initialized in the order given in the initializer.
-/// - no references to fields are allowed to be created inside of the initializer.
-///
-/// This initializer is for initializing data in-place that might later be moved. If you want to
-/// pin-initialize, use [`pin_init!`].
-///
-/// [`try_init!`]: crate::try_init!
-// For a detailed example of how this macro works, see the module documentation of the hidden
-// module `__internal` inside of `init/__internal.rs`.
-#[macro_export]
-macro_rules! init {
- ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
- $($fields:tt)*
- }) => {
- $crate::__init_internal!(
- @this($($this)?),
- @typ($t $(::<$($generics),*>)?),
- @fields($($fields)*),
- @error(::core::convert::Infallible),
- @data(InitData, /*no use_data*/),
- @has_data(HasInitData, __init_data),
- @construct_closure(init_from_closure),
- @munch_fields($($fields)*),
- )
- }
-}
-
-/// Construct an in-place fallible initializer for `struct`s.
-///
-/// This macro defaults the error to [`Error`]. If you need [`Infallible`], then use
-/// [`init!`].
-///
-/// The syntax is identical to [`try_pin_init!`]. If you want to specify a custom error,
-/// append `? $type` after the `struct` initializer.
-/// The safety caveats from [`try_pin_init!`] also apply:
-/// - `unsafe` code must guarantee either full initialization or return an error and allow
-/// deallocation of the memory.
-/// - the fields are initialized in the order given in the initializer.
-/// - no references to fields are allowed to be created inside of the initializer.
-///
-/// # Examples
-///
-/// ```rust,ignore
-/// use kernel::{alloc::KBox, init::{PinInit, zeroed}, error::Error};
-/// struct BigBuf {
-/// big: KBox<[u8; 1024 * 1024 * 1024]>,
-/// small: [u8; 1024 * 1024],
-/// }
-///
-/// impl BigBuf {
-/// fn new() -> impl Init<Self, Error> {
-/// try_init!(Self {
-/// big: KBox::init(zeroed(), GFP_KERNEL)?,
-/// small: [0; 1024 * 1024],
-/// }? Error)
-/// }
-/// }
-/// ```
-// For a detailed example of how this macro works, see the module documentation of the hidden
-// module `__internal` inside of `init/__internal.rs`.
-#[macro_export]
-macro_rules! try_init {
- ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
- $($fields:tt)*
- }) => {
- $crate::__init_internal!(
- @this($($this)?),
- @typ($t $(::<$($generics),*>)?),
- @fields($($fields)*),
- @error($crate::error::Error),
- @data(InitData, /*no use_data*/),
- @has_data(HasInitData, __init_data),
- @construct_closure(init_from_closure),
- @munch_fields($($fields)*),
- )
- };
- ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
- $($fields:tt)*
- }? $err:ty) => {
- $crate::__init_internal!(
- @this($($this)?),
- @typ($t $(::<$($generics),*>)?),
- @fields($($fields)*),
- @error($err),
- @data(InitData, /*no use_data*/),
- @has_data(HasInitData, __init_data),
- @construct_closure(init_from_closure),
- @munch_fields($($fields)*),
- )
- };
-}
-
-/// Asserts that a field on a struct using `#[pin_data]` is marked with `#[pin]` ie. that it is
-/// structurally pinned.
-///
-/// # Example
-///
-/// This will succeed:
-/// ```ignore
-/// use kernel::assert_pinned;
-/// #[pin_data]
-/// struct MyStruct {
-/// #[pin]
-/// some_field: u64,
-/// }
-///
-/// assert_pinned!(MyStruct, some_field, u64);
-/// ```
-///
-/// This will fail:
-// TODO: replace with `compile_fail` when supported.
-/// ```ignore
-/// use kernel::assert_pinned;
-/// #[pin_data]
-/// struct MyStruct {
-/// some_field: u64,
-/// }
-///
-/// assert_pinned!(MyStruct, some_field, u64);
-/// ```
-///
-/// Some uses of the macro may trigger the `can't use generic parameters from outer item` error. To
-/// work around this, you may pass the `inline` parameter to the macro. The `inline` parameter can
-/// only be used when the macro is invoked from a function body.
-/// ```ignore
-/// use kernel::assert_pinned;
-/// #[pin_data]
-/// struct Foo<T> {
-/// #[pin]
-/// elem: T,
-/// }
-///
-/// impl<T> Foo<T> {
-/// fn project(self: Pin<&mut Self>) -> Pin<&mut T> {
-/// assert_pinned!(Foo<T>, elem, T, inline);
-///
-/// // SAFETY: The field is structurally pinned.
-/// unsafe { self.map_unchecked_mut(|me| &mut me.elem) }
-/// }
-/// }
-/// ```
-#[macro_export]
-macro_rules! assert_pinned {
- ($ty:ty, $field:ident, $field_ty:ty, inline) => {
- let _ = move |ptr: *mut $field_ty| {
- // SAFETY: This code is unreachable.
- let data = unsafe { <$ty as $crate::init::__internal::HasPinData>::__pin_data() };
- let init = $crate::init::__internal::AlwaysFail::<$field_ty>::new();
- // SAFETY: This code is unreachable.
- unsafe { data.$field(ptr, init) }.ok();
- };
- };
-
- ($ty:ty, $field:ident, $field_ty:ty) => {
- const _: () = {
- $crate::assert_pinned!($ty, $field, $field_ty, inline);
- };
- };
-}
-
-/// A pin-initializer for the type `T`.
-///
-/// To use this initializer, you will need a suitable memory location that can hold a `T`. This can
-/// be [`KBox<T>`], [`Arc<T>`], [`UniqueArc<T>`] or even the stack (see [`stack_pin_init!`]). Use
-/// the [`InPlaceInit::pin_init`] function of a smart pointer like [`Arc<T>`] on this.
-///
-/// Also see the [module description](self).
-///
-/// # Safety
-///
-/// When implementing this trait you will need to take great care. Also there are probably very few
-/// cases where a manual implementation is necessary. Use [`pin_init_from_closure`] where possible.
-///
-/// The [`PinInit::__pinned_init`] function:
-/// - returns `Ok(())` if it initialized every field of `slot`,
-/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
-/// - `slot` can be deallocated without UB occurring,
-/// - `slot` does not need to be dropped,
-/// - `slot` is not partially initialized.
-/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`.
-///
-/// [`Arc<T>`]: crate::sync::Arc
-/// [`Arc::pin_init`]: crate::sync::Arc::pin_init
-#[must_use = "An initializer must be used in order to create its value."]
-pub unsafe trait PinInit<T: ?Sized, E = Infallible>: Sized {
- /// Initializes `slot`.
- ///
- /// # Safety
- ///
- /// - `slot` is a valid pointer to uninitialized memory.
- /// - the caller does not touch `slot` when `Err` is returned, they are only permitted to
- /// deallocate.
- /// - `slot` will not move until it is dropped, i.e. it will be pinned.
- unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E>;
-
- /// First initializes the value using `self` then calls the function `f` with the initialized
- /// value.
- ///
- /// If `f` returns an error the value is dropped and the initializer will forward the error.
- ///
- /// # Examples
- ///
- /// ```rust,ignore
- /// # #![expect(clippy::disallowed_names)]
- /// use kernel::{types::Opaque, init::pin_init_from_closure};
- /// #[repr(C)]
- /// struct RawFoo([u8; 16]);
- /// extern "C" {
- /// fn init_foo(_: *mut RawFoo);
- /// }
- ///
- /// #[pin_data]
- /// struct Foo {
- /// #[pin]
- /// raw: Opaque<RawFoo>,
- /// }
- ///
- /// impl Foo {
- /// fn setup(self: Pin<&mut Self>) {
- /// pr_info!("Setting up foo");
- /// }
- /// }
- ///
- /// let foo = pin_init!(Foo {
- /// // SAFETY: TODO.
- /// raw <- unsafe {
- /// Opaque::ffi_init(|s| {
- /// init_foo(s);
- /// })
- /// },
- /// }).pin_chain(|foo| {
- /// foo.setup();
- /// Ok(())
- /// });
- /// ```
- fn pin_chain<F>(self, f: F) -> ChainPinInit<Self, F, T, E>
- where
- F: FnOnce(Pin<&mut T>) -> Result<(), E>,
- {
- ChainPinInit(self, f, PhantomData)
- }
-}
-
-/// An initializer returned by [`PinInit::pin_chain`].
-pub struct ChainPinInit<I, F, T: ?Sized, E>(I, F, __internal::Invariant<(E, KBox<T>)>);
-
-// SAFETY: The `__pinned_init` function is implemented such that it
-// - returns `Ok(())` on successful initialization,
-// - returns `Err(err)` on error and in this case `slot` will be dropped.
-// - considers `slot` pinned.
-unsafe impl<T: ?Sized, E, I, F> PinInit<T, E> for ChainPinInit<I, F, T, E>
-where
- I: PinInit<T, E>,
- F: FnOnce(Pin<&mut T>) -> Result<(), E>,
-{
- unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
- // SAFETY: All requirements fulfilled since this function is `__pinned_init`.
- unsafe { self.0.__pinned_init(slot)? };
- // SAFETY: The above call initialized `slot` and we still have unique access.
- let val = unsafe { &mut *slot };
- // SAFETY: `slot` is considered pinned.
- let val = unsafe { Pin::new_unchecked(val) };
- // SAFETY: `slot` was initialized above.
- (self.1)(val).inspect_err(|_| unsafe { core::ptr::drop_in_place(slot) })
- }
-}
-
-/// An initializer for `T`.
-///
-/// To use this initializer, you will need a suitable memory location that can hold a `T`. This can
-/// be [`KBox<T>`], [`Arc<T>`], [`UniqueArc<T>`] or even the stack (see [`stack_pin_init!`]). Use
-/// the [`InPlaceInit::init`] function of a smart pointer like [`Arc<T>`] on this. Because
-/// [`PinInit<T, E>`] is a super trait, you can use every function that takes it as well.
-///
-/// Also see the [module description](self).
-///
-/// # Safety
-///
-/// When implementing this trait you will need to take great care. Also there are probably very few
-/// cases where a manual implementation is necessary. Use [`init_from_closure`] where possible.
-///
-/// The [`Init::__init`] function:
-/// - returns `Ok(())` if it initialized every field of `slot`,
-/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
-/// - `slot` can be deallocated without UB occurring,
-/// - `slot` does not need to be dropped,
-/// - `slot` is not partially initialized.
-/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`.
-///
-/// The `__pinned_init` function from the supertrait [`PinInit`] needs to execute the exact same
-/// code as `__init`.
-///
-/// Contrary to its supertype [`PinInit<T, E>`] the caller is allowed to
-/// move the pointee after initialization.
-///
-/// [`Arc<T>`]: crate::sync::Arc
-#[must_use = "An initializer must be used in order to create its value."]
-pub unsafe trait Init<T: ?Sized, E = Infallible>: PinInit<T, E> {
- /// Initializes `slot`.
- ///
- /// # Safety
- ///
- /// - `slot` is a valid pointer to uninitialized memory.
- /// - the caller does not touch `slot` when `Err` is returned, they are only permitted to
- /// deallocate.
- unsafe fn __init(self, slot: *mut T) -> Result<(), E>;
-
- /// First initializes the value using `self` then calls the function `f` with the initialized
- /// value.
- ///
- /// If `f` returns an error the value is dropped and the initializer will forward the error.
- ///
- /// # Examples
- ///
- /// ```rust,ignore
- /// # #![expect(clippy::disallowed_names)]
- /// use kernel::{types::Opaque, init::{self, init_from_closure}};
- /// struct Foo {
- /// buf: [u8; 1_000_000],
- /// }
- ///
- /// impl Foo {
- /// fn setup(&mut self) {
- /// pr_info!("Setting up foo");
- /// }
- /// }
- ///
- /// let foo = init!(Foo {
- /// buf <- init::zeroed()
- /// }).chain(|foo| {
- /// foo.setup();
- /// Ok(())
- /// });
- /// ```
- fn chain<F>(self, f: F) -> ChainInit<Self, F, T, E>
- where
- F: FnOnce(&mut T) -> Result<(), E>,
- {
- ChainInit(self, f, PhantomData)
- }
-}
-
-/// An initializer returned by [`Init::chain`].
-pub struct ChainInit<I, F, T: ?Sized, E>(I, F, __internal::Invariant<(E, KBox<T>)>);
-
-// SAFETY: The `__init` function is implemented such that it
-// - returns `Ok(())` on successful initialization,
-// - returns `Err(err)` on error and in this case `slot` will be dropped.
-unsafe impl<T: ?Sized, E, I, F> Init<T, E> for ChainInit<I, F, T, E>
-where
- I: Init<T, E>,
- F: FnOnce(&mut T) -> Result<(), E>,
-{
- unsafe fn __init(self, slot: *mut T) -> Result<(), E> {
- // SAFETY: All requirements fulfilled since this function is `__init`.
- unsafe { self.0.__pinned_init(slot)? };
- // SAFETY: The above call initialized `slot` and we still have unique access.
- (self.1)(unsafe { &mut *slot }).inspect_err(|_|
- // SAFETY: `slot` was initialized above.
- unsafe { core::ptr::drop_in_place(slot) })
- }
-}
-
-// SAFETY: `__pinned_init` behaves exactly the same as `__init`.
-unsafe impl<T: ?Sized, E, I, F> PinInit<T, E> for ChainInit<I, F, T, E>
-where
- I: Init<T, E>,
- F: FnOnce(&mut T) -> Result<(), E>,
-{
- unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
- // SAFETY: `__init` has less strict requirements compared to `__pinned_init`.
- unsafe { self.__init(slot) }
- }
-}
-
-/// Creates a new [`PinInit<T, E>`] from the given closure.
-///
-/// # Safety
-///
-/// The closure:
-/// - returns `Ok(())` if it initialized every field of `slot`,
-/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
-/// - `slot` can be deallocated without UB occurring,
-/// - `slot` does not need to be dropped,
-/// - `slot` is not partially initialized.
-/// - may assume that the `slot` does not move if `T: !Unpin`,
-/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`.
-#[inline]
-pub const unsafe fn pin_init_from_closure<T: ?Sized, E>(
- f: impl FnOnce(*mut T) -> Result<(), E>,
-) -> impl PinInit<T, E> {
- __internal::InitClosure(f, PhantomData)
-}
-
-/// Creates a new [`Init<T, E>`] from the given closure.
-///
-/// # Safety
-///
-/// The closure:
-/// - returns `Ok(())` if it initialized every field of `slot`,
-/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
-/// - `slot` can be deallocated without UB occurring,
-/// - `slot` does not need to be dropped,
-/// - `slot` is not partially initialized.
-/// - the `slot` may move after initialization.
-/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`.
-#[inline]
-pub const unsafe fn init_from_closure<T: ?Sized, E>(
- f: impl FnOnce(*mut T) -> Result<(), E>,
-) -> impl Init<T, E> {
- __internal::InitClosure(f, PhantomData)
-}
-
-/// An initializer that leaves the memory uninitialized.
-///
-/// The initializer is a no-op. The `slot` memory is not changed.
-#[inline]
-pub fn uninit<T, E>() -> impl Init<MaybeUninit<T>, E> {
- // SAFETY: The memory is allowed to be uninitialized.
- unsafe { init_from_closure(|_| Ok(())) }
-}
-
-/// Initializes an array by initializing each element via the provided initializer.
-///
-/// # Examples
-///
-/// ```rust,ignore
-/// use kernel::{alloc::KBox, error::Error, init::init_array_from_fn};
-/// let array: KBox<[usize; 1_000]> =
-/// KBox::init::<Error>(init_array_from_fn(|i| i), GFP_KERNEL)?;
-/// assert_eq!(array.len(), 1_000);
-/// # Ok::<(), Error>(())
-/// ```
-pub fn init_array_from_fn<I, const N: usize, T, E>(
- mut make_init: impl FnMut(usize) -> I,
-) -> impl Init<[T; N], E>
-where
- I: Init<T, E>,
-{
- let init = move |slot: *mut [T; N]| {
- let slot = slot.cast::<T>();
- // Counts the number of initialized elements and when dropped drops that many elements from
- // `slot`.
- let mut init_count = ScopeGuard::new_with_data(0, |i| {
- // We now free every element that has been initialized before.
- // SAFETY: The loop initialized exactly the values from 0..i and since we
- // return `Err` below, the caller will consider the memory at `slot` as
- // uninitialized.
- unsafe { ptr::drop_in_place(ptr::slice_from_raw_parts_mut(slot, i)) };
- });
- for i in 0..N {
- let init = make_init(i);
- // SAFETY: Since 0 <= `i` < N, it is still in bounds of `[T; N]`.
- let ptr = unsafe { slot.add(i) };
- // SAFETY: The pointer is derived from `slot` and thus satisfies the `__init`
- // requirements.
- unsafe { init.__init(ptr) }?;
- *init_count += 1;
- }
- init_count.dismiss();
- Ok(())
- };
- // SAFETY: The initializer above initializes every element of the array. On failure it drops
- // any initialized elements and returns `Err`.
- unsafe { init_from_closure(init) }
-}
-
-/// Initializes an array by initializing each element via the provided initializer.
-///
-/// # Examples
-///
-/// ```rust,ignore
-/// use kernel::{sync::{Arc, Mutex}, init::pin_init_array_from_fn, new_mutex};
-/// let array: Arc<[Mutex<usize>; 1_000]> =
-/// Arc::pin_init(pin_init_array_from_fn(|i| new_mutex!(i)), GFP_KERNEL)?;
-/// assert_eq!(array.len(), 1_000);
-/// # Ok::<(), Error>(())
-/// ```
-pub fn pin_init_array_from_fn<I, const N: usize, T, E>(
- mut make_init: impl FnMut(usize) -> I,
-) -> impl PinInit<[T; N], E>
-where
- I: PinInit<T, E>,
-{
- let init = move |slot: *mut [T; N]| {
- let slot = slot.cast::<T>();
- // Counts the number of initialized elements and when dropped drops that many elements from
- // `slot`.
- let mut init_count = ScopeGuard::new_with_data(0, |i| {
- // We now free every element that has been initialized before.
- // SAFETY: The loop initialized exactly the values from 0..i and since we
- // return `Err` below, the caller will consider the memory at `slot` as
- // uninitialized.
- unsafe { ptr::drop_in_place(ptr::slice_from_raw_parts_mut(slot, i)) };
- });
- for i in 0..N {
- let init = make_init(i);
- // SAFETY: Since 0 <= `i` < N, it is still in bounds of `[T; N]`.
- let ptr = unsafe { slot.add(i) };
- // SAFETY: The pointer is derived from `slot` and thus satisfies the `__init`
- // requirements.
- unsafe { init.__pinned_init(ptr) }?;
- *init_count += 1;
- }
- init_count.dismiss();
- Ok(())
- };
- // SAFETY: The initializer above initializes every element of the array. On failure it drops
- // any initialized elements and returns `Err`.
- unsafe { pin_init_from_closure(init) }
-}
-
-// SAFETY: Every type can be initialized by-value.
-unsafe impl<T, E> Init<T, E> for T {
- unsafe fn __init(self, slot: *mut T) -> Result<(), E> {
- // SAFETY: TODO.
- unsafe { slot.write(self) };
- Ok(())
- }
-}
-
-// SAFETY: Every type can be initialized by-value. `__pinned_init` calls `__init`.
-unsafe impl<T, E> PinInit<T, E> for T {
- unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
- // SAFETY: TODO.
- unsafe { self.__init(slot) }
- }
-}
-
-/// Smart pointer that can initialize memory in-place.
-pub trait InPlaceInit<T>: Sized {
- /// Pinned version of `Self`.
- ///
- /// If a type already implicitly pins its pointee, `Pin<Self>` is unnecessary. In this case use
- /// `Self`, otherwise just use `Pin<Self>`.
- type PinnedSelf;
-
- /// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this
- /// type.
- ///
- /// If `T: !Unpin` it will not be able to move afterwards.
- fn try_pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> Result<Self::PinnedSelf, E>
- where
- E: From<AllocError>;
-
- /// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this
- /// type.
- ///
- /// If `T: !Unpin` it will not be able to move afterwards.
- fn pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> error::Result<Self::PinnedSelf>
- where
- Error: From<E>,
- {
- // SAFETY: We delegate to `init` and only change the error type.
- let init = unsafe {
- pin_init_from_closure(|slot| init.__pinned_init(slot).map_err(|e| Error::from(e)))
- };
- Self::try_pin_init(init, flags)
- }
-
- /// Use the given initializer to in-place initialize a `T`.
- fn try_init<E>(init: impl Init<T, E>, flags: Flags) -> Result<Self, E>
- where
- E: From<AllocError>;
-
- /// Use the given initializer to in-place initialize a `T`.
- fn init<E>(init: impl Init<T, E>, flags: Flags) -> error::Result<Self>
- where
- Error: From<E>,
- {
- // SAFETY: We delegate to `init` and only change the error type.
- let init = unsafe {
- init_from_closure(|slot| init.__pinned_init(slot).map_err(|e| Error::from(e)))
- };
- Self::try_init(init, flags)
- }
-}
-
-impl<T> InPlaceInit<T> for Arc<T> {
- type PinnedSelf = Self;
-
- #[inline]
- fn try_pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> Result<Self::PinnedSelf, E>
- where
- E: From<AllocError>,
- {
- UniqueArc::try_pin_init(init, flags).map(|u| u.into())
- }
-
- #[inline]
- fn try_init<E>(init: impl Init<T, E>, flags: Flags) -> Result<Self, E>
- where
- E: From<AllocError>,
- {
- UniqueArc::try_init(init, flags).map(|u| u.into())
- }
-}
-
-impl<T> InPlaceInit<T> for UniqueArc<T> {
- type PinnedSelf = Pin<Self>;
-
- #[inline]
- fn try_pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> Result<Self::PinnedSelf, E>
- where
- E: From<AllocError>,
- {
- UniqueArc::new_uninit(flags)?.write_pin_init(init)
- }
-
- #[inline]
- fn try_init<E>(init: impl Init<T, E>, flags: Flags) -> Result<Self, E>
- where
- E: From<AllocError>,
- {
- UniqueArc::new_uninit(flags)?.write_init(init)
- }
-}
-
-/// Smart pointer containing uninitialized memory and that can write a value.
-pub trait InPlaceWrite<T> {
- /// The type `Self` turns into when the contents are initialized.
- type Initialized;
-
- /// Use the given initializer to write a value into `self`.
- ///
- /// Does not drop the current value and considers it as uninitialized memory.
- fn write_init<E>(self, init: impl Init<T, E>) -> Result<Self::Initialized, E>;
-
- /// Use the given pin-initializer to write a value into `self`.
- ///
- /// Does not drop the current value and considers it as uninitialized memory.
- fn write_pin_init<E>(self, init: impl PinInit<T, E>) -> Result<Pin<Self::Initialized>, E>;
-}
-
-impl<T> InPlaceWrite<T> for UniqueArc<MaybeUninit<T>> {
- type Initialized = UniqueArc<T>;
-
- fn write_init<E>(mut self, init: impl Init<T, E>) -> Result<Self::Initialized, E> {
- let slot = self.as_mut_ptr();
- // SAFETY: When init errors/panics, slot will get deallocated but not dropped,
- // slot is valid.
- unsafe { init.__init(slot)? };
- // SAFETY: All fields have been initialized.
- Ok(unsafe { self.assume_init() })
- }
-
- fn write_pin_init<E>(mut self, init: impl PinInit<T, E>) -> Result<Pin<Self::Initialized>, E> {
- let slot = self.as_mut_ptr();
- // SAFETY: When init errors/panics, slot will get deallocated but not dropped,
- // slot is valid and will not be moved, because we pin it later.
- unsafe { init.__pinned_init(slot)? };
- // SAFETY: All fields have been initialized.
- Ok(unsafe { self.assume_init() }.into())
- }
-}
-
-/// Trait facilitating pinned destruction.
-///
-/// Use [`pinned_drop`] to implement this trait safely:
-///
-/// ```rust,ignore
-/// # use kernel::sync::Mutex;
-/// use kernel::macros::pinned_drop;
-/// use core::pin::Pin;
-/// #[pin_data(PinnedDrop)]
-/// struct Foo {
-/// #[pin]
-/// mtx: Mutex<usize>,
-/// }
-///
-/// #[pinned_drop]
-/// impl PinnedDrop for Foo {
-/// fn drop(self: Pin<&mut Self>) {
-/// pr_info!("Foo is being dropped!");
-/// }
-/// }
-/// ```
-///
-/// # Safety
-///
-/// This trait must be implemented via the [`pinned_drop`] proc-macro attribute on the impl.
-///
-/// [`pinned_drop`]: kernel::macros::pinned_drop
-pub unsafe trait PinnedDrop: __internal::HasPinData {
- /// Executes the pinned destructor of this type.
- ///
- /// While this function is marked safe, it is actually unsafe to call it manually. For this
- /// reason it takes an additional parameter. This type can only be constructed by `unsafe` code
- /// and thus prevents this function from being called where it should not.
- ///
- /// This extra parameter will be generated by the `#[pinned_drop]` proc-macro attribute
- /// automatically.
- fn drop(self: Pin<&mut Self>, only_call_from_drop: __internal::OnlyCallFromDrop);
-}
-
-/// Marker trait for types that can be initialized by writing just zeroes.
-///
-/// # Safety
-///
-/// The bit pattern consisting of only zeroes is a valid bit pattern for this type. In other words,
-/// this is not UB:
-///
-/// ```rust,ignore
-/// let val: Self = unsafe { core::mem::zeroed() };
-/// ```
-pub unsafe trait Zeroable {}
-
-/// Create a new zeroed T.
-///
-/// The returned initializer will write `0x00` to every byte of the given `slot`.
-#[inline]
-pub fn zeroed<T: Zeroable>() -> impl Init<T> {
- // SAFETY: Because `T: Zeroable`, all bytes zero is a valid bit pattern for `T`
- // and because we write all zeroes, the memory is initialized.
- unsafe {
- init_from_closure(|slot: *mut T| {
- slot.write_bytes(0, 1);
- Ok(())
- })
- }
-}
-
-macro_rules! impl_zeroable {
- ($($({$($generics:tt)*})? $t:ty, )*) => {
- // SAFETY: Safety comments written in the macro invocation.
- $(unsafe impl$($($generics)*)? Zeroable for $t {})*
- };
-}
-
-impl_zeroable! {
- // SAFETY: All primitives that are allowed to be zero.
- bool,
- char,
- u8, u16, u32, u64, u128, usize,
- i8, i16, i32, i64, i128, isize,
- f32, f64,
-
- // Note: do not add uninhabited types (such as `!` or `core::convert::Infallible`) to this list;
- // creating an instance of an uninhabited type is immediate undefined behavior. For more on
- // uninhabited/empty types, consult The Rustonomicon:
- // <https://doc.rust-lang.org/stable/nomicon/exotic-sizes.html#empty-types>. The Rust Reference
- // also has information on undefined behavior:
- // <https://doc.rust-lang.org/stable/reference/behavior-considered-undefined.html>.
- //
- // SAFETY: These are inhabited ZSTs; there is nothing to zero and a valid value exists.
- {<T: ?Sized>} PhantomData<T>, core::marker::PhantomPinned, (),
-
- // SAFETY: Type is allowed to take any value, including all zeros.
- {<T>} MaybeUninit<T>,
- // SAFETY: Type is allowed to take any value, including all zeros.
- {<T>} Opaque<T>,
-
- // SAFETY: `T: Zeroable` and `UnsafeCell` is `repr(transparent)`.
- {<T: ?Sized + Zeroable>} UnsafeCell<T>,
-
- // SAFETY: All zeros is equivalent to `None` (option layout optimization guarantee).
- Option<NonZeroU8>, Option<NonZeroU16>, Option<NonZeroU32>, Option<NonZeroU64>,
- Option<NonZeroU128>, Option<NonZeroUsize>,
- Option<NonZeroI8>, Option<NonZeroI16>, Option<NonZeroI32>, Option<NonZeroI64>,
- Option<NonZeroI128>, Option<NonZeroIsize>,
-
- // SAFETY: All zeros is equivalent to `None` (option layout optimization guarantee).
- //
- // In this case we are allowed to use `T: ?Sized`, since all zeros is the `None` variant.
- {<T: ?Sized>} Option<NonNull<T>>,
- {<T: ?Sized>} Option<KBox<T>>,
-
- // SAFETY: `null` pointer is valid.
- //
- // We cannot use `T: ?Sized`, since the VTABLE pointer part of fat pointers is not allowed to be
- // null.
- //
- // When `Pointee` gets stabilized, we could use
- // `T: ?Sized where <T as Pointee>::Metadata: Zeroable`
- {<T>} *mut T, {<T>} *const T,
-
- // SAFETY: `null` pointer is valid and the metadata part of these fat pointers is allowed to be
- // zero.
- {<T>} *mut [T], {<T>} *const [T], *mut str, *const str,
-
- // SAFETY: `T` is `Zeroable`.
- {<const N: usize, T: Zeroable>} [T; N], {<T: Zeroable>} Wrapping<T>,
-}
-
-macro_rules! impl_tuple_zeroable {
- ($(,)?) => {};
- ($first:ident, $($t:ident),* $(,)?) => {
- // SAFETY: All elements are zeroable and padding can be zero.
- unsafe impl<$first: Zeroable, $($t: Zeroable),*> Zeroable for ($first, $($t),*) {}
- impl_tuple_zeroable!($($t),* ,);
- }
-}
-
-impl_tuple_zeroable!(A, B, C, D, E, F, G, H, I, J);
+++ /dev/null
-// SPDX-License-Identifier: Apache-2.0 OR MIT
-
-//! This module contains API-internal items for pin-init.
-//!
-//! These items must not be used outside of
-//! - `kernel/init.rs`
-//! - `macros/pin_data.rs`
-//! - `macros/pinned_drop.rs`
-
-use super::*;
-
-/// See the [nomicon] for what subtyping is. See also [this table].
-///
-/// [nomicon]: https://doc.rust-lang.org/nomicon/subtyping.html
-/// [this table]: https://doc.rust-lang.org/nomicon/phantom-data.html#table-of-phantomdata-patterns
-pub(super) type Invariant<T> = PhantomData<fn(*mut T) -> *mut T>;
-
-/// Module-internal type implementing `PinInit` and `Init`.
-///
-/// It is unsafe to create this type, since the closure needs to fulfill the same safety
-/// requirement as the `__pinned_init`/`__init` functions.
-pub(crate) struct InitClosure<F, T: ?Sized, E>(pub(crate) F, pub(crate) Invariant<(E, T)>);
-
-// SAFETY: While constructing the `InitClosure`, the user promised that it upholds the
-// `__init` invariants.
-unsafe impl<T: ?Sized, F, E> Init<T, E> for InitClosure<F, T, E>
-where
- F: FnOnce(*mut T) -> Result<(), E>,
-{
- #[inline]
- unsafe fn __init(self, slot: *mut T) -> Result<(), E> {
- (self.0)(slot)
- }
-}
-
-// SAFETY: While constructing the `InitClosure`, the user promised that it upholds the
-// `__pinned_init` invariants.
-unsafe impl<T: ?Sized, F, E> PinInit<T, E> for InitClosure<F, T, E>
-where
- F: FnOnce(*mut T) -> Result<(), E>,
-{
- #[inline]
- unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
- (self.0)(slot)
- }
-}
-
-/// This trait is only implemented via the `#[pin_data]` proc-macro. It is used to facilitate
-/// the pin projections within the initializers.
-///
-/// # Safety
-///
-/// Only the `init` module is allowed to use this trait.
-pub unsafe trait HasPinData {
- type PinData: PinData;
-
- #[expect(clippy::missing_safety_doc)]
- unsafe fn __pin_data() -> Self::PinData;
-}
-
-/// Marker trait for pinning data of structs.
-///
-/// # Safety
-///
-/// Only the `init` module is allowed to use this trait.
-pub unsafe trait PinData: Copy {
- type Datee: ?Sized + HasPinData;
-
- /// Type inference helper function.
- fn make_closure<F, O, E>(self, f: F) -> F
- where
- F: FnOnce(*mut Self::Datee) -> Result<O, E>,
- {
- f
- }
-}
-
-/// This trait is automatically implemented for every type. It aims to provide the same type
-/// inference help as `HasPinData`.
-///
-/// # Safety
-///
-/// Only the `init` module is allowed to use this trait.
-pub unsafe trait HasInitData {
- type InitData: InitData;
-
- #[expect(clippy::missing_safety_doc)]
- unsafe fn __init_data() -> Self::InitData;
-}
-
-/// Same function as `PinData`, but for arbitrary data.
-///
-/// # Safety
-///
-/// Only the `init` module is allowed to use this trait.
-pub unsafe trait InitData: Copy {
- type Datee: ?Sized + HasInitData;
-
- /// Type inference helper function.
- fn make_closure<F, O, E>(self, f: F) -> F
- where
- F: FnOnce(*mut Self::Datee) -> Result<O, E>,
- {
- f
- }
-}
-
-pub struct AllData<T: ?Sized>(PhantomData<fn(KBox<T>) -> KBox<T>>);
-
-impl<T: ?Sized> Clone for AllData<T> {
- fn clone(&self) -> Self {
- *self
- }
-}
-
-impl<T: ?Sized> Copy for AllData<T> {}
-
-// SAFETY: TODO.
-unsafe impl<T: ?Sized> InitData for AllData<T> {
- type Datee = T;
-}
-
-// SAFETY: TODO.
-unsafe impl<T: ?Sized> HasInitData for T {
- type InitData = AllData<T>;
-
- unsafe fn __init_data() -> Self::InitData {
- AllData(PhantomData)
- }
-}
-
-/// Stack initializer helper type. Use [`stack_pin_init`] instead of this primitive.
-///
-/// # Invariants
-///
-/// If `self.is_init` is true, then `self.value` is initialized.
-///
-/// [`stack_pin_init`]: kernel::stack_pin_init
-pub struct StackInit<T> {
- value: MaybeUninit<T>,
- is_init: bool,
-}
-
-impl<T> Drop for StackInit<T> {
- #[inline]
- fn drop(&mut self) {
- if self.is_init {
- // SAFETY: As we are being dropped, we only call this once. And since `self.is_init` is
- // true, `self.value` is initialized.
- unsafe { self.value.assume_init_drop() };
- }
- }
-}
-
-impl<T> StackInit<T> {
- /// Creates a new [`StackInit<T>`] that is uninitialized. Use [`stack_pin_init`] instead of this
- /// primitive.
- ///
- /// [`stack_pin_init`]: kernel::stack_pin_init
- #[inline]
- pub fn uninit() -> Self {
- Self {
- value: MaybeUninit::uninit(),
- is_init: false,
- }
- }
-
- /// Initializes the contents and returns the result.
- #[inline]
- pub fn init<E>(self: Pin<&mut Self>, init: impl PinInit<T, E>) -> Result<Pin<&mut T>, E> {
- // SAFETY: We never move out of `this`.
- let this = unsafe { Pin::into_inner_unchecked(self) };
- // The value is currently initialized, so it needs to be dropped before we can reuse
- // the memory (this is a safety guarantee of `Pin`).
- if this.is_init {
- this.is_init = false;
- // SAFETY: `this.is_init` was true and therefore `this.value` is initialized.
- unsafe { this.value.assume_init_drop() };
- }
- // SAFETY: The memory slot is valid and this type ensures that it will stay pinned.
- unsafe { init.__pinned_init(this.value.as_mut_ptr())? };
- // INVARIANT: `this.value` is initialized above.
- this.is_init = true;
- // SAFETY: The slot is now pinned, since we will never give access to `&mut T`.
- Ok(unsafe { Pin::new_unchecked(this.value.assume_init_mut()) })
- }
-}
-
-/// When a value of this type is dropped, it drops a `T`.
-///
-/// Can be forgotten to prevent the drop.
-pub struct DropGuard<T: ?Sized> {
- ptr: *mut T,
-}
-
-impl<T: ?Sized> DropGuard<T> {
- /// Creates a new [`DropGuard<T>`]. It will [`ptr::drop_in_place`] `ptr` when it gets dropped.
- ///
- /// # Safety
- ///
- /// `ptr` must be a valid pointer.
- ///
- /// It is the callers responsibility that `self` will only get dropped if the pointee of `ptr`:
- /// - has not been dropped,
- /// - is not accessible by any other means,
- /// - will not be dropped by any other means.
- #[inline]
- pub unsafe fn new(ptr: *mut T) -> Self {
- Self { ptr }
- }
-}
-
-impl<T: ?Sized> Drop for DropGuard<T> {
- #[inline]
- fn drop(&mut self) {
- // SAFETY: A `DropGuard` can only be constructed using the unsafe `new` function
- // ensuring that this operation is safe.
- unsafe { ptr::drop_in_place(self.ptr) }
- }
-}
-
-/// Token used by `PinnedDrop` to prevent calling the function without creating this unsafely
-/// created struct. This is needed, because the `drop` function is safe, but should not be called
-/// manually.
-pub struct OnlyCallFromDrop(());
-
-impl OnlyCallFromDrop {
- /// # Safety
- ///
- /// This function should only be called from the [`Drop::drop`] function and only be used to
- /// delegate the destruction to the pinned destructor [`PinnedDrop::drop`] of the same type.
- pub unsafe fn new() -> Self {
- Self(())
- }
-}
-
-/// Initializer that always fails.
-///
-/// Used by [`assert_pinned!`].
-///
-/// [`assert_pinned!`]: crate::assert_pinned
-pub struct AlwaysFail<T: ?Sized> {
- _t: PhantomData<T>,
-}
-
-impl<T: ?Sized> AlwaysFail<T> {
- /// Creates a new initializer that always fails.
- pub fn new() -> Self {
- Self { _t: PhantomData }
- }
-}
-
-impl<T: ?Sized> Default for AlwaysFail<T> {
- fn default() -> Self {
- Self::new()
- }
-}
-
-// SAFETY: `__pinned_init` always fails, which is always okay.
-unsafe impl<T: ?Sized> PinInit<T, ()> for AlwaysFail<T> {
- unsafe fn __pinned_init(self, _slot: *mut T) -> Result<(), ()> {
- Err(())
- }
-}
+++ /dev/null
-// SPDX-License-Identifier: Apache-2.0 OR MIT
-
-//! This module provides the macros that actually implement the proc-macros `pin_data` and
-//! `pinned_drop`. It also contains `__init_internal` the implementation of the `{try_}{pin_}init!`
-//! macros.
-//!
-//! These macros should never be called directly, since they expect their input to be
-//! in a certain format which is internal. If used incorrectly, these macros can lead to UB even in
-//! safe code! Use the public facing macros instead.
-//!
-//! This architecture has been chosen because the kernel does not yet have access to `syn` which
-//! would make matters a lot easier for implementing these as proc-macros.
-//!
-//! # Macro expansion example
-//!
-//! This section is intended for readers trying to understand the macros in this module and the
-//! `pin_init!` macros from `init.rs`.
-//!
-//! We will look at the following example:
-//!
-//! ```rust,ignore
-//! # use kernel::init::*;
-//! # use core::pin::Pin;
-//! #[pin_data]
-//! #[repr(C)]
-//! struct Bar<T> {
-//! #[pin]
-//! t: T,
-//! pub x: usize,
-//! }
-//!
-//! impl<T> Bar<T> {
-//! fn new(t: T) -> impl PinInit<Self> {
-//! pin_init!(Self { t, x: 0 })
-//! }
-//! }
-//!
-//! #[pin_data(PinnedDrop)]
-//! struct Foo {
-//! a: usize,
-//! #[pin]
-//! b: Bar<u32>,
-//! }
-//!
-//! #[pinned_drop]
-//! impl PinnedDrop for Foo {
-//! fn drop(self: Pin<&mut Self>) {
-//! pr_info!("{self:p} is getting dropped.");
-//! }
-//! }
-//!
-//! let a = 42;
-//! let initializer = pin_init!(Foo {
-//! a,
-//! b <- Bar::new(36),
-//! });
-//! ```
-//!
-//! This example includes the most common and important features of the pin-init API.
-//!
-//! Below you can find individual section about the different macro invocations. Here are some
-//! general things we need to take into account when designing macros:
-//! - use global paths, similarly to file paths, these start with the separator: `::core::panic!()`
-//! this ensures that the correct item is used, since users could define their own `mod core {}`
-//! and then their own `panic!` inside to execute arbitrary code inside of our macro.
-//! - macro `unsafe` hygiene: we need to ensure that we do not expand arbitrary, user-supplied
-//! expressions inside of an `unsafe` block in the macro, because this would allow users to do
-//! `unsafe` operations without an associated `unsafe` block.
-//!
-//! ## `#[pin_data]` on `Bar`
-//!
-//! This macro is used to specify which fields are structurally pinned and which fields are not. It
-//! is placed on the struct definition and allows `#[pin]` to be placed on the fields.
-//!
-//! Here is the definition of `Bar` from our example:
-//!
-//! ```rust,ignore
-//! # use kernel::init::*;
-//! #[pin_data]
-//! #[repr(C)]
-//! struct Bar<T> {
-//! #[pin]
-//! t: T,
-//! pub x: usize,
-//! }
-//! ```
-//!
-//! This expands to the following code:
-//!
-//! ```rust,ignore
-//! // Firstly the normal definition of the struct, attributes are preserved:
-//! #[repr(C)]
-//! struct Bar<T> {
-//! t: T,
-//! pub x: usize,
-//! }
-//! // Then an anonymous constant is defined, this is because we do not want any code to access the
-//! // types that we define inside:
-//! const _: () = {
-//! // We define the pin-data carrying struct, it is a ZST and needs to have the same generics,
-//! // since we need to implement access functions for each field and thus need to know its
-//! // type.
-//! struct __ThePinData<T> {
-//! __phantom: ::core::marker::PhantomData<fn(Bar<T>) -> Bar<T>>,
-//! }
-//! // We implement `Copy` for the pin-data struct, since all functions it defines will take
-//! // `self` by value.
-//! impl<T> ::core::clone::Clone for __ThePinData<T> {
-//! fn clone(&self) -> Self {
-//! *self
-//! }
-//! }
-//! impl<T> ::core::marker::Copy for __ThePinData<T> {}
-//! // For every field of `Bar`, the pin-data struct will define a function with the same name
-//! // and accessor (`pub` or `pub(crate)` etc.). This function will take a pointer to the
-//! // field (`slot`) and a `PinInit` or `Init` depending on the projection kind of the field
-//! // (if pinning is structural for the field, then `PinInit` otherwise `Init`).
-//! #[allow(dead_code)]
-//! impl<T> __ThePinData<T> {
-//! unsafe fn t<E>(
-//! self,
-//! slot: *mut T,
-//! // Since `t` is `#[pin]`, this is `PinInit`.
-//! init: impl ::kernel::init::PinInit<T, E>,
-//! ) -> ::core::result::Result<(), E> {
-//! unsafe { ::kernel::init::PinInit::__pinned_init(init, slot) }
-//! }
-//! pub unsafe fn x<E>(
-//! self,
-//! slot: *mut usize,
-//! // Since `x` is not `#[pin]`, this is `Init`.
-//! init: impl ::kernel::init::Init<usize, E>,
-//! ) -> ::core::result::Result<(), E> {
-//! unsafe { ::kernel::init::Init::__init(init, slot) }
-//! }
-//! }
-//! // Implement the internal `HasPinData` trait that associates `Bar` with the pin-data struct
-//! // that we constructed above.
-//! unsafe impl<T> ::kernel::init::__internal::HasPinData for Bar<T> {
-//! type PinData = __ThePinData<T>;
-//! unsafe fn __pin_data() -> Self::PinData {
-//! __ThePinData {
-//! __phantom: ::core::marker::PhantomData,
-//! }
-//! }
-//! }
-//! // Implement the internal `PinData` trait that marks the pin-data struct as a pin-data
-//! // struct. This is important to ensure that no user can implement a rogue `__pin_data`
-//! // function without using `unsafe`.
-//! unsafe impl<T> ::kernel::init::__internal::PinData for __ThePinData<T> {
-//! type Datee = Bar<T>;
-//! }
-//! // Now we only want to implement `Unpin` for `Bar` when every structurally pinned field is
-//! // `Unpin`. In other words, whether `Bar` is `Unpin` only depends on structurally pinned
-//! // fields (those marked with `#[pin]`). These fields will be listed in this struct, in our
-//! // case no such fields exist, hence this is almost empty. The two phantomdata fields exist
-//! // for two reasons:
-//! // - `__phantom`: every generic must be used, since we cannot really know which generics
-//! // are used, we declare all and then use everything here once.
-//! // - `__phantom_pin`: uses the `'__pin` lifetime and ensures that this struct is invariant
-//! // over it. The lifetime is needed to work around the limitation that trait bounds must
-//! // not be trivial, e.g. the user has a `#[pin] PhantomPinned` field -- this is
-//! // unconditionally `!Unpin` and results in an error. The lifetime tricks the compiler
-//! // into accepting these bounds regardless.
-//! #[allow(dead_code)]
-//! struct __Unpin<'__pin, T> {
-//! __phantom_pin: ::core::marker::PhantomData<fn(&'__pin ()) -> &'__pin ()>,
-//! __phantom: ::core::marker::PhantomData<fn(Bar<T>) -> Bar<T>>,
-//! // Our only `#[pin]` field is `t`.
-//! t: T,
-//! }
-//! #[doc(hidden)]
-//! impl<'__pin, T> ::core::marker::Unpin for Bar<T>
-//! where
-//! __Unpin<'__pin, T>: ::core::marker::Unpin,
-//! {}
-//! // Now we need to ensure that `Bar` does not implement `Drop`, since that would give users
-//! // access to `&mut self` inside of `drop` even if the struct was pinned. This could lead to
-//! // UB with only safe code, so we disallow this by giving a trait implementation error using
-//! // a direct impl and a blanket implementation.
-//! trait MustNotImplDrop {}
-//! // Normally `Drop` bounds do not have the correct semantics, but for this purpose they do
-//! // (normally people want to know if a type has any kind of drop glue at all, here we want
-//! // to know if it has any kind of custom drop glue, which is exactly what this bound does).
-//! #[expect(drop_bounds)]
-//! impl<T: ::core::ops::Drop> MustNotImplDrop for T {}
-//! impl<T> MustNotImplDrop for Bar<T> {}
-//! // Here comes a convenience check, if one implemented `PinnedDrop`, but forgot to add it to
-//! // `#[pin_data]`, then this will error with the same mechanic as above, this is not needed
-//! // for safety, but a good sanity check, since no normal code calls `PinnedDrop::drop`.
-//! #[expect(non_camel_case_types)]
-//! trait UselessPinnedDropImpl_you_need_to_specify_PinnedDrop {}
-//! impl<
-//! T: ::kernel::init::PinnedDrop,
-//! > UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for T {}
-//! impl<T> UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for Bar<T> {}
-//! };
-//! ```
-//!
-//! ## `pin_init!` in `impl Bar`
-//!
-//! This macro creates an pin-initializer for the given struct. It requires that the struct is
-//! annotated by `#[pin_data]`.
-//!
-//! Here is the impl on `Bar` defining the new function:
-//!
-//! ```rust,ignore
-//! impl<T> Bar<T> {
-//! fn new(t: T) -> impl PinInit<Self> {
-//! pin_init!(Self { t, x: 0 })
-//! }
-//! }
-//! ```
-//!
-//! This expands to the following code:
-//!
-//! ```rust,ignore
-//! impl<T> Bar<T> {
-//! fn new(t: T) -> impl PinInit<Self> {
-//! {
-//! // We do not want to allow arbitrary returns, so we declare this type as the `Ok`
-//! // return type and shadow it later when we insert the arbitrary user code. That way
-//! // there will be no possibility of returning without `unsafe`.
-//! struct __InitOk;
-//! // Get the data about fields from the supplied type.
-//! // - the function is unsafe, hence the unsafe block
-//! // - we `use` the `HasPinData` trait in the block, it is only available in that
-//! // scope.
-//! let data = unsafe {
-//! use ::kernel::init::__internal::HasPinData;
-//! Self::__pin_data()
-//! };
-//! // Ensure that `data` really is of type `PinData` and help with type inference:
-//! let init = ::kernel::init::__internal::PinData::make_closure::<
-//! _,
-//! __InitOk,
-//! ::core::convert::Infallible,
-//! >(data, move |slot| {
-//! {
-//! // Shadow the structure so it cannot be used to return early. If a user
-//! // tries to write `return Ok(__InitOk)`, then they get a type error,
-//! // since that will refer to this struct instead of the one defined
-//! // above.
-//! struct __InitOk;
-//! // This is the expansion of `t,`, which is syntactic sugar for `t: t,`.
-//! {
-//! unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).t), t) };
-//! }
-//! // Since initialization could fail later (not in this case, since the
-//! // error type is `Infallible`) we will need to drop this field if there
-//! // is an error later. This `DropGuard` will drop the field when it gets
-//! // dropped and has not yet been forgotten.
-//! let __t_guard = unsafe {
-//! ::pinned_init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).t))
-//! };
-//! // Expansion of `x: 0,`:
-//! // Since this can be an arbitrary expression we cannot place it inside
-//! // of the `unsafe` block, so we bind it here.
-//! {
-//! let x = 0;
-//! unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).x), x) };
-//! }
-//! // We again create a `DropGuard`.
-//! let __x_guard = unsafe {
-//! ::kernel::init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).x))
-//! };
-//! // Since initialization has successfully completed, we can now forget
-//! // the guards. This is not `mem::forget`, since we only have
-//! // `&DropGuard`.
-//! ::core::mem::forget(__x_guard);
-//! ::core::mem::forget(__t_guard);
-//! // Here we use the type checker to ensure that every field has been
-//! // initialized exactly once, since this is `if false` it will never get
-//! // executed, but still type-checked.
-//! // Additionally we abuse `slot` to automatically infer the correct type
-//! // for the struct. This is also another check that every field is
-//! // accessible from this scope.
-//! #[allow(unreachable_code, clippy::diverging_sub_expression)]
-//! let _ = || {
-//! unsafe {
-//! ::core::ptr::write(
-//! slot,
-//! Self {
-//! // We only care about typecheck finding every field
-//! // here, the expression does not matter, just conjure
-//! // one using `panic!()`:
-//! t: ::core::panic!(),
-//! x: ::core::panic!(),
-//! },
-//! );
-//! };
-//! };
-//! }
-//! // We leave the scope above and gain access to the previously shadowed
-//! // `__InitOk` that we need to return.
-//! Ok(__InitOk)
-//! });
-//! // Change the return type from `__InitOk` to `()`.
-//! let init = move |
-//! slot,
-//! | -> ::core::result::Result<(), ::core::convert::Infallible> {
-//! init(slot).map(|__InitOk| ())
-//! };
-//! // Construct the initializer.
-//! let init = unsafe {
-//! ::kernel::init::pin_init_from_closure::<
-//! _,
-//! ::core::convert::Infallible,
-//! >(init)
-//! };
-//! init
-//! }
-//! }
-//! }
-//! ```
-//!
-//! ## `#[pin_data]` on `Foo`
-//!
-//! Since we already took a look at `#[pin_data]` on `Bar`, this section will only explain the
-//! differences/new things in the expansion of the `Foo` definition:
-//!
-//! ```rust,ignore
-//! #[pin_data(PinnedDrop)]
-//! struct Foo {
-//! a: usize,
-//! #[pin]
-//! b: Bar<u32>,
-//! }
-//! ```
-//!
-//! This expands to the following code:
-//!
-//! ```rust,ignore
-//! struct Foo {
-//! a: usize,
-//! b: Bar<u32>,
-//! }
-//! const _: () = {
-//! struct __ThePinData {
-//! __phantom: ::core::marker::PhantomData<fn(Foo) -> Foo>,
-//! }
-//! impl ::core::clone::Clone for __ThePinData {
-//! fn clone(&self) -> Self {
-//! *self
-//! }
-//! }
-//! impl ::core::marker::Copy for __ThePinData {}
-//! #[allow(dead_code)]
-//! impl __ThePinData {
-//! unsafe fn b<E>(
-//! self,
-//! slot: *mut Bar<u32>,
-//! init: impl ::kernel::init::PinInit<Bar<u32>, E>,
-//! ) -> ::core::result::Result<(), E> {
-//! unsafe { ::kernel::init::PinInit::__pinned_init(init, slot) }
-//! }
-//! unsafe fn a<E>(
-//! self,
-//! slot: *mut usize,
-//! init: impl ::kernel::init::Init<usize, E>,
-//! ) -> ::core::result::Result<(), E> {
-//! unsafe { ::kernel::init::Init::__init(init, slot) }
-//! }
-//! }
-//! unsafe impl ::kernel::init::__internal::HasPinData for Foo {
-//! type PinData = __ThePinData;
-//! unsafe fn __pin_data() -> Self::PinData {
-//! __ThePinData {
-//! __phantom: ::core::marker::PhantomData,
-//! }
-//! }
-//! }
-//! unsafe impl ::kernel::init::__internal::PinData for __ThePinData {
-//! type Datee = Foo;
-//! }
-//! #[allow(dead_code)]
-//! struct __Unpin<'__pin> {
-//! __phantom_pin: ::core::marker::PhantomData<fn(&'__pin ()) -> &'__pin ()>,
-//! __phantom: ::core::marker::PhantomData<fn(Foo) -> Foo>,
-//! b: Bar<u32>,
-//! }
-//! #[doc(hidden)]
-//! impl<'__pin> ::core::marker::Unpin for Foo
-//! where
-//! __Unpin<'__pin>: ::core::marker::Unpin,
-//! {}
-//! // Since we specified `PinnedDrop` as the argument to `#[pin_data]`, we expect `Foo` to
-//! // implement `PinnedDrop`. Thus we do not need to prevent `Drop` implementations like
-//! // before, instead we implement `Drop` here and delegate to `PinnedDrop`.
-//! impl ::core::ops::Drop for Foo {
-//! fn drop(&mut self) {
-//! // Since we are getting dropped, no one else has a reference to `self` and thus we
-//! // can assume that we never move.
-//! let pinned = unsafe { ::core::pin::Pin::new_unchecked(self) };
-//! // Create the unsafe token that proves that we are inside of a destructor, this
-//! // type is only allowed to be created in a destructor.
-//! let token = unsafe { ::kernel::init::__internal::OnlyCallFromDrop::new() };
-//! ::kernel::init::PinnedDrop::drop(pinned, token);
-//! }
-//! }
-//! };
-//! ```
-//!
-//! ## `#[pinned_drop]` on `impl PinnedDrop for Foo`
-//!
-//! This macro is used to implement the `PinnedDrop` trait, since that trait is `unsafe` and has an
-//! extra parameter that should not be used at all. The macro hides that parameter.
-//!
-//! Here is the `PinnedDrop` impl for `Foo`:
-//!
-//! ```rust,ignore
-//! #[pinned_drop]
-//! impl PinnedDrop for Foo {
-//! fn drop(self: Pin<&mut Self>) {
-//! pr_info!("{self:p} is getting dropped.");
-//! }
-//! }
-//! ```
-//!
-//! This expands to the following code:
-//!
-//! ```rust,ignore
-//! // `unsafe`, full path and the token parameter are added, everything else stays the same.
-//! unsafe impl ::kernel::init::PinnedDrop for Foo {
-//! fn drop(self: Pin<&mut Self>, _: ::kernel::init::__internal::OnlyCallFromDrop) {
-//! pr_info!("{self:p} is getting dropped.");
-//! }
-//! }
-//! ```
-//!
-//! ## `pin_init!` on `Foo`
-//!
-//! Since we already took a look at `pin_init!` on `Bar`, this section will only show the expansion
-//! of `pin_init!` on `Foo`:
-//!
-//! ```rust,ignore
-//! let a = 42;
-//! let initializer = pin_init!(Foo {
-//! a,
-//! b <- Bar::new(36),
-//! });
-//! ```
-//!
-//! This expands to the following code:
-//!
-//! ```rust,ignore
-//! let a = 42;
-//! let initializer = {
-//! struct __InitOk;
-//! let data = unsafe {
-//! use ::kernel::init::__internal::HasPinData;
-//! Foo::__pin_data()
-//! };
-//! let init = ::kernel::init::__internal::PinData::make_closure::<
-//! _,
-//! __InitOk,
-//! ::core::convert::Infallible,
-//! >(data, move |slot| {
-//! {
-//! struct __InitOk;
-//! {
-//! unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).a), a) };
-//! }
-//! let __a_guard = unsafe {
-//! ::kernel::init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).a))
-//! };
-//! let init = Bar::new(36);
-//! unsafe { data.b(::core::addr_of_mut!((*slot).b), b)? };
-//! let __b_guard = unsafe {
-//! ::kernel::init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).b))
-//! };
-//! ::core::mem::forget(__b_guard);
-//! ::core::mem::forget(__a_guard);
-//! #[allow(unreachable_code, clippy::diverging_sub_expression)]
-//! let _ = || {
-//! unsafe {
-//! ::core::ptr::write(
-//! slot,
-//! Foo {
-//! a: ::core::panic!(),
-//! b: ::core::panic!(),
-//! },
-//! );
-//! };
-//! };
-//! }
-//! Ok(__InitOk)
-//! });
-//! let init = move |
-//! slot,
-//! | -> ::core::result::Result<(), ::core::convert::Infallible> {
-//! init(slot).map(|__InitOk| ())
-//! };
-//! let init = unsafe {
-//! ::kernel::init::pin_init_from_closure::<_, ::core::convert::Infallible>(init)
-//! };
-//! init
-//! };
-//! ```
-
-/// Creates a `unsafe impl<...> PinnedDrop for $type` block.
-///
-/// See [`PinnedDrop`] for more information.
-#[doc(hidden)]
-#[macro_export]
-macro_rules! __pinned_drop {
- (
- @impl_sig($($impl_sig:tt)*),
- @impl_body(
- $(#[$($attr:tt)*])*
- fn drop($($sig:tt)*) {
- $($inner:tt)*
- }
- ),
- ) => {
- // SAFETY: TODO.
- unsafe $($impl_sig)* {
- // Inherit all attributes and the type/ident tokens for the signature.
- $(#[$($attr)*])*
- fn drop($($sig)*, _: $crate::init::__internal::OnlyCallFromDrop) {
- $($inner)*
- }
- }
- }
-}
-
-/// This macro first parses the struct definition such that it separates pinned and not pinned
-/// fields. Afterwards it declares the struct and implement the `PinData` trait safely.
-#[doc(hidden)]
-#[macro_export]
-macro_rules! __pin_data {
- // Proc-macro entry point, this is supplied by the proc-macro pre-parsing.
- (parse_input:
- @args($($pinned_drop:ident)?),
- @sig(
- $(#[$($struct_attr:tt)*])*
- $vis:vis struct $name:ident
- $(where $($whr:tt)*)?
- ),
- @impl_generics($($impl_generics:tt)*),
- @ty_generics($($ty_generics:tt)*),
- @decl_generics($($decl_generics:tt)*),
- @body({ $($fields:tt)* }),
- ) => {
- // We now use token munching to iterate through all of the fields. While doing this we
- // identify fields marked with `#[pin]`, these fields are the 'pinned fields'. The user
- // wants these to be structurally pinned. The rest of the fields are the
- // 'not pinned fields'. Additionally we collect all fields, since we need them in the right
- // order to declare the struct.
- //
- // In this call we also put some explaining comments for the parameters.
- $crate::__pin_data!(find_pinned_fields:
- // Attributes on the struct itself, these will just be propagated to be put onto the
- // struct definition.
- @struct_attrs($(#[$($struct_attr)*])*),
- // The visibility of the struct.
- @vis($vis),
- // The name of the struct.
- @name($name),
- // The 'impl generics', the generics that will need to be specified on the struct inside
- // of an `impl<$ty_generics>` block.
- @impl_generics($($impl_generics)*),
- // The 'ty generics', the generics that will need to be specified on the impl blocks.
- @ty_generics($($ty_generics)*),
- // The 'decl generics', the generics that need to be specified on the struct
- // definition.
- @decl_generics($($decl_generics)*),
- // The where clause of any impl block and the declaration.
- @where($($($whr)*)?),
- // The remaining fields tokens that need to be processed.
- // We add a `,` at the end to ensure correct parsing.
- @fields_munch($($fields)* ,),
- // The pinned fields.
- @pinned(),
- // The not pinned fields.
- @not_pinned(),
- // All fields.
- @fields(),
- // The accumulator containing all attributes already parsed.
- @accum(),
- // Contains `yes` or `` to indicate if `#[pin]` was found on the current field.
- @is_pinned(),
- // The proc-macro argument, this should be `PinnedDrop` or ``.
- @pinned_drop($($pinned_drop)?),
- );
- };
- (find_pinned_fields:
- @struct_attrs($($struct_attrs:tt)*),
- @vis($vis:vis),
- @name($name:ident),
- @impl_generics($($impl_generics:tt)*),
- @ty_generics($($ty_generics:tt)*),
- @decl_generics($($decl_generics:tt)*),
- @where($($whr:tt)*),
- // We found a PhantomPinned field, this should generally be pinned!
- @fields_munch($field:ident : $($($(::)?core::)?marker::)?PhantomPinned, $($rest:tt)*),
- @pinned($($pinned:tt)*),
- @not_pinned($($not_pinned:tt)*),
- @fields($($fields:tt)*),
- @accum($($accum:tt)*),
- // This field is not pinned.
- @is_pinned(),
- @pinned_drop($($pinned_drop:ident)?),
- ) => {
- ::core::compile_error!(concat!(
- "The field `",
- stringify!($field),
- "` of type `PhantomPinned` only has an effect, if it has the `#[pin]` attribute.",
- ));
- $crate::__pin_data!(find_pinned_fields:
- @struct_attrs($($struct_attrs)*),
- @vis($vis),
- @name($name),
- @impl_generics($($impl_generics)*),
- @ty_generics($($ty_generics)*),
- @decl_generics($($decl_generics)*),
- @where($($whr)*),
- @fields_munch($($rest)*),
- @pinned($($pinned)* $($accum)* $field: ::core::marker::PhantomPinned,),
- @not_pinned($($not_pinned)*),
- @fields($($fields)* $($accum)* $field: ::core::marker::PhantomPinned,),
- @accum(),
- @is_pinned(),
- @pinned_drop($($pinned_drop)?),
- );
- };
- (find_pinned_fields:
- @struct_attrs($($struct_attrs:tt)*),
- @vis($vis:vis),
- @name($name:ident),
- @impl_generics($($impl_generics:tt)*),
- @ty_generics($($ty_generics:tt)*),
- @decl_generics($($decl_generics:tt)*),
- @where($($whr:tt)*),
- // We reached the field declaration.
- @fields_munch($field:ident : $type:ty, $($rest:tt)*),
- @pinned($($pinned:tt)*),
- @not_pinned($($not_pinned:tt)*),
- @fields($($fields:tt)*),
- @accum($($accum:tt)*),
- // This field is pinned.
- @is_pinned(yes),
- @pinned_drop($($pinned_drop:ident)?),
- ) => {
- $crate::__pin_data!(find_pinned_fields:
- @struct_attrs($($struct_attrs)*),
- @vis($vis),
- @name($name),
- @impl_generics($($impl_generics)*),
- @ty_generics($($ty_generics)*),
- @decl_generics($($decl_generics)*),
- @where($($whr)*),
- @fields_munch($($rest)*),
- @pinned($($pinned)* $($accum)* $field: $type,),
- @not_pinned($($not_pinned)*),
- @fields($($fields)* $($accum)* $field: $type,),
- @accum(),
- @is_pinned(),
- @pinned_drop($($pinned_drop)?),
- );
- };
- (find_pinned_fields:
- @struct_attrs($($struct_attrs:tt)*),
- @vis($vis:vis),
- @name($name:ident),
- @impl_generics($($impl_generics:tt)*),
- @ty_generics($($ty_generics:tt)*),
- @decl_generics($($decl_generics:tt)*),
- @where($($whr:tt)*),
- // We reached the field declaration.
- @fields_munch($field:ident : $type:ty, $($rest:tt)*),
- @pinned($($pinned:tt)*),
- @not_pinned($($not_pinned:tt)*),
- @fields($($fields:tt)*),
- @accum($($accum:tt)*),
- // This field is not pinned.
- @is_pinned(),
- @pinned_drop($($pinned_drop:ident)?),
- ) => {
- $crate::__pin_data!(find_pinned_fields:
- @struct_attrs($($struct_attrs)*),
- @vis($vis),
- @name($name),
- @impl_generics($($impl_generics)*),
- @ty_generics($($ty_generics)*),
- @decl_generics($($decl_generics)*),
- @where($($whr)*),
- @fields_munch($($rest)*),
- @pinned($($pinned)*),
- @not_pinned($($not_pinned)* $($accum)* $field: $type,),
- @fields($($fields)* $($accum)* $field: $type,),
- @accum(),
- @is_pinned(),
- @pinned_drop($($pinned_drop)?),
- );
- };
- (find_pinned_fields:
- @struct_attrs($($struct_attrs:tt)*),
- @vis($vis:vis),
- @name($name:ident),
- @impl_generics($($impl_generics:tt)*),
- @ty_generics($($ty_generics:tt)*),
- @decl_generics($($decl_generics:tt)*),
- @where($($whr:tt)*),
- // We found the `#[pin]` attr.
- @fields_munch(#[pin] $($rest:tt)*),
- @pinned($($pinned:tt)*),
- @not_pinned($($not_pinned:tt)*),
- @fields($($fields:tt)*),
- @accum($($accum:tt)*),
- @is_pinned($($is_pinned:ident)?),
- @pinned_drop($($pinned_drop:ident)?),
- ) => {
- $crate::__pin_data!(find_pinned_fields:
- @struct_attrs($($struct_attrs)*),
- @vis($vis),
- @name($name),
- @impl_generics($($impl_generics)*),
- @ty_generics($($ty_generics)*),
- @decl_generics($($decl_generics)*),
- @where($($whr)*),
- @fields_munch($($rest)*),
- // We do not include `#[pin]` in the list of attributes, since it is not actually an
- // attribute that is defined somewhere.
- @pinned($($pinned)*),
- @not_pinned($($not_pinned)*),
- @fields($($fields)*),
- @accum($($accum)*),
- // Set this to `yes`.
- @is_pinned(yes),
- @pinned_drop($($pinned_drop)?),
- );
- };
- (find_pinned_fields:
- @struct_attrs($($struct_attrs:tt)*),
- @vis($vis:vis),
- @name($name:ident),
- @impl_generics($($impl_generics:tt)*),
- @ty_generics($($ty_generics:tt)*),
- @decl_generics($($decl_generics:tt)*),
- @where($($whr:tt)*),
- // We reached the field declaration with visibility, for simplicity we only munch the
- // visibility and put it into `$accum`.
- @fields_munch($fvis:vis $field:ident $($rest:tt)*),
- @pinned($($pinned:tt)*),
- @not_pinned($($not_pinned:tt)*),
- @fields($($fields:tt)*),
- @accum($($accum:tt)*),
- @is_pinned($($is_pinned:ident)?),
- @pinned_drop($($pinned_drop:ident)?),
- ) => {
- $crate::__pin_data!(find_pinned_fields:
- @struct_attrs($($struct_attrs)*),
- @vis($vis),
- @name($name),
- @impl_generics($($impl_generics)*),
- @ty_generics($($ty_generics)*),
- @decl_generics($($decl_generics)*),
- @where($($whr)*),
- @fields_munch($field $($rest)*),
- @pinned($($pinned)*),
- @not_pinned($($not_pinned)*),
- @fields($($fields)*),
- @accum($($accum)* $fvis),
- @is_pinned($($is_pinned)?),
- @pinned_drop($($pinned_drop)?),
- );
- };
- (find_pinned_fields:
- @struct_attrs($($struct_attrs:tt)*),
- @vis($vis:vis),
- @name($name:ident),
- @impl_generics($($impl_generics:tt)*),
- @ty_generics($($ty_generics:tt)*),
- @decl_generics($($decl_generics:tt)*),
- @where($($whr:tt)*),
- // Some other attribute, just put it into `$accum`.
- @fields_munch(#[$($attr:tt)*] $($rest:tt)*),
- @pinned($($pinned:tt)*),
- @not_pinned($($not_pinned:tt)*),
- @fields($($fields:tt)*),
- @accum($($accum:tt)*),
- @is_pinned($($is_pinned:ident)?),
- @pinned_drop($($pinned_drop:ident)?),
- ) => {
- $crate::__pin_data!(find_pinned_fields:
- @struct_attrs($($struct_attrs)*),
- @vis($vis),
- @name($name),
- @impl_generics($($impl_generics)*),
- @ty_generics($($ty_generics)*),
- @decl_generics($($decl_generics)*),
- @where($($whr)*),
- @fields_munch($($rest)*),
- @pinned($($pinned)*),
- @not_pinned($($not_pinned)*),
- @fields($($fields)*),
- @accum($($accum)* #[$($attr)*]),
- @is_pinned($($is_pinned)?),
- @pinned_drop($($pinned_drop)?),
- );
- };
- (find_pinned_fields:
- @struct_attrs($($struct_attrs:tt)*),
- @vis($vis:vis),
- @name($name:ident),
- @impl_generics($($impl_generics:tt)*),
- @ty_generics($($ty_generics:tt)*),
- @decl_generics($($decl_generics:tt)*),
- @where($($whr:tt)*),
- // We reached the end of the fields, plus an optional additional comma, since we added one
- // before and the user is also allowed to put a trailing comma.
- @fields_munch($(,)?),
- @pinned($($pinned:tt)*),
- @not_pinned($($not_pinned:tt)*),
- @fields($($fields:tt)*),
- @accum(),
- @is_pinned(),
- @pinned_drop($($pinned_drop:ident)?),
- ) => {
- // Declare the struct with all fields in the correct order.
- $($struct_attrs)*
- $vis struct $name <$($decl_generics)*>
- where $($whr)*
- {
- $($fields)*
- }
-
- // We put the rest into this const item, because it then will not be accessible to anything
- // outside.
- const _: () = {
- // We declare this struct which will host all of the projection function for our type.
- // it will be invariant over all generic parameters which are inherited from the
- // struct.
- $vis struct __ThePinData<$($impl_generics)*>
- where $($whr)*
- {
- __phantom: ::core::marker::PhantomData<
- fn($name<$($ty_generics)*>) -> $name<$($ty_generics)*>
- >,
- }
-
- impl<$($impl_generics)*> ::core::clone::Clone for __ThePinData<$($ty_generics)*>
- where $($whr)*
- {
- fn clone(&self) -> Self { *self }
- }
-
- impl<$($impl_generics)*> ::core::marker::Copy for __ThePinData<$($ty_generics)*>
- where $($whr)*
- {}
-
- // Make all projection functions.
- $crate::__pin_data!(make_pin_data:
- @pin_data(__ThePinData),
- @impl_generics($($impl_generics)*),
- @ty_generics($($ty_generics)*),
- @where($($whr)*),
- @pinned($($pinned)*),
- @not_pinned($($not_pinned)*),
- );
-
- // SAFETY: We have added the correct projection functions above to `__ThePinData` and
- // we also use the least restrictive generics possible.
- unsafe impl<$($impl_generics)*>
- $crate::init::__internal::HasPinData for $name<$($ty_generics)*>
- where $($whr)*
- {
- type PinData = __ThePinData<$($ty_generics)*>;
-
- unsafe fn __pin_data() -> Self::PinData {
- __ThePinData { __phantom: ::core::marker::PhantomData }
- }
- }
-
- // SAFETY: TODO.
- unsafe impl<$($impl_generics)*>
- $crate::init::__internal::PinData for __ThePinData<$($ty_generics)*>
- where $($whr)*
- {
- type Datee = $name<$($ty_generics)*>;
- }
-
- // This struct will be used for the unpin analysis. Since only structurally pinned
- // fields are relevant whether the struct should implement `Unpin`.
- #[allow(dead_code)]
- struct __Unpin <'__pin, $($impl_generics)*>
- where $($whr)*
- {
- __phantom_pin: ::core::marker::PhantomData<fn(&'__pin ()) -> &'__pin ()>,
- __phantom: ::core::marker::PhantomData<
- fn($name<$($ty_generics)*>) -> $name<$($ty_generics)*>
- >,
- // Only the pinned fields.
- $($pinned)*
- }
-
- #[doc(hidden)]
- impl<'__pin, $($impl_generics)*> ::core::marker::Unpin for $name<$($ty_generics)*>
- where
- __Unpin<'__pin, $($ty_generics)*>: ::core::marker::Unpin,
- $($whr)*
- {}
-
- // We need to disallow normal `Drop` implementation, the exact behavior depends on
- // whether `PinnedDrop` was specified as the parameter.
- $crate::__pin_data!(drop_prevention:
- @name($name),
- @impl_generics($($impl_generics)*),
- @ty_generics($($ty_generics)*),
- @where($($whr)*),
- @pinned_drop($($pinned_drop)?),
- );
- };
- };
- // When no `PinnedDrop` was specified, then we have to prevent implementing drop.
- (drop_prevention:
- @name($name:ident),
- @impl_generics($($impl_generics:tt)*),
- @ty_generics($($ty_generics:tt)*),
- @where($($whr:tt)*),
- @pinned_drop(),
- ) => {
- // We prevent this by creating a trait that will be implemented for all types implementing
- // `Drop`. Additionally we will implement this trait for the struct leading to a conflict,
- // if it also implements `Drop`
- trait MustNotImplDrop {}
- #[expect(drop_bounds)]
- impl<T: ::core::ops::Drop> MustNotImplDrop for T {}
- impl<$($impl_generics)*> MustNotImplDrop for $name<$($ty_generics)*>
- where $($whr)* {}
- // We also take care to prevent users from writing a useless `PinnedDrop` implementation.
- // They might implement `PinnedDrop` correctly for the struct, but forget to give
- // `PinnedDrop` as the parameter to `#[pin_data]`.
- #[expect(non_camel_case_types)]
- trait UselessPinnedDropImpl_you_need_to_specify_PinnedDrop {}
- impl<T: $crate::init::PinnedDrop>
- UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for T {}
- impl<$($impl_generics)*>
- UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for $name<$($ty_generics)*>
- where $($whr)* {}
- };
- // When `PinnedDrop` was specified we just implement `Drop` and delegate.
- (drop_prevention:
- @name($name:ident),
- @impl_generics($($impl_generics:tt)*),
- @ty_generics($($ty_generics:tt)*),
- @where($($whr:tt)*),
- @pinned_drop(PinnedDrop),
- ) => {
- impl<$($impl_generics)*> ::core::ops::Drop for $name<$($ty_generics)*>
- where $($whr)*
- {
- fn drop(&mut self) {
- // SAFETY: Since this is a destructor, `self` will not move after this function
- // terminates, since it is inaccessible.
- let pinned = unsafe { ::core::pin::Pin::new_unchecked(self) };
- // SAFETY: Since this is a drop function, we can create this token to call the
- // pinned destructor of this type.
- let token = unsafe { $crate::init::__internal::OnlyCallFromDrop::new() };
- $crate::init::PinnedDrop::drop(pinned, token);
- }
- }
- };
- // If some other parameter was specified, we emit a readable error.
- (drop_prevention:
- @name($name:ident),
- @impl_generics($($impl_generics:tt)*),
- @ty_generics($($ty_generics:tt)*),
- @where($($whr:tt)*),
- @pinned_drop($($rest:tt)*),
- ) => {
- compile_error!(
- "Wrong parameters to `#[pin_data]`, expected nothing or `PinnedDrop`, got '{}'.",
- stringify!($($rest)*),
- );
- };
- (make_pin_data:
- @pin_data($pin_data:ident),
- @impl_generics($($impl_generics:tt)*),
- @ty_generics($($ty_generics:tt)*),
- @where($($whr:tt)*),
- @pinned($($(#[$($p_attr:tt)*])* $pvis:vis $p_field:ident : $p_type:ty),* $(,)?),
- @not_pinned($($(#[$($attr:tt)*])* $fvis:vis $field:ident : $type:ty),* $(,)?),
- ) => {
- // For every field, we create a projection function according to its projection type. If a
- // field is structurally pinned, then it must be initialized via `PinInit`, if it is not
- // structurally pinned, then it can be initialized via `Init`.
- //
- // The functions are `unsafe` to prevent accidentally calling them.
- #[allow(dead_code)]
- #[expect(clippy::missing_safety_doc)]
- impl<$($impl_generics)*> $pin_data<$($ty_generics)*>
- where $($whr)*
- {
- $(
- $(#[$($p_attr)*])*
- $pvis unsafe fn $p_field<E>(
- self,
- slot: *mut $p_type,
- init: impl $crate::init::PinInit<$p_type, E>,
- ) -> ::core::result::Result<(), E> {
- // SAFETY: TODO.
- unsafe { $crate::init::PinInit::__pinned_init(init, slot) }
- }
- )*
- $(
- $(#[$($attr)*])*
- $fvis unsafe fn $field<E>(
- self,
- slot: *mut $type,
- init: impl $crate::init::Init<$type, E>,
- ) -> ::core::result::Result<(), E> {
- // SAFETY: TODO.
- unsafe { $crate::init::Init::__init(init, slot) }
- }
- )*
- }
- };
-}
-
-/// The internal init macro. Do not call manually!
-///
-/// This is called by the `{try_}{pin_}init!` macros with various inputs.
-///
-/// This macro has multiple internal call configurations, these are always the very first ident:
-/// - nothing: this is the base case and called by the `{try_}{pin_}init!` macros.
-/// - `with_update_parsed`: when the `..Zeroable::zeroed()` syntax has been handled.
-/// - `init_slot`: recursively creates the code that initializes all fields in `slot`.
-/// - `make_initializer`: recursively create the struct initializer that guarantees that every
-/// field has been initialized exactly once.
-#[doc(hidden)]
-#[macro_export]
-macro_rules! __init_internal {
- (
- @this($($this:ident)?),
- @typ($t:path),
- @fields($($fields:tt)*),
- @error($err:ty),
- // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData`
- // case.
- @data($data:ident, $($use_data:ident)?),
- // `HasPinData` or `HasInitData`.
- @has_data($has_data:ident, $get_data:ident),
- // `pin_init_from_closure` or `init_from_closure`.
- @construct_closure($construct_closure:ident),
- @munch_fields(),
- ) => {
- $crate::__init_internal!(with_update_parsed:
- @this($($this)?),
- @typ($t),
- @fields($($fields)*),
- @error($err),
- @data($data, $($use_data)?),
- @has_data($has_data, $get_data),
- @construct_closure($construct_closure),
- @zeroed(), // Nothing means default behavior.
- )
- };
- (
- @this($($this:ident)?),
- @typ($t:path),
- @fields($($fields:tt)*),
- @error($err:ty),
- // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData`
- // case.
- @data($data:ident, $($use_data:ident)?),
- // `HasPinData` or `HasInitData`.
- @has_data($has_data:ident, $get_data:ident),
- // `pin_init_from_closure` or `init_from_closure`.
- @construct_closure($construct_closure:ident),
- @munch_fields(..Zeroable::zeroed()),
- ) => {
- $crate::__init_internal!(with_update_parsed:
- @this($($this)?),
- @typ($t),
- @fields($($fields)*),
- @error($err),
- @data($data, $($use_data)?),
- @has_data($has_data, $get_data),
- @construct_closure($construct_closure),
- @zeroed(()), // `()` means zero all fields not mentioned.
- )
- };
- (
- @this($($this:ident)?),
- @typ($t:path),
- @fields($($fields:tt)*),
- @error($err:ty),
- // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData`
- // case.
- @data($data:ident, $($use_data:ident)?),
- // `HasPinData` or `HasInitData`.
- @has_data($has_data:ident, $get_data:ident),
- // `pin_init_from_closure` or `init_from_closure`.
- @construct_closure($construct_closure:ident),
- @munch_fields($ignore:tt $($rest:tt)*),
- ) => {
- $crate::__init_internal!(
- @this($($this)?),
- @typ($t),
- @fields($($fields)*),
- @error($err),
- @data($data, $($use_data)?),
- @has_data($has_data, $get_data),
- @construct_closure($construct_closure),
- @munch_fields($($rest)*),
- )
- };
- (with_update_parsed:
- @this($($this:ident)?),
- @typ($t:path),
- @fields($($fields:tt)*),
- @error($err:ty),
- // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData`
- // case.
- @data($data:ident, $($use_data:ident)?),
- // `HasPinData` or `HasInitData`.
- @has_data($has_data:ident, $get_data:ident),
- // `pin_init_from_closure` or `init_from_closure`.
- @construct_closure($construct_closure:ident),
- @zeroed($($init_zeroed:expr)?),
- ) => {{
- // We do not want to allow arbitrary returns, so we declare this type as the `Ok` return
- // type and shadow it later when we insert the arbitrary user code. That way there will be
- // no possibility of returning without `unsafe`.
- struct __InitOk;
- // Get the data about fields from the supplied type.
- //
- // SAFETY: TODO.
- let data = unsafe {
- use $crate::init::__internal::$has_data;
- // Here we abuse `paste!` to retokenize `$t`. Declarative macros have some internal
- // information that is associated to already parsed fragments, so a path fragment
- // cannot be used in this position. Doing the retokenization results in valid rust
- // code.
- ::kernel::macros::paste!($t::$get_data())
- };
- // Ensure that `data` really is of type `$data` and help with type inference:
- let init = $crate::init::__internal::$data::make_closure::<_, __InitOk, $err>(
- data,
- move |slot| {
- {
- // Shadow the structure so it cannot be used to return early.
- struct __InitOk;
- // If `$init_zeroed` is present we should zero the slot now and not emit an
- // error when fields are missing (since they will be zeroed). We also have to
- // check that the type actually implements `Zeroable`.
- $({
- fn assert_zeroable<T: $crate::init::Zeroable>(_: *mut T) {}
- // Ensure that the struct is indeed `Zeroable`.
- assert_zeroable(slot);
- // SAFETY: The type implements `Zeroable` by the check above.
- unsafe { ::core::ptr::write_bytes(slot, 0, 1) };
- $init_zeroed // This will be `()` if set.
- })?
- // Create the `this` so it can be referenced by the user inside of the
- // expressions creating the individual fields.
- $(let $this = unsafe { ::core::ptr::NonNull::new_unchecked(slot) };)?
- // Initialize every field.
- $crate::__init_internal!(init_slot($($use_data)?):
- @data(data),
- @slot(slot),
- @guards(),
- @munch_fields($($fields)*,),
- );
- // We use unreachable code to ensure that all fields have been mentioned exactly
- // once, this struct initializer will still be type-checked and complain with a
- // very natural error message if a field is forgotten/mentioned more than once.
- #[allow(unreachable_code, clippy::diverging_sub_expression)]
- let _ = || {
- $crate::__init_internal!(make_initializer:
- @slot(slot),
- @type_name($t),
- @munch_fields($($fields)*,),
- @acc(),
- );
- };
- }
- Ok(__InitOk)
- }
- );
- let init = move |slot| -> ::core::result::Result<(), $err> {
- init(slot).map(|__InitOk| ())
- };
- // SAFETY: TODO.
- let init = unsafe { $crate::init::$construct_closure::<_, $err>(init) };
- init
- }};
- (init_slot($($use_data:ident)?):
- @data($data:ident),
- @slot($slot:ident),
- @guards($($guards:ident,)*),
- @munch_fields($(..Zeroable::zeroed())? $(,)?),
- ) => {
- // Endpoint of munching, no fields are left. If execution reaches this point, all fields
- // have been initialized. Therefore we can now dismiss the guards by forgetting them.
- $(::core::mem::forget($guards);)*
- };
- (init_slot($use_data:ident): // `use_data` is present, so we use the `data` to init fields.
- @data($data:ident),
- @slot($slot:ident),
- @guards($($guards:ident,)*),
- // In-place initialization syntax.
- @munch_fields($field:ident <- $val:expr, $($rest:tt)*),
- ) => {
- let init = $val;
- // Call the initializer.
- //
- // SAFETY: `slot` is valid, because we are inside of an initializer closure, we
- // return when an error/panic occurs.
- // We also use the `data` to require the correct trait (`Init` or `PinInit`) for `$field`.
- unsafe { $data.$field(::core::ptr::addr_of_mut!((*$slot).$field), init)? };
- // Create the drop guard:
- //
- // We rely on macro hygiene to make it impossible for users to access this local variable.
- // We use `paste!` to create new hygiene for `$field`.
- ::kernel::macros::paste! {
- // SAFETY: We forget the guard later when initialization has succeeded.
- let [< __ $field _guard >] = unsafe {
- $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))
- };
-
- $crate::__init_internal!(init_slot($use_data):
- @data($data),
- @slot($slot),
- @guards([< __ $field _guard >], $($guards,)*),
- @munch_fields($($rest)*),
- );
- }
- };
- (init_slot(): // No `use_data`, so we use `Init::__init` directly.
- @data($data:ident),
- @slot($slot:ident),
- @guards($($guards:ident,)*),
- // In-place initialization syntax.
- @munch_fields($field:ident <- $val:expr, $($rest:tt)*),
- ) => {
- let init = $val;
- // Call the initializer.
- //
- // SAFETY: `slot` is valid, because we are inside of an initializer closure, we
- // return when an error/panic occurs.
- unsafe { $crate::init::Init::__init(init, ::core::ptr::addr_of_mut!((*$slot).$field))? };
- // Create the drop guard:
- //
- // We rely on macro hygiene to make it impossible for users to access this local variable.
- // We use `paste!` to create new hygiene for `$field`.
- ::kernel::macros::paste! {
- // SAFETY: We forget the guard later when initialization has succeeded.
- let [< __ $field _guard >] = unsafe {
- $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))
- };
-
- $crate::__init_internal!(init_slot():
- @data($data),
- @slot($slot),
- @guards([< __ $field _guard >], $($guards,)*),
- @munch_fields($($rest)*),
- );
- }
- };
- (init_slot($($use_data:ident)?):
- @data($data:ident),
- @slot($slot:ident),
- @guards($($guards:ident,)*),
- // Init by-value.
- @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*),
- ) => {
- {
- $(let $field = $val;)?
- // Initialize the field.
- //
- // SAFETY: The memory at `slot` is uninitialized.
- unsafe { ::core::ptr::write(::core::ptr::addr_of_mut!((*$slot).$field), $field) };
- }
- // Create the drop guard:
- //
- // We rely on macro hygiene to make it impossible for users to access this local variable.
- // We use `paste!` to create new hygiene for `$field`.
- ::kernel::macros::paste! {
- // SAFETY: We forget the guard later when initialization has succeeded.
- let [< __ $field _guard >] = unsafe {
- $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))
- };
-
- $crate::__init_internal!(init_slot($($use_data)?):
- @data($data),
- @slot($slot),
- @guards([< __ $field _guard >], $($guards,)*),
- @munch_fields($($rest)*),
- );
- }
- };
- (make_initializer:
- @slot($slot:ident),
- @type_name($t:path),
- @munch_fields(..Zeroable::zeroed() $(,)?),
- @acc($($acc:tt)*),
- ) => {
- // Endpoint, nothing more to munch, create the initializer. Since the users specified
- // `..Zeroable::zeroed()`, the slot will already have been zeroed and all field that have
- // not been overwritten are thus zero and initialized. We still check that all fields are
- // actually accessible by using the struct update syntax ourselves.
- // We are inside of a closure that is never executed and thus we can abuse `slot` to
- // get the correct type inference here:
- #[allow(unused_assignments)]
- unsafe {
- let mut zeroed = ::core::mem::zeroed();
- // We have to use type inference here to make zeroed have the correct type. This does
- // not get executed, so it has no effect.
- ::core::ptr::write($slot, zeroed);
- zeroed = ::core::mem::zeroed();
- // Here we abuse `paste!` to retokenize `$t`. Declarative macros have some internal
- // information that is associated to already parsed fragments, so a path fragment
- // cannot be used in this position. Doing the retokenization results in valid rust
- // code.
- ::kernel::macros::paste!(
- ::core::ptr::write($slot, $t {
- $($acc)*
- ..zeroed
- });
- );
- }
- };
- (make_initializer:
- @slot($slot:ident),
- @type_name($t:path),
- @munch_fields($(,)?),
- @acc($($acc:tt)*),
- ) => {
- // Endpoint, nothing more to munch, create the initializer.
- // Since we are in the closure that is never called, this will never get executed.
- // We abuse `slot` to get the correct type inference here:
- //
- // SAFETY: TODO.
- unsafe {
- // Here we abuse `paste!` to retokenize `$t`. Declarative macros have some internal
- // information that is associated to already parsed fragments, so a path fragment
- // cannot be used in this position. Doing the retokenization results in valid rust
- // code.
- ::kernel::macros::paste!(
- ::core::ptr::write($slot, $t {
- $($acc)*
- });
- );
- }
- };
- (make_initializer:
- @slot($slot:ident),
- @type_name($t:path),
- @munch_fields($field:ident <- $val:expr, $($rest:tt)*),
- @acc($($acc:tt)*),
- ) => {
- $crate::__init_internal!(make_initializer:
- @slot($slot),
- @type_name($t),
- @munch_fields($($rest)*),
- @acc($($acc)* $field: ::core::panic!(),),
- );
- };
- (make_initializer:
- @slot($slot:ident),
- @type_name($t:path),
- @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*),
- @acc($($acc:tt)*),
- ) => {
- $crate::__init_internal!(make_initializer:
- @slot($slot),
- @type_name($t),
- @munch_fields($($rest)*),
- @acc($($acc)* $field: ::core::panic!(),),
- );
- };
-}
-
-#[doc(hidden)]
-#[macro_export]
-macro_rules! __derive_zeroable {
- (parse_input:
- @sig(
- $(#[$($struct_attr:tt)*])*
- $vis:vis struct $name:ident
- $(where $($whr:tt)*)?
- ),
- @impl_generics($($impl_generics:tt)*),
- @ty_generics($($ty_generics:tt)*),
- @body({
- $(
- $(#[$($field_attr:tt)*])*
- $field:ident : $field_ty:ty
- ),* $(,)?
- }),
- ) => {
- // SAFETY: Every field type implements `Zeroable` and padding bytes may be zero.
- #[automatically_derived]
- unsafe impl<$($impl_generics)*> $crate::init::Zeroable for $name<$($ty_generics)*>
- where
- $($($whr)*)?
- {}
- const _: () = {
- fn assert_zeroable<T: ?::core::marker::Sized + $crate::init::Zeroable>() {}
- fn ensure_zeroable<$($impl_generics)*>()
- where $($($whr)*)?
- {
- $(assert_zeroable::<$field_ty>();)*
- }
- };
- };
-}
#[cfg(CONFIG_RUST_FW_LOADER_ABSTRACTIONS)]
pub mod firmware;
pub mod fs;
+#[path = "../pin-init/src/lib.rs"]
pub mod init;
pub mod io;
pub mod ioctl;
}
}
-/// Parsed generics.
-///
-/// See the field documentation for an explanation what each of the fields represents.
-///
-/// # Examples
-///
-/// ```rust,ignore
-/// # let input = todo!();
-/// let (Generics { decl_generics, impl_generics, ty_generics }, rest) = parse_generics(input);
-/// quote! {
-/// struct Foo<$($decl_generics)*> {
-/// // ...
-/// }
-///
-/// impl<$impl_generics> Foo<$ty_generics> {
-/// fn foo() {
-/// // ...
-/// }
-/// }
-/// }
-/// ```
-pub(crate) struct Generics {
- /// The generics with bounds and default values (e.g. `T: Clone, const N: usize = 0`).
- ///
- /// Use this on type definitions e.g. `struct Foo<$decl_generics> ...` (or `union`/`enum`).
- pub(crate) decl_generics: Vec<TokenTree>,
- /// The generics with bounds (e.g. `T: Clone, const N: usize`).
- ///
- /// Use this on `impl` blocks e.g. `impl<$impl_generics> Trait for ...`.
- pub(crate) impl_generics: Vec<TokenTree>,
- /// The generics without bounds and without default values (e.g. `T, N`).
- ///
- /// Use this when you use the type that is declared with these generics e.g.
- /// `Foo<$ty_generics>`.
- pub(crate) ty_generics: Vec<TokenTree>,
-}
-
-/// Parses the given `TokenStream` into `Generics` and the rest.
-///
-/// The generics are not present in the rest, but a where clause might remain.
-pub(crate) fn parse_generics(input: TokenStream) -> (Generics, Vec<TokenTree>) {
- // The generics with bounds and default values.
- let mut decl_generics = vec![];
- // `impl_generics`, the declared generics with their bounds.
- let mut impl_generics = vec![];
- // Only the names of the generics, without any bounds.
- let mut ty_generics = vec![];
- // Tokens not related to the generics e.g. the `where` token and definition.
- let mut rest = vec![];
- // The current level of `<`.
- let mut nesting = 0;
- let mut toks = input.into_iter();
- // If we are at the beginning of a generic parameter.
- let mut at_start = true;
- let mut skip_until_comma = false;
- while let Some(tt) = toks.next() {
- if nesting == 1 && matches!(&tt, TokenTree::Punct(p) if p.as_char() == '>') {
- // Found the end of the generics.
- break;
- } else if nesting >= 1 {
- decl_generics.push(tt.clone());
- }
- match tt.clone() {
- TokenTree::Punct(p) if p.as_char() == '<' => {
- if nesting >= 1 && !skip_until_comma {
- // This is inside of the generics and part of some bound.
- impl_generics.push(tt);
- }
- nesting += 1;
- }
- TokenTree::Punct(p) if p.as_char() == '>' => {
- // This is a parsing error, so we just end it here.
- if nesting == 0 {
- break;
- } else {
- nesting -= 1;
- if nesting >= 1 && !skip_until_comma {
- // We are still inside of the generics and part of some bound.
- impl_generics.push(tt);
- }
- }
- }
- TokenTree::Punct(p) if skip_until_comma && p.as_char() == ',' => {
- if nesting == 1 {
- impl_generics.push(tt.clone());
- impl_generics.push(tt);
- skip_until_comma = false;
- }
- }
- _ if !skip_until_comma => {
- match nesting {
- // If we haven't entered the generics yet, we still want to keep these tokens.
- 0 => rest.push(tt),
- 1 => {
- // Here depending on the token, it might be a generic variable name.
- match tt.clone() {
- TokenTree::Ident(i) if at_start && i.to_string() == "const" => {
- let Some(name) = toks.next() else {
- // Parsing error.
- break;
- };
- impl_generics.push(tt);
- impl_generics.push(name.clone());
- ty_generics.push(name.clone());
- decl_generics.push(name);
- at_start = false;
- }
- TokenTree::Ident(_) if at_start => {
- impl_generics.push(tt.clone());
- ty_generics.push(tt);
- at_start = false;
- }
- TokenTree::Punct(p) if p.as_char() == ',' => {
- impl_generics.push(tt.clone());
- ty_generics.push(tt);
- at_start = true;
- }
- // Lifetimes begin with `'`.
- TokenTree::Punct(p) if p.as_char() == '\'' && at_start => {
- impl_generics.push(tt.clone());
- ty_generics.push(tt);
- }
- // Generics can have default values, we skip these.
- TokenTree::Punct(p) if p.as_char() == '=' => {
- skip_until_comma = true;
- }
- _ => impl_generics.push(tt),
- }
- }
- _ => impl_generics.push(tt),
- }
- }
- _ => {}
- }
- }
- rest.extend(toks);
- (
- Generics {
- impl_generics,
- decl_generics,
- ty_generics,
- },
- rest,
- )
-}
-
/// Given a function declaration, finds the name of the function.
pub(crate) fn function_name(input: TokenStream) -> Option<Ident> {
let mut input = input.into_iter();
}
None
}
+
+include!("../pin-init/internal/src/helpers.rs");
mod helpers;
mod module;
mod paste;
+#[path = "../pin-init/internal/src/pin_data.rs"]
mod pin_data;
+#[path = "../pin-init/internal/src/pinned_drop.rs"]
mod pinned_drop;
mod vtable;
+#[path = "../pin-init/internal/src/zeroable.rs"]
mod zeroable;
use proc_macro::TokenStream;
concat_idents::concat_idents(ts)
}
-/// Used to specify the pinning information of the fields of a struct.
-///
-/// This is somewhat similar in purpose as
-/// [pin-project-lite](https://crates.io/crates/pin-project-lite).
-/// Place this macro on a struct definition and then `#[pin]` in front of the attributes of each
-/// field you want to structurally pin.
-///
-/// This macro enables the use of the [`pin_init!`] macro. When pin-initializing a `struct`,
-/// then `#[pin]` directs the type of initializer that is required.
-///
-/// If your `struct` implements `Drop`, then you need to add `PinnedDrop` as arguments to this
-/// macro, and change your `Drop` implementation to `PinnedDrop` annotated with
-/// `#[`[`macro@pinned_drop`]`]`, since dropping pinned values requires extra care.
-///
-/// # Examples
-///
-/// ```ignore
-/// # #![feature(lint_reasons)]
-/// # use kernel::prelude::*;
-/// # use std::{sync::Mutex, process::Command};
-/// # use kernel::macros::pin_data;
-/// #[pin_data]
-/// struct DriverData {
-/// #[pin]
-/// queue: Mutex<KVec<Command>>,
-/// buf: KBox<[u8; 1024 * 1024]>,
-/// }
-/// ```
-///
-/// ```ignore
-/// # #![feature(lint_reasons)]
-/// # use kernel::prelude::*;
-/// # use std::{sync::Mutex, process::Command};
-/// # use core::pin::Pin;
-/// # pub struct Info;
-/// # mod bindings {
-/// # pub unsafe fn destroy_info(_ptr: *mut super::Info) {}
-/// # }
-/// use kernel::macros::{pin_data, pinned_drop};
-///
-/// #[pin_data(PinnedDrop)]
-/// struct DriverData {
-/// #[pin]
-/// queue: Mutex<KVec<Command>>,
-/// buf: KBox<[u8; 1024 * 1024]>,
-/// raw_info: *mut Info,
-/// }
-///
-/// #[pinned_drop]
-/// impl PinnedDrop for DriverData {
-/// fn drop(self: Pin<&mut Self>) {
-/// unsafe { bindings::destroy_info(self.raw_info) };
-/// }
-/// }
-/// # fn main() {}
-/// ```
-///
-/// [`pin_init!`]: ../kernel/macro.pin_init.html
-// ^ cannot use direct link, since `kernel` is not a dependency of `macros`.
-#[proc_macro_attribute]
-pub fn pin_data(inner: TokenStream, item: TokenStream) -> TokenStream {
- pin_data::pin_data(inner, item)
-}
-
-/// Used to implement `PinnedDrop` safely.
-///
-/// Only works on structs that are annotated via `#[`[`macro@pin_data`]`]`.
-///
-/// # Examples
-///
-/// ```ignore
-/// # #![feature(lint_reasons)]
-/// # use kernel::prelude::*;
-/// # use macros::{pin_data, pinned_drop};
-/// # use std::{sync::Mutex, process::Command};
-/// # use core::pin::Pin;
-/// # mod bindings {
-/// # pub struct Info;
-/// # pub unsafe fn destroy_info(_ptr: *mut Info) {}
-/// # }
-/// #[pin_data(PinnedDrop)]
-/// struct DriverData {
-/// #[pin]
-/// queue: Mutex<KVec<Command>>,
-/// buf: KBox<[u8; 1024 * 1024]>,
-/// raw_info: *mut bindings::Info,
-/// }
-///
-/// #[pinned_drop]
-/// impl PinnedDrop for DriverData {
-/// fn drop(self: Pin<&mut Self>) {
-/// unsafe { bindings::destroy_info(self.raw_info) };
-/// }
-/// }
-/// ```
-#[proc_macro_attribute]
-pub fn pinned_drop(args: TokenStream, input: TokenStream) -> TokenStream {
- pinned_drop::pinned_drop(args, input)
-}
-
/// Paste identifiers together.
///
/// Within the `paste!` macro, identifiers inside `[<` and `>]` are concatenated together to form a
tokens.into_iter().collect()
}
-/// Derives the [`Zeroable`] trait for the given struct.
-///
-/// This can only be used for structs where every field implements the [`Zeroable`] trait.
-///
-/// # Examples
-///
-/// ```ignore
-/// use kernel::macros::Zeroable;
-///
-/// #[derive(Zeroable)]
-/// pub struct DriverData {
-/// id: i64,
-/// buf_ptr: *mut u8,
-/// len: usize,
-/// }
-/// ```
-#[proc_macro_derive(Zeroable)]
-pub fn derive_zeroable(input: TokenStream) -> TokenStream {
- zeroable::derive(input)
-}
+include!("../pin-init/internal/src/lib.rs");
+++ /dev/null
-// SPDX-License-Identifier: Apache-2.0 OR MIT
-
-use crate::helpers::{parse_generics, Generics};
-use proc_macro::{Group, Punct, Spacing, TokenStream, TokenTree};
-
-pub(crate) fn pin_data(args: TokenStream, input: TokenStream) -> TokenStream {
- // This proc-macro only does some pre-parsing and then delegates the actual parsing to
- // `kernel::__pin_data!`.
-
- let (
- Generics {
- impl_generics,
- decl_generics,
- ty_generics,
- },
- rest,
- ) = parse_generics(input);
- // The struct definition might contain the `Self` type. Since `__pin_data!` will define a new
- // type with the same generics and bounds, this poses a problem, since `Self` will refer to the
- // new type as opposed to this struct definition. Therefore we have to replace `Self` with the
- // concrete name.
-
- // Errors that occur when replacing `Self` with `struct_name`.
- let mut errs = TokenStream::new();
- // The name of the struct with ty_generics.
- let struct_name = rest
- .iter()
- .skip_while(|tt| !matches!(tt, TokenTree::Ident(i) if i.to_string() == "struct"))
- .nth(1)
- .and_then(|tt| match tt {
- TokenTree::Ident(_) => {
- let tt = tt.clone();
- let mut res = vec![tt];
- if !ty_generics.is_empty() {
- // We add this, so it is maximally compatible with e.g. `Self::CONST` which
- // will be replaced by `StructName::<$generics>::CONST`.
- res.push(TokenTree::Punct(Punct::new(':', Spacing::Joint)));
- res.push(TokenTree::Punct(Punct::new(':', Spacing::Alone)));
- res.push(TokenTree::Punct(Punct::new('<', Spacing::Alone)));
- res.extend(ty_generics.iter().cloned());
- res.push(TokenTree::Punct(Punct::new('>', Spacing::Alone)));
- }
- Some(res)
- }
- _ => None,
- })
- .unwrap_or_else(|| {
- // If we did not find the name of the struct then we will use `Self` as the replacement
- // and add a compile error to ensure it does not compile.
- errs.extend(
- "::core::compile_error!(\"Could not locate type name.\");"
- .parse::<TokenStream>()
- .unwrap(),
- );
- "Self".parse::<TokenStream>().unwrap().into_iter().collect()
- });
- let impl_generics = impl_generics
- .into_iter()
- .flat_map(|tt| replace_self_and_deny_type_defs(&struct_name, tt, &mut errs))
- .collect::<Vec<_>>();
- let mut rest = rest
- .into_iter()
- .flat_map(|tt| {
- // We ignore top level `struct` tokens, since they would emit a compile error.
- if matches!(&tt, TokenTree::Ident(i) if i.to_string() == "struct") {
- vec![tt]
- } else {
- replace_self_and_deny_type_defs(&struct_name, tt, &mut errs)
- }
- })
- .collect::<Vec<_>>();
- // This should be the body of the struct `{...}`.
- let last = rest.pop();
- let mut quoted = quote!(::kernel::__pin_data! {
- parse_input:
- @args(#args),
- @sig(#(#rest)*),
- @impl_generics(#(#impl_generics)*),
- @ty_generics(#(#ty_generics)*),
- @decl_generics(#(#decl_generics)*),
- @body(#last),
- });
- quoted.extend(errs);
- quoted
-}
-
-/// Replaces `Self` with `struct_name` and errors on `enum`, `trait`, `struct` `union` and `impl`
-/// keywords.
-///
-/// The error is appended to `errs` to allow normal parsing to continue.
-fn replace_self_and_deny_type_defs(
- struct_name: &Vec<TokenTree>,
- tt: TokenTree,
- errs: &mut TokenStream,
-) -> Vec<TokenTree> {
- match tt {
- TokenTree::Ident(ref i)
- if i.to_string() == "enum"
- || i.to_string() == "trait"
- || i.to_string() == "struct"
- || i.to_string() == "union"
- || i.to_string() == "impl" =>
- {
- errs.extend(
- format!(
- "::core::compile_error!(\"Cannot use `{i}` inside of struct definition with \
- `#[pin_data]`.\");"
- )
- .parse::<TokenStream>()
- .unwrap()
- .into_iter()
- .map(|mut tok| {
- tok.set_span(tt.span());
- tok
- }),
- );
- vec![tt]
- }
- TokenTree::Ident(i) if i.to_string() == "Self" => struct_name.clone(),
- TokenTree::Literal(_) | TokenTree::Punct(_) | TokenTree::Ident(_) => vec![tt],
- TokenTree::Group(g) => vec![TokenTree::Group(Group::new(
- g.delimiter(),
- g.stream()
- .into_iter()
- .flat_map(|tt| replace_self_and_deny_type_defs(struct_name, tt, errs))
- .collect(),
- ))],
- }
-}
+++ /dev/null
-// SPDX-License-Identifier: Apache-2.0 OR MIT
-
-use proc_macro::{TokenStream, TokenTree};
-
-pub(crate) fn pinned_drop(_args: TokenStream, input: TokenStream) -> TokenStream {
- let mut toks = input.into_iter().collect::<Vec<_>>();
- assert!(!toks.is_empty());
- // Ensure that we have an `impl` item.
- assert!(matches!(&toks[0], TokenTree::Ident(i) if i.to_string() == "impl"));
- // Ensure that we are implementing `PinnedDrop`.
- let mut nesting: usize = 0;
- let mut pinned_drop_idx = None;
- for (i, tt) in toks.iter().enumerate() {
- match tt {
- TokenTree::Punct(p) if p.as_char() == '<' => {
- nesting += 1;
- }
- TokenTree::Punct(p) if p.as_char() == '>' => {
- nesting = nesting.checked_sub(1).unwrap();
- continue;
- }
- _ => {}
- }
- if i >= 1 && nesting == 0 {
- // Found the end of the generics, this should be `PinnedDrop`.
- assert!(
- matches!(tt, TokenTree::Ident(i) if i.to_string() == "PinnedDrop"),
- "expected 'PinnedDrop', found: '{:?}'",
- tt
- );
- pinned_drop_idx = Some(i);
- break;
- }
- }
- let idx = pinned_drop_idx
- .unwrap_or_else(|| panic!("Expected an `impl` block implementing `PinnedDrop`."));
- // Fully qualify the `PinnedDrop`, as to avoid any tampering.
- toks.splice(idx..idx, quote!(::kernel::init::));
- // Take the `{}` body and call the declarative macro.
- if let Some(TokenTree::Group(last)) = toks.pop() {
- let last = last.stream();
- quote!(::kernel::__pinned_drop! {
- @impl_sig(#(#toks)*),
- @impl_body(#last),
- })
- } else {
- TokenStream::from_iter(toks)
- }
-}
+++ /dev/null
-// SPDX-License-Identifier: GPL-2.0
-
-use crate::helpers::{parse_generics, Generics};
-use proc_macro::{TokenStream, TokenTree};
-
-pub(crate) fn derive(input: TokenStream) -> TokenStream {
- let (
- Generics {
- impl_generics,
- decl_generics: _,
- ty_generics,
- },
- mut rest,
- ) = parse_generics(input);
- // This should be the body of the struct `{...}`.
- let last = rest.pop();
- // Now we insert `Zeroable` as a bound for every generic parameter in `impl_generics`.
- let mut new_impl_generics = Vec::with_capacity(impl_generics.len());
- // Are we inside of a generic where we want to add `Zeroable`?
- let mut in_generic = !impl_generics.is_empty();
- // Have we already inserted `Zeroable`?
- let mut inserted = false;
- // Level of `<>` nestings.
- let mut nested = 0;
- for tt in impl_generics {
- match &tt {
- // If we find a `,`, then we have finished a generic/constant/lifetime parameter.
- TokenTree::Punct(p) if nested == 0 && p.as_char() == ',' => {
- if in_generic && !inserted {
- new_impl_generics.extend(quote! { : ::kernel::init::Zeroable });
- }
- in_generic = true;
- inserted = false;
- new_impl_generics.push(tt);
- }
- // If we find `'`, then we are entering a lifetime.
- TokenTree::Punct(p) if nested == 0 && p.as_char() == '\'' => {
- in_generic = false;
- new_impl_generics.push(tt);
- }
- TokenTree::Punct(p) if nested == 0 && p.as_char() == ':' => {
- new_impl_generics.push(tt);
- if in_generic {
- new_impl_generics.extend(quote! { ::kernel::init::Zeroable + });
- inserted = true;
- }
- }
- TokenTree::Punct(p) if p.as_char() == '<' => {
- nested += 1;
- new_impl_generics.push(tt);
- }
- TokenTree::Punct(p) if p.as_char() == '>' => {
- assert!(nested > 0);
- nested -= 1;
- new_impl_generics.push(tt);
- }
- _ => new_impl_generics.push(tt),
- }
- }
- assert_eq!(nested, 0);
- if in_generic && !inserted {
- new_impl_generics.extend(quote! { : ::kernel::init::Zeroable });
- }
- quote! {
- ::kernel::__derive_zeroable!(
- parse_input:
- @sig(#(#rest)*),
- @impl_generics(#(#new_impl_generics)*),
- @ty_generics(#(#ty_generics)*),
- @body(#last),
- );
- }
-}
--- /dev/null
+// SPDX-License-Identifier: Apache-2.0 OR MIT
+
+/// Parsed generics.
+///
+/// See the field documentation for an explanation what each of the fields represents.
+///
+/// # Examples
+///
+/// ```rust,ignore
+/// # let input = todo!();
+/// let (Generics { decl_generics, impl_generics, ty_generics }, rest) = parse_generics(input);
+/// quote! {
+/// struct Foo<$($decl_generics)*> {
+/// // ...
+/// }
+///
+/// impl<$impl_generics> Foo<$ty_generics> {
+/// fn foo() {
+/// // ...
+/// }
+/// }
+/// }
+/// ```
+pub(crate) struct Generics {
+ /// The generics with bounds and default values (e.g. `T: Clone, const N: usize = 0`).
+ ///
+ /// Use this on type definitions e.g. `struct Foo<$decl_generics> ...` (or `union`/`enum`).
+ pub(crate) decl_generics: Vec<TokenTree>,
+ /// The generics with bounds (e.g. `T: Clone, const N: usize`).
+ ///
+ /// Use this on `impl` blocks e.g. `impl<$impl_generics> Trait for ...`.
+ pub(crate) impl_generics: Vec<TokenTree>,
+ /// The generics without bounds and without default values (e.g. `T, N`).
+ ///
+ /// Use this when you use the type that is declared with these generics e.g.
+ /// `Foo<$ty_generics>`.
+ pub(crate) ty_generics: Vec<TokenTree>,
+}
+
+/// Parses the given `TokenStream` into `Generics` and the rest.
+///
+/// The generics are not present in the rest, but a where clause might remain.
+pub(crate) fn parse_generics(input: TokenStream) -> (Generics, Vec<TokenTree>) {
+ // The generics with bounds and default values.
+ let mut decl_generics = vec![];
+ // `impl_generics`, the declared generics with their bounds.
+ let mut impl_generics = vec![];
+ // Only the names of the generics, without any bounds.
+ let mut ty_generics = vec![];
+ // Tokens not related to the generics e.g. the `where` token and definition.
+ let mut rest = vec![];
+ // The current level of `<`.
+ let mut nesting = 0;
+ let mut toks = input.into_iter();
+ // If we are at the beginning of a generic parameter.
+ let mut at_start = true;
+ let mut skip_until_comma = false;
+ while let Some(tt) = toks.next() {
+ if nesting == 1 && matches!(&tt, TokenTree::Punct(p) if p.as_char() == '>') {
+ // Found the end of the generics.
+ break;
+ } else if nesting >= 1 {
+ decl_generics.push(tt.clone());
+ }
+ match tt.clone() {
+ TokenTree::Punct(p) if p.as_char() == '<' => {
+ if nesting >= 1 && !skip_until_comma {
+ // This is inside of the generics and part of some bound.
+ impl_generics.push(tt);
+ }
+ nesting += 1;
+ }
+ TokenTree::Punct(p) if p.as_char() == '>' => {
+ // This is a parsing error, so we just end it here.
+ if nesting == 0 {
+ break;
+ } else {
+ nesting -= 1;
+ if nesting >= 1 && !skip_until_comma {
+ // We are still inside of the generics and part of some bound.
+ impl_generics.push(tt);
+ }
+ }
+ }
+ TokenTree::Punct(p) if skip_until_comma && p.as_char() == ',' => {
+ if nesting == 1 {
+ impl_generics.push(tt.clone());
+ impl_generics.push(tt);
+ skip_until_comma = false;
+ }
+ }
+ _ if !skip_until_comma => {
+ match nesting {
+ // If we haven't entered the generics yet, we still want to keep these tokens.
+ 0 => rest.push(tt),
+ 1 => {
+ // Here depending on the token, it might be a generic variable name.
+ match tt.clone() {
+ TokenTree::Ident(i) if at_start && i.to_string() == "const" => {
+ let Some(name) = toks.next() else {
+ // Parsing error.
+ break;
+ };
+ impl_generics.push(tt);
+ impl_generics.push(name.clone());
+ ty_generics.push(name.clone());
+ decl_generics.push(name);
+ at_start = false;
+ }
+ TokenTree::Ident(_) if at_start => {
+ impl_generics.push(tt.clone());
+ ty_generics.push(tt);
+ at_start = false;
+ }
+ TokenTree::Punct(p) if p.as_char() == ',' => {
+ impl_generics.push(tt.clone());
+ ty_generics.push(tt);
+ at_start = true;
+ }
+ // Lifetimes begin with `'`.
+ TokenTree::Punct(p) if p.as_char() == '\'' && at_start => {
+ impl_generics.push(tt.clone());
+ ty_generics.push(tt);
+ }
+ // Generics can have default values, we skip these.
+ TokenTree::Punct(p) if p.as_char() == '=' => {
+ skip_until_comma = true;
+ }
+ _ => impl_generics.push(tt),
+ }
+ }
+ _ => impl_generics.push(tt),
+ }
+ }
+ _ => {}
+ }
+ }
+ rest.extend(toks);
+ (
+ Generics {
+ impl_generics,
+ decl_generics,
+ ty_generics,
+ },
+ rest,
+ )
+}
--- /dev/null
+// SPDX-License-Identifier: Apache-2.0 OR MIT
+
+/// Used to specify the pinning information of the fields of a struct.
+///
+/// This is somewhat similar in purpose as
+/// [pin-project-lite](https://crates.io/crates/pin-project-lite).
+/// Place this macro on a struct definition and then `#[pin]` in front of the attributes of each
+/// field you want to structurally pin.
+///
+/// This macro enables the use of the [`pin_init!`] macro. When pin-initializing a `struct`,
+/// then `#[pin]` directs the type of initializer that is required.
+///
+/// If your `struct` implements `Drop`, then you need to add `PinnedDrop` as arguments to this
+/// macro, and change your `Drop` implementation to `PinnedDrop` annotated with
+/// `#[`[`macro@pinned_drop`]`]`, since dropping pinned values requires extra care.
+///
+/// # Examples
+///
+/// ```ignore
+/// # #![feature(lint_reasons)]
+/// # use kernel::prelude::*;
+/// # use std::{sync::Mutex, process::Command};
+/// # use kernel::macros::pin_data;
+/// #[pin_data]
+/// struct DriverData {
+/// #[pin]
+/// queue: Mutex<KVec<Command>>,
+/// buf: KBox<[u8; 1024 * 1024]>,
+/// }
+/// ```
+///
+/// ```ignore
+/// # #![feature(lint_reasons)]
+/// # use kernel::prelude::*;
+/// # use std::{sync::Mutex, process::Command};
+/// # use core::pin::Pin;
+/// # pub struct Info;
+/// # mod bindings {
+/// # pub unsafe fn destroy_info(_ptr: *mut super::Info) {}
+/// # }
+/// use kernel::macros::{pin_data, pinned_drop};
+///
+/// #[pin_data(PinnedDrop)]
+/// struct DriverData {
+/// #[pin]
+/// queue: Mutex<KVec<Command>>,
+/// buf: KBox<[u8; 1024 * 1024]>,
+/// raw_info: *mut Info,
+/// }
+///
+/// #[pinned_drop]
+/// impl PinnedDrop for DriverData {
+/// fn drop(self: Pin<&mut Self>) {
+/// unsafe { bindings::destroy_info(self.raw_info) };
+/// }
+/// }
+/// # fn main() {}
+/// ```
+///
+/// [`pin_init!`]: ../kernel/macro.pin_init.html
+// ^ cannot use direct link, since `kernel` is not a dependency of `macros`.
+#[proc_macro_attribute]
+pub fn pin_data(inner: TokenStream, item: TokenStream) -> TokenStream {
+ pin_data::pin_data(inner, item)
+}
+
+/// Used to implement `PinnedDrop` safely.
+///
+/// Only works on structs that are annotated via `#[`[`macro@pin_data`]`]`.
+///
+/// # Examples
+///
+/// ```ignore
+/// # #![feature(lint_reasons)]
+/// # use kernel::prelude::*;
+/// # use macros::{pin_data, pinned_drop};
+/// # use std::{sync::Mutex, process::Command};
+/// # use core::pin::Pin;
+/// # mod bindings {
+/// # pub struct Info;
+/// # pub unsafe fn destroy_info(_ptr: *mut Info) {}
+/// # }
+/// #[pin_data(PinnedDrop)]
+/// struct DriverData {
+/// #[pin]
+/// queue: Mutex<KVec<Command>>,
+/// buf: KBox<[u8; 1024 * 1024]>,
+/// raw_info: *mut bindings::Info,
+/// }
+///
+/// #[pinned_drop]
+/// impl PinnedDrop for DriverData {
+/// fn drop(self: Pin<&mut Self>) {
+/// unsafe { bindings::destroy_info(self.raw_info) };
+/// }
+/// }
+/// ```
+#[proc_macro_attribute]
+pub fn pinned_drop(args: TokenStream, input: TokenStream) -> TokenStream {
+ pinned_drop::pinned_drop(args, input)
+}
+
+/// Derives the [`Zeroable`] trait for the given struct.
+///
+/// This can only be used for structs where every field implements the [`Zeroable`] trait.
+///
+/// # Examples
+///
+/// ```ignore
+/// use kernel::macros::Zeroable;
+///
+/// #[derive(Zeroable)]
+/// pub struct DriverData {
+/// id: i64,
+/// buf_ptr: *mut u8,
+/// len: usize,
+/// }
+/// ```
+#[proc_macro_derive(Zeroable)]
+pub fn derive_zeroable(input: TokenStream) -> TokenStream {
+ zeroable::derive(input)
+}
--- /dev/null
+// SPDX-License-Identifier: Apache-2.0 OR MIT
+
+use crate::helpers::{parse_generics, Generics};
+use proc_macro::{Group, Punct, Spacing, TokenStream, TokenTree};
+
+pub(crate) fn pin_data(args: TokenStream, input: TokenStream) -> TokenStream {
+ // This proc-macro only does some pre-parsing and then delegates the actual parsing to
+ // `kernel::__pin_data!`.
+
+ let (
+ Generics {
+ impl_generics,
+ decl_generics,
+ ty_generics,
+ },
+ rest,
+ ) = parse_generics(input);
+ // The struct definition might contain the `Self` type. Since `__pin_data!` will define a new
+ // type with the same generics and bounds, this poses a problem, since `Self` will refer to the
+ // new type as opposed to this struct definition. Therefore we have to replace `Self` with the
+ // concrete name.
+
+ // Errors that occur when replacing `Self` with `struct_name`.
+ let mut errs = TokenStream::new();
+ // The name of the struct with ty_generics.
+ let struct_name = rest
+ .iter()
+ .skip_while(|tt| !matches!(tt, TokenTree::Ident(i) if i.to_string() == "struct"))
+ .nth(1)
+ .and_then(|tt| match tt {
+ TokenTree::Ident(_) => {
+ let tt = tt.clone();
+ let mut res = vec![tt];
+ if !ty_generics.is_empty() {
+ // We add this, so it is maximally compatible with e.g. `Self::CONST` which
+ // will be replaced by `StructName::<$generics>::CONST`.
+ res.push(TokenTree::Punct(Punct::new(':', Spacing::Joint)));
+ res.push(TokenTree::Punct(Punct::new(':', Spacing::Alone)));
+ res.push(TokenTree::Punct(Punct::new('<', Spacing::Alone)));
+ res.extend(ty_generics.iter().cloned());
+ res.push(TokenTree::Punct(Punct::new('>', Spacing::Alone)));
+ }
+ Some(res)
+ }
+ _ => None,
+ })
+ .unwrap_or_else(|| {
+ // If we did not find the name of the struct then we will use `Self` as the replacement
+ // and add a compile error to ensure it does not compile.
+ errs.extend(
+ "::core::compile_error!(\"Could not locate type name.\");"
+ .parse::<TokenStream>()
+ .unwrap(),
+ );
+ "Self".parse::<TokenStream>().unwrap().into_iter().collect()
+ });
+ let impl_generics = impl_generics
+ .into_iter()
+ .flat_map(|tt| replace_self_and_deny_type_defs(&struct_name, tt, &mut errs))
+ .collect::<Vec<_>>();
+ let mut rest = rest
+ .into_iter()
+ .flat_map(|tt| {
+ // We ignore top level `struct` tokens, since they would emit a compile error.
+ if matches!(&tt, TokenTree::Ident(i) if i.to_string() == "struct") {
+ vec![tt]
+ } else {
+ replace_self_and_deny_type_defs(&struct_name, tt, &mut errs)
+ }
+ })
+ .collect::<Vec<_>>();
+ // This should be the body of the struct `{...}`.
+ let last = rest.pop();
+ let mut quoted = quote!(::kernel::__pin_data! {
+ parse_input:
+ @args(#args),
+ @sig(#(#rest)*),
+ @impl_generics(#(#impl_generics)*),
+ @ty_generics(#(#ty_generics)*),
+ @decl_generics(#(#decl_generics)*),
+ @body(#last),
+ });
+ quoted.extend(errs);
+ quoted
+}
+
+/// Replaces `Self` with `struct_name` and errors on `enum`, `trait`, `struct` `union` and `impl`
+/// keywords.
+///
+/// The error is appended to `errs` to allow normal parsing to continue.
+fn replace_self_and_deny_type_defs(
+ struct_name: &Vec<TokenTree>,
+ tt: TokenTree,
+ errs: &mut TokenStream,
+) -> Vec<TokenTree> {
+ match tt {
+ TokenTree::Ident(ref i)
+ if i.to_string() == "enum"
+ || i.to_string() == "trait"
+ || i.to_string() == "struct"
+ || i.to_string() == "union"
+ || i.to_string() == "impl" =>
+ {
+ errs.extend(
+ format!(
+ "::core::compile_error!(\"Cannot use `{i}` inside of struct definition with \
+ `#[pin_data]`.\");"
+ )
+ .parse::<TokenStream>()
+ .unwrap()
+ .into_iter()
+ .map(|mut tok| {
+ tok.set_span(tt.span());
+ tok
+ }),
+ );
+ vec![tt]
+ }
+ TokenTree::Ident(i) if i.to_string() == "Self" => struct_name.clone(),
+ TokenTree::Literal(_) | TokenTree::Punct(_) | TokenTree::Ident(_) => vec![tt],
+ TokenTree::Group(g) => vec![TokenTree::Group(Group::new(
+ g.delimiter(),
+ g.stream()
+ .into_iter()
+ .flat_map(|tt| replace_self_and_deny_type_defs(struct_name, tt, errs))
+ .collect(),
+ ))],
+ }
+}
--- /dev/null
+// SPDX-License-Identifier: Apache-2.0 OR MIT
+
+use proc_macro::{TokenStream, TokenTree};
+
+pub(crate) fn pinned_drop(_args: TokenStream, input: TokenStream) -> TokenStream {
+ let mut toks = input.into_iter().collect::<Vec<_>>();
+ assert!(!toks.is_empty());
+ // Ensure that we have an `impl` item.
+ assert!(matches!(&toks[0], TokenTree::Ident(i) if i.to_string() == "impl"));
+ // Ensure that we are implementing `PinnedDrop`.
+ let mut nesting: usize = 0;
+ let mut pinned_drop_idx = None;
+ for (i, tt) in toks.iter().enumerate() {
+ match tt {
+ TokenTree::Punct(p) if p.as_char() == '<' => {
+ nesting += 1;
+ }
+ TokenTree::Punct(p) if p.as_char() == '>' => {
+ nesting = nesting.checked_sub(1).unwrap();
+ continue;
+ }
+ _ => {}
+ }
+ if i >= 1 && nesting == 0 {
+ // Found the end of the generics, this should be `PinnedDrop`.
+ assert!(
+ matches!(tt, TokenTree::Ident(i) if i.to_string() == "PinnedDrop"),
+ "expected 'PinnedDrop', found: '{:?}'",
+ tt
+ );
+ pinned_drop_idx = Some(i);
+ break;
+ }
+ }
+ let idx = pinned_drop_idx
+ .unwrap_or_else(|| panic!("Expected an `impl` block implementing `PinnedDrop`."));
+ // Fully qualify the `PinnedDrop`, as to avoid any tampering.
+ toks.splice(idx..idx, quote!(::kernel::init::));
+ // Take the `{}` body and call the declarative macro.
+ if let Some(TokenTree::Group(last)) = toks.pop() {
+ let last = last.stream();
+ quote!(::kernel::__pinned_drop! {
+ @impl_sig(#(#toks)*),
+ @impl_body(#last),
+ })
+ } else {
+ TokenStream::from_iter(toks)
+ }
+}
--- /dev/null
+// SPDX-License-Identifier: GPL-2.0
+
+use crate::helpers::{parse_generics, Generics};
+use proc_macro::{TokenStream, TokenTree};
+
+pub(crate) fn derive(input: TokenStream) -> TokenStream {
+ let (
+ Generics {
+ impl_generics,
+ decl_generics: _,
+ ty_generics,
+ },
+ mut rest,
+ ) = parse_generics(input);
+ // This should be the body of the struct `{...}`.
+ let last = rest.pop();
+ // Now we insert `Zeroable` as a bound for every generic parameter in `impl_generics`.
+ let mut new_impl_generics = Vec::with_capacity(impl_generics.len());
+ // Are we inside of a generic where we want to add `Zeroable`?
+ let mut in_generic = !impl_generics.is_empty();
+ // Have we already inserted `Zeroable`?
+ let mut inserted = false;
+ // Level of `<>` nestings.
+ let mut nested = 0;
+ for tt in impl_generics {
+ match &tt {
+ // If we find a `,`, then we have finished a generic/constant/lifetime parameter.
+ TokenTree::Punct(p) if nested == 0 && p.as_char() == ',' => {
+ if in_generic && !inserted {
+ new_impl_generics.extend(quote! { : ::kernel::init::Zeroable });
+ }
+ in_generic = true;
+ inserted = false;
+ new_impl_generics.push(tt);
+ }
+ // If we find `'`, then we are entering a lifetime.
+ TokenTree::Punct(p) if nested == 0 && p.as_char() == '\'' => {
+ in_generic = false;
+ new_impl_generics.push(tt);
+ }
+ TokenTree::Punct(p) if nested == 0 && p.as_char() == ':' => {
+ new_impl_generics.push(tt);
+ if in_generic {
+ new_impl_generics.extend(quote! { ::kernel::init::Zeroable + });
+ inserted = true;
+ }
+ }
+ TokenTree::Punct(p) if p.as_char() == '<' => {
+ nested += 1;
+ new_impl_generics.push(tt);
+ }
+ TokenTree::Punct(p) if p.as_char() == '>' => {
+ assert!(nested > 0);
+ nested -= 1;
+ new_impl_generics.push(tt);
+ }
+ _ => new_impl_generics.push(tt),
+ }
+ }
+ assert_eq!(nested, 0);
+ if in_generic && !inserted {
+ new_impl_generics.extend(quote! { : ::kernel::init::Zeroable });
+ }
+ quote! {
+ ::kernel::__derive_zeroable!(
+ parse_input:
+ @sig(#(#rest)*),
+ @impl_generics(#(#new_impl_generics)*),
+ @ty_generics(#(#ty_generics)*),
+ @body(#last),
+ );
+ }
+}
--- /dev/null
+// SPDX-License-Identifier: Apache-2.0 OR MIT
+
+//! This module contains API-internal items for pin-init.
+//!
+//! These items must not be used outside of
+//! - `kernel/init.rs`
+//! - `macros/pin_data.rs`
+//! - `macros/pinned_drop.rs`
+
+use super::*;
+
+/// See the [nomicon] for what subtyping is. See also [this table].
+///
+/// [nomicon]: https://doc.rust-lang.org/nomicon/subtyping.html
+/// [this table]: https://doc.rust-lang.org/nomicon/phantom-data.html#table-of-phantomdata-patterns
+pub(super) type Invariant<T> = PhantomData<fn(*mut T) -> *mut T>;
+
+/// Module-internal type implementing `PinInit` and `Init`.
+///
+/// It is unsafe to create this type, since the closure needs to fulfill the same safety
+/// requirement as the `__pinned_init`/`__init` functions.
+pub(crate) struct InitClosure<F, T: ?Sized, E>(pub(crate) F, pub(crate) Invariant<(E, T)>);
+
+// SAFETY: While constructing the `InitClosure`, the user promised that it upholds the
+// `__init` invariants.
+unsafe impl<T: ?Sized, F, E> Init<T, E> for InitClosure<F, T, E>
+where
+ F: FnOnce(*mut T) -> Result<(), E>,
+{
+ #[inline]
+ unsafe fn __init(self, slot: *mut T) -> Result<(), E> {
+ (self.0)(slot)
+ }
+}
+
+// SAFETY: While constructing the `InitClosure`, the user promised that it upholds the
+// `__pinned_init` invariants.
+unsafe impl<T: ?Sized, F, E> PinInit<T, E> for InitClosure<F, T, E>
+where
+ F: FnOnce(*mut T) -> Result<(), E>,
+{
+ #[inline]
+ unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
+ (self.0)(slot)
+ }
+}
+
+/// This trait is only implemented via the `#[pin_data]` proc-macro. It is used to facilitate
+/// the pin projections within the initializers.
+///
+/// # Safety
+///
+/// Only the `init` module is allowed to use this trait.
+pub unsafe trait HasPinData {
+ type PinData: PinData;
+
+ #[expect(clippy::missing_safety_doc)]
+ unsafe fn __pin_data() -> Self::PinData;
+}
+
+/// Marker trait for pinning data of structs.
+///
+/// # Safety
+///
+/// Only the `init` module is allowed to use this trait.
+pub unsafe trait PinData: Copy {
+ type Datee: ?Sized + HasPinData;
+
+ /// Type inference helper function.
+ fn make_closure<F, O, E>(self, f: F) -> F
+ where
+ F: FnOnce(*mut Self::Datee) -> Result<O, E>,
+ {
+ f
+ }
+}
+
+/// This trait is automatically implemented for every type. It aims to provide the same type
+/// inference help as `HasPinData`.
+///
+/// # Safety
+///
+/// Only the `init` module is allowed to use this trait.
+pub unsafe trait HasInitData {
+ type InitData: InitData;
+
+ #[expect(clippy::missing_safety_doc)]
+ unsafe fn __init_data() -> Self::InitData;
+}
+
+/// Same function as `PinData`, but for arbitrary data.
+///
+/// # Safety
+///
+/// Only the `init` module is allowed to use this trait.
+pub unsafe trait InitData: Copy {
+ type Datee: ?Sized + HasInitData;
+
+ /// Type inference helper function.
+ fn make_closure<F, O, E>(self, f: F) -> F
+ where
+ F: FnOnce(*mut Self::Datee) -> Result<O, E>,
+ {
+ f
+ }
+}
+
+pub struct AllData<T: ?Sized>(PhantomData<fn(KBox<T>) -> KBox<T>>);
+
+impl<T: ?Sized> Clone for AllData<T> {
+ fn clone(&self) -> Self {
+ *self
+ }
+}
+
+impl<T: ?Sized> Copy for AllData<T> {}
+
+// SAFETY: TODO.
+unsafe impl<T: ?Sized> InitData for AllData<T> {
+ type Datee = T;
+}
+
+// SAFETY: TODO.
+unsafe impl<T: ?Sized> HasInitData for T {
+ type InitData = AllData<T>;
+
+ unsafe fn __init_data() -> Self::InitData {
+ AllData(PhantomData)
+ }
+}
+
+/// Stack initializer helper type. Use [`stack_pin_init`] instead of this primitive.
+///
+/// # Invariants
+///
+/// If `self.is_init` is true, then `self.value` is initialized.
+///
+/// [`stack_pin_init`]: kernel::stack_pin_init
+pub struct StackInit<T> {
+ value: MaybeUninit<T>,
+ is_init: bool,
+}
+
+impl<T> Drop for StackInit<T> {
+ #[inline]
+ fn drop(&mut self) {
+ if self.is_init {
+ // SAFETY: As we are being dropped, we only call this once. And since `self.is_init` is
+ // true, `self.value` is initialized.
+ unsafe { self.value.assume_init_drop() };
+ }
+ }
+}
+
+impl<T> StackInit<T> {
+ /// Creates a new [`StackInit<T>`] that is uninitialized. Use [`stack_pin_init`] instead of this
+ /// primitive.
+ ///
+ /// [`stack_pin_init`]: kernel::stack_pin_init
+ #[inline]
+ pub fn uninit() -> Self {
+ Self {
+ value: MaybeUninit::uninit(),
+ is_init: false,
+ }
+ }
+
+ /// Initializes the contents and returns the result.
+ #[inline]
+ pub fn init<E>(self: Pin<&mut Self>, init: impl PinInit<T, E>) -> Result<Pin<&mut T>, E> {
+ // SAFETY: We never move out of `this`.
+ let this = unsafe { Pin::into_inner_unchecked(self) };
+ // The value is currently initialized, so it needs to be dropped before we can reuse
+ // the memory (this is a safety guarantee of `Pin`).
+ if this.is_init {
+ this.is_init = false;
+ // SAFETY: `this.is_init` was true and therefore `this.value` is initialized.
+ unsafe { this.value.assume_init_drop() };
+ }
+ // SAFETY: The memory slot is valid and this type ensures that it will stay pinned.
+ unsafe { init.__pinned_init(this.value.as_mut_ptr())? };
+ // INVARIANT: `this.value` is initialized above.
+ this.is_init = true;
+ // SAFETY: The slot is now pinned, since we will never give access to `&mut T`.
+ Ok(unsafe { Pin::new_unchecked(this.value.assume_init_mut()) })
+ }
+}
+
+/// When a value of this type is dropped, it drops a `T`.
+///
+/// Can be forgotten to prevent the drop.
+pub struct DropGuard<T: ?Sized> {
+ ptr: *mut T,
+}
+
+impl<T: ?Sized> DropGuard<T> {
+ /// Creates a new [`DropGuard<T>`]. It will [`ptr::drop_in_place`] `ptr` when it gets dropped.
+ ///
+ /// # Safety
+ ///
+ /// `ptr` must be a valid pointer.
+ ///
+ /// It is the callers responsibility that `self` will only get dropped if the pointee of `ptr`:
+ /// - has not been dropped,
+ /// - is not accessible by any other means,
+ /// - will not be dropped by any other means.
+ #[inline]
+ pub unsafe fn new(ptr: *mut T) -> Self {
+ Self { ptr }
+ }
+}
+
+impl<T: ?Sized> Drop for DropGuard<T> {
+ #[inline]
+ fn drop(&mut self) {
+ // SAFETY: A `DropGuard` can only be constructed using the unsafe `new` function
+ // ensuring that this operation is safe.
+ unsafe { ptr::drop_in_place(self.ptr) }
+ }
+}
+
+/// Token used by `PinnedDrop` to prevent calling the function without creating this unsafely
+/// created struct. This is needed, because the `drop` function is safe, but should not be called
+/// manually.
+pub struct OnlyCallFromDrop(());
+
+impl OnlyCallFromDrop {
+ /// # Safety
+ ///
+ /// This function should only be called from the [`Drop::drop`] function and only be used to
+ /// delegate the destruction to the pinned destructor [`PinnedDrop::drop`] of the same type.
+ pub unsafe fn new() -> Self {
+ Self(())
+ }
+}
+
+/// Initializer that always fails.
+///
+/// Used by [`assert_pinned!`].
+///
+/// [`assert_pinned!`]: crate::assert_pinned
+pub struct AlwaysFail<T: ?Sized> {
+ _t: PhantomData<T>,
+}
+
+impl<T: ?Sized> AlwaysFail<T> {
+ /// Creates a new initializer that always fails.
+ pub fn new() -> Self {
+ Self { _t: PhantomData }
+ }
+}
+
+impl<T: ?Sized> Default for AlwaysFail<T> {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+// SAFETY: `__pinned_init` always fails, which is always okay.
+unsafe impl<T: ?Sized> PinInit<T, ()> for AlwaysFail<T> {
+ unsafe fn __pinned_init(self, _slot: *mut T) -> Result<(), ()> {
+ Err(())
+ }
+}
--- /dev/null
+// SPDX-License-Identifier: Apache-2.0 OR MIT
+
+//! API to safely and fallibly initialize pinned `struct`s using in-place constructors.
+//!
+//! It also allows in-place initialization of big `struct`s that would otherwise produce a stack
+//! overflow.
+//!
+//! Most `struct`s from the [`sync`] module need to be pinned, because they contain self-referential
+//! `struct`s from C. [Pinning][pinning] is Rust's way of ensuring data does not move.
+//!
+//! # Overview
+//!
+//! To initialize a `struct` with an in-place constructor you will need two things:
+//! - an in-place constructor,
+//! - a memory location that can hold your `struct` (this can be the [stack], an [`Arc<T>`],
+//! [`UniqueArc<T>`], [`KBox<T>`] or any other smart pointer that implements [`InPlaceInit`]).
+//!
+//! To get an in-place constructor there are generally three options:
+//! - directly creating an in-place constructor using the [`pin_init!`] macro,
+//! - a custom function/macro returning an in-place constructor provided by someone else,
+//! - using the unsafe function [`pin_init_from_closure()`] to manually create an initializer.
+//!
+//! Aside from pinned initialization, this API also supports in-place construction without pinning,
+//! the macros/types/functions are generally named like the pinned variants without the `pin`
+//! prefix.
+//!
+//! # Examples
+//!
+//! ## Using the [`pin_init!`] macro
+//!
+//! If you want to use [`PinInit`], then you will have to annotate your `struct` with
+//! `#[`[`pin_data`]`]`. It is a macro that uses `#[pin]` as a marker for
+//! [structurally pinned fields]. After doing this, you can then create an in-place constructor via
+//! [`pin_init!`]. The syntax is almost the same as normal `struct` initializers. The difference is
+//! that you need to write `<-` instead of `:` for fields that you want to initialize in-place.
+//!
+//! ```rust,ignore
+//! # #![expect(clippy::disallowed_names)]
+//! use kernel::sync::{new_mutex, Mutex};
+//! # use core::pin::Pin;
+//! #[pin_data]
+//! struct Foo {
+//! #[pin]
+//! a: Mutex<usize>,
+//! b: u32,
+//! }
+//!
+//! let foo = pin_init!(Foo {
+//! a <- new_mutex!(42, "Foo::a"),
+//! b: 24,
+//! });
+//! ```
+//!
+//! `foo` now is of the type [`impl PinInit<Foo>`]. We can now use any smart pointer that we like
+//! (or just the stack) to actually initialize a `Foo`:
+//!
+//! ```rust,ignore
+//! # #![expect(clippy::disallowed_names)]
+//! # use kernel::sync::{new_mutex, Mutex};
+//! # use core::pin::Pin;
+//! # #[pin_data]
+//! # struct Foo {
+//! # #[pin]
+//! # a: Mutex<usize>,
+//! # b: u32,
+//! # }
+//! # let foo = pin_init!(Foo {
+//! # a <- new_mutex!(42, "Foo::a"),
+//! # b: 24,
+//! # });
+//! let foo: Result<Pin<KBox<Foo>>> = KBox::pin_init(foo, GFP_KERNEL);
+//! ```
+//!
+//! For more information see the [`pin_init!`] macro.
+//!
+//! ## Using a custom function/macro that returns an initializer
+//!
+//! Many types from the kernel supply a function/macro that returns an initializer, because the
+//! above method only works for types where you can access the fields.
+//!
+//! ```rust,ignore
+//! # use kernel::sync::{new_mutex, Arc, Mutex};
+//! let mtx: Result<Arc<Mutex<usize>>> =
+//! Arc::pin_init(new_mutex!(42, "example::mtx"), GFP_KERNEL);
+//! ```
+//!
+//! To declare an init macro/function you just return an [`impl PinInit<T, E>`]:
+//!
+//! ```rust,ignore
+//! # use kernel::{sync::Mutex, new_mutex, init::PinInit, try_pin_init};
+//! #[pin_data]
+//! struct DriverData {
+//! #[pin]
+//! status: Mutex<i32>,
+//! buffer: KBox<[u8; 1_000_000]>,
+//! }
+//!
+//! impl DriverData {
+//! fn new() -> impl PinInit<Self, Error> {
+//! try_pin_init!(Self {
+//! status <- new_mutex!(0, "DriverData::status"),
+//! buffer: KBox::init(kernel::init::zeroed(), GFP_KERNEL)?,
+//! })
+//! }
+//! }
+//! ```
+//!
+//! ## Manual creation of an initializer
+//!
+//! Often when working with primitives the previous approaches are not sufficient. That is where
+//! [`pin_init_from_closure()`] comes in. This `unsafe` function allows you to create a
+//! [`impl PinInit<T, E>`] directly from a closure. Of course you have to ensure that the closure
+//! actually does the initialization in the correct way. Here are the things to look out for
+//! (we are calling the parameter to the closure `slot`):
+//! - when the closure returns `Ok(())`, then it has completed the initialization successfully, so
+//! `slot` now contains a valid bit pattern for the type `T`,
+//! - when the closure returns `Err(e)`, then the caller may deallocate the memory at `slot`, so
+//! you need to take care to clean up anything if your initialization fails mid-way,
+//! - you may assume that `slot` will stay pinned even after the closure returns until `drop` of
+//! `slot` gets called.
+//!
+//! ```rust,ignore
+//! # #![expect(unreachable_pub, clippy::disallowed_names)]
+//! use kernel::{init, types::Opaque};
+//! use core::{ptr::addr_of_mut, marker::PhantomPinned, pin::Pin};
+//! # mod bindings {
+//! # #![expect(non_camel_case_types)]
+//! # #![expect(clippy::missing_safety_doc)]
+//! # pub struct foo;
+//! # pub unsafe fn init_foo(_ptr: *mut foo) {}
+//! # pub unsafe fn destroy_foo(_ptr: *mut foo) {}
+//! # pub unsafe fn enable_foo(_ptr: *mut foo, _flags: u32) -> i32 { 0 }
+//! # }
+//! # // `Error::from_errno` is `pub(crate)` in the `kernel` crate, thus provide a workaround.
+//! # trait FromErrno {
+//! # fn from_errno(errno: kernel::ffi::c_int) -> Error {
+//! # // Dummy error that can be constructed outside the `kernel` crate.
+//! # Error::from(core::fmt::Error)
+//! # }
+//! # }
+//! # impl FromErrno for Error {}
+//! /// # Invariants
+//! ///
+//! /// `foo` is always initialized
+//! #[pin_data(PinnedDrop)]
+//! pub struct RawFoo {
+//! #[pin]
+//! foo: Opaque<bindings::foo>,
+//! #[pin]
+//! _p: PhantomPinned,
+//! }
+//!
+//! impl RawFoo {
+//! pub fn new(flags: u32) -> impl PinInit<Self, Error> {
+//! // SAFETY:
+//! // - when the closure returns `Ok(())`, then it has successfully initialized and
+//! // enabled `foo`,
+//! // - when it returns `Err(e)`, then it has cleaned up before
+//! unsafe {
+//! init::pin_init_from_closure(move |slot: *mut Self| {
+//! // `slot` contains uninit memory, avoid creating a reference.
+//! let foo = addr_of_mut!((*slot).foo);
+//!
+//! // Initialize the `foo`
+//! bindings::init_foo(Opaque::raw_get(foo));
+//!
+//! // Try to enable it.
+//! let err = bindings::enable_foo(Opaque::raw_get(foo), flags);
+//! if err != 0 {
+//! // Enabling has failed, first clean up the foo and then return the error.
+//! bindings::destroy_foo(Opaque::raw_get(foo));
+//! return Err(Error::from_errno(err));
+//! }
+//!
+//! // All fields of `RawFoo` have been initialized, since `_p` is a ZST.
+//! Ok(())
+//! })
+//! }
+//! }
+//! }
+//!
+//! #[pinned_drop]
+//! impl PinnedDrop for RawFoo {
+//! fn drop(self: Pin<&mut Self>) {
+//! // SAFETY: Since `foo` is initialized, destroying is safe.
+//! unsafe { bindings::destroy_foo(self.foo.get()) };
+//! }
+//! }
+//! ```
+//!
+//! For the special case where initializing a field is a single FFI-function call that cannot fail,
+//! there exist the helper function [`Opaque::ffi_init`]. This function initialize a single
+//! [`Opaque`] field by just delegating to the supplied closure. You can use these in combination
+//! with [`pin_init!`].
+//!
+//! For more information on how to use [`pin_init_from_closure()`], take a look at the uses inside
+//! the `kernel` crate. The [`sync`] module is a good starting point.
+//!
+//! [`sync`]: kernel::sync
+//! [pinning]: https://doc.rust-lang.org/std/pin/index.html
+//! [structurally pinned fields]:
+//! https://doc.rust-lang.org/std/pin/index.html#pinning-is-structural-for-field
+//! [stack]: crate::stack_pin_init
+//! [`Arc<T>`]: crate::sync::Arc
+//! [`impl PinInit<Foo>`]: PinInit
+//! [`impl PinInit<T, E>`]: PinInit
+//! [`impl Init<T, E>`]: Init
+//! [`Opaque`]: kernel::types::Opaque
+//! [`Opaque::ffi_init`]: kernel::types::Opaque::ffi_init
+//! [`pin_data`]: ::macros::pin_data
+//! [`pin_init!`]: crate::pin_init!
+
+use crate::{
+ alloc::{AllocError, Flags, KBox},
+ error::{self, Error},
+ sync::Arc,
+ sync::UniqueArc,
+ types::{Opaque, ScopeGuard},
+};
+use core::{
+ cell::UnsafeCell,
+ convert::Infallible,
+ marker::PhantomData,
+ mem::MaybeUninit,
+ num::*,
+ pin::Pin,
+ ptr::{self, NonNull},
+};
+
+#[doc(hidden)]
+pub mod __internal;
+#[doc(hidden)]
+pub mod macros;
+
+/// Initialize and pin a type directly on the stack.
+///
+/// # Examples
+///
+/// ```rust,ignore
+/// # #![expect(clippy::disallowed_names)]
+/// # use kernel::{init, macros::pin_data, pin_init, stack_pin_init, init::*, sync::Mutex, new_mutex};
+/// # use core::pin::Pin;
+/// #[pin_data]
+/// struct Foo {
+/// #[pin]
+/// a: Mutex<usize>,
+/// b: Bar,
+/// }
+///
+/// #[pin_data]
+/// struct Bar {
+/// x: u32,
+/// }
+///
+/// stack_pin_init!(let foo = pin_init!(Foo {
+/// a <- new_mutex!(42),
+/// b: Bar {
+/// x: 64,
+/// },
+/// }));
+/// let foo: Pin<&mut Foo> = foo;
+/// pr_info!("a: {}", &*foo.a.lock());
+/// ```
+///
+/// # Syntax
+///
+/// A normal `let` binding with optional type annotation. The expression is expected to implement
+/// [`PinInit`]/[`Init`] with the error type [`Infallible`]. If you want to use a different error
+/// type, then use [`stack_try_pin_init!`].
+///
+/// [`stack_try_pin_init!`]: crate::stack_try_pin_init!
+#[macro_export]
+macro_rules! stack_pin_init {
+ (let $var:ident $(: $t:ty)? = $val:expr) => {
+ let val = $val;
+ let mut $var = ::core::pin::pin!($crate::init::__internal::StackInit$(::<$t>)?::uninit());
+ let mut $var = match $crate::init::__internal::StackInit::init($var, val) {
+ Ok(res) => res,
+ Err(x) => {
+ let x: ::core::convert::Infallible = x;
+ match x {}
+ }
+ };
+ };
+}
+
+/// Initialize and pin a type directly on the stack.
+///
+/// # Examples
+///
+/// ```rust,ignore
+/// # #![expect(clippy::disallowed_names)]
+/// # use kernel::{
+/// # init,
+/// # pin_init,
+/// # stack_try_pin_init,
+/// # init::*,
+/// # sync::Mutex,
+/// # new_mutex,
+/// # alloc::AllocError,
+/// # };
+/// # use macros::pin_data;
+/// # use core::pin::Pin;
+/// #[pin_data]
+/// struct Foo {
+/// #[pin]
+/// a: Mutex<usize>,
+/// b: KBox<Bar>,
+/// }
+///
+/// struct Bar {
+/// x: u32,
+/// }
+///
+/// stack_try_pin_init!(let foo: Result<Pin<&mut Foo>, AllocError> = pin_init!(Foo {
+/// a <- new_mutex!(42),
+/// b: KBox::new(Bar {
+/// x: 64,
+/// }, GFP_KERNEL)?,
+/// }));
+/// let foo = foo.unwrap();
+/// pr_info!("a: {}", &*foo.a.lock());
+/// ```
+///
+/// ```rust,ignore
+/// # #![expect(clippy::disallowed_names)]
+/// # use kernel::{
+/// # init,
+/// # pin_init,
+/// # stack_try_pin_init,
+/// # init::*,
+/// # sync::Mutex,
+/// # new_mutex,
+/// # alloc::AllocError,
+/// # };
+/// # use macros::pin_data;
+/// # use core::pin::Pin;
+/// #[pin_data]
+/// struct Foo {
+/// #[pin]
+/// a: Mutex<usize>,
+/// b: KBox<Bar>,
+/// }
+///
+/// struct Bar {
+/// x: u32,
+/// }
+///
+/// stack_try_pin_init!(let foo: Pin<&mut Foo> =? pin_init!(Foo {
+/// a <- new_mutex!(42),
+/// b: KBox::new(Bar {
+/// x: 64,
+/// }, GFP_KERNEL)?,
+/// }));
+/// pr_info!("a: {}", &*foo.a.lock());
+/// # Ok::<_, AllocError>(())
+/// ```
+///
+/// # Syntax
+///
+/// A normal `let` binding with optional type annotation. The expression is expected to implement
+/// [`PinInit`]/[`Init`]. This macro assigns a result to the given variable, adding a `?` after the
+/// `=` will propagate this error.
+#[macro_export]
+macro_rules! stack_try_pin_init {
+ (let $var:ident $(: $t:ty)? = $val:expr) => {
+ let val = $val;
+ let mut $var = ::core::pin::pin!($crate::init::__internal::StackInit$(::<$t>)?::uninit());
+ let mut $var = $crate::init::__internal::StackInit::init($var, val);
+ };
+ (let $var:ident $(: $t:ty)? =? $val:expr) => {
+ let val = $val;
+ let mut $var = ::core::pin::pin!($crate::init::__internal::StackInit$(::<$t>)?::uninit());
+ let mut $var = $crate::init::__internal::StackInit::init($var, val)?;
+ };
+}
+
+/// Construct an in-place, pinned initializer for `struct`s.
+///
+/// This macro defaults the error to [`Infallible`]. If you need [`Error`], then use
+/// [`try_pin_init!`].
+///
+/// The syntax is almost identical to that of a normal `struct` initializer:
+///
+/// ```rust,ignore
+/// # use kernel::{init, pin_init, macros::pin_data, init::*};
+/// # use core::pin::Pin;
+/// #[pin_data]
+/// struct Foo {
+/// a: usize,
+/// b: Bar,
+/// }
+///
+/// #[pin_data]
+/// struct Bar {
+/// x: u32,
+/// }
+///
+/// # fn demo() -> impl PinInit<Foo> {
+/// let a = 42;
+///
+/// let initializer = pin_init!(Foo {
+/// a,
+/// b: Bar {
+/// x: 64,
+/// },
+/// });
+/// # initializer }
+/// # KBox::pin_init(demo(), GFP_KERNEL).unwrap();
+/// ```
+///
+/// Arbitrary Rust expressions can be used to set the value of a variable.
+///
+/// The fields are initialized in the order that they appear in the initializer. So it is possible
+/// to read already initialized fields using raw pointers.
+///
+/// IMPORTANT: You are not allowed to create references to fields of the struct inside of the
+/// initializer.
+///
+/// # Init-functions
+///
+/// When working with this API it is often desired to let others construct your types without
+/// giving access to all fields. This is where you would normally write a plain function `new`
+/// that would return a new instance of your type. With this API that is also possible.
+/// However, there are a few extra things to keep in mind.
+///
+/// To create an initializer function, simply declare it like this:
+///
+/// ```rust,ignore
+/// # use kernel::{init, pin_init, init::*};
+/// # use core::pin::Pin;
+/// # #[pin_data]
+/// # struct Foo {
+/// # a: usize,
+/// # b: Bar,
+/// # }
+/// # #[pin_data]
+/// # struct Bar {
+/// # x: u32,
+/// # }
+/// impl Foo {
+/// fn new() -> impl PinInit<Self> {
+/// pin_init!(Self {
+/// a: 42,
+/// b: Bar {
+/// x: 64,
+/// },
+/// })
+/// }
+/// }
+/// ```
+///
+/// Users of `Foo` can now create it like this:
+///
+/// ```rust,ignore
+/// # #![expect(clippy::disallowed_names)]
+/// # use kernel::{init, pin_init, macros::pin_data, init::*};
+/// # use core::pin::Pin;
+/// # #[pin_data]
+/// # struct Foo {
+/// # a: usize,
+/// # b: Bar,
+/// # }
+/// # #[pin_data]
+/// # struct Bar {
+/// # x: u32,
+/// # }
+/// # impl Foo {
+/// # fn new() -> impl PinInit<Self> {
+/// # pin_init!(Self {
+/// # a: 42,
+/// # b: Bar {
+/// # x: 64,
+/// # },
+/// # })
+/// # }
+/// # }
+/// let foo = KBox::pin_init(Foo::new(), GFP_KERNEL);
+/// ```
+///
+/// They can also easily embed it into their own `struct`s:
+///
+/// ```rust,ignore
+/// # use kernel::{init, pin_init, macros::pin_data, init::*};
+/// # use core::pin::Pin;
+/// # #[pin_data]
+/// # struct Foo {
+/// # a: usize,
+/// # b: Bar,
+/// # }
+/// # #[pin_data]
+/// # struct Bar {
+/// # x: u32,
+/// # }
+/// # impl Foo {
+/// # fn new() -> impl PinInit<Self> {
+/// # pin_init!(Self {
+/// # a: 42,
+/// # b: Bar {
+/// # x: 64,
+/// # },
+/// # })
+/// # }
+/// # }
+/// #[pin_data]
+/// struct FooContainer {
+/// #[pin]
+/// foo1: Foo,
+/// #[pin]
+/// foo2: Foo,
+/// other: u32,
+/// }
+///
+/// impl FooContainer {
+/// fn new(other: u32) -> impl PinInit<Self> {
+/// pin_init!(Self {
+/// foo1 <- Foo::new(),
+/// foo2 <- Foo::new(),
+/// other,
+/// })
+/// }
+/// }
+/// ```
+///
+/// Here we see that when using `pin_init!` with `PinInit`, one needs to write `<-` instead of `:`.
+/// This signifies that the given field is initialized in-place. As with `struct` initializers, just
+/// writing the field (in this case `other`) without `:` or `<-` means `other: other,`.
+///
+/// # Syntax
+///
+/// As already mentioned in the examples above, inside of `pin_init!` a `struct` initializer with
+/// the following modifications is expected:
+/// - Fields that you want to initialize in-place have to use `<-` instead of `:`.
+/// - In front of the initializer you can write `&this in` to have access to a [`NonNull<Self>`]
+/// pointer named `this` inside of the initializer.
+/// - Using struct update syntax one can place `..Zeroable::zeroed()` at the very end of the
+/// struct, this initializes every field with 0 and then runs all initializers specified in the
+/// body. This can only be done if [`Zeroable`] is implemented for the struct.
+///
+/// For instance:
+///
+/// ```rust,ignore
+/// # use kernel::{macros::{Zeroable, pin_data}, pin_init};
+/// # use core::{ptr::addr_of_mut, marker::PhantomPinned};
+/// #[pin_data]
+/// #[derive(Zeroable)]
+/// struct Buf {
+/// // `ptr` points into `buf`.
+/// ptr: *mut u8,
+/// buf: [u8; 64],
+/// #[pin]
+/// pin: PhantomPinned,
+/// }
+/// pin_init!(&this in Buf {
+/// buf: [0; 64],
+/// // SAFETY: TODO.
+/// ptr: unsafe { addr_of_mut!((*this.as_ptr()).buf).cast() },
+/// pin: PhantomPinned,
+/// });
+/// pin_init!(Buf {
+/// buf: [1; 64],
+/// ..Zeroable::zeroed()
+/// });
+/// ```
+///
+/// [`try_pin_init!`]: kernel::try_pin_init
+/// [`NonNull<Self>`]: core::ptr::NonNull
+// For a detailed example of how this macro works, see the module documentation of the hidden
+// module `__internal` inside of `init/__internal.rs`.
+#[macro_export]
+macro_rules! pin_init {
+ ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
+ $($fields:tt)*
+ }) => {
+ $crate::__init_internal!(
+ @this($($this)?),
+ @typ($t $(::<$($generics),*>)?),
+ @fields($($fields)*),
+ @error(::core::convert::Infallible),
+ @data(PinData, use_data),
+ @has_data(HasPinData, __pin_data),
+ @construct_closure(pin_init_from_closure),
+ @munch_fields($($fields)*),
+ )
+ };
+}
+
+/// Construct an in-place, fallible pinned initializer for `struct`s.
+///
+/// If the initialization can complete without error (or [`Infallible`]), then use [`pin_init!`].
+///
+/// You can use the `?` operator or use `return Err(err)` inside the initializer to stop
+/// initialization and return the error.
+///
+/// IMPORTANT: if you have `unsafe` code inside of the initializer you have to ensure that when
+/// initialization fails, the memory can be safely deallocated without any further modifications.
+///
+/// This macro defaults the error to [`Error`].
+///
+/// The syntax is identical to [`pin_init!`] with the following exception: you can append `? $type`
+/// after the `struct` initializer to specify the error type you want to use.
+///
+/// # Examples
+///
+/// ```rust,ignore
+/// use kernel::{init::{self, PinInit}, error::Error};
+/// #[pin_data]
+/// struct BigBuf {
+/// big: KBox<[u8; 1024 * 1024 * 1024]>,
+/// small: [u8; 1024 * 1024],
+/// ptr: *mut u8,
+/// }
+///
+/// impl BigBuf {
+/// fn new() -> impl PinInit<Self, Error> {
+/// try_pin_init!(Self {
+/// big: KBox::init(init::zeroed(), GFP_KERNEL)?,
+/// small: [0; 1024 * 1024],
+/// ptr: core::ptr::null_mut(),
+/// }? Error)
+/// }
+/// }
+/// ```
+// For a detailed example of how this macro works, see the module documentation of the hidden
+// module `__internal` inside of `init/__internal.rs`.
+#[macro_export]
+macro_rules! try_pin_init {
+ ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
+ $($fields:tt)*
+ }) => {
+ $crate::__init_internal!(
+ @this($($this)?),
+ @typ($t $(::<$($generics),*>)? ),
+ @fields($($fields)*),
+ @error($crate::error::Error),
+ @data(PinData, use_data),
+ @has_data(HasPinData, __pin_data),
+ @construct_closure(pin_init_from_closure),
+ @munch_fields($($fields)*),
+ )
+ };
+ ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
+ $($fields:tt)*
+ }? $err:ty) => {
+ $crate::__init_internal!(
+ @this($($this)?),
+ @typ($t $(::<$($generics),*>)? ),
+ @fields($($fields)*),
+ @error($err),
+ @data(PinData, use_data),
+ @has_data(HasPinData, __pin_data),
+ @construct_closure(pin_init_from_closure),
+ @munch_fields($($fields)*),
+ )
+ };
+}
+
+/// Construct an in-place initializer for `struct`s.
+///
+/// This macro defaults the error to [`Infallible`]. If you need [`Error`], then use
+/// [`try_init!`].
+///
+/// The syntax is identical to [`pin_init!`] and its safety caveats also apply:
+/// - `unsafe` code must guarantee either full initialization or return an error and allow
+/// deallocation of the memory.
+/// - the fields are initialized in the order given in the initializer.
+/// - no references to fields are allowed to be created inside of the initializer.
+///
+/// This initializer is for initializing data in-place that might later be moved. If you want to
+/// pin-initialize, use [`pin_init!`].
+///
+/// [`try_init!`]: crate::try_init!
+// For a detailed example of how this macro works, see the module documentation of the hidden
+// module `__internal` inside of `init/__internal.rs`.
+#[macro_export]
+macro_rules! init {
+ ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
+ $($fields:tt)*
+ }) => {
+ $crate::__init_internal!(
+ @this($($this)?),
+ @typ($t $(::<$($generics),*>)?),
+ @fields($($fields)*),
+ @error(::core::convert::Infallible),
+ @data(InitData, /*no use_data*/),
+ @has_data(HasInitData, __init_data),
+ @construct_closure(init_from_closure),
+ @munch_fields($($fields)*),
+ )
+ }
+}
+
+/// Construct an in-place fallible initializer for `struct`s.
+///
+/// This macro defaults the error to [`Error`]. If you need [`Infallible`], then use
+/// [`init!`].
+///
+/// The syntax is identical to [`try_pin_init!`]. If you want to specify a custom error,
+/// append `? $type` after the `struct` initializer.
+/// The safety caveats from [`try_pin_init!`] also apply:
+/// - `unsafe` code must guarantee either full initialization or return an error and allow
+/// deallocation of the memory.
+/// - the fields are initialized in the order given in the initializer.
+/// - no references to fields are allowed to be created inside of the initializer.
+///
+/// # Examples
+///
+/// ```rust,ignore
+/// use kernel::{alloc::KBox, init::{PinInit, zeroed}, error::Error};
+/// struct BigBuf {
+/// big: KBox<[u8; 1024 * 1024 * 1024]>,
+/// small: [u8; 1024 * 1024],
+/// }
+///
+/// impl BigBuf {
+/// fn new() -> impl Init<Self, Error> {
+/// try_init!(Self {
+/// big: KBox::init(zeroed(), GFP_KERNEL)?,
+/// small: [0; 1024 * 1024],
+/// }? Error)
+/// }
+/// }
+/// ```
+// For a detailed example of how this macro works, see the module documentation of the hidden
+// module `__internal` inside of `init/__internal.rs`.
+#[macro_export]
+macro_rules! try_init {
+ ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
+ $($fields:tt)*
+ }) => {
+ $crate::__init_internal!(
+ @this($($this)?),
+ @typ($t $(::<$($generics),*>)?),
+ @fields($($fields)*),
+ @error($crate::error::Error),
+ @data(InitData, /*no use_data*/),
+ @has_data(HasInitData, __init_data),
+ @construct_closure(init_from_closure),
+ @munch_fields($($fields)*),
+ )
+ };
+ ($(&$this:ident in)? $t:ident $(::<$($generics:ty),* $(,)?>)? {
+ $($fields:tt)*
+ }? $err:ty) => {
+ $crate::__init_internal!(
+ @this($($this)?),
+ @typ($t $(::<$($generics),*>)?),
+ @fields($($fields)*),
+ @error($err),
+ @data(InitData, /*no use_data*/),
+ @has_data(HasInitData, __init_data),
+ @construct_closure(init_from_closure),
+ @munch_fields($($fields)*),
+ )
+ };
+}
+
+/// Asserts that a field on a struct using `#[pin_data]` is marked with `#[pin]` ie. that it is
+/// structurally pinned.
+///
+/// # Example
+///
+/// This will succeed:
+/// ```ignore
+/// use kernel::assert_pinned;
+/// #[pin_data]
+/// struct MyStruct {
+/// #[pin]
+/// some_field: u64,
+/// }
+///
+/// assert_pinned!(MyStruct, some_field, u64);
+/// ```
+///
+/// This will fail:
+// TODO: replace with `compile_fail` when supported.
+/// ```ignore
+/// use kernel::assert_pinned;
+/// #[pin_data]
+/// struct MyStruct {
+/// some_field: u64,
+/// }
+///
+/// assert_pinned!(MyStruct, some_field, u64);
+/// ```
+///
+/// Some uses of the macro may trigger the `can't use generic parameters from outer item` error. To
+/// work around this, you may pass the `inline` parameter to the macro. The `inline` parameter can
+/// only be used when the macro is invoked from a function body.
+/// ```ignore
+/// use kernel::assert_pinned;
+/// #[pin_data]
+/// struct Foo<T> {
+/// #[pin]
+/// elem: T,
+/// }
+///
+/// impl<T> Foo<T> {
+/// fn project(self: Pin<&mut Self>) -> Pin<&mut T> {
+/// assert_pinned!(Foo<T>, elem, T, inline);
+///
+/// // SAFETY: The field is structurally pinned.
+/// unsafe { self.map_unchecked_mut(|me| &mut me.elem) }
+/// }
+/// }
+/// ```
+#[macro_export]
+macro_rules! assert_pinned {
+ ($ty:ty, $field:ident, $field_ty:ty, inline) => {
+ let _ = move |ptr: *mut $field_ty| {
+ // SAFETY: This code is unreachable.
+ let data = unsafe { <$ty as $crate::init::__internal::HasPinData>::__pin_data() };
+ let init = $crate::init::__internal::AlwaysFail::<$field_ty>::new();
+ // SAFETY: This code is unreachable.
+ unsafe { data.$field(ptr, init) }.ok();
+ };
+ };
+
+ ($ty:ty, $field:ident, $field_ty:ty) => {
+ const _: () = {
+ $crate::assert_pinned!($ty, $field, $field_ty, inline);
+ };
+ };
+}
+
+/// A pin-initializer for the type `T`.
+///
+/// To use this initializer, you will need a suitable memory location that can hold a `T`. This can
+/// be [`KBox<T>`], [`Arc<T>`], [`UniqueArc<T>`] or even the stack (see [`stack_pin_init!`]). Use
+/// the [`InPlaceInit::pin_init`] function of a smart pointer like [`Arc<T>`] on this.
+///
+/// Also see the [module description](self).
+///
+/// # Safety
+///
+/// When implementing this trait you will need to take great care. Also there are probably very few
+/// cases where a manual implementation is necessary. Use [`pin_init_from_closure`] where possible.
+///
+/// The [`PinInit::__pinned_init`] function:
+/// - returns `Ok(())` if it initialized every field of `slot`,
+/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
+/// - `slot` can be deallocated without UB occurring,
+/// - `slot` does not need to be dropped,
+/// - `slot` is not partially initialized.
+/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`.
+///
+/// [`Arc<T>`]: crate::sync::Arc
+/// [`Arc::pin_init`]: crate::sync::Arc::pin_init
+#[must_use = "An initializer must be used in order to create its value."]
+pub unsafe trait PinInit<T: ?Sized, E = Infallible>: Sized {
+ /// Initializes `slot`.
+ ///
+ /// # Safety
+ ///
+ /// - `slot` is a valid pointer to uninitialized memory.
+ /// - the caller does not touch `slot` when `Err` is returned, they are only permitted to
+ /// deallocate.
+ /// - `slot` will not move until it is dropped, i.e. it will be pinned.
+ unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E>;
+
+ /// First initializes the value using `self` then calls the function `f` with the initialized
+ /// value.
+ ///
+ /// If `f` returns an error the value is dropped and the initializer will forward the error.
+ ///
+ /// # Examples
+ ///
+ /// ```rust,ignore
+ /// # #![expect(clippy::disallowed_names)]
+ /// use kernel::{types::Opaque, init::pin_init_from_closure};
+ /// #[repr(C)]
+ /// struct RawFoo([u8; 16]);
+ /// extern "C" {
+ /// fn init_foo(_: *mut RawFoo);
+ /// }
+ ///
+ /// #[pin_data]
+ /// struct Foo {
+ /// #[pin]
+ /// raw: Opaque<RawFoo>,
+ /// }
+ ///
+ /// impl Foo {
+ /// fn setup(self: Pin<&mut Self>) {
+ /// pr_info!("Setting up foo");
+ /// }
+ /// }
+ ///
+ /// let foo = pin_init!(Foo {
+ /// // SAFETY: TODO.
+ /// raw <- unsafe {
+ /// Opaque::ffi_init(|s| {
+ /// init_foo(s);
+ /// })
+ /// },
+ /// }).pin_chain(|foo| {
+ /// foo.setup();
+ /// Ok(())
+ /// });
+ /// ```
+ fn pin_chain<F>(self, f: F) -> ChainPinInit<Self, F, T, E>
+ where
+ F: FnOnce(Pin<&mut T>) -> Result<(), E>,
+ {
+ ChainPinInit(self, f, PhantomData)
+ }
+}
+
+/// An initializer returned by [`PinInit::pin_chain`].
+pub struct ChainPinInit<I, F, T: ?Sized, E>(I, F, __internal::Invariant<(E, KBox<T>)>);
+
+// SAFETY: The `__pinned_init` function is implemented such that it
+// - returns `Ok(())` on successful initialization,
+// - returns `Err(err)` on error and in this case `slot` will be dropped.
+// - considers `slot` pinned.
+unsafe impl<T: ?Sized, E, I, F> PinInit<T, E> for ChainPinInit<I, F, T, E>
+where
+ I: PinInit<T, E>,
+ F: FnOnce(Pin<&mut T>) -> Result<(), E>,
+{
+ unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
+ // SAFETY: All requirements fulfilled since this function is `__pinned_init`.
+ unsafe { self.0.__pinned_init(slot)? };
+ // SAFETY: The above call initialized `slot` and we still have unique access.
+ let val = unsafe { &mut *slot };
+ // SAFETY: `slot` is considered pinned.
+ let val = unsafe { Pin::new_unchecked(val) };
+ // SAFETY: `slot` was initialized above.
+ (self.1)(val).inspect_err(|_| unsafe { core::ptr::drop_in_place(slot) })
+ }
+}
+
+/// An initializer for `T`.
+///
+/// To use this initializer, you will need a suitable memory location that can hold a `T`. This can
+/// be [`KBox<T>`], [`Arc<T>`], [`UniqueArc<T>`] or even the stack (see [`stack_pin_init!`]). Use
+/// the [`InPlaceInit::init`] function of a smart pointer like [`Arc<T>`] on this. Because
+/// [`PinInit<T, E>`] is a super trait, you can use every function that takes it as well.
+///
+/// Also see the [module description](self).
+///
+/// # Safety
+///
+/// When implementing this trait you will need to take great care. Also there are probably very few
+/// cases where a manual implementation is necessary. Use [`init_from_closure`] where possible.
+///
+/// The [`Init::__init`] function:
+/// - returns `Ok(())` if it initialized every field of `slot`,
+/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
+/// - `slot` can be deallocated without UB occurring,
+/// - `slot` does not need to be dropped,
+/// - `slot` is not partially initialized.
+/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`.
+///
+/// The `__pinned_init` function from the supertrait [`PinInit`] needs to execute the exact same
+/// code as `__init`.
+///
+/// Contrary to its supertype [`PinInit<T, E>`] the caller is allowed to
+/// move the pointee after initialization.
+///
+/// [`Arc<T>`]: crate::sync::Arc
+#[must_use = "An initializer must be used in order to create its value."]
+pub unsafe trait Init<T: ?Sized, E = Infallible>: PinInit<T, E> {
+ /// Initializes `slot`.
+ ///
+ /// # Safety
+ ///
+ /// - `slot` is a valid pointer to uninitialized memory.
+ /// - the caller does not touch `slot` when `Err` is returned, they are only permitted to
+ /// deallocate.
+ unsafe fn __init(self, slot: *mut T) -> Result<(), E>;
+
+ /// First initializes the value using `self` then calls the function `f` with the initialized
+ /// value.
+ ///
+ /// If `f` returns an error the value is dropped and the initializer will forward the error.
+ ///
+ /// # Examples
+ ///
+ /// ```rust,ignore
+ /// # #![expect(clippy::disallowed_names)]
+ /// use kernel::{types::Opaque, init::{self, init_from_closure}};
+ /// struct Foo {
+ /// buf: [u8; 1_000_000],
+ /// }
+ ///
+ /// impl Foo {
+ /// fn setup(&mut self) {
+ /// pr_info!("Setting up foo");
+ /// }
+ /// }
+ ///
+ /// let foo = init!(Foo {
+ /// buf <- init::zeroed()
+ /// }).chain(|foo| {
+ /// foo.setup();
+ /// Ok(())
+ /// });
+ /// ```
+ fn chain<F>(self, f: F) -> ChainInit<Self, F, T, E>
+ where
+ F: FnOnce(&mut T) -> Result<(), E>,
+ {
+ ChainInit(self, f, PhantomData)
+ }
+}
+
+/// An initializer returned by [`Init::chain`].
+pub struct ChainInit<I, F, T: ?Sized, E>(I, F, __internal::Invariant<(E, KBox<T>)>);
+
+// SAFETY: The `__init` function is implemented such that it
+// - returns `Ok(())` on successful initialization,
+// - returns `Err(err)` on error and in this case `slot` will be dropped.
+unsafe impl<T: ?Sized, E, I, F> Init<T, E> for ChainInit<I, F, T, E>
+where
+ I: Init<T, E>,
+ F: FnOnce(&mut T) -> Result<(), E>,
+{
+ unsafe fn __init(self, slot: *mut T) -> Result<(), E> {
+ // SAFETY: All requirements fulfilled since this function is `__init`.
+ unsafe { self.0.__pinned_init(slot)? };
+ // SAFETY: The above call initialized `slot` and we still have unique access.
+ (self.1)(unsafe { &mut *slot }).inspect_err(|_|
+ // SAFETY: `slot` was initialized above.
+ unsafe { core::ptr::drop_in_place(slot) })
+ }
+}
+
+// SAFETY: `__pinned_init` behaves exactly the same as `__init`.
+unsafe impl<T: ?Sized, E, I, F> PinInit<T, E> for ChainInit<I, F, T, E>
+where
+ I: Init<T, E>,
+ F: FnOnce(&mut T) -> Result<(), E>,
+{
+ unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
+ // SAFETY: `__init` has less strict requirements compared to `__pinned_init`.
+ unsafe { self.__init(slot) }
+ }
+}
+
+/// Creates a new [`PinInit<T, E>`] from the given closure.
+///
+/// # Safety
+///
+/// The closure:
+/// - returns `Ok(())` if it initialized every field of `slot`,
+/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
+/// - `slot` can be deallocated without UB occurring,
+/// - `slot` does not need to be dropped,
+/// - `slot` is not partially initialized.
+/// - may assume that the `slot` does not move if `T: !Unpin`,
+/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`.
+#[inline]
+pub const unsafe fn pin_init_from_closure<T: ?Sized, E>(
+ f: impl FnOnce(*mut T) -> Result<(), E>,
+) -> impl PinInit<T, E> {
+ __internal::InitClosure(f, PhantomData)
+}
+
+/// Creates a new [`Init<T, E>`] from the given closure.
+///
+/// # Safety
+///
+/// The closure:
+/// - returns `Ok(())` if it initialized every field of `slot`,
+/// - returns `Err(err)` if it encountered an error and then cleaned `slot`, this means:
+/// - `slot` can be deallocated without UB occurring,
+/// - `slot` does not need to be dropped,
+/// - `slot` is not partially initialized.
+/// - the `slot` may move after initialization.
+/// - while constructing the `T` at `slot` it upholds the pinning invariants of `T`.
+#[inline]
+pub const unsafe fn init_from_closure<T: ?Sized, E>(
+ f: impl FnOnce(*mut T) -> Result<(), E>,
+) -> impl Init<T, E> {
+ __internal::InitClosure(f, PhantomData)
+}
+
+/// An initializer that leaves the memory uninitialized.
+///
+/// The initializer is a no-op. The `slot` memory is not changed.
+#[inline]
+pub fn uninit<T, E>() -> impl Init<MaybeUninit<T>, E> {
+ // SAFETY: The memory is allowed to be uninitialized.
+ unsafe { init_from_closure(|_| Ok(())) }
+}
+
+/// Initializes an array by initializing each element via the provided initializer.
+///
+/// # Examples
+///
+/// ```rust,ignore
+/// use kernel::{alloc::KBox, error::Error, init::init_array_from_fn};
+/// let array: KBox<[usize; 1_000]> =
+/// KBox::init::<Error>(init_array_from_fn(|i| i), GFP_KERNEL)?;
+/// assert_eq!(array.len(), 1_000);
+/// # Ok::<(), Error>(())
+/// ```
+pub fn init_array_from_fn<I, const N: usize, T, E>(
+ mut make_init: impl FnMut(usize) -> I,
+) -> impl Init<[T; N], E>
+where
+ I: Init<T, E>,
+{
+ let init = move |slot: *mut [T; N]| {
+ let slot = slot.cast::<T>();
+ // Counts the number of initialized elements and when dropped drops that many elements from
+ // `slot`.
+ let mut init_count = ScopeGuard::new_with_data(0, |i| {
+ // We now free every element that has been initialized before.
+ // SAFETY: The loop initialized exactly the values from 0..i and since we
+ // return `Err` below, the caller will consider the memory at `slot` as
+ // uninitialized.
+ unsafe { ptr::drop_in_place(ptr::slice_from_raw_parts_mut(slot, i)) };
+ });
+ for i in 0..N {
+ let init = make_init(i);
+ // SAFETY: Since 0 <= `i` < N, it is still in bounds of `[T; N]`.
+ let ptr = unsafe { slot.add(i) };
+ // SAFETY: The pointer is derived from `slot` and thus satisfies the `__init`
+ // requirements.
+ unsafe { init.__init(ptr) }?;
+ *init_count += 1;
+ }
+ init_count.dismiss();
+ Ok(())
+ };
+ // SAFETY: The initializer above initializes every element of the array. On failure it drops
+ // any initialized elements and returns `Err`.
+ unsafe { init_from_closure(init) }
+}
+
+/// Initializes an array by initializing each element via the provided initializer.
+///
+/// # Examples
+///
+/// ```rust,ignore
+/// use kernel::{sync::{Arc, Mutex}, init::pin_init_array_from_fn, new_mutex};
+/// let array: Arc<[Mutex<usize>; 1_000]> =
+/// Arc::pin_init(pin_init_array_from_fn(|i| new_mutex!(i)), GFP_KERNEL)?;
+/// assert_eq!(array.len(), 1_000);
+/// # Ok::<(), Error>(())
+/// ```
+pub fn pin_init_array_from_fn<I, const N: usize, T, E>(
+ mut make_init: impl FnMut(usize) -> I,
+) -> impl PinInit<[T; N], E>
+where
+ I: PinInit<T, E>,
+{
+ let init = move |slot: *mut [T; N]| {
+ let slot = slot.cast::<T>();
+ // Counts the number of initialized elements and when dropped drops that many elements from
+ // `slot`.
+ let mut init_count = ScopeGuard::new_with_data(0, |i| {
+ // We now free every element that has been initialized before.
+ // SAFETY: The loop initialized exactly the values from 0..i and since we
+ // return `Err` below, the caller will consider the memory at `slot` as
+ // uninitialized.
+ unsafe { ptr::drop_in_place(ptr::slice_from_raw_parts_mut(slot, i)) };
+ });
+ for i in 0..N {
+ let init = make_init(i);
+ // SAFETY: Since 0 <= `i` < N, it is still in bounds of `[T; N]`.
+ let ptr = unsafe { slot.add(i) };
+ // SAFETY: The pointer is derived from `slot` and thus satisfies the `__init`
+ // requirements.
+ unsafe { init.__pinned_init(ptr) }?;
+ *init_count += 1;
+ }
+ init_count.dismiss();
+ Ok(())
+ };
+ // SAFETY: The initializer above initializes every element of the array. On failure it drops
+ // any initialized elements and returns `Err`.
+ unsafe { pin_init_from_closure(init) }
+}
+
+// SAFETY: Every type can be initialized by-value.
+unsafe impl<T, E> Init<T, E> for T {
+ unsafe fn __init(self, slot: *mut T) -> Result<(), E> {
+ // SAFETY: TODO.
+ unsafe { slot.write(self) };
+ Ok(())
+ }
+}
+
+// SAFETY: Every type can be initialized by-value. `__pinned_init` calls `__init`.
+unsafe impl<T, E> PinInit<T, E> for T {
+ unsafe fn __pinned_init(self, slot: *mut T) -> Result<(), E> {
+ // SAFETY: TODO.
+ unsafe { self.__init(slot) }
+ }
+}
+
+/// Smart pointer that can initialize memory in-place.
+pub trait InPlaceInit<T>: Sized {
+ /// Pinned version of `Self`.
+ ///
+ /// If a type already implicitly pins its pointee, `Pin<Self>` is unnecessary. In this case use
+ /// `Self`, otherwise just use `Pin<Self>`.
+ type PinnedSelf;
+
+ /// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this
+ /// type.
+ ///
+ /// If `T: !Unpin` it will not be able to move afterwards.
+ fn try_pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> Result<Self::PinnedSelf, E>
+ where
+ E: From<AllocError>;
+
+ /// Use the given pin-initializer to pin-initialize a `T` inside of a new smart pointer of this
+ /// type.
+ ///
+ /// If `T: !Unpin` it will not be able to move afterwards.
+ fn pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> error::Result<Self::PinnedSelf>
+ where
+ Error: From<E>,
+ {
+ // SAFETY: We delegate to `init` and only change the error type.
+ let init = unsafe {
+ pin_init_from_closure(|slot| init.__pinned_init(slot).map_err(|e| Error::from(e)))
+ };
+ Self::try_pin_init(init, flags)
+ }
+
+ /// Use the given initializer to in-place initialize a `T`.
+ fn try_init<E>(init: impl Init<T, E>, flags: Flags) -> Result<Self, E>
+ where
+ E: From<AllocError>;
+
+ /// Use the given initializer to in-place initialize a `T`.
+ fn init<E>(init: impl Init<T, E>, flags: Flags) -> error::Result<Self>
+ where
+ Error: From<E>,
+ {
+ // SAFETY: We delegate to `init` and only change the error type.
+ let init = unsafe {
+ init_from_closure(|slot| init.__pinned_init(slot).map_err(|e| Error::from(e)))
+ };
+ Self::try_init(init, flags)
+ }
+}
+
+impl<T> InPlaceInit<T> for Arc<T> {
+ type PinnedSelf = Self;
+
+ #[inline]
+ fn try_pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> Result<Self::PinnedSelf, E>
+ where
+ E: From<AllocError>,
+ {
+ UniqueArc::try_pin_init(init, flags).map(|u| u.into())
+ }
+
+ #[inline]
+ fn try_init<E>(init: impl Init<T, E>, flags: Flags) -> Result<Self, E>
+ where
+ E: From<AllocError>,
+ {
+ UniqueArc::try_init(init, flags).map(|u| u.into())
+ }
+}
+
+impl<T> InPlaceInit<T> for UniqueArc<T> {
+ type PinnedSelf = Pin<Self>;
+
+ #[inline]
+ fn try_pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> Result<Self::PinnedSelf, E>
+ where
+ E: From<AllocError>,
+ {
+ UniqueArc::new_uninit(flags)?.write_pin_init(init)
+ }
+
+ #[inline]
+ fn try_init<E>(init: impl Init<T, E>, flags: Flags) -> Result<Self, E>
+ where
+ E: From<AllocError>,
+ {
+ UniqueArc::new_uninit(flags)?.write_init(init)
+ }
+}
+
+/// Smart pointer containing uninitialized memory and that can write a value.
+pub trait InPlaceWrite<T> {
+ /// The type `Self` turns into when the contents are initialized.
+ type Initialized;
+
+ /// Use the given initializer to write a value into `self`.
+ ///
+ /// Does not drop the current value and considers it as uninitialized memory.
+ fn write_init<E>(self, init: impl Init<T, E>) -> Result<Self::Initialized, E>;
+
+ /// Use the given pin-initializer to write a value into `self`.
+ ///
+ /// Does not drop the current value and considers it as uninitialized memory.
+ fn write_pin_init<E>(self, init: impl PinInit<T, E>) -> Result<Pin<Self::Initialized>, E>;
+}
+
+impl<T> InPlaceWrite<T> for UniqueArc<MaybeUninit<T>> {
+ type Initialized = UniqueArc<T>;
+
+ fn write_init<E>(mut self, init: impl Init<T, E>) -> Result<Self::Initialized, E> {
+ let slot = self.as_mut_ptr();
+ // SAFETY: When init errors/panics, slot will get deallocated but not dropped,
+ // slot is valid.
+ unsafe { init.__init(slot)? };
+ // SAFETY: All fields have been initialized.
+ Ok(unsafe { self.assume_init() })
+ }
+
+ fn write_pin_init<E>(mut self, init: impl PinInit<T, E>) -> Result<Pin<Self::Initialized>, E> {
+ let slot = self.as_mut_ptr();
+ // SAFETY: When init errors/panics, slot will get deallocated but not dropped,
+ // slot is valid and will not be moved, because we pin it later.
+ unsafe { init.__pinned_init(slot)? };
+ // SAFETY: All fields have been initialized.
+ Ok(unsafe { self.assume_init() }.into())
+ }
+}
+
+/// Trait facilitating pinned destruction.
+///
+/// Use [`pinned_drop`] to implement this trait safely:
+///
+/// ```rust,ignore
+/// # use kernel::sync::Mutex;
+/// use kernel::macros::pinned_drop;
+/// use core::pin::Pin;
+/// #[pin_data(PinnedDrop)]
+/// struct Foo {
+/// #[pin]
+/// mtx: Mutex<usize>,
+/// }
+///
+/// #[pinned_drop]
+/// impl PinnedDrop for Foo {
+/// fn drop(self: Pin<&mut Self>) {
+/// pr_info!("Foo is being dropped!");
+/// }
+/// }
+/// ```
+///
+/// # Safety
+///
+/// This trait must be implemented via the [`pinned_drop`] proc-macro attribute on the impl.
+///
+/// [`pinned_drop`]: kernel::macros::pinned_drop
+pub unsafe trait PinnedDrop: __internal::HasPinData {
+ /// Executes the pinned destructor of this type.
+ ///
+ /// While this function is marked safe, it is actually unsafe to call it manually. For this
+ /// reason it takes an additional parameter. This type can only be constructed by `unsafe` code
+ /// and thus prevents this function from being called where it should not.
+ ///
+ /// This extra parameter will be generated by the `#[pinned_drop]` proc-macro attribute
+ /// automatically.
+ fn drop(self: Pin<&mut Self>, only_call_from_drop: __internal::OnlyCallFromDrop);
+}
+
+/// Marker trait for types that can be initialized by writing just zeroes.
+///
+/// # Safety
+///
+/// The bit pattern consisting of only zeroes is a valid bit pattern for this type. In other words,
+/// this is not UB:
+///
+/// ```rust,ignore
+/// let val: Self = unsafe { core::mem::zeroed() };
+/// ```
+pub unsafe trait Zeroable {}
+
+/// Create a new zeroed T.
+///
+/// The returned initializer will write `0x00` to every byte of the given `slot`.
+#[inline]
+pub fn zeroed<T: Zeroable>() -> impl Init<T> {
+ // SAFETY: Because `T: Zeroable`, all bytes zero is a valid bit pattern for `T`
+ // and because we write all zeroes, the memory is initialized.
+ unsafe {
+ init_from_closure(|slot: *mut T| {
+ slot.write_bytes(0, 1);
+ Ok(())
+ })
+ }
+}
+
+macro_rules! impl_zeroable {
+ ($($({$($generics:tt)*})? $t:ty, )*) => {
+ // SAFETY: Safety comments written in the macro invocation.
+ $(unsafe impl$($($generics)*)? Zeroable for $t {})*
+ };
+}
+
+impl_zeroable! {
+ // SAFETY: All primitives that are allowed to be zero.
+ bool,
+ char,
+ u8, u16, u32, u64, u128, usize,
+ i8, i16, i32, i64, i128, isize,
+ f32, f64,
+
+ // Note: do not add uninhabited types (such as `!` or `core::convert::Infallible`) to this list;
+ // creating an instance of an uninhabited type is immediate undefined behavior. For more on
+ // uninhabited/empty types, consult The Rustonomicon:
+ // <https://doc.rust-lang.org/stable/nomicon/exotic-sizes.html#empty-types>. The Rust Reference
+ // also has information on undefined behavior:
+ // <https://doc.rust-lang.org/stable/reference/behavior-considered-undefined.html>.
+ //
+ // SAFETY: These are inhabited ZSTs; there is nothing to zero and a valid value exists.
+ {<T: ?Sized>} PhantomData<T>, core::marker::PhantomPinned, (),
+
+ // SAFETY: Type is allowed to take any value, including all zeros.
+ {<T>} MaybeUninit<T>,
+ // SAFETY: Type is allowed to take any value, including all zeros.
+ {<T>} Opaque<T>,
+
+ // SAFETY: `T: Zeroable` and `UnsafeCell` is `repr(transparent)`.
+ {<T: ?Sized + Zeroable>} UnsafeCell<T>,
+
+ // SAFETY: All zeros is equivalent to `None` (option layout optimization guarantee).
+ Option<NonZeroU8>, Option<NonZeroU16>, Option<NonZeroU32>, Option<NonZeroU64>,
+ Option<NonZeroU128>, Option<NonZeroUsize>,
+ Option<NonZeroI8>, Option<NonZeroI16>, Option<NonZeroI32>, Option<NonZeroI64>,
+ Option<NonZeroI128>, Option<NonZeroIsize>,
+
+ // SAFETY: All zeros is equivalent to `None` (option layout optimization guarantee).
+ //
+ // In this case we are allowed to use `T: ?Sized`, since all zeros is the `None` variant.
+ {<T: ?Sized>} Option<NonNull<T>>,
+ {<T: ?Sized>} Option<KBox<T>>,
+
+ // SAFETY: `null` pointer is valid.
+ //
+ // We cannot use `T: ?Sized`, since the VTABLE pointer part of fat pointers is not allowed to be
+ // null.
+ //
+ // When `Pointee` gets stabilized, we could use
+ // `T: ?Sized where <T as Pointee>::Metadata: Zeroable`
+ {<T>} *mut T, {<T>} *const T,
+
+ // SAFETY: `null` pointer is valid and the metadata part of these fat pointers is allowed to be
+ // zero.
+ {<T>} *mut [T], {<T>} *const [T], *mut str, *const str,
+
+ // SAFETY: `T` is `Zeroable`.
+ {<const N: usize, T: Zeroable>} [T; N], {<T: Zeroable>} Wrapping<T>,
+}
+
+macro_rules! impl_tuple_zeroable {
+ ($(,)?) => {};
+ ($first:ident, $($t:ident),* $(,)?) => {
+ // SAFETY: All elements are zeroable and padding can be zero.
+ unsafe impl<$first: Zeroable, $($t: Zeroable),*> Zeroable for ($first, $($t),*) {}
+ impl_tuple_zeroable!($($t),* ,);
+ }
+}
+
+impl_tuple_zeroable!(A, B, C, D, E, F, G, H, I, J);
--- /dev/null
+// SPDX-License-Identifier: Apache-2.0 OR MIT
+
+//! This module provides the macros that actually implement the proc-macros `pin_data` and
+//! `pinned_drop`. It also contains `__init_internal` the implementation of the `{try_}{pin_}init!`
+//! macros.
+//!
+//! These macros should never be called directly, since they expect their input to be
+//! in a certain format which is internal. If used incorrectly, these macros can lead to UB even in
+//! safe code! Use the public facing macros instead.
+//!
+//! This architecture has been chosen because the kernel does not yet have access to `syn` which
+//! would make matters a lot easier for implementing these as proc-macros.
+//!
+//! # Macro expansion example
+//!
+//! This section is intended for readers trying to understand the macros in this module and the
+//! `pin_init!` macros from `init.rs`.
+//!
+//! We will look at the following example:
+//!
+//! ```rust,ignore
+//! # use kernel::init::*;
+//! # use core::pin::Pin;
+//! #[pin_data]
+//! #[repr(C)]
+//! struct Bar<T> {
+//! #[pin]
+//! t: T,
+//! pub x: usize,
+//! }
+//!
+//! impl<T> Bar<T> {
+//! fn new(t: T) -> impl PinInit<Self> {
+//! pin_init!(Self { t, x: 0 })
+//! }
+//! }
+//!
+//! #[pin_data(PinnedDrop)]
+//! struct Foo {
+//! a: usize,
+//! #[pin]
+//! b: Bar<u32>,
+//! }
+//!
+//! #[pinned_drop]
+//! impl PinnedDrop for Foo {
+//! fn drop(self: Pin<&mut Self>) {
+//! pr_info!("{self:p} is getting dropped.");
+//! }
+//! }
+//!
+//! let a = 42;
+//! let initializer = pin_init!(Foo {
+//! a,
+//! b <- Bar::new(36),
+//! });
+//! ```
+//!
+//! This example includes the most common and important features of the pin-init API.
+//!
+//! Below you can find individual section about the different macro invocations. Here are some
+//! general things we need to take into account when designing macros:
+//! - use global paths, similarly to file paths, these start with the separator: `::core::panic!()`
+//! this ensures that the correct item is used, since users could define their own `mod core {}`
+//! and then their own `panic!` inside to execute arbitrary code inside of our macro.
+//! - macro `unsafe` hygiene: we need to ensure that we do not expand arbitrary, user-supplied
+//! expressions inside of an `unsafe` block in the macro, because this would allow users to do
+//! `unsafe` operations without an associated `unsafe` block.
+//!
+//! ## `#[pin_data]` on `Bar`
+//!
+//! This macro is used to specify which fields are structurally pinned and which fields are not. It
+//! is placed on the struct definition and allows `#[pin]` to be placed on the fields.
+//!
+//! Here is the definition of `Bar` from our example:
+//!
+//! ```rust,ignore
+//! # use kernel::init::*;
+//! #[pin_data]
+//! #[repr(C)]
+//! struct Bar<T> {
+//! #[pin]
+//! t: T,
+//! pub x: usize,
+//! }
+//! ```
+//!
+//! This expands to the following code:
+//!
+//! ```rust,ignore
+//! // Firstly the normal definition of the struct, attributes are preserved:
+//! #[repr(C)]
+//! struct Bar<T> {
+//! t: T,
+//! pub x: usize,
+//! }
+//! // Then an anonymous constant is defined, this is because we do not want any code to access the
+//! // types that we define inside:
+//! const _: () = {
+//! // We define the pin-data carrying struct, it is a ZST and needs to have the same generics,
+//! // since we need to implement access functions for each field and thus need to know its
+//! // type.
+//! struct __ThePinData<T> {
+//! __phantom: ::core::marker::PhantomData<fn(Bar<T>) -> Bar<T>>,
+//! }
+//! // We implement `Copy` for the pin-data struct, since all functions it defines will take
+//! // `self` by value.
+//! impl<T> ::core::clone::Clone for __ThePinData<T> {
+//! fn clone(&self) -> Self {
+//! *self
+//! }
+//! }
+//! impl<T> ::core::marker::Copy for __ThePinData<T> {}
+//! // For every field of `Bar`, the pin-data struct will define a function with the same name
+//! // and accessor (`pub` or `pub(crate)` etc.). This function will take a pointer to the
+//! // field (`slot`) and a `PinInit` or `Init` depending on the projection kind of the field
+//! // (if pinning is structural for the field, then `PinInit` otherwise `Init`).
+//! #[allow(dead_code)]
+//! impl<T> __ThePinData<T> {
+//! unsafe fn t<E>(
+//! self,
+//! slot: *mut T,
+//! // Since `t` is `#[pin]`, this is `PinInit`.
+//! init: impl ::kernel::init::PinInit<T, E>,
+//! ) -> ::core::result::Result<(), E> {
+//! unsafe { ::kernel::init::PinInit::__pinned_init(init, slot) }
+//! }
+//! pub unsafe fn x<E>(
+//! self,
+//! slot: *mut usize,
+//! // Since `x` is not `#[pin]`, this is `Init`.
+//! init: impl ::kernel::init::Init<usize, E>,
+//! ) -> ::core::result::Result<(), E> {
+//! unsafe { ::kernel::init::Init::__init(init, slot) }
+//! }
+//! }
+//! // Implement the internal `HasPinData` trait that associates `Bar` with the pin-data struct
+//! // that we constructed above.
+//! unsafe impl<T> ::kernel::init::__internal::HasPinData for Bar<T> {
+//! type PinData = __ThePinData<T>;
+//! unsafe fn __pin_data() -> Self::PinData {
+//! __ThePinData {
+//! __phantom: ::core::marker::PhantomData,
+//! }
+//! }
+//! }
+//! // Implement the internal `PinData` trait that marks the pin-data struct as a pin-data
+//! // struct. This is important to ensure that no user can implement a rogue `__pin_data`
+//! // function without using `unsafe`.
+//! unsafe impl<T> ::kernel::init::__internal::PinData for __ThePinData<T> {
+//! type Datee = Bar<T>;
+//! }
+//! // Now we only want to implement `Unpin` for `Bar` when every structurally pinned field is
+//! // `Unpin`. In other words, whether `Bar` is `Unpin` only depends on structurally pinned
+//! // fields (those marked with `#[pin]`). These fields will be listed in this struct, in our
+//! // case no such fields exist, hence this is almost empty. The two phantomdata fields exist
+//! // for two reasons:
+//! // - `__phantom`: every generic must be used, since we cannot really know which generics
+//! // are used, we declare all and then use everything here once.
+//! // - `__phantom_pin`: uses the `'__pin` lifetime and ensures that this struct is invariant
+//! // over it. The lifetime is needed to work around the limitation that trait bounds must
+//! // not be trivial, e.g. the user has a `#[pin] PhantomPinned` field -- this is
+//! // unconditionally `!Unpin` and results in an error. The lifetime tricks the compiler
+//! // into accepting these bounds regardless.
+//! #[allow(dead_code)]
+//! struct __Unpin<'__pin, T> {
+//! __phantom_pin: ::core::marker::PhantomData<fn(&'__pin ()) -> &'__pin ()>,
+//! __phantom: ::core::marker::PhantomData<fn(Bar<T>) -> Bar<T>>,
+//! // Our only `#[pin]` field is `t`.
+//! t: T,
+//! }
+//! #[doc(hidden)]
+//! impl<'__pin, T> ::core::marker::Unpin for Bar<T>
+//! where
+//! __Unpin<'__pin, T>: ::core::marker::Unpin,
+//! {}
+//! // Now we need to ensure that `Bar` does not implement `Drop`, since that would give users
+//! // access to `&mut self` inside of `drop` even if the struct was pinned. This could lead to
+//! // UB with only safe code, so we disallow this by giving a trait implementation error using
+//! // a direct impl and a blanket implementation.
+//! trait MustNotImplDrop {}
+//! // Normally `Drop` bounds do not have the correct semantics, but for this purpose they do
+//! // (normally people want to know if a type has any kind of drop glue at all, here we want
+//! // to know if it has any kind of custom drop glue, which is exactly what this bound does).
+//! #[expect(drop_bounds)]
+//! impl<T: ::core::ops::Drop> MustNotImplDrop for T {}
+//! impl<T> MustNotImplDrop for Bar<T> {}
+//! // Here comes a convenience check, if one implemented `PinnedDrop`, but forgot to add it to
+//! // `#[pin_data]`, then this will error with the same mechanic as above, this is not needed
+//! // for safety, but a good sanity check, since no normal code calls `PinnedDrop::drop`.
+//! #[expect(non_camel_case_types)]
+//! trait UselessPinnedDropImpl_you_need_to_specify_PinnedDrop {}
+//! impl<
+//! T: ::kernel::init::PinnedDrop,
+//! > UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for T {}
+//! impl<T> UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for Bar<T> {}
+//! };
+//! ```
+//!
+//! ## `pin_init!` in `impl Bar`
+//!
+//! This macro creates an pin-initializer for the given struct. It requires that the struct is
+//! annotated by `#[pin_data]`.
+//!
+//! Here is the impl on `Bar` defining the new function:
+//!
+//! ```rust,ignore
+//! impl<T> Bar<T> {
+//! fn new(t: T) -> impl PinInit<Self> {
+//! pin_init!(Self { t, x: 0 })
+//! }
+//! }
+//! ```
+//!
+//! This expands to the following code:
+//!
+//! ```rust,ignore
+//! impl<T> Bar<T> {
+//! fn new(t: T) -> impl PinInit<Self> {
+//! {
+//! // We do not want to allow arbitrary returns, so we declare this type as the `Ok`
+//! // return type and shadow it later when we insert the arbitrary user code. That way
+//! // there will be no possibility of returning without `unsafe`.
+//! struct __InitOk;
+//! // Get the data about fields from the supplied type.
+//! // - the function is unsafe, hence the unsafe block
+//! // - we `use` the `HasPinData` trait in the block, it is only available in that
+//! // scope.
+//! let data = unsafe {
+//! use ::kernel::init::__internal::HasPinData;
+//! Self::__pin_data()
+//! };
+//! // Ensure that `data` really is of type `PinData` and help with type inference:
+//! let init = ::kernel::init::__internal::PinData::make_closure::<
+//! _,
+//! __InitOk,
+//! ::core::convert::Infallible,
+//! >(data, move |slot| {
+//! {
+//! // Shadow the structure so it cannot be used to return early. If a user
+//! // tries to write `return Ok(__InitOk)`, then they get a type error,
+//! // since that will refer to this struct instead of the one defined
+//! // above.
+//! struct __InitOk;
+//! // This is the expansion of `t,`, which is syntactic sugar for `t: t,`.
+//! {
+//! unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).t), t) };
+//! }
+//! // Since initialization could fail later (not in this case, since the
+//! // error type is `Infallible`) we will need to drop this field if there
+//! // is an error later. This `DropGuard` will drop the field when it gets
+//! // dropped and has not yet been forgotten.
+//! let __t_guard = unsafe {
+//! ::pinned_init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).t))
+//! };
+//! // Expansion of `x: 0,`:
+//! // Since this can be an arbitrary expression we cannot place it inside
+//! // of the `unsafe` block, so we bind it here.
+//! {
+//! let x = 0;
+//! unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).x), x) };
+//! }
+//! // We again create a `DropGuard`.
+//! let __x_guard = unsafe {
+//! ::kernel::init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).x))
+//! };
+//! // Since initialization has successfully completed, we can now forget
+//! // the guards. This is not `mem::forget`, since we only have
+//! // `&DropGuard`.
+//! ::core::mem::forget(__x_guard);
+//! ::core::mem::forget(__t_guard);
+//! // Here we use the type checker to ensure that every field has been
+//! // initialized exactly once, since this is `if false` it will never get
+//! // executed, but still type-checked.
+//! // Additionally we abuse `slot` to automatically infer the correct type
+//! // for the struct. This is also another check that every field is
+//! // accessible from this scope.
+//! #[allow(unreachable_code, clippy::diverging_sub_expression)]
+//! let _ = || {
+//! unsafe {
+//! ::core::ptr::write(
+//! slot,
+//! Self {
+//! // We only care about typecheck finding every field
+//! // here, the expression does not matter, just conjure
+//! // one using `panic!()`:
+//! t: ::core::panic!(),
+//! x: ::core::panic!(),
+//! },
+//! );
+//! };
+//! };
+//! }
+//! // We leave the scope above and gain access to the previously shadowed
+//! // `__InitOk` that we need to return.
+//! Ok(__InitOk)
+//! });
+//! // Change the return type from `__InitOk` to `()`.
+//! let init = move |
+//! slot,
+//! | -> ::core::result::Result<(), ::core::convert::Infallible> {
+//! init(slot).map(|__InitOk| ())
+//! };
+//! // Construct the initializer.
+//! let init = unsafe {
+//! ::kernel::init::pin_init_from_closure::<
+//! _,
+//! ::core::convert::Infallible,
+//! >(init)
+//! };
+//! init
+//! }
+//! }
+//! }
+//! ```
+//!
+//! ## `#[pin_data]` on `Foo`
+//!
+//! Since we already took a look at `#[pin_data]` on `Bar`, this section will only explain the
+//! differences/new things in the expansion of the `Foo` definition:
+//!
+//! ```rust,ignore
+//! #[pin_data(PinnedDrop)]
+//! struct Foo {
+//! a: usize,
+//! #[pin]
+//! b: Bar<u32>,
+//! }
+//! ```
+//!
+//! This expands to the following code:
+//!
+//! ```rust,ignore
+//! struct Foo {
+//! a: usize,
+//! b: Bar<u32>,
+//! }
+//! const _: () = {
+//! struct __ThePinData {
+//! __phantom: ::core::marker::PhantomData<fn(Foo) -> Foo>,
+//! }
+//! impl ::core::clone::Clone for __ThePinData {
+//! fn clone(&self) -> Self {
+//! *self
+//! }
+//! }
+//! impl ::core::marker::Copy for __ThePinData {}
+//! #[allow(dead_code)]
+//! impl __ThePinData {
+//! unsafe fn b<E>(
+//! self,
+//! slot: *mut Bar<u32>,
+//! init: impl ::kernel::init::PinInit<Bar<u32>, E>,
+//! ) -> ::core::result::Result<(), E> {
+//! unsafe { ::kernel::init::PinInit::__pinned_init(init, slot) }
+//! }
+//! unsafe fn a<E>(
+//! self,
+//! slot: *mut usize,
+//! init: impl ::kernel::init::Init<usize, E>,
+//! ) -> ::core::result::Result<(), E> {
+//! unsafe { ::kernel::init::Init::__init(init, slot) }
+//! }
+//! }
+//! unsafe impl ::kernel::init::__internal::HasPinData for Foo {
+//! type PinData = __ThePinData;
+//! unsafe fn __pin_data() -> Self::PinData {
+//! __ThePinData {
+//! __phantom: ::core::marker::PhantomData,
+//! }
+//! }
+//! }
+//! unsafe impl ::kernel::init::__internal::PinData for __ThePinData {
+//! type Datee = Foo;
+//! }
+//! #[allow(dead_code)]
+//! struct __Unpin<'__pin> {
+//! __phantom_pin: ::core::marker::PhantomData<fn(&'__pin ()) -> &'__pin ()>,
+//! __phantom: ::core::marker::PhantomData<fn(Foo) -> Foo>,
+//! b: Bar<u32>,
+//! }
+//! #[doc(hidden)]
+//! impl<'__pin> ::core::marker::Unpin for Foo
+//! where
+//! __Unpin<'__pin>: ::core::marker::Unpin,
+//! {}
+//! // Since we specified `PinnedDrop` as the argument to `#[pin_data]`, we expect `Foo` to
+//! // implement `PinnedDrop`. Thus we do not need to prevent `Drop` implementations like
+//! // before, instead we implement `Drop` here and delegate to `PinnedDrop`.
+//! impl ::core::ops::Drop for Foo {
+//! fn drop(&mut self) {
+//! // Since we are getting dropped, no one else has a reference to `self` and thus we
+//! // can assume that we never move.
+//! let pinned = unsafe { ::core::pin::Pin::new_unchecked(self) };
+//! // Create the unsafe token that proves that we are inside of a destructor, this
+//! // type is only allowed to be created in a destructor.
+//! let token = unsafe { ::kernel::init::__internal::OnlyCallFromDrop::new() };
+//! ::kernel::init::PinnedDrop::drop(pinned, token);
+//! }
+//! }
+//! };
+//! ```
+//!
+//! ## `#[pinned_drop]` on `impl PinnedDrop for Foo`
+//!
+//! This macro is used to implement the `PinnedDrop` trait, since that trait is `unsafe` and has an
+//! extra parameter that should not be used at all. The macro hides that parameter.
+//!
+//! Here is the `PinnedDrop` impl for `Foo`:
+//!
+//! ```rust,ignore
+//! #[pinned_drop]
+//! impl PinnedDrop for Foo {
+//! fn drop(self: Pin<&mut Self>) {
+//! pr_info!("{self:p} is getting dropped.");
+//! }
+//! }
+//! ```
+//!
+//! This expands to the following code:
+//!
+//! ```rust,ignore
+//! // `unsafe`, full path and the token parameter are added, everything else stays the same.
+//! unsafe impl ::kernel::init::PinnedDrop for Foo {
+//! fn drop(self: Pin<&mut Self>, _: ::kernel::init::__internal::OnlyCallFromDrop) {
+//! pr_info!("{self:p} is getting dropped.");
+//! }
+//! }
+//! ```
+//!
+//! ## `pin_init!` on `Foo`
+//!
+//! Since we already took a look at `pin_init!` on `Bar`, this section will only show the expansion
+//! of `pin_init!` on `Foo`:
+//!
+//! ```rust,ignore
+//! let a = 42;
+//! let initializer = pin_init!(Foo {
+//! a,
+//! b <- Bar::new(36),
+//! });
+//! ```
+//!
+//! This expands to the following code:
+//!
+//! ```rust,ignore
+//! let a = 42;
+//! let initializer = {
+//! struct __InitOk;
+//! let data = unsafe {
+//! use ::kernel::init::__internal::HasPinData;
+//! Foo::__pin_data()
+//! };
+//! let init = ::kernel::init::__internal::PinData::make_closure::<
+//! _,
+//! __InitOk,
+//! ::core::convert::Infallible,
+//! >(data, move |slot| {
+//! {
+//! struct __InitOk;
+//! {
+//! unsafe { ::core::ptr::write(::core::addr_of_mut!((*slot).a), a) };
+//! }
+//! let __a_guard = unsafe {
+//! ::kernel::init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).a))
+//! };
+//! let init = Bar::new(36);
+//! unsafe { data.b(::core::addr_of_mut!((*slot).b), b)? };
+//! let __b_guard = unsafe {
+//! ::kernel::init::__internal::DropGuard::new(::core::addr_of_mut!((*slot).b))
+//! };
+//! ::core::mem::forget(__b_guard);
+//! ::core::mem::forget(__a_guard);
+//! #[allow(unreachable_code, clippy::diverging_sub_expression)]
+//! let _ = || {
+//! unsafe {
+//! ::core::ptr::write(
+//! slot,
+//! Foo {
+//! a: ::core::panic!(),
+//! b: ::core::panic!(),
+//! },
+//! );
+//! };
+//! };
+//! }
+//! Ok(__InitOk)
+//! });
+//! let init = move |
+//! slot,
+//! | -> ::core::result::Result<(), ::core::convert::Infallible> {
+//! init(slot).map(|__InitOk| ())
+//! };
+//! let init = unsafe {
+//! ::kernel::init::pin_init_from_closure::<_, ::core::convert::Infallible>(init)
+//! };
+//! init
+//! };
+//! ```
+
+/// Creates a `unsafe impl<...> PinnedDrop for $type` block.
+///
+/// See [`PinnedDrop`] for more information.
+#[doc(hidden)]
+#[macro_export]
+macro_rules! __pinned_drop {
+ (
+ @impl_sig($($impl_sig:tt)*),
+ @impl_body(
+ $(#[$($attr:tt)*])*
+ fn drop($($sig:tt)*) {
+ $($inner:tt)*
+ }
+ ),
+ ) => {
+ // SAFETY: TODO.
+ unsafe $($impl_sig)* {
+ // Inherit all attributes and the type/ident tokens for the signature.
+ $(#[$($attr)*])*
+ fn drop($($sig)*, _: $crate::init::__internal::OnlyCallFromDrop) {
+ $($inner)*
+ }
+ }
+ }
+}
+
+/// This macro first parses the struct definition such that it separates pinned and not pinned
+/// fields. Afterwards it declares the struct and implement the `PinData` trait safely.
+#[doc(hidden)]
+#[macro_export]
+macro_rules! __pin_data {
+ // Proc-macro entry point, this is supplied by the proc-macro pre-parsing.
+ (parse_input:
+ @args($($pinned_drop:ident)?),
+ @sig(
+ $(#[$($struct_attr:tt)*])*
+ $vis:vis struct $name:ident
+ $(where $($whr:tt)*)?
+ ),
+ @impl_generics($($impl_generics:tt)*),
+ @ty_generics($($ty_generics:tt)*),
+ @decl_generics($($decl_generics:tt)*),
+ @body({ $($fields:tt)* }),
+ ) => {
+ // We now use token munching to iterate through all of the fields. While doing this we
+ // identify fields marked with `#[pin]`, these fields are the 'pinned fields'. The user
+ // wants these to be structurally pinned. The rest of the fields are the
+ // 'not pinned fields'. Additionally we collect all fields, since we need them in the right
+ // order to declare the struct.
+ //
+ // In this call we also put some explaining comments for the parameters.
+ $crate::__pin_data!(find_pinned_fields:
+ // Attributes on the struct itself, these will just be propagated to be put onto the
+ // struct definition.
+ @struct_attrs($(#[$($struct_attr)*])*),
+ // The visibility of the struct.
+ @vis($vis),
+ // The name of the struct.
+ @name($name),
+ // The 'impl generics', the generics that will need to be specified on the struct inside
+ // of an `impl<$ty_generics>` block.
+ @impl_generics($($impl_generics)*),
+ // The 'ty generics', the generics that will need to be specified on the impl blocks.
+ @ty_generics($($ty_generics)*),
+ // The 'decl generics', the generics that need to be specified on the struct
+ // definition.
+ @decl_generics($($decl_generics)*),
+ // The where clause of any impl block and the declaration.
+ @where($($($whr)*)?),
+ // The remaining fields tokens that need to be processed.
+ // We add a `,` at the end to ensure correct parsing.
+ @fields_munch($($fields)* ,),
+ // The pinned fields.
+ @pinned(),
+ // The not pinned fields.
+ @not_pinned(),
+ // All fields.
+ @fields(),
+ // The accumulator containing all attributes already parsed.
+ @accum(),
+ // Contains `yes` or `` to indicate if `#[pin]` was found on the current field.
+ @is_pinned(),
+ // The proc-macro argument, this should be `PinnedDrop` or ``.
+ @pinned_drop($($pinned_drop)?),
+ );
+ };
+ (find_pinned_fields:
+ @struct_attrs($($struct_attrs:tt)*),
+ @vis($vis:vis),
+ @name($name:ident),
+ @impl_generics($($impl_generics:tt)*),
+ @ty_generics($($ty_generics:tt)*),
+ @decl_generics($($decl_generics:tt)*),
+ @where($($whr:tt)*),
+ // We found a PhantomPinned field, this should generally be pinned!
+ @fields_munch($field:ident : $($($(::)?core::)?marker::)?PhantomPinned, $($rest:tt)*),
+ @pinned($($pinned:tt)*),
+ @not_pinned($($not_pinned:tt)*),
+ @fields($($fields:tt)*),
+ @accum($($accum:tt)*),
+ // This field is not pinned.
+ @is_pinned(),
+ @pinned_drop($($pinned_drop:ident)?),
+ ) => {
+ ::core::compile_error!(concat!(
+ "The field `",
+ stringify!($field),
+ "` of type `PhantomPinned` only has an effect, if it has the `#[pin]` attribute.",
+ ));
+ $crate::__pin_data!(find_pinned_fields:
+ @struct_attrs($($struct_attrs)*),
+ @vis($vis),
+ @name($name),
+ @impl_generics($($impl_generics)*),
+ @ty_generics($($ty_generics)*),
+ @decl_generics($($decl_generics)*),
+ @where($($whr)*),
+ @fields_munch($($rest)*),
+ @pinned($($pinned)* $($accum)* $field: ::core::marker::PhantomPinned,),
+ @not_pinned($($not_pinned)*),
+ @fields($($fields)* $($accum)* $field: ::core::marker::PhantomPinned,),
+ @accum(),
+ @is_pinned(),
+ @pinned_drop($($pinned_drop)?),
+ );
+ };
+ (find_pinned_fields:
+ @struct_attrs($($struct_attrs:tt)*),
+ @vis($vis:vis),
+ @name($name:ident),
+ @impl_generics($($impl_generics:tt)*),
+ @ty_generics($($ty_generics:tt)*),
+ @decl_generics($($decl_generics:tt)*),
+ @where($($whr:tt)*),
+ // We reached the field declaration.
+ @fields_munch($field:ident : $type:ty, $($rest:tt)*),
+ @pinned($($pinned:tt)*),
+ @not_pinned($($not_pinned:tt)*),
+ @fields($($fields:tt)*),
+ @accum($($accum:tt)*),
+ // This field is pinned.
+ @is_pinned(yes),
+ @pinned_drop($($pinned_drop:ident)?),
+ ) => {
+ $crate::__pin_data!(find_pinned_fields:
+ @struct_attrs($($struct_attrs)*),
+ @vis($vis),
+ @name($name),
+ @impl_generics($($impl_generics)*),
+ @ty_generics($($ty_generics)*),
+ @decl_generics($($decl_generics)*),
+ @where($($whr)*),
+ @fields_munch($($rest)*),
+ @pinned($($pinned)* $($accum)* $field: $type,),
+ @not_pinned($($not_pinned)*),
+ @fields($($fields)* $($accum)* $field: $type,),
+ @accum(),
+ @is_pinned(),
+ @pinned_drop($($pinned_drop)?),
+ );
+ };
+ (find_pinned_fields:
+ @struct_attrs($($struct_attrs:tt)*),
+ @vis($vis:vis),
+ @name($name:ident),
+ @impl_generics($($impl_generics:tt)*),
+ @ty_generics($($ty_generics:tt)*),
+ @decl_generics($($decl_generics:tt)*),
+ @where($($whr:tt)*),
+ // We reached the field declaration.
+ @fields_munch($field:ident : $type:ty, $($rest:tt)*),
+ @pinned($($pinned:tt)*),
+ @not_pinned($($not_pinned:tt)*),
+ @fields($($fields:tt)*),
+ @accum($($accum:tt)*),
+ // This field is not pinned.
+ @is_pinned(),
+ @pinned_drop($($pinned_drop:ident)?),
+ ) => {
+ $crate::__pin_data!(find_pinned_fields:
+ @struct_attrs($($struct_attrs)*),
+ @vis($vis),
+ @name($name),
+ @impl_generics($($impl_generics)*),
+ @ty_generics($($ty_generics)*),
+ @decl_generics($($decl_generics)*),
+ @where($($whr)*),
+ @fields_munch($($rest)*),
+ @pinned($($pinned)*),
+ @not_pinned($($not_pinned)* $($accum)* $field: $type,),
+ @fields($($fields)* $($accum)* $field: $type,),
+ @accum(),
+ @is_pinned(),
+ @pinned_drop($($pinned_drop)?),
+ );
+ };
+ (find_pinned_fields:
+ @struct_attrs($($struct_attrs:tt)*),
+ @vis($vis:vis),
+ @name($name:ident),
+ @impl_generics($($impl_generics:tt)*),
+ @ty_generics($($ty_generics:tt)*),
+ @decl_generics($($decl_generics:tt)*),
+ @where($($whr:tt)*),
+ // We found the `#[pin]` attr.
+ @fields_munch(#[pin] $($rest:tt)*),
+ @pinned($($pinned:tt)*),
+ @not_pinned($($not_pinned:tt)*),
+ @fields($($fields:tt)*),
+ @accum($($accum:tt)*),
+ @is_pinned($($is_pinned:ident)?),
+ @pinned_drop($($pinned_drop:ident)?),
+ ) => {
+ $crate::__pin_data!(find_pinned_fields:
+ @struct_attrs($($struct_attrs)*),
+ @vis($vis),
+ @name($name),
+ @impl_generics($($impl_generics)*),
+ @ty_generics($($ty_generics)*),
+ @decl_generics($($decl_generics)*),
+ @where($($whr)*),
+ @fields_munch($($rest)*),
+ // We do not include `#[pin]` in the list of attributes, since it is not actually an
+ // attribute that is defined somewhere.
+ @pinned($($pinned)*),
+ @not_pinned($($not_pinned)*),
+ @fields($($fields)*),
+ @accum($($accum)*),
+ // Set this to `yes`.
+ @is_pinned(yes),
+ @pinned_drop($($pinned_drop)?),
+ );
+ };
+ (find_pinned_fields:
+ @struct_attrs($($struct_attrs:tt)*),
+ @vis($vis:vis),
+ @name($name:ident),
+ @impl_generics($($impl_generics:tt)*),
+ @ty_generics($($ty_generics:tt)*),
+ @decl_generics($($decl_generics:tt)*),
+ @where($($whr:tt)*),
+ // We reached the field declaration with visibility, for simplicity we only munch the
+ // visibility and put it into `$accum`.
+ @fields_munch($fvis:vis $field:ident $($rest:tt)*),
+ @pinned($($pinned:tt)*),
+ @not_pinned($($not_pinned:tt)*),
+ @fields($($fields:tt)*),
+ @accum($($accum:tt)*),
+ @is_pinned($($is_pinned:ident)?),
+ @pinned_drop($($pinned_drop:ident)?),
+ ) => {
+ $crate::__pin_data!(find_pinned_fields:
+ @struct_attrs($($struct_attrs)*),
+ @vis($vis),
+ @name($name),
+ @impl_generics($($impl_generics)*),
+ @ty_generics($($ty_generics)*),
+ @decl_generics($($decl_generics)*),
+ @where($($whr)*),
+ @fields_munch($field $($rest)*),
+ @pinned($($pinned)*),
+ @not_pinned($($not_pinned)*),
+ @fields($($fields)*),
+ @accum($($accum)* $fvis),
+ @is_pinned($($is_pinned)?),
+ @pinned_drop($($pinned_drop)?),
+ );
+ };
+ (find_pinned_fields:
+ @struct_attrs($($struct_attrs:tt)*),
+ @vis($vis:vis),
+ @name($name:ident),
+ @impl_generics($($impl_generics:tt)*),
+ @ty_generics($($ty_generics:tt)*),
+ @decl_generics($($decl_generics:tt)*),
+ @where($($whr:tt)*),
+ // Some other attribute, just put it into `$accum`.
+ @fields_munch(#[$($attr:tt)*] $($rest:tt)*),
+ @pinned($($pinned:tt)*),
+ @not_pinned($($not_pinned:tt)*),
+ @fields($($fields:tt)*),
+ @accum($($accum:tt)*),
+ @is_pinned($($is_pinned:ident)?),
+ @pinned_drop($($pinned_drop:ident)?),
+ ) => {
+ $crate::__pin_data!(find_pinned_fields:
+ @struct_attrs($($struct_attrs)*),
+ @vis($vis),
+ @name($name),
+ @impl_generics($($impl_generics)*),
+ @ty_generics($($ty_generics)*),
+ @decl_generics($($decl_generics)*),
+ @where($($whr)*),
+ @fields_munch($($rest)*),
+ @pinned($($pinned)*),
+ @not_pinned($($not_pinned)*),
+ @fields($($fields)*),
+ @accum($($accum)* #[$($attr)*]),
+ @is_pinned($($is_pinned)?),
+ @pinned_drop($($pinned_drop)?),
+ );
+ };
+ (find_pinned_fields:
+ @struct_attrs($($struct_attrs:tt)*),
+ @vis($vis:vis),
+ @name($name:ident),
+ @impl_generics($($impl_generics:tt)*),
+ @ty_generics($($ty_generics:tt)*),
+ @decl_generics($($decl_generics:tt)*),
+ @where($($whr:tt)*),
+ // We reached the end of the fields, plus an optional additional comma, since we added one
+ // before and the user is also allowed to put a trailing comma.
+ @fields_munch($(,)?),
+ @pinned($($pinned:tt)*),
+ @not_pinned($($not_pinned:tt)*),
+ @fields($($fields:tt)*),
+ @accum(),
+ @is_pinned(),
+ @pinned_drop($($pinned_drop:ident)?),
+ ) => {
+ // Declare the struct with all fields in the correct order.
+ $($struct_attrs)*
+ $vis struct $name <$($decl_generics)*>
+ where $($whr)*
+ {
+ $($fields)*
+ }
+
+ // We put the rest into this const item, because it then will not be accessible to anything
+ // outside.
+ const _: () = {
+ // We declare this struct which will host all of the projection function for our type.
+ // it will be invariant over all generic parameters which are inherited from the
+ // struct.
+ $vis struct __ThePinData<$($impl_generics)*>
+ where $($whr)*
+ {
+ __phantom: ::core::marker::PhantomData<
+ fn($name<$($ty_generics)*>) -> $name<$($ty_generics)*>
+ >,
+ }
+
+ impl<$($impl_generics)*> ::core::clone::Clone for __ThePinData<$($ty_generics)*>
+ where $($whr)*
+ {
+ fn clone(&self) -> Self { *self }
+ }
+
+ impl<$($impl_generics)*> ::core::marker::Copy for __ThePinData<$($ty_generics)*>
+ where $($whr)*
+ {}
+
+ // Make all projection functions.
+ $crate::__pin_data!(make_pin_data:
+ @pin_data(__ThePinData),
+ @impl_generics($($impl_generics)*),
+ @ty_generics($($ty_generics)*),
+ @where($($whr)*),
+ @pinned($($pinned)*),
+ @not_pinned($($not_pinned)*),
+ );
+
+ // SAFETY: We have added the correct projection functions above to `__ThePinData` and
+ // we also use the least restrictive generics possible.
+ unsafe impl<$($impl_generics)*>
+ $crate::init::__internal::HasPinData for $name<$($ty_generics)*>
+ where $($whr)*
+ {
+ type PinData = __ThePinData<$($ty_generics)*>;
+
+ unsafe fn __pin_data() -> Self::PinData {
+ __ThePinData { __phantom: ::core::marker::PhantomData }
+ }
+ }
+
+ // SAFETY: TODO.
+ unsafe impl<$($impl_generics)*>
+ $crate::init::__internal::PinData for __ThePinData<$($ty_generics)*>
+ where $($whr)*
+ {
+ type Datee = $name<$($ty_generics)*>;
+ }
+
+ // This struct will be used for the unpin analysis. Since only structurally pinned
+ // fields are relevant whether the struct should implement `Unpin`.
+ #[allow(dead_code)]
+ struct __Unpin <'__pin, $($impl_generics)*>
+ where $($whr)*
+ {
+ __phantom_pin: ::core::marker::PhantomData<fn(&'__pin ()) -> &'__pin ()>,
+ __phantom: ::core::marker::PhantomData<
+ fn($name<$($ty_generics)*>) -> $name<$($ty_generics)*>
+ >,
+ // Only the pinned fields.
+ $($pinned)*
+ }
+
+ #[doc(hidden)]
+ impl<'__pin, $($impl_generics)*> ::core::marker::Unpin for $name<$($ty_generics)*>
+ where
+ __Unpin<'__pin, $($ty_generics)*>: ::core::marker::Unpin,
+ $($whr)*
+ {}
+
+ // We need to disallow normal `Drop` implementation, the exact behavior depends on
+ // whether `PinnedDrop` was specified as the parameter.
+ $crate::__pin_data!(drop_prevention:
+ @name($name),
+ @impl_generics($($impl_generics)*),
+ @ty_generics($($ty_generics)*),
+ @where($($whr)*),
+ @pinned_drop($($pinned_drop)?),
+ );
+ };
+ };
+ // When no `PinnedDrop` was specified, then we have to prevent implementing drop.
+ (drop_prevention:
+ @name($name:ident),
+ @impl_generics($($impl_generics:tt)*),
+ @ty_generics($($ty_generics:tt)*),
+ @where($($whr:tt)*),
+ @pinned_drop(),
+ ) => {
+ // We prevent this by creating a trait that will be implemented for all types implementing
+ // `Drop`. Additionally we will implement this trait for the struct leading to a conflict,
+ // if it also implements `Drop`
+ trait MustNotImplDrop {}
+ #[expect(drop_bounds)]
+ impl<T: ::core::ops::Drop> MustNotImplDrop for T {}
+ impl<$($impl_generics)*> MustNotImplDrop for $name<$($ty_generics)*>
+ where $($whr)* {}
+ // We also take care to prevent users from writing a useless `PinnedDrop` implementation.
+ // They might implement `PinnedDrop` correctly for the struct, but forget to give
+ // `PinnedDrop` as the parameter to `#[pin_data]`.
+ #[expect(non_camel_case_types)]
+ trait UselessPinnedDropImpl_you_need_to_specify_PinnedDrop {}
+ impl<T: $crate::init::PinnedDrop>
+ UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for T {}
+ impl<$($impl_generics)*>
+ UselessPinnedDropImpl_you_need_to_specify_PinnedDrop for $name<$($ty_generics)*>
+ where $($whr)* {}
+ };
+ // When `PinnedDrop` was specified we just implement `Drop` and delegate.
+ (drop_prevention:
+ @name($name:ident),
+ @impl_generics($($impl_generics:tt)*),
+ @ty_generics($($ty_generics:tt)*),
+ @where($($whr:tt)*),
+ @pinned_drop(PinnedDrop),
+ ) => {
+ impl<$($impl_generics)*> ::core::ops::Drop for $name<$($ty_generics)*>
+ where $($whr)*
+ {
+ fn drop(&mut self) {
+ // SAFETY: Since this is a destructor, `self` will not move after this function
+ // terminates, since it is inaccessible.
+ let pinned = unsafe { ::core::pin::Pin::new_unchecked(self) };
+ // SAFETY: Since this is a drop function, we can create this token to call the
+ // pinned destructor of this type.
+ let token = unsafe { $crate::init::__internal::OnlyCallFromDrop::new() };
+ $crate::init::PinnedDrop::drop(pinned, token);
+ }
+ }
+ };
+ // If some other parameter was specified, we emit a readable error.
+ (drop_prevention:
+ @name($name:ident),
+ @impl_generics($($impl_generics:tt)*),
+ @ty_generics($($ty_generics:tt)*),
+ @where($($whr:tt)*),
+ @pinned_drop($($rest:tt)*),
+ ) => {
+ compile_error!(
+ "Wrong parameters to `#[pin_data]`, expected nothing or `PinnedDrop`, got '{}'.",
+ stringify!($($rest)*),
+ );
+ };
+ (make_pin_data:
+ @pin_data($pin_data:ident),
+ @impl_generics($($impl_generics:tt)*),
+ @ty_generics($($ty_generics:tt)*),
+ @where($($whr:tt)*),
+ @pinned($($(#[$($p_attr:tt)*])* $pvis:vis $p_field:ident : $p_type:ty),* $(,)?),
+ @not_pinned($($(#[$($attr:tt)*])* $fvis:vis $field:ident : $type:ty),* $(,)?),
+ ) => {
+ // For every field, we create a projection function according to its projection type. If a
+ // field is structurally pinned, then it must be initialized via `PinInit`, if it is not
+ // structurally pinned, then it can be initialized via `Init`.
+ //
+ // The functions are `unsafe` to prevent accidentally calling them.
+ #[allow(dead_code)]
+ #[expect(clippy::missing_safety_doc)]
+ impl<$($impl_generics)*> $pin_data<$($ty_generics)*>
+ where $($whr)*
+ {
+ $(
+ $(#[$($p_attr)*])*
+ $pvis unsafe fn $p_field<E>(
+ self,
+ slot: *mut $p_type,
+ init: impl $crate::init::PinInit<$p_type, E>,
+ ) -> ::core::result::Result<(), E> {
+ // SAFETY: TODO.
+ unsafe { $crate::init::PinInit::__pinned_init(init, slot) }
+ }
+ )*
+ $(
+ $(#[$($attr)*])*
+ $fvis unsafe fn $field<E>(
+ self,
+ slot: *mut $type,
+ init: impl $crate::init::Init<$type, E>,
+ ) -> ::core::result::Result<(), E> {
+ // SAFETY: TODO.
+ unsafe { $crate::init::Init::__init(init, slot) }
+ }
+ )*
+ }
+ };
+}
+
+/// The internal init macro. Do not call manually!
+///
+/// This is called by the `{try_}{pin_}init!` macros with various inputs.
+///
+/// This macro has multiple internal call configurations, these are always the very first ident:
+/// - nothing: this is the base case and called by the `{try_}{pin_}init!` macros.
+/// - `with_update_parsed`: when the `..Zeroable::zeroed()` syntax has been handled.
+/// - `init_slot`: recursively creates the code that initializes all fields in `slot`.
+/// - `make_initializer`: recursively create the struct initializer that guarantees that every
+/// field has been initialized exactly once.
+#[doc(hidden)]
+#[macro_export]
+macro_rules! __init_internal {
+ (
+ @this($($this:ident)?),
+ @typ($t:path),
+ @fields($($fields:tt)*),
+ @error($err:ty),
+ // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData`
+ // case.
+ @data($data:ident, $($use_data:ident)?),
+ // `HasPinData` or `HasInitData`.
+ @has_data($has_data:ident, $get_data:ident),
+ // `pin_init_from_closure` or `init_from_closure`.
+ @construct_closure($construct_closure:ident),
+ @munch_fields(),
+ ) => {
+ $crate::__init_internal!(with_update_parsed:
+ @this($($this)?),
+ @typ($t),
+ @fields($($fields)*),
+ @error($err),
+ @data($data, $($use_data)?),
+ @has_data($has_data, $get_data),
+ @construct_closure($construct_closure),
+ @zeroed(), // Nothing means default behavior.
+ )
+ };
+ (
+ @this($($this:ident)?),
+ @typ($t:path),
+ @fields($($fields:tt)*),
+ @error($err:ty),
+ // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData`
+ // case.
+ @data($data:ident, $($use_data:ident)?),
+ // `HasPinData` or `HasInitData`.
+ @has_data($has_data:ident, $get_data:ident),
+ // `pin_init_from_closure` or `init_from_closure`.
+ @construct_closure($construct_closure:ident),
+ @munch_fields(..Zeroable::zeroed()),
+ ) => {
+ $crate::__init_internal!(with_update_parsed:
+ @this($($this)?),
+ @typ($t),
+ @fields($($fields)*),
+ @error($err),
+ @data($data, $($use_data)?),
+ @has_data($has_data, $get_data),
+ @construct_closure($construct_closure),
+ @zeroed(()), // `()` means zero all fields not mentioned.
+ )
+ };
+ (
+ @this($($this:ident)?),
+ @typ($t:path),
+ @fields($($fields:tt)*),
+ @error($err:ty),
+ // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData`
+ // case.
+ @data($data:ident, $($use_data:ident)?),
+ // `HasPinData` or `HasInitData`.
+ @has_data($has_data:ident, $get_data:ident),
+ // `pin_init_from_closure` or `init_from_closure`.
+ @construct_closure($construct_closure:ident),
+ @munch_fields($ignore:tt $($rest:tt)*),
+ ) => {
+ $crate::__init_internal!(
+ @this($($this)?),
+ @typ($t),
+ @fields($($fields)*),
+ @error($err),
+ @data($data, $($use_data)?),
+ @has_data($has_data, $get_data),
+ @construct_closure($construct_closure),
+ @munch_fields($($rest)*),
+ )
+ };
+ (with_update_parsed:
+ @this($($this:ident)?),
+ @typ($t:path),
+ @fields($($fields:tt)*),
+ @error($err:ty),
+ // Either `PinData` or `InitData`, `$use_data` should only be present in the `PinData`
+ // case.
+ @data($data:ident, $($use_data:ident)?),
+ // `HasPinData` or `HasInitData`.
+ @has_data($has_data:ident, $get_data:ident),
+ // `pin_init_from_closure` or `init_from_closure`.
+ @construct_closure($construct_closure:ident),
+ @zeroed($($init_zeroed:expr)?),
+ ) => {{
+ // We do not want to allow arbitrary returns, so we declare this type as the `Ok` return
+ // type and shadow it later when we insert the arbitrary user code. That way there will be
+ // no possibility of returning without `unsafe`.
+ struct __InitOk;
+ // Get the data about fields from the supplied type.
+ //
+ // SAFETY: TODO.
+ let data = unsafe {
+ use $crate::init::__internal::$has_data;
+ // Here we abuse `paste!` to retokenize `$t`. Declarative macros have some internal
+ // information that is associated to already parsed fragments, so a path fragment
+ // cannot be used in this position. Doing the retokenization results in valid rust
+ // code.
+ ::kernel::macros::paste!($t::$get_data())
+ };
+ // Ensure that `data` really is of type `$data` and help with type inference:
+ let init = $crate::init::__internal::$data::make_closure::<_, __InitOk, $err>(
+ data,
+ move |slot| {
+ {
+ // Shadow the structure so it cannot be used to return early.
+ struct __InitOk;
+ // If `$init_zeroed` is present we should zero the slot now and not emit an
+ // error when fields are missing (since they will be zeroed). We also have to
+ // check that the type actually implements `Zeroable`.
+ $({
+ fn assert_zeroable<T: $crate::init::Zeroable>(_: *mut T) {}
+ // Ensure that the struct is indeed `Zeroable`.
+ assert_zeroable(slot);
+ // SAFETY: The type implements `Zeroable` by the check above.
+ unsafe { ::core::ptr::write_bytes(slot, 0, 1) };
+ $init_zeroed // This will be `()` if set.
+ })?
+ // Create the `this` so it can be referenced by the user inside of the
+ // expressions creating the individual fields.
+ $(let $this = unsafe { ::core::ptr::NonNull::new_unchecked(slot) };)?
+ // Initialize every field.
+ $crate::__init_internal!(init_slot($($use_data)?):
+ @data(data),
+ @slot(slot),
+ @guards(),
+ @munch_fields($($fields)*,),
+ );
+ // We use unreachable code to ensure that all fields have been mentioned exactly
+ // once, this struct initializer will still be type-checked and complain with a
+ // very natural error message if a field is forgotten/mentioned more than once.
+ #[allow(unreachable_code, clippy::diverging_sub_expression)]
+ let _ = || {
+ $crate::__init_internal!(make_initializer:
+ @slot(slot),
+ @type_name($t),
+ @munch_fields($($fields)*,),
+ @acc(),
+ );
+ };
+ }
+ Ok(__InitOk)
+ }
+ );
+ let init = move |slot| -> ::core::result::Result<(), $err> {
+ init(slot).map(|__InitOk| ())
+ };
+ // SAFETY: TODO.
+ let init = unsafe { $crate::init::$construct_closure::<_, $err>(init) };
+ init
+ }};
+ (init_slot($($use_data:ident)?):
+ @data($data:ident),
+ @slot($slot:ident),
+ @guards($($guards:ident,)*),
+ @munch_fields($(..Zeroable::zeroed())? $(,)?),
+ ) => {
+ // Endpoint of munching, no fields are left. If execution reaches this point, all fields
+ // have been initialized. Therefore we can now dismiss the guards by forgetting them.
+ $(::core::mem::forget($guards);)*
+ };
+ (init_slot($use_data:ident): // `use_data` is present, so we use the `data` to init fields.
+ @data($data:ident),
+ @slot($slot:ident),
+ @guards($($guards:ident,)*),
+ // In-place initialization syntax.
+ @munch_fields($field:ident <- $val:expr, $($rest:tt)*),
+ ) => {
+ let init = $val;
+ // Call the initializer.
+ //
+ // SAFETY: `slot` is valid, because we are inside of an initializer closure, we
+ // return when an error/panic occurs.
+ // We also use the `data` to require the correct trait (`Init` or `PinInit`) for `$field`.
+ unsafe { $data.$field(::core::ptr::addr_of_mut!((*$slot).$field), init)? };
+ // Create the drop guard:
+ //
+ // We rely on macro hygiene to make it impossible for users to access this local variable.
+ // We use `paste!` to create new hygiene for `$field`.
+ ::kernel::macros::paste! {
+ // SAFETY: We forget the guard later when initialization has succeeded.
+ let [< __ $field _guard >] = unsafe {
+ $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))
+ };
+
+ $crate::__init_internal!(init_slot($use_data):
+ @data($data),
+ @slot($slot),
+ @guards([< __ $field _guard >], $($guards,)*),
+ @munch_fields($($rest)*),
+ );
+ }
+ };
+ (init_slot(): // No `use_data`, so we use `Init::__init` directly.
+ @data($data:ident),
+ @slot($slot:ident),
+ @guards($($guards:ident,)*),
+ // In-place initialization syntax.
+ @munch_fields($field:ident <- $val:expr, $($rest:tt)*),
+ ) => {
+ let init = $val;
+ // Call the initializer.
+ //
+ // SAFETY: `slot` is valid, because we are inside of an initializer closure, we
+ // return when an error/panic occurs.
+ unsafe { $crate::init::Init::__init(init, ::core::ptr::addr_of_mut!((*$slot).$field))? };
+ // Create the drop guard:
+ //
+ // We rely on macro hygiene to make it impossible for users to access this local variable.
+ // We use `paste!` to create new hygiene for `$field`.
+ ::kernel::macros::paste! {
+ // SAFETY: We forget the guard later when initialization has succeeded.
+ let [< __ $field _guard >] = unsafe {
+ $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))
+ };
+
+ $crate::__init_internal!(init_slot():
+ @data($data),
+ @slot($slot),
+ @guards([< __ $field _guard >], $($guards,)*),
+ @munch_fields($($rest)*),
+ );
+ }
+ };
+ (init_slot($($use_data:ident)?):
+ @data($data:ident),
+ @slot($slot:ident),
+ @guards($($guards:ident,)*),
+ // Init by-value.
+ @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*),
+ ) => {
+ {
+ $(let $field = $val;)?
+ // Initialize the field.
+ //
+ // SAFETY: The memory at `slot` is uninitialized.
+ unsafe { ::core::ptr::write(::core::ptr::addr_of_mut!((*$slot).$field), $field) };
+ }
+ // Create the drop guard:
+ //
+ // We rely on macro hygiene to make it impossible for users to access this local variable.
+ // We use `paste!` to create new hygiene for `$field`.
+ ::kernel::macros::paste! {
+ // SAFETY: We forget the guard later when initialization has succeeded.
+ let [< __ $field _guard >] = unsafe {
+ $crate::init::__internal::DropGuard::new(::core::ptr::addr_of_mut!((*$slot).$field))
+ };
+
+ $crate::__init_internal!(init_slot($($use_data)?):
+ @data($data),
+ @slot($slot),
+ @guards([< __ $field _guard >], $($guards,)*),
+ @munch_fields($($rest)*),
+ );
+ }
+ };
+ (make_initializer:
+ @slot($slot:ident),
+ @type_name($t:path),
+ @munch_fields(..Zeroable::zeroed() $(,)?),
+ @acc($($acc:tt)*),
+ ) => {
+ // Endpoint, nothing more to munch, create the initializer. Since the users specified
+ // `..Zeroable::zeroed()`, the slot will already have been zeroed and all field that have
+ // not been overwritten are thus zero and initialized. We still check that all fields are
+ // actually accessible by using the struct update syntax ourselves.
+ // We are inside of a closure that is never executed and thus we can abuse `slot` to
+ // get the correct type inference here:
+ #[allow(unused_assignments)]
+ unsafe {
+ let mut zeroed = ::core::mem::zeroed();
+ // We have to use type inference here to make zeroed have the correct type. This does
+ // not get executed, so it has no effect.
+ ::core::ptr::write($slot, zeroed);
+ zeroed = ::core::mem::zeroed();
+ // Here we abuse `paste!` to retokenize `$t`. Declarative macros have some internal
+ // information that is associated to already parsed fragments, so a path fragment
+ // cannot be used in this position. Doing the retokenization results in valid rust
+ // code.
+ ::kernel::macros::paste!(
+ ::core::ptr::write($slot, $t {
+ $($acc)*
+ ..zeroed
+ });
+ );
+ }
+ };
+ (make_initializer:
+ @slot($slot:ident),
+ @type_name($t:path),
+ @munch_fields($(,)?),
+ @acc($($acc:tt)*),
+ ) => {
+ // Endpoint, nothing more to munch, create the initializer.
+ // Since we are in the closure that is never called, this will never get executed.
+ // We abuse `slot` to get the correct type inference here:
+ //
+ // SAFETY: TODO.
+ unsafe {
+ // Here we abuse `paste!` to retokenize `$t`. Declarative macros have some internal
+ // information that is associated to already parsed fragments, so a path fragment
+ // cannot be used in this position. Doing the retokenization results in valid rust
+ // code.
+ ::kernel::macros::paste!(
+ ::core::ptr::write($slot, $t {
+ $($acc)*
+ });
+ );
+ }
+ };
+ (make_initializer:
+ @slot($slot:ident),
+ @type_name($t:path),
+ @munch_fields($field:ident <- $val:expr, $($rest:tt)*),
+ @acc($($acc:tt)*),
+ ) => {
+ $crate::__init_internal!(make_initializer:
+ @slot($slot),
+ @type_name($t),
+ @munch_fields($($rest)*),
+ @acc($($acc)* $field: ::core::panic!(),),
+ );
+ };
+ (make_initializer:
+ @slot($slot:ident),
+ @type_name($t:path),
+ @munch_fields($field:ident $(: $val:expr)?, $($rest:tt)*),
+ @acc($($acc:tt)*),
+ ) => {
+ $crate::__init_internal!(make_initializer:
+ @slot($slot),
+ @type_name($t),
+ @munch_fields($($rest)*),
+ @acc($($acc)* $field: ::core::panic!(),),
+ );
+ };
+}
+
+#[doc(hidden)]
+#[macro_export]
+macro_rules! __derive_zeroable {
+ (parse_input:
+ @sig(
+ $(#[$($struct_attr:tt)*])*
+ $vis:vis struct $name:ident
+ $(where $($whr:tt)*)?
+ ),
+ @impl_generics($($impl_generics:tt)*),
+ @ty_generics($($ty_generics:tt)*),
+ @body({
+ $(
+ $(#[$($field_attr:tt)*])*
+ $field:ident : $field_ty:ty
+ ),* $(,)?
+ }),
+ ) => {
+ // SAFETY: Every field type implements `Zeroable` and padding bytes may be zero.
+ #[automatically_derived]
+ unsafe impl<$($impl_generics)*> $crate::init::Zeroable for $name<$($ty_generics)*>
+ where
+ $($($whr)*)?
+ {}
+ const _: () = {
+ fn assert_zeroable<T: ?::core::marker::Sized + $crate::init::Zeroable>() {}
+ fn ensure_zeroable<$($impl_generics)*>()
+ where $($($whr)*)?
+ {
+ $(assert_zeroable::<$field_ty>();)*
+ }
+ };
+ };
+}
projects / linux-block.git / commitdiff