1 // SPDX-License-Identifier: GPL-2.0
3 //! A reference-counted pointer.
5 //! This module implements a way for users to create reference-counted objects and pointers to
6 //! them. Such a pointer automatically increments and decrements the count, and drops the
7 //! underlying object when it reaches zero. It is also safe to use concurrently from multiple
10 //! It is different from the standard library's [`Arc`] in a few ways:
11 //! 1. It is backed by the kernel's `refcount_t` type.
12 //! 2. It does not support weak references, which allows it to be half the size.
13 //! 3. It saturates the reference count instead of aborting when it goes over a threshold.
14 //! 4. It does not provide a `get_mut` method, so the ref counted object is pinned.
16 //! [`Arc`]: https://doc.rust-lang.org/std/sync/struct.Arc.html
21 init::{self, InPlaceInit, Init, PinInit},
23 types::{ForeignOwnable, Opaque},
25 use alloc::boxed::Box;
29 marker::{PhantomData, Unsize},
30 mem::{ManuallyDrop, MaybeUninit},
31 ops::{Deref, DerefMut},
39 /// A reference-counted pointer to an instance of `T`.
41 /// The reference count is incremented when new instances of [`Arc`] are created, and decremented
42 /// when they are dropped. When the count reaches zero, the underlying `T` is also dropped.
46 /// The reference count on an instance of [`Arc`] is always non-zero.
47 /// The object pointed to by [`Arc`] is always pinned.
52 /// use kernel::sync::Arc;
59 /// // Create a ref-counted instance of `Example`.
60 /// let obj = Arc::try_new(Example { a: 10, b: 20 })?;
62 /// // Get a new pointer to `obj` and increment the refcount.
63 /// let cloned = obj.clone();
65 /// // Assert that both `obj` and `cloned` point to the same underlying object.
66 /// assert!(core::ptr::eq(&*obj, &*cloned));
68 /// // Destroy `obj` and decrement its refcount.
71 /// // Check that the values are still accessible through `cloned`.
72 /// assert_eq!(cloned.a, 10);
73 /// assert_eq!(cloned.b, 20);
75 /// // The refcount drops to zero when `cloned` goes out of scope, and the memory is freed.
78 /// Using `Arc<T>` as the type of `self`:
81 /// use kernel::sync::Arc;
89 /// fn take_over(self: Arc<Self>) {
93 /// fn use_reference(self: &Arc<Self>) {
98 /// let obj = Arc::try_new(Example { a: 10, b: 20 })?;
99 /// obj.use_reference();
103 /// Coercion from `Arc<Example>` to `Arc<dyn MyTrait>`:
106 /// use kernel::sync::{Arc, ArcBorrow};
109 /// // Trait has a function whose `self` type is `Arc<Self>`.
110 /// fn example1(self: Arc<Self>) {}
112 /// // Trait has a function whose `self` type is `ArcBorrow<'_, Self>`.
113 /// fn example2(self: ArcBorrow<'_, Self>) {}
117 /// impl MyTrait for Example {}
119 /// // `obj` has type `Arc<Example>`.
120 /// let obj: Arc<Example> = Arc::try_new(Example)?;
122 /// // `coerced` has type `Arc<dyn MyTrait>`.
123 /// let coerced: Arc<dyn MyTrait> = obj;
125 pub struct Arc<T: ?Sized> {
126 ptr: NonNull<ArcInner<T>>,
127 _p: PhantomData<ArcInner<T>>,
132 struct ArcInner<T: ?Sized> {
133 refcount: Opaque<bindings::refcount_t>,
137 // This is to allow [`Arc`] (and variants) to be used as the type of `self`.
138 impl<T: ?Sized> core::ops::Receiver for Arc<T> {}
140 // This is to allow coercion from `Arc<T>` to `Arc<U>` if `T` can be converted to the
141 // dynamically-sized type (DST) `U`.
142 impl<T: ?Sized + Unsize<U>, U: ?Sized> core::ops::CoerceUnsized<Arc<U>> for Arc<T> {}
144 // This is to allow `Arc<U>` to be dispatched on when `Arc<T>` can be coerced into `Arc<U>`.
145 impl<T: ?Sized + Unsize<U>, U: ?Sized> core::ops::DispatchFromDyn<Arc<U>> for Arc<T> {}
147 // SAFETY: It is safe to send `Arc<T>` to another thread when the underlying `T` is `Sync` because
148 // it effectively means sharing `&T` (which is safe because `T` is `Sync`); additionally, it needs
149 // `T` to be `Send` because any thread that has an `Arc<T>` may ultimately access `T` directly, for
150 // example, when the reference count reaches zero and `T` is dropped.
151 unsafe impl<T: ?Sized + Sync + Send> Send for Arc<T> {}
153 // SAFETY: It is safe to send `&Arc<T>` to another thread when the underlying `T` is `Sync` for the
154 // same reason as above. `T` needs to be `Send` as well because a thread can clone an `&Arc<T>`
155 // into an `Arc<T>`, which may lead to `T` being accessed by the same reasoning as above.
156 unsafe impl<T: ?Sized + Sync + Send> Sync for Arc<T> {}
159 /// Constructs a new reference counted instance of `T`.
160 pub fn try_new(contents: T) -> Result<Self, AllocError> {
161 // INVARIANT: The refcount is initialised to a non-zero value.
162 let value = ArcInner {
163 // SAFETY: There are no safety requirements for this FFI call.
164 refcount: Opaque::new(unsafe { bindings::REFCOUNT_INIT(1) }),
168 let inner = Box::try_new(value)?;
170 // SAFETY: We just created `inner` with a reference count of 1, which is owned by the new
172 Ok(unsafe { Self::from_inner(Box::leak(inner).into()) })
175 /// Use the given initializer to in-place initialize a `T`.
177 /// If `T: !Unpin` it will not be able to move afterwards.
179 pub fn pin_init<E>(init: impl PinInit<T, E>) -> error::Result<Self>
183 UniqueArc::pin_init(init).map(|u| u.into())
186 /// Use the given initializer to in-place initialize a `T`.
188 /// This is equivalent to [`Arc<T>::pin_init`], since an [`Arc`] is always pinned.
190 pub fn init<E>(init: impl Init<T, E>) -> error::Result<Self>
194 UniqueArc::init(init).map(|u| u.into())
198 impl<T: ?Sized> Arc<T> {
199 /// Constructs a new [`Arc`] from an existing [`ArcInner`].
203 /// The caller must ensure that `inner` points to a valid location and has a non-zero reference
204 /// count, one of which will be owned by the new [`Arc`] instance.
205 unsafe fn from_inner(inner: NonNull<ArcInner<T>>) -> Self {
206 // INVARIANT: By the safety requirements, the invariants hold.
213 /// Returns an [`ArcBorrow`] from the given [`Arc`].
215 /// This is useful when the argument of a function call is an [`ArcBorrow`] (e.g., in a method
216 /// receiver), but we have an [`Arc`] instead. Getting an [`ArcBorrow`] is free when optimised.
218 pub fn as_arc_borrow(&self) -> ArcBorrow<'_, T> {
219 // SAFETY: The constraint that the lifetime of the shared reference must outlive that of
220 // the returned `ArcBorrow` ensures that the object remains alive and that no mutable
221 // reference can be created.
222 unsafe { ArcBorrow::new(self.ptr) }
225 /// Compare whether two [`Arc`] pointers reference the same underlying object.
226 pub fn ptr_eq(this: &Self, other: &Self) -> bool {
227 core::ptr::eq(this.ptr.as_ptr(), other.ptr.as_ptr())
231 impl<T: 'static> ForeignOwnable for Arc<T> {
232 type Borrowed<'a> = ArcBorrow<'a, T>;
234 fn into_foreign(self) -> *const core::ffi::c_void {
235 ManuallyDrop::new(self).ptr.as_ptr() as _
238 unsafe fn borrow<'a>(ptr: *const core::ffi::c_void) -> ArcBorrow<'a, T> {
239 // SAFETY: By the safety requirement of this function, we know that `ptr` came from
240 // a previous call to `Arc::into_foreign`.
241 let inner = NonNull::new(ptr as *mut ArcInner<T>).unwrap();
243 // SAFETY: The safety requirements of `from_foreign` ensure that the object remains alive
244 // for the lifetime of the returned value. Additionally, the safety requirements of
245 // `ForeignOwnable::borrow_mut` ensure that no new mutable references are created.
246 unsafe { ArcBorrow::new(inner) }
249 unsafe fn from_foreign(ptr: *const core::ffi::c_void) -> Self {
250 // SAFETY: By the safety requirement of this function, we know that `ptr` came from
251 // a previous call to `Arc::into_foreign`, which guarantees that `ptr` is valid and
252 // holds a reference count increment that is transferrable to us.
253 unsafe { Self::from_inner(NonNull::new(ptr as _).unwrap()) }
257 impl<T: ?Sized> Deref for Arc<T> {
260 fn deref(&self) -> &Self::Target {
261 // SAFETY: By the type invariant, there is necessarily a reference to the object, so it is
262 // safe to dereference it.
263 unsafe { &self.ptr.as_ref().data }
267 impl<T: ?Sized> Clone for Arc<T> {
268 fn clone(&self) -> Self {
269 // INVARIANT: C `refcount_inc` saturates the refcount, so it cannot overflow to zero.
270 // SAFETY: By the type invariant, there is necessarily a reference to the object, so it is
271 // safe to increment the refcount.
272 unsafe { bindings::refcount_inc(self.ptr.as_ref().refcount.get()) };
274 // SAFETY: We just incremented the refcount. This increment is now owned by the new `Arc`.
275 unsafe { Self::from_inner(self.ptr) }
279 impl<T: ?Sized> Drop for Arc<T> {
281 // SAFETY: By the type invariant, there is necessarily a reference to the object. We cannot
282 // touch `refcount` after it's decremented to a non-zero value because another thread/CPU
283 // may concurrently decrement it to zero and free it. It is ok to have a raw pointer to
284 // freed/invalid memory as long as it is never dereferenced.
285 let refcount = unsafe { self.ptr.as_ref() }.refcount.get();
287 // INVARIANT: If the refcount reaches zero, there are no other instances of `Arc`, and
288 // this instance is being dropped, so the broken invariant is not observable.
289 // SAFETY: Also by the type invariant, we are allowed to decrement the refcount.
290 let is_zero = unsafe { bindings::refcount_dec_and_test(refcount) };
292 // The count reached zero, we must free the memory.
294 // SAFETY: The pointer was initialised from the result of `Box::leak`.
295 unsafe { Box::from_raw(self.ptr.as_ptr()) };
300 impl<T: ?Sized> From<UniqueArc<T>> for Arc<T> {
301 fn from(item: UniqueArc<T>) -> Self {
306 impl<T: ?Sized> From<Pin<UniqueArc<T>>> for Arc<T> {
307 fn from(item: Pin<UniqueArc<T>>) -> Self {
308 // SAFETY: The type invariants of `Arc` guarantee that the data is pinned.
309 unsafe { Pin::into_inner_unchecked(item).inner }
313 /// A borrowed reference to an [`Arc`] instance.
315 /// For cases when one doesn't ever need to increment the refcount on the allocation, it is simpler
316 /// to use just `&T`, which we can trivially get from an `Arc<T>` instance.
318 /// However, when one may need to increment the refcount, it is preferable to use an `ArcBorrow<T>`
319 /// over `&Arc<T>` because the latter results in a double-indirection: a pointer (shared reference)
320 /// to a pointer (`Arc<T>`) to the object (`T`). An [`ArcBorrow`] eliminates this double
321 /// indirection while still allowing one to increment the refcount and getting an `Arc<T>` when/if
326 /// There are no mutable references to the underlying [`Arc`], and it remains valid for the
327 /// lifetime of the [`ArcBorrow`] instance.
332 /// use crate::sync::{Arc, ArcBorrow};
336 /// fn do_something(e: ArcBorrow<'_, Example>) -> Arc<Example> {
340 /// let obj = Arc::try_new(Example)?;
341 /// let cloned = do_something(obj.as_arc_borrow());
343 /// // Assert that both `obj` and `cloned` point to the same underlying object.
344 /// assert!(core::ptr::eq(&*obj, &*cloned));
347 /// Using `ArcBorrow<T>` as the type of `self`:
350 /// use crate::sync::{Arc, ArcBorrow};
358 /// fn use_reference(self: ArcBorrow<'_, Self>) {
363 /// let obj = Arc::try_new(Example { a: 10, b: 20 })?;
364 /// obj.as_arc_borrow().use_reference();
366 pub struct ArcBorrow<'a, T: ?Sized + 'a> {
367 inner: NonNull<ArcInner<T>>,
368 _p: PhantomData<&'a ()>,
371 // This is to allow [`ArcBorrow`] (and variants) to be used as the type of `self`.
372 impl<T: ?Sized> core::ops::Receiver for ArcBorrow<'_, T> {}
374 // This is to allow `ArcBorrow<U>` to be dispatched on when `ArcBorrow<T>` can be coerced into
376 impl<T: ?Sized + Unsize<U>, U: ?Sized> core::ops::DispatchFromDyn<ArcBorrow<'_, U>>
381 impl<T: ?Sized> Clone for ArcBorrow<'_, T> {
382 fn clone(&self) -> Self {
387 impl<T: ?Sized> Copy for ArcBorrow<'_, T> {}
389 impl<T: ?Sized> ArcBorrow<'_, T> {
390 /// Creates a new [`ArcBorrow`] instance.
394 /// Callers must ensure the following for the lifetime of the returned [`ArcBorrow`] instance:
395 /// 1. That `inner` remains valid;
396 /// 2. That no mutable references to `inner` are created.
397 unsafe fn new(inner: NonNull<ArcInner<T>>) -> Self {
398 // INVARIANT: The safety requirements guarantee the invariants.
406 impl<T: ?Sized> From<ArcBorrow<'_, T>> for Arc<T> {
407 fn from(b: ArcBorrow<'_, T>) -> Self {
408 // SAFETY: The existence of `b` guarantees that the refcount is non-zero. `ManuallyDrop`
409 // guarantees that `drop` isn't called, so it's ok that the temporary `Arc` doesn't own the
411 ManuallyDrop::new(unsafe { Arc::from_inner(b.inner) })
417 impl<T: ?Sized> Deref for ArcBorrow<'_, T> {
420 fn deref(&self) -> &Self::Target {
421 // SAFETY: By the type invariant, the underlying object is still alive with no mutable
422 // references to it, so it is safe to create a shared reference.
423 unsafe { &self.inner.as_ref().data }
427 /// A refcounted object that is known to have a refcount of 1.
429 /// It is mutable and can be converted to an [`Arc`] so that it can be shared.
433 /// `inner` always has a reference count of 1.
437 /// In the following example, we make changes to the inner object before turning it into an
438 /// `Arc<Test>` object (after which point, it cannot be mutated directly). Note that `x.into()`
442 /// use kernel::sync::{Arc, UniqueArc};
449 /// fn test() -> Result<Arc<Example>> {
450 /// let mut x = UniqueArc::try_new(Example { a: 10, b: 20 })?;
456 /// # test().unwrap();
459 /// In the following example we first allocate memory for a ref-counted `Example` but we don't
460 /// initialise it on allocation. We do initialise it later with a call to [`UniqueArc::write`],
461 /// followed by a conversion to `Arc<Example>`. This is particularly useful when allocation happens
462 /// in one context (e.g., sleepable) and initialisation in another (e.g., atomic):
465 /// use kernel::sync::{Arc, UniqueArc};
472 /// fn test() -> Result<Arc<Example>> {
473 /// let x = UniqueArc::try_new_uninit()?;
474 /// Ok(x.write(Example { a: 10, b: 20 }).into())
477 /// # test().unwrap();
480 /// In the last example below, the caller gets a pinned instance of `Example` while converting to
481 /// `Arc<Example>`; this is useful in scenarios where one needs a pinned reference during
482 /// initialisation, for example, when initialising fields that are wrapped in locks.
485 /// use kernel::sync::{Arc, UniqueArc};
492 /// fn test() -> Result<Arc<Example>> {
493 /// let mut pinned = Pin::from(UniqueArc::try_new(Example { a: 10, b: 20 })?);
494 /// // We can modify `pinned` because it is `Unpin`.
495 /// pinned.as_mut().a += 1;
496 /// Ok(pinned.into())
499 /// # test().unwrap();
501 pub struct UniqueArc<T: ?Sized> {
505 impl<T> UniqueArc<T> {
506 /// Tries to allocate a new [`UniqueArc`] instance.
507 pub fn try_new(value: T) -> Result<Self, AllocError> {
509 // INVARIANT: The newly-created object has a ref-count of 1.
510 inner: Arc::try_new(value)?,
514 /// Tries to allocate a new [`UniqueArc`] instance whose contents are not initialised yet.
515 pub fn try_new_uninit() -> Result<UniqueArc<MaybeUninit<T>>, AllocError> {
516 // INVARIANT: The refcount is initialised to a non-zero value.
517 let inner = Box::try_init::<AllocError>(try_init!(ArcInner {
518 // SAFETY: There are no safety requirements for this FFI call.
519 refcount: Opaque::new(unsafe { bindings::REFCOUNT_INIT(1) }),
520 data <- init::uninit::<T, AllocError>(),
523 // INVARIANT: The newly-created object has a ref-count of 1.
524 // SAFETY: The pointer from the `Box` is valid.
525 inner: unsafe { Arc::from_inner(Box::leak(inner).into()) },
530 impl<T> UniqueArc<MaybeUninit<T>> {
531 /// Converts a `UniqueArc<MaybeUninit<T>>` into a `UniqueArc<T>` by writing a value into it.
532 pub fn write(mut self, value: T) -> UniqueArc<T> {
533 self.deref_mut().write(value);
534 // SAFETY: We just wrote the value to be initialized.
535 unsafe { self.assume_init() }
538 /// Unsafely assume that `self` is initialized.
542 /// The caller guarantees that the value behind this pointer has been initialized. It is
543 /// *immediate* UB to call this when the value is not initialized.
544 pub unsafe fn assume_init(self) -> UniqueArc<T> {
545 let inner = ManuallyDrop::new(self).inner.ptr;
547 // SAFETY: The new `Arc` is taking over `ptr` from `self.inner` (which won't be
548 // dropped). The types are compatible because `MaybeUninit<T>` is compatible with `T`.
549 inner: unsafe { Arc::from_inner(inner.cast()) },
553 /// Initialize `self` using the given initializer.
554 pub fn init_with<E>(mut self, init: impl Init<T, E>) -> core::result::Result<UniqueArc<T>, E> {
555 // SAFETY: The supplied pointer is valid for initialization.
556 match unsafe { init.__init(self.as_mut_ptr()) } {
557 // SAFETY: Initialization completed successfully.
558 Ok(()) => Ok(unsafe { self.assume_init() }),
559 Err(err) => Err(err),
563 /// Pin-initialize `self` using the given pin-initializer.
564 pub fn pin_init_with<E>(
566 init: impl PinInit<T, E>,
567 ) -> core::result::Result<Pin<UniqueArc<T>>, E> {
568 // SAFETY: The supplied pointer is valid for initialization and we will later pin the value
569 // to ensure it does not move.
570 match unsafe { init.__pinned_init(self.as_mut_ptr()) } {
571 // SAFETY: Initialization completed successfully.
572 Ok(()) => Ok(unsafe { self.assume_init() }.into()),
573 Err(err) => Err(err),
578 impl<T: ?Sized> From<UniqueArc<T>> for Pin<UniqueArc<T>> {
579 fn from(obj: UniqueArc<T>) -> Self {
580 // SAFETY: It is not possible to move/replace `T` inside a `Pin<UniqueArc<T>>` (unless `T`
581 // is `Unpin`), so it is ok to convert it to `Pin<UniqueArc<T>>`.
582 unsafe { Pin::new_unchecked(obj) }
586 impl<T: ?Sized> Deref for UniqueArc<T> {
589 fn deref(&self) -> &Self::Target {
594 impl<T: ?Sized> DerefMut for UniqueArc<T> {
595 fn deref_mut(&mut self) -> &mut Self::Target {
596 // SAFETY: By the `Arc` type invariant, there is necessarily a reference to the object, so
597 // it is safe to dereference it. Additionally, we know there is only one reference when
598 // it's inside a `UniqueArc`, so it is safe to get a mutable reference.
599 unsafe { &mut self.inner.ptr.as_mut().data }
603 impl<T: fmt::Display + ?Sized> fmt::Display for UniqueArc<T> {
604 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
605 fmt::Display::fmt(self.deref(), f)
609 impl<T: fmt::Display + ?Sized> fmt::Display for Arc<T> {
610 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
611 fmt::Display::fmt(self.deref(), f)
615 impl<T: fmt::Debug + ?Sized> fmt::Debug for UniqueArc<T> {
616 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
617 fmt::Debug::fmt(self.deref(), f)
621 impl<T: fmt::Debug + ?Sized> fmt::Debug for Arc<T> {
622 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
623 fmt::Debug::fmt(self.deref(), f)