Commit | Line | Data |
---|---|---|
9dc04365 WAF |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | ||
3 | //! A reference-counted pointer. | |
4 | //! | |
5 | //! This module implements a way for users to create reference-counted objects and pointers to | |
6 | //! them. Such a pointer automatically increments and decrements the count, and drops the | |
7 | //! underlying object when it reaches zero. It is also safe to use concurrently from multiple | |
8 | //! threads. | |
9 | //! | |
10 | //! It is different from the standard library's [`Arc`] in a few ways: | |
11 | //! 1. It is backed by the kernel's `refcount_t` type. | |
12 | //! 2. It does not support weak references, which allows it to be half the size. | |
13 | //! 3. It saturates the reference count instead of aborting when it goes over a threshold. | |
14 | //! 4. It does not provide a `get_mut` method, so the ref counted object is pinned. | |
15 | //! | |
16 | //! [`Arc`]: https://doc.rust-lang.org/std/sync/struct.Arc.html | |
17 | ||
0c7ae432 | 18 | use crate::{ |
2c109285 | 19 | alloc::{box_ext::BoxExt, AllocError, Flags}, |
0c7ae432 | 20 | bindings, |
92c4a1e7 | 21 | error::{self, Error}, |
701608bd BL |
22 | init::{self, InPlaceInit, Init, PinInit}, |
23 | try_init, | |
0c7ae432 WAF |
24 | types::{ForeignOwnable, Opaque}, |
25 | }; | |
9dc04365 | 26 | use alloc::boxed::Box; |
f75cb6fc | 27 | use core::{ |
2c109285 | 28 | alloc::Layout, |
00140a83 | 29 | fmt, |
f75cb6fc | 30 | marker::{PhantomData, Unsize}, |
70e42ebb WAF |
31 | mem::{ManuallyDrop, MaybeUninit}, |
32 | ops::{Deref, DerefMut}, | |
33 | pin::Pin, | |
44f2e626 | 34 | ptr::NonNull, |
f75cb6fc | 35 | }; |
701608bd | 36 | use macros::pin_data; |
9dc04365 | 37 | |
1edd0337 AL |
38 | mod std_vendor; |
39 | ||
9dc04365 WAF |
40 | /// A reference-counted pointer to an instance of `T`. |
41 | /// | |
42 | /// The reference count is incremented when new instances of [`Arc`] are created, and decremented | |
43 | /// when they are dropped. When the count reaches zero, the underlying `T` is also dropped. | |
44 | /// | |
45 | /// # Invariants | |
46 | /// | |
47 | /// The reference count on an instance of [`Arc`] is always non-zero. | |
48 | /// The object pointed to by [`Arc`] is always pinned. | |
49 | /// | |
50 | /// # Examples | |
51 | /// | |
52 | /// ``` | |
53 | /// use kernel::sync::Arc; | |
54 | /// | |
55 | /// struct Example { | |
56 | /// a: u32, | |
57 | /// b: u32, | |
58 | /// } | |
59 | /// | |
ebf2b8a7 | 60 | /// // Create a refcounted instance of `Example`. |
cc41670e | 61 | /// let obj = Arc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?; |
9dc04365 WAF |
62 | /// |
63 | /// // Get a new pointer to `obj` and increment the refcount. | |
64 | /// let cloned = obj.clone(); | |
65 | /// | |
66 | /// // Assert that both `obj` and `cloned` point to the same underlying object. | |
67 | /// assert!(core::ptr::eq(&*obj, &*cloned)); | |
68 | /// | |
69 | /// // Destroy `obj` and decrement its refcount. | |
70 | /// drop(obj); | |
71 | /// | |
72 | /// // Check that the values are still accessible through `cloned`. | |
73 | /// assert_eq!(cloned.a, 10); | |
74 | /// assert_eq!(cloned.b, 20); | |
75 | /// | |
76 | /// // The refcount drops to zero when `cloned` goes out of scope, and the memory is freed. | |
bfa7dff0 | 77 | /// # Ok::<(), Error>(()) |
9dc04365 | 78 | /// ``` |
53528772 WAF |
79 | /// |
80 | /// Using `Arc<T>` as the type of `self`: | |
81 | /// | |
82 | /// ``` | |
83 | /// use kernel::sync::Arc; | |
84 | /// | |
85 | /// struct Example { | |
86 | /// a: u32, | |
87 | /// b: u32, | |
88 | /// } | |
89 | /// | |
90 | /// impl Example { | |
91 | /// fn take_over(self: Arc<Self>) { | |
92 | /// // ... | |
93 | /// } | |
94 | /// | |
95 | /// fn use_reference(self: &Arc<Self>) { | |
96 | /// // ... | |
97 | /// } | |
98 | /// } | |
99 | /// | |
cc41670e | 100 | /// let obj = Arc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?; |
53528772 WAF |
101 | /// obj.use_reference(); |
102 | /// obj.take_over(); | |
bfa7dff0 | 103 | /// # Ok::<(), Error>(()) |
53528772 | 104 | /// ``` |
f75cb6fc WAF |
105 | /// |
106 | /// Coercion from `Arc<Example>` to `Arc<dyn MyTrait>`: | |
107 | /// | |
108 | /// ``` | |
0748424a WAF |
109 | /// use kernel::sync::{Arc, ArcBorrow}; |
110 | /// | |
111 | /// trait MyTrait { | |
112 | /// // Trait has a function whose `self` type is `Arc<Self>`. | |
113 | /// fn example1(self: Arc<Self>) {} | |
f75cb6fc | 114 | /// |
0748424a WAF |
115 | /// // Trait has a function whose `self` type is `ArcBorrow<'_, Self>`. |
116 | /// fn example2(self: ArcBorrow<'_, Self>) {} | |
117 | /// } | |
f75cb6fc WAF |
118 | /// |
119 | /// struct Example; | |
120 | /// impl MyTrait for Example {} | |
121 | /// | |
122 | /// // `obj` has type `Arc<Example>`. | |
cc41670e | 123 | /// let obj: Arc<Example> = Arc::new(Example, GFP_KERNEL)?; |
f75cb6fc WAF |
124 | /// |
125 | /// // `coerced` has type `Arc<dyn MyTrait>`. | |
126 | /// let coerced: Arc<dyn MyTrait> = obj; | |
bfa7dff0 | 127 | /// # Ok::<(), Error>(()) |
f75cb6fc | 128 | /// ``` |
9dc04365 WAF |
129 | pub struct Arc<T: ?Sized> { |
130 | ptr: NonNull<ArcInner<T>>, | |
131 | _p: PhantomData<ArcInner<T>>, | |
132 | } | |
133 | ||
701608bd | 134 | #[pin_data] |
9dc04365 WAF |
135 | #[repr(C)] |
136 | struct ArcInner<T: ?Sized> { | |
137 | refcount: Opaque<bindings::refcount_t>, | |
138 | data: T, | |
139 | } | |
140 | ||
51f6af86 AR |
141 | impl<T: ?Sized> ArcInner<T> { |
142 | /// Converts a pointer to the contents of an [`Arc`] into a pointer to the [`ArcInner`]. | |
143 | /// | |
144 | /// # Safety | |
145 | /// | |
146 | /// `ptr` must have been returned by a previous call to [`Arc::into_raw`], and the `Arc` must | |
147 | /// not yet have been destroyed. | |
148 | unsafe fn container_of(ptr: *const T) -> NonNull<ArcInner<T>> { | |
149 | let refcount_layout = Layout::new::<bindings::refcount_t>(); | |
150 | // SAFETY: The caller guarantees that the pointer is valid. | |
151 | let val_layout = Layout::for_value(unsafe { &*ptr }); | |
152 | // SAFETY: We're computing the layout of a real struct that existed when compiling this | |
153 | // binary, so its layout is not so large that it can trigger arithmetic overflow. | |
154 | let val_offset = unsafe { refcount_layout.extend(val_layout).unwrap_unchecked().1 }; | |
155 | ||
156 | // Pointer casts leave the metadata unchanged. This is okay because the metadata of `T` and | |
157 | // `ArcInner<T>` is the same since `ArcInner` is a struct with `T` as its last field. | |
158 | // | |
159 | // This is documented at: | |
160 | // <https://doc.rust-lang.org/std/ptr/trait.Pointee.html>. | |
161 | let ptr = ptr as *const ArcInner<T>; | |
162 | ||
163 | // SAFETY: The pointer is in-bounds of an allocation both before and after offsetting the | |
164 | // pointer, since it originates from a previous call to `Arc::into_raw` on an `Arc` that is | |
165 | // still valid. | |
166 | let ptr = unsafe { ptr.byte_sub(val_offset) }; | |
167 | ||
168 | // SAFETY: The pointer can't be null since you can't have an `ArcInner<T>` value at the null | |
169 | // address. | |
170 | unsafe { NonNull::new_unchecked(ptr.cast_mut()) } | |
171 | } | |
172 | } | |
173 | ||
53528772 WAF |
174 | // This is to allow [`Arc`] (and variants) to be used as the type of `self`. |
175 | impl<T: ?Sized> core::ops::Receiver for Arc<T> {} | |
176 | ||
f75cb6fc WAF |
177 | // This is to allow coercion from `Arc<T>` to `Arc<U>` if `T` can be converted to the |
178 | // dynamically-sized type (DST) `U`. | |
179 | impl<T: ?Sized + Unsize<U>, U: ?Sized> core::ops::CoerceUnsized<Arc<U>> for Arc<T> {} | |
180 | ||
0748424a WAF |
181 | // This is to allow `Arc<U>` to be dispatched on when `Arc<T>` can be coerced into `Arc<U>`. |
182 | impl<T: ?Sized + Unsize<U>, U: ?Sized> core::ops::DispatchFromDyn<Arc<U>> for Arc<T> {} | |
183 | ||
9dc04365 WAF |
184 | // SAFETY: It is safe to send `Arc<T>` to another thread when the underlying `T` is `Sync` because |
185 | // it effectively means sharing `&T` (which is safe because `T` is `Sync`); additionally, it needs | |
f8110cd1 AR |
186 | // `T` to be `Send` because any thread that has an `Arc<T>` may ultimately access `T` using a |
187 | // mutable reference when the reference count reaches zero and `T` is dropped. | |
9dc04365 WAF |
188 | unsafe impl<T: ?Sized + Sync + Send> Send for Arc<T> {} |
189 | ||
d701e061 AR |
190 | // SAFETY: It is safe to send `&Arc<T>` to another thread when the underlying `T` is `Sync` |
191 | // because it effectively means sharing `&T` (which is safe because `T` is `Sync`); additionally, | |
192 | // it needs `T` to be `Send` because any thread that has a `&Arc<T>` may clone it and get an | |
193 | // `Arc<T>` on that thread, so the thread may ultimately access `T` using a mutable reference when | |
194 | // the reference count reaches zero and `T` is dropped. | |
9dc04365 WAF |
195 | unsafe impl<T: ?Sized + Sync + Send> Sync for Arc<T> {} |
196 | ||
197 | impl<T> Arc<T> { | |
198 | /// Constructs a new reference counted instance of `T`. | |
cc41670e | 199 | pub fn new(contents: T, flags: Flags) -> Result<Self, AllocError> { |
9dc04365 WAF |
200 | // INVARIANT: The refcount is initialised to a non-zero value. |
201 | let value = ArcInner { | |
202 | // SAFETY: There are no safety requirements for this FFI call. | |
203 | refcount: Opaque::new(unsafe { bindings::REFCOUNT_INIT(1) }), | |
204 | data: contents, | |
205 | }; | |
206 | ||
cc41670e | 207 | let inner = <Box<_> as BoxExt<_>>::new(value, flags)?; |
9dc04365 WAF |
208 | |
209 | // SAFETY: We just created `inner` with a reference count of 1, which is owned by the new | |
210 | // `Arc` object. | |
211 | Ok(unsafe { Self::from_inner(Box::leak(inner).into()) }) | |
212 | } | |
92c4a1e7 BL |
213 | |
214 | /// Use the given initializer to in-place initialize a `T`. | |
215 | /// | |
216 | /// If `T: !Unpin` it will not be able to move afterwards. | |
217 | #[inline] | |
c34aa00d | 218 | pub fn pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> error::Result<Self> |
92c4a1e7 BL |
219 | where |
220 | Error: From<E>, | |
221 | { | |
c34aa00d | 222 | UniqueArc::pin_init(init, flags).map(|u| u.into()) |
92c4a1e7 BL |
223 | } |
224 | ||
225 | /// Use the given initializer to in-place initialize a `T`. | |
226 | /// | |
eed7a146 | 227 | /// This is equivalent to [`Arc<T>::pin_init`], since an [`Arc`] is always pinned. |
92c4a1e7 | 228 | #[inline] |
c34aa00d | 229 | pub fn init<E>(init: impl Init<T, E>, flags: Flags) -> error::Result<Self> |
92c4a1e7 BL |
230 | where |
231 | Error: From<E>, | |
232 | { | |
c34aa00d | 233 | UniqueArc::init(init, flags).map(|u| u.into()) |
92c4a1e7 | 234 | } |
9dc04365 WAF |
235 | } |
236 | ||
237 | impl<T: ?Sized> Arc<T> { | |
238 | /// Constructs a new [`Arc`] from an existing [`ArcInner`]. | |
239 | /// | |
240 | /// # Safety | |
241 | /// | |
242 | /// The caller must ensure that `inner` points to a valid location and has a non-zero reference | |
243 | /// count, one of which will be owned by the new [`Arc`] instance. | |
244 | unsafe fn from_inner(inner: NonNull<ArcInner<T>>) -> Self { | |
245 | // INVARIANT: By the safety requirements, the invariants hold. | |
246 | Arc { | |
247 | ptr: inner, | |
248 | _p: PhantomData, | |
249 | } | |
250 | } | |
17f67160 | 251 | |
a8321776 WAF |
252 | /// Convert the [`Arc`] into a raw pointer. |
253 | /// | |
254 | /// The raw pointer has ownership of the refcount that this Arc object owned. | |
255 | pub fn into_raw(self) -> *const T { | |
256 | let ptr = self.ptr.as_ptr(); | |
257 | core::mem::forget(self); | |
258 | // SAFETY: The pointer is valid. | |
259 | unsafe { core::ptr::addr_of!((*ptr).data) } | |
260 | } | |
261 | ||
262 | /// Recreates an [`Arc`] instance previously deconstructed via [`Arc::into_raw`]. | |
263 | /// | |
264 | /// # Safety | |
265 | /// | |
266 | /// `ptr` must have been returned by a previous call to [`Arc::into_raw`]. Additionally, it | |
267 | /// must not be called more than once for each previous call to [`Arc::into_raw`]. | |
268 | pub unsafe fn from_raw(ptr: *const T) -> Self { | |
51f6af86 AR |
269 | // SAFETY: The caller promises that this pointer originates from a call to `into_raw` on an |
270 | // `Arc` that is still valid. | |
271 | let ptr = unsafe { ArcInner::container_of(ptr) }; | |
a8321776 WAF |
272 | |
273 | // SAFETY: By the safety requirements we know that `ptr` came from `Arc::into_raw`, so the | |
274 | // reference count held then will be owned by the new `Arc` object. | |
51f6af86 | 275 | unsafe { Self::from_inner(ptr) } |
a8321776 WAF |
276 | } |
277 | ||
17f67160 WAF |
278 | /// Returns an [`ArcBorrow`] from the given [`Arc`]. |
279 | /// | |
280 | /// This is useful when the argument of a function call is an [`ArcBorrow`] (e.g., in a method | |
281 | /// receiver), but we have an [`Arc`] instead. Getting an [`ArcBorrow`] is free when optimised. | |
282 | #[inline] | |
283 | pub fn as_arc_borrow(&self) -> ArcBorrow<'_, T> { | |
284 | // SAFETY: The constraint that the lifetime of the shared reference must outlive that of | |
285 | // the returned `ArcBorrow` ensures that the object remains alive and that no mutable | |
286 | // reference can be created. | |
287 | unsafe { ArcBorrow::new(self.ptr) } | |
288 | } | |
bd780aea AR |
289 | |
290 | /// Compare whether two [`Arc`] pointers reference the same underlying object. | |
291 | pub fn ptr_eq(this: &Self, other: &Self) -> bool { | |
292 | core::ptr::eq(this.ptr.as_ptr(), other.ptr.as_ptr()) | |
293 | } | |
a0a4e170 AR |
294 | |
295 | /// Converts this [`Arc`] into a [`UniqueArc`], or destroys it if it is not unique. | |
296 | /// | |
297 | /// When this destroys the `Arc`, it does so while properly avoiding races. This means that | |
298 | /// this method will never call the destructor of the value. | |
299 | /// | |
300 | /// # Examples | |
301 | /// | |
302 | /// ``` | |
303 | /// use kernel::sync::{Arc, UniqueArc}; | |
304 | /// | |
305 | /// let arc = Arc::new(42, GFP_KERNEL)?; | |
306 | /// let unique_arc = arc.into_unique_or_drop(); | |
307 | /// | |
308 | /// // The above conversion should succeed since refcount of `arc` is 1. | |
309 | /// assert!(unique_arc.is_some()); | |
310 | /// | |
311 | /// assert_eq!(*(unique_arc.unwrap()), 42); | |
312 | /// | |
313 | /// # Ok::<(), Error>(()) | |
314 | /// ``` | |
315 | /// | |
316 | /// ``` | |
317 | /// use kernel::sync::{Arc, UniqueArc}; | |
318 | /// | |
319 | /// let arc = Arc::new(42, GFP_KERNEL)?; | |
320 | /// let another = arc.clone(); | |
321 | /// | |
322 | /// let unique_arc = arc.into_unique_or_drop(); | |
323 | /// | |
324 | /// // The above conversion should fail since refcount of `arc` is >1. | |
325 | /// assert!(unique_arc.is_none()); | |
326 | /// | |
327 | /// # Ok::<(), Error>(()) | |
328 | /// ``` | |
329 | pub fn into_unique_or_drop(self) -> Option<Pin<UniqueArc<T>>> { | |
330 | // We will manually manage the refcount in this method, so we disable the destructor. | |
331 | let me = ManuallyDrop::new(self); | |
332 | // SAFETY: We own a refcount, so the pointer is still valid. | |
333 | let refcount = unsafe { me.ptr.as_ref() }.refcount.get(); | |
334 | ||
335 | // If the refcount reaches a non-zero value, then we have destroyed this `Arc` and will | |
336 | // return without further touching the `Arc`. If the refcount reaches zero, then there are | |
337 | // no other arcs, and we can create a `UniqueArc`. | |
338 | // | |
339 | // SAFETY: We own a refcount, so the pointer is not dangling. | |
340 | let is_zero = unsafe { bindings::refcount_dec_and_test(refcount) }; | |
341 | if is_zero { | |
342 | // SAFETY: We have exclusive access to the arc, so we can perform unsynchronized | |
343 | // accesses to the refcount. | |
344 | unsafe { core::ptr::write(refcount, bindings::REFCOUNT_INIT(1)) }; | |
345 | ||
346 | // INVARIANT: We own the only refcount to this arc, so we may create a `UniqueArc`. We | |
347 | // must pin the `UniqueArc` because the values was previously in an `Arc`, and they pin | |
348 | // their values. | |
349 | Some(Pin::from(UniqueArc { | |
350 | inner: ManuallyDrop::into_inner(me), | |
351 | })) | |
352 | } else { | |
353 | None | |
354 | } | |
355 | } | |
9dc04365 WAF |
356 | } |
357 | ||
0c7ae432 WAF |
358 | impl<T: 'static> ForeignOwnable for Arc<T> { |
359 | type Borrowed<'a> = ArcBorrow<'a, T>; | |
360 | ||
361 | fn into_foreign(self) -> *const core::ffi::c_void { | |
362 | ManuallyDrop::new(self).ptr.as_ptr() as _ | |
363 | } | |
364 | ||
365 | unsafe fn borrow<'a>(ptr: *const core::ffi::c_void) -> ArcBorrow<'a, T> { | |
366 | // SAFETY: By the safety requirement of this function, we know that `ptr` came from | |
367 | // a previous call to `Arc::into_foreign`. | |
368 | let inner = NonNull::new(ptr as *mut ArcInner<T>).unwrap(); | |
369 | ||
370 | // SAFETY: The safety requirements of `from_foreign` ensure that the object remains alive | |
1d24eb2d | 371 | // for the lifetime of the returned value. |
0c7ae432 WAF |
372 | unsafe { ArcBorrow::new(inner) } |
373 | } | |
374 | ||
375 | unsafe fn from_foreign(ptr: *const core::ffi::c_void) -> Self { | |
376 | // SAFETY: By the safety requirement of this function, we know that `ptr` came from | |
377 | // a previous call to `Arc::into_foreign`, which guarantees that `ptr` is valid and | |
378 | // holds a reference count increment that is transferrable to us. | |
379 | unsafe { Self::from_inner(NonNull::new(ptr as _).unwrap()) } | |
380 | } | |
381 | } | |
382 | ||
9dc04365 WAF |
383 | impl<T: ?Sized> Deref for Arc<T> { |
384 | type Target = T; | |
385 | ||
386 | fn deref(&self) -> &Self::Target { | |
387 | // SAFETY: By the type invariant, there is necessarily a reference to the object, so it is | |
388 | // safe to dereference it. | |
389 | unsafe { &self.ptr.as_ref().data } | |
390 | } | |
391 | } | |
392 | ||
47329ba1 AR |
393 | impl<T: ?Sized> AsRef<T> for Arc<T> { |
394 | fn as_ref(&self) -> &T { | |
395 | self.deref() | |
396 | } | |
397 | } | |
398 | ||
9dc04365 WAF |
399 | impl<T: ?Sized> Clone for Arc<T> { |
400 | fn clone(&self) -> Self { | |
401 | // INVARIANT: C `refcount_inc` saturates the refcount, so it cannot overflow to zero. | |
402 | // SAFETY: By the type invariant, there is necessarily a reference to the object, so it is | |
403 | // safe to increment the refcount. | |
404 | unsafe { bindings::refcount_inc(self.ptr.as_ref().refcount.get()) }; | |
405 | ||
406 | // SAFETY: We just incremented the refcount. This increment is now owned by the new `Arc`. | |
407 | unsafe { Self::from_inner(self.ptr) } | |
408 | } | |
409 | } | |
410 | ||
411 | impl<T: ?Sized> Drop for Arc<T> { | |
412 | fn drop(&mut self) { | |
413 | // SAFETY: By the type invariant, there is necessarily a reference to the object. We cannot | |
414 | // touch `refcount` after it's decremented to a non-zero value because another thread/CPU | |
415 | // may concurrently decrement it to zero and free it. It is ok to have a raw pointer to | |
416 | // freed/invalid memory as long as it is never dereferenced. | |
417 | let refcount = unsafe { self.ptr.as_ref() }.refcount.get(); | |
418 | ||
419 | // INVARIANT: If the refcount reaches zero, there are no other instances of `Arc`, and | |
420 | // this instance is being dropped, so the broken invariant is not observable. | |
421 | // SAFETY: Also by the type invariant, we are allowed to decrement the refcount. | |
422 | let is_zero = unsafe { bindings::refcount_dec_and_test(refcount) }; | |
423 | if is_zero { | |
424 | // The count reached zero, we must free the memory. | |
425 | // | |
426 | // SAFETY: The pointer was initialised from the result of `Box::leak`. | |
828176d0 | 427 | unsafe { drop(Box::from_raw(self.ptr.as_ptr())) }; |
9dc04365 WAF |
428 | } |
429 | } | |
430 | } | |
17f67160 | 431 | |
70e42ebb WAF |
432 | impl<T: ?Sized> From<UniqueArc<T>> for Arc<T> { |
433 | fn from(item: UniqueArc<T>) -> Self { | |
434 | item.inner | |
435 | } | |
436 | } | |
437 | ||
438 | impl<T: ?Sized> From<Pin<UniqueArc<T>>> for Arc<T> { | |
439 | fn from(item: Pin<UniqueArc<T>>) -> Self { | |
440 | // SAFETY: The type invariants of `Arc` guarantee that the data is pinned. | |
441 | unsafe { Pin::into_inner_unchecked(item).inner } | |
442 | } | |
443 | } | |
444 | ||
17f67160 WAF |
445 | /// A borrowed reference to an [`Arc`] instance. |
446 | /// | |
447 | /// For cases when one doesn't ever need to increment the refcount on the allocation, it is simpler | |
4c799d1d | 448 | /// to use just `&T`, which we can trivially get from an [`Arc<T>`] instance. |
17f67160 WAF |
449 | /// |
450 | /// However, when one may need to increment the refcount, it is preferable to use an `ArcBorrow<T>` | |
451 | /// over `&Arc<T>` because the latter results in a double-indirection: a pointer (shared reference) | |
4c799d1d VO |
452 | /// to a pointer ([`Arc<T>`]) to the object (`T`). An [`ArcBorrow`] eliminates this double |
453 | /// indirection while still allowing one to increment the refcount and getting an [`Arc<T>`] when/if | |
17f67160 WAF |
454 | /// needed. |
455 | /// | |
456 | /// # Invariants | |
457 | /// | |
458 | /// There are no mutable references to the underlying [`Arc`], and it remains valid for the | |
459 | /// lifetime of the [`ArcBorrow`] instance. | |
460 | /// | |
461 | /// # Example | |
462 | /// | |
463 | /// ``` | |
bfa7dff0 | 464 | /// use kernel::sync::{Arc, ArcBorrow}; |
17f67160 WAF |
465 | /// |
466 | /// struct Example; | |
467 | /// | |
468 | /// fn do_something(e: ArcBorrow<'_, Example>) -> Arc<Example> { | |
469 | /// e.into() | |
470 | /// } | |
471 | /// | |
cc41670e | 472 | /// let obj = Arc::new(Example, GFP_KERNEL)?; |
17f67160 WAF |
473 | /// let cloned = do_something(obj.as_arc_borrow()); |
474 | /// | |
475 | /// // Assert that both `obj` and `cloned` point to the same underlying object. | |
476 | /// assert!(core::ptr::eq(&*obj, &*cloned)); | |
bfa7dff0 | 477 | /// # Ok::<(), Error>(()) |
17f67160 | 478 | /// ``` |
92a655ae WAF |
479 | /// |
480 | /// Using `ArcBorrow<T>` as the type of `self`: | |
481 | /// | |
482 | /// ``` | |
bfa7dff0 | 483 | /// use kernel::sync::{Arc, ArcBorrow}; |
92a655ae WAF |
484 | /// |
485 | /// struct Example { | |
486 | /// a: u32, | |
487 | /// b: u32, | |
488 | /// } | |
489 | /// | |
490 | /// impl Example { | |
491 | /// fn use_reference(self: ArcBorrow<'_, Self>) { | |
492 | /// // ... | |
493 | /// } | |
494 | /// } | |
495 | /// | |
cc41670e | 496 | /// let obj = Arc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?; |
92a655ae | 497 | /// obj.as_arc_borrow().use_reference(); |
bfa7dff0 | 498 | /// # Ok::<(), Error>(()) |
92a655ae | 499 | /// ``` |
17f67160 WAF |
500 | pub struct ArcBorrow<'a, T: ?Sized + 'a> { |
501 | inner: NonNull<ArcInner<T>>, | |
502 | _p: PhantomData<&'a ()>, | |
503 | } | |
504 | ||
92a655ae WAF |
505 | // This is to allow [`ArcBorrow`] (and variants) to be used as the type of `self`. |
506 | impl<T: ?Sized> core::ops::Receiver for ArcBorrow<'_, T> {} | |
507 | ||
0748424a WAF |
508 | // This is to allow `ArcBorrow<U>` to be dispatched on when `ArcBorrow<T>` can be coerced into |
509 | // `ArcBorrow<U>`. | |
510 | impl<T: ?Sized + Unsize<U>, U: ?Sized> core::ops::DispatchFromDyn<ArcBorrow<'_, U>> | |
511 | for ArcBorrow<'_, T> | |
512 | { | |
513 | } | |
514 | ||
17f67160 WAF |
515 | impl<T: ?Sized> Clone for ArcBorrow<'_, T> { |
516 | fn clone(&self) -> Self { | |
517 | *self | |
518 | } | |
519 | } | |
520 | ||
521 | impl<T: ?Sized> Copy for ArcBorrow<'_, T> {} | |
522 | ||
523 | impl<T: ?Sized> ArcBorrow<'_, T> { | |
524 | /// Creates a new [`ArcBorrow`] instance. | |
525 | /// | |
526 | /// # Safety | |
527 | /// | |
528 | /// Callers must ensure the following for the lifetime of the returned [`ArcBorrow`] instance: | |
529 | /// 1. That `inner` remains valid; | |
530 | /// 2. That no mutable references to `inner` are created. | |
531 | unsafe fn new(inner: NonNull<ArcInner<T>>) -> Self { | |
532 | // INVARIANT: The safety requirements guarantee the invariants. | |
533 | Self { | |
534 | inner, | |
535 | _p: PhantomData, | |
536 | } | |
537 | } | |
51f6af86 AR |
538 | |
539 | /// Creates an [`ArcBorrow`] to an [`Arc`] that has previously been deconstructed with | |
540 | /// [`Arc::into_raw`]. | |
541 | /// | |
542 | /// # Safety | |
543 | /// | |
544 | /// * The provided pointer must originate from a call to [`Arc::into_raw`]. | |
545 | /// * For the duration of the lifetime annotated on this `ArcBorrow`, the reference count must | |
546 | /// not hit zero. | |
547 | /// * For the duration of the lifetime annotated on this `ArcBorrow`, there must not be a | |
548 | /// [`UniqueArc`] reference to this value. | |
549 | pub unsafe fn from_raw(ptr: *const T) -> Self { | |
550 | // SAFETY: The caller promises that this pointer originates from a call to `into_raw` on an | |
551 | // `Arc` that is still valid. | |
552 | let ptr = unsafe { ArcInner::container_of(ptr) }; | |
553 | ||
554 | // SAFETY: The caller promises that the value remains valid since the reference count must | |
555 | // not hit zero, and no mutable reference will be created since that would involve a | |
556 | // `UniqueArc`. | |
557 | unsafe { Self::new(ptr) } | |
558 | } | |
17f67160 WAF |
559 | } |
560 | ||
561 | impl<T: ?Sized> From<ArcBorrow<'_, T>> for Arc<T> { | |
562 | fn from(b: ArcBorrow<'_, T>) -> Self { | |
563 | // SAFETY: The existence of `b` guarantees that the refcount is non-zero. `ManuallyDrop` | |
564 | // guarantees that `drop` isn't called, so it's ok that the temporary `Arc` doesn't own the | |
565 | // increment. | |
566 | ManuallyDrop::new(unsafe { Arc::from_inner(b.inner) }) | |
567 | .deref() | |
568 | .clone() | |
569 | } | |
570 | } | |
571 | ||
572 | impl<T: ?Sized> Deref for ArcBorrow<'_, T> { | |
573 | type Target = T; | |
574 | ||
575 | fn deref(&self) -> &Self::Target { | |
576 | // SAFETY: By the type invariant, the underlying object is still alive with no mutable | |
577 | // references to it, so it is safe to create a shared reference. | |
578 | unsafe { &self.inner.as_ref().data } | |
579 | } | |
580 | } | |
70e42ebb WAF |
581 | |
582 | /// A refcounted object that is known to have a refcount of 1. | |
583 | /// | |
584 | /// It is mutable and can be converted to an [`Arc`] so that it can be shared. | |
585 | /// | |
586 | /// # Invariants | |
587 | /// | |
588 | /// `inner` always has a reference count of 1. | |
589 | /// | |
590 | /// # Examples | |
591 | /// | |
592 | /// In the following example, we make changes to the inner object before turning it into an | |
593 | /// `Arc<Test>` object (after which point, it cannot be mutated directly). Note that `x.into()` | |
594 | /// cannot fail. | |
595 | /// | |
596 | /// ``` | |
597 | /// use kernel::sync::{Arc, UniqueArc}; | |
598 | /// | |
599 | /// struct Example { | |
600 | /// a: u32, | |
601 | /// b: u32, | |
602 | /// } | |
603 | /// | |
604 | /// fn test() -> Result<Arc<Example>> { | |
cc41670e | 605 | /// let mut x = UniqueArc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?; |
70e42ebb WAF |
606 | /// x.a += 1; |
607 | /// x.b += 1; | |
608 | /// Ok(x.into()) | |
609 | /// } | |
610 | /// | |
611 | /// # test().unwrap(); | |
612 | /// ``` | |
613 | /// | |
ebf2b8a7 | 614 | /// In the following example we first allocate memory for a refcounted `Example` but we don't |
70e42ebb WAF |
615 | /// initialise it on allocation. We do initialise it later with a call to [`UniqueArc::write`], |
616 | /// followed by a conversion to `Arc<Example>`. This is particularly useful when allocation happens | |
617 | /// in one context (e.g., sleepable) and initialisation in another (e.g., atomic): | |
618 | /// | |
619 | /// ``` | |
620 | /// use kernel::sync::{Arc, UniqueArc}; | |
621 | /// | |
622 | /// struct Example { | |
623 | /// a: u32, | |
624 | /// b: u32, | |
625 | /// } | |
626 | /// | |
627 | /// fn test() -> Result<Arc<Example>> { | |
cc41670e | 628 | /// let x = UniqueArc::new_uninit(GFP_KERNEL)?; |
70e42ebb WAF |
629 | /// Ok(x.write(Example { a: 10, b: 20 }).into()) |
630 | /// } | |
631 | /// | |
632 | /// # test().unwrap(); | |
633 | /// ``` | |
634 | /// | |
635 | /// In the last example below, the caller gets a pinned instance of `Example` while converting to | |
636 | /// `Arc<Example>`; this is useful in scenarios where one needs a pinned reference during | |
637 | /// initialisation, for example, when initialising fields that are wrapped in locks. | |
638 | /// | |
639 | /// ``` | |
640 | /// use kernel::sync::{Arc, UniqueArc}; | |
641 | /// | |
642 | /// struct Example { | |
643 | /// a: u32, | |
644 | /// b: u32, | |
645 | /// } | |
646 | /// | |
647 | /// fn test() -> Result<Arc<Example>> { | |
cc41670e | 648 | /// let mut pinned = Pin::from(UniqueArc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?); |
70e42ebb WAF |
649 | /// // We can modify `pinned` because it is `Unpin`. |
650 | /// pinned.as_mut().a += 1; | |
651 | /// Ok(pinned.into()) | |
652 | /// } | |
653 | /// | |
654 | /// # test().unwrap(); | |
655 | /// ``` | |
656 | pub struct UniqueArc<T: ?Sized> { | |
657 | inner: Arc<T>, | |
658 | } | |
659 | ||
660 | impl<T> UniqueArc<T> { | |
661 | /// Tries to allocate a new [`UniqueArc`] instance. | |
cc41670e | 662 | pub fn new(value: T, flags: Flags) -> Result<Self, AllocError> { |
70e42ebb | 663 | Ok(Self { |
ebf2b8a7 | 664 | // INVARIANT: The newly-created object has a refcount of 1. |
cc41670e | 665 | inner: Arc::new(value, flags)?, |
70e42ebb WAF |
666 | }) |
667 | } | |
668 | ||
669 | /// Tries to allocate a new [`UniqueArc`] instance whose contents are not initialised yet. | |
c34aa00d | 670 | pub fn new_uninit(flags: Flags) -> Result<UniqueArc<MaybeUninit<T>>, AllocError> { |
701608bd | 671 | // INVARIANT: The refcount is initialised to a non-zero value. |
c34aa00d WAF |
672 | let inner = Box::try_init::<AllocError>( |
673 | try_init!(ArcInner { | |
674 | // SAFETY: There are no safety requirements for this FFI call. | |
675 | refcount: Opaque::new(unsafe { bindings::REFCOUNT_INIT(1) }), | |
676 | data <- init::uninit::<T, AllocError>(), | |
677 | }? AllocError), | |
678 | flags, | |
679 | )?; | |
701608bd | 680 | Ok(UniqueArc { |
ebf2b8a7 | 681 | // INVARIANT: The newly-created object has a refcount of 1. |
701608bd BL |
682 | // SAFETY: The pointer from the `Box` is valid. |
683 | inner: unsafe { Arc::from_inner(Box::leak(inner).into()) }, | |
70e42ebb WAF |
684 | }) |
685 | } | |
686 | } | |
687 | ||
688 | impl<T> UniqueArc<MaybeUninit<T>> { | |
689 | /// Converts a `UniqueArc<MaybeUninit<T>>` into a `UniqueArc<T>` by writing a value into it. | |
690 | pub fn write(mut self, value: T) -> UniqueArc<T> { | |
691 | self.deref_mut().write(value); | |
31d95c2f AL |
692 | // SAFETY: We just wrote the value to be initialized. |
693 | unsafe { self.assume_init() } | |
694 | } | |
695 | ||
696 | /// Unsafely assume that `self` is initialized. | |
697 | /// | |
698 | /// # Safety | |
699 | /// | |
700 | /// The caller guarantees that the value behind this pointer has been initialized. It is | |
701 | /// *immediate* UB to call this when the value is not initialized. | |
702 | pub unsafe fn assume_init(self) -> UniqueArc<T> { | |
70e42ebb WAF |
703 | let inner = ManuallyDrop::new(self).inner.ptr; |
704 | UniqueArc { | |
705 | // SAFETY: The new `Arc` is taking over `ptr` from `self.inner` (which won't be | |
706 | // dropped). The types are compatible because `MaybeUninit<T>` is compatible with `T`. | |
707 | inner: unsafe { Arc::from_inner(inner.cast()) }, | |
708 | } | |
709 | } | |
1944caa8 BL |
710 | |
711 | /// Initialize `self` using the given initializer. | |
712 | pub fn init_with<E>(mut self, init: impl Init<T, E>) -> core::result::Result<UniqueArc<T>, E> { | |
713 | // SAFETY: The supplied pointer is valid for initialization. | |
714 | match unsafe { init.__init(self.as_mut_ptr()) } { | |
715 | // SAFETY: Initialization completed successfully. | |
716 | Ok(()) => Ok(unsafe { self.assume_init() }), | |
717 | Err(err) => Err(err), | |
718 | } | |
719 | } | |
720 | ||
721 | /// Pin-initialize `self` using the given pin-initializer. | |
722 | pub fn pin_init_with<E>( | |
723 | mut self, | |
724 | init: impl PinInit<T, E>, | |
725 | ) -> core::result::Result<Pin<UniqueArc<T>>, E> { | |
726 | // SAFETY: The supplied pointer is valid for initialization and we will later pin the value | |
727 | // to ensure it does not move. | |
728 | match unsafe { init.__pinned_init(self.as_mut_ptr()) } { | |
729 | // SAFETY: Initialization completed successfully. | |
730 | Ok(()) => Ok(unsafe { self.assume_init() }.into()), | |
731 | Err(err) => Err(err), | |
732 | } | |
733 | } | |
70e42ebb WAF |
734 | } |
735 | ||
736 | impl<T: ?Sized> From<UniqueArc<T>> for Pin<UniqueArc<T>> { | |
737 | fn from(obj: UniqueArc<T>) -> Self { | |
738 | // SAFETY: It is not possible to move/replace `T` inside a `Pin<UniqueArc<T>>` (unless `T` | |
739 | // is `Unpin`), so it is ok to convert it to `Pin<UniqueArc<T>>`. | |
740 | unsafe { Pin::new_unchecked(obj) } | |
741 | } | |
742 | } | |
743 | ||
744 | impl<T: ?Sized> Deref for UniqueArc<T> { | |
745 | type Target = T; | |
746 | ||
747 | fn deref(&self) -> &Self::Target { | |
748 | self.inner.deref() | |
749 | } | |
750 | } | |
751 | ||
752 | impl<T: ?Sized> DerefMut for UniqueArc<T> { | |
753 | fn deref_mut(&mut self) -> &mut Self::Target { | |
754 | // SAFETY: By the `Arc` type invariant, there is necessarily a reference to the object, so | |
755 | // it is safe to dereference it. Additionally, we know there is only one reference when | |
756 | // it's inside a `UniqueArc`, so it is safe to get a mutable reference. | |
757 | unsafe { &mut self.inner.ptr.as_mut().data } | |
758 | } | |
759 | } | |
00140a83 BF |
760 | |
761 | impl<T: fmt::Display + ?Sized> fmt::Display for UniqueArc<T> { | |
762 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { | |
763 | fmt::Display::fmt(self.deref(), f) | |
764 | } | |
765 | } | |
766 | ||
767 | impl<T: fmt::Display + ?Sized> fmt::Display for Arc<T> { | |
768 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { | |
769 | fmt::Display::fmt(self.deref(), f) | |
770 | } | |
771 | } | |
772 | ||
773 | impl<T: fmt::Debug + ?Sized> fmt::Debug for UniqueArc<T> { | |
774 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { | |
775 | fmt::Debug::fmt(self.deref(), f) | |
776 | } | |
777 | } | |
778 | ||
779 | impl<T: fmt::Debug + ?Sized> fmt::Debug for Arc<T> { | |
780 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { | |
781 | fmt::Debug::fmt(self.deref(), f) | |
782 | } | |
783 | } |