Commit | Line | Data |
---|---|---|
9dc04365 WAF |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | ||
3 | //! A reference-counted pointer. | |
4 | //! | |
5 | //! This module implements a way for users to create reference-counted objects and pointers to | |
6 | //! them. Such a pointer automatically increments and decrements the count, and drops the | |
7 | //! underlying object when it reaches zero. It is also safe to use concurrently from multiple | |
8 | //! threads. | |
9 | //! | |
10 | //! It is different from the standard library's [`Arc`] in a few ways: | |
11 | //! 1. It is backed by the kernel's `refcount_t` type. | |
12 | //! 2. It does not support weak references, which allows it to be half the size. | |
13 | //! 3. It saturates the reference count instead of aborting when it goes over a threshold. | |
14 | //! 4. It does not provide a `get_mut` method, so the ref counted object is pinned. | |
15 | //! | |
16 | //! [`Arc`]: https://doc.rust-lang.org/std/sync/struct.Arc.html | |
17 | ||
0c7ae432 | 18 | use crate::{ |
2c109285 | 19 | alloc::{box_ext::BoxExt, AllocError, Flags}, |
92c4a1e7 | 20 | error::{self, Error}, |
701608bd BL |
21 | init::{self, InPlaceInit, Init, PinInit}, |
22 | try_init, | |
0c7ae432 WAF |
23 | types::{ForeignOwnable, Opaque}, |
24 | }; | |
9dc04365 | 25 | use alloc::boxed::Box; |
f75cb6fc | 26 | use core::{ |
2c109285 | 27 | alloc::Layout, |
00140a83 | 28 | fmt, |
f75cb6fc | 29 | marker::{PhantomData, Unsize}, |
70e42ebb WAF |
30 | mem::{ManuallyDrop, MaybeUninit}, |
31 | ops::{Deref, DerefMut}, | |
32 | pin::Pin, | |
44f2e626 | 33 | ptr::NonNull, |
f75cb6fc | 34 | }; |
701608bd | 35 | use macros::pin_data; |
9dc04365 | 36 | |
1edd0337 AL |
37 | mod std_vendor; |
38 | ||
9dc04365 WAF |
39 | /// A reference-counted pointer to an instance of `T`. |
40 | /// | |
41 | /// The reference count is incremented when new instances of [`Arc`] are created, and decremented | |
42 | /// when they are dropped. When the count reaches zero, the underlying `T` is also dropped. | |
43 | /// | |
44 | /// # Invariants | |
45 | /// | |
46 | /// The reference count on an instance of [`Arc`] is always non-zero. | |
47 | /// The object pointed to by [`Arc`] is always pinned. | |
48 | /// | |
49 | /// # Examples | |
50 | /// | |
51 | /// ``` | |
52 | /// use kernel::sync::Arc; | |
53 | /// | |
54 | /// struct Example { | |
55 | /// a: u32, | |
56 | /// b: u32, | |
57 | /// } | |
58 | /// | |
ebf2b8a7 | 59 | /// // Create a refcounted instance of `Example`. |
cc41670e | 60 | /// let obj = Arc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?; |
9dc04365 WAF |
61 | /// |
62 | /// // Get a new pointer to `obj` and increment the refcount. | |
63 | /// let cloned = obj.clone(); | |
64 | /// | |
65 | /// // Assert that both `obj` and `cloned` point to the same underlying object. | |
66 | /// assert!(core::ptr::eq(&*obj, &*cloned)); | |
67 | /// | |
68 | /// // Destroy `obj` and decrement its refcount. | |
69 | /// drop(obj); | |
70 | /// | |
71 | /// // Check that the values are still accessible through `cloned`. | |
72 | /// assert_eq!(cloned.a, 10); | |
73 | /// assert_eq!(cloned.b, 20); | |
74 | /// | |
75 | /// // The refcount drops to zero when `cloned` goes out of scope, and the memory is freed. | |
bfa7dff0 | 76 | /// # Ok::<(), Error>(()) |
9dc04365 | 77 | /// ``` |
53528772 WAF |
78 | /// |
79 | /// Using `Arc<T>` as the type of `self`: | |
80 | /// | |
81 | /// ``` | |
82 | /// use kernel::sync::Arc; | |
83 | /// | |
84 | /// struct Example { | |
85 | /// a: u32, | |
86 | /// b: u32, | |
87 | /// } | |
88 | /// | |
89 | /// impl Example { | |
90 | /// fn take_over(self: Arc<Self>) { | |
91 | /// // ... | |
92 | /// } | |
93 | /// | |
94 | /// fn use_reference(self: &Arc<Self>) { | |
95 | /// // ... | |
96 | /// } | |
97 | /// } | |
98 | /// | |
cc41670e | 99 | /// let obj = Arc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?; |
53528772 WAF |
100 | /// obj.use_reference(); |
101 | /// obj.take_over(); | |
bfa7dff0 | 102 | /// # Ok::<(), Error>(()) |
53528772 | 103 | /// ``` |
f75cb6fc WAF |
104 | /// |
105 | /// Coercion from `Arc<Example>` to `Arc<dyn MyTrait>`: | |
106 | /// | |
107 | /// ``` | |
0748424a WAF |
108 | /// use kernel::sync::{Arc, ArcBorrow}; |
109 | /// | |
110 | /// trait MyTrait { | |
111 | /// // Trait has a function whose `self` type is `Arc<Self>`. | |
112 | /// fn example1(self: Arc<Self>) {} | |
f75cb6fc | 113 | /// |
0748424a WAF |
114 | /// // Trait has a function whose `self` type is `ArcBorrow<'_, Self>`. |
115 | /// fn example2(self: ArcBorrow<'_, Self>) {} | |
116 | /// } | |
f75cb6fc WAF |
117 | /// |
118 | /// struct Example; | |
119 | /// impl MyTrait for Example {} | |
120 | /// | |
121 | /// // `obj` has type `Arc<Example>`. | |
cc41670e | 122 | /// let obj: Arc<Example> = Arc::new(Example, GFP_KERNEL)?; |
f75cb6fc WAF |
123 | /// |
124 | /// // `coerced` has type `Arc<dyn MyTrait>`. | |
125 | /// let coerced: Arc<dyn MyTrait> = obj; | |
bfa7dff0 | 126 | /// # Ok::<(), Error>(()) |
f75cb6fc | 127 | /// ``` |
9dc04365 WAF |
128 | pub struct Arc<T: ?Sized> { |
129 | ptr: NonNull<ArcInner<T>>, | |
130 | _p: PhantomData<ArcInner<T>>, | |
131 | } | |
132 | ||
701608bd | 133 | #[pin_data] |
9dc04365 WAF |
134 | #[repr(C)] |
135 | struct ArcInner<T: ?Sized> { | |
136 | refcount: Opaque<bindings::refcount_t>, | |
137 | data: T, | |
138 | } | |
139 | ||
51f6af86 AR |
140 | impl<T: ?Sized> ArcInner<T> { |
141 | /// Converts a pointer to the contents of an [`Arc`] into a pointer to the [`ArcInner`]. | |
142 | /// | |
143 | /// # Safety | |
144 | /// | |
145 | /// `ptr` must have been returned by a previous call to [`Arc::into_raw`], and the `Arc` must | |
146 | /// not yet have been destroyed. | |
147 | unsafe fn container_of(ptr: *const T) -> NonNull<ArcInner<T>> { | |
148 | let refcount_layout = Layout::new::<bindings::refcount_t>(); | |
149 | // SAFETY: The caller guarantees that the pointer is valid. | |
150 | let val_layout = Layout::for_value(unsafe { &*ptr }); | |
151 | // SAFETY: We're computing the layout of a real struct that existed when compiling this | |
152 | // binary, so its layout is not so large that it can trigger arithmetic overflow. | |
153 | let val_offset = unsafe { refcount_layout.extend(val_layout).unwrap_unchecked().1 }; | |
154 | ||
155 | // Pointer casts leave the metadata unchanged. This is okay because the metadata of `T` and | |
156 | // `ArcInner<T>` is the same since `ArcInner` is a struct with `T` as its last field. | |
157 | // | |
158 | // This is documented at: | |
159 | // <https://doc.rust-lang.org/std/ptr/trait.Pointee.html>. | |
160 | let ptr = ptr as *const ArcInner<T>; | |
161 | ||
162 | // SAFETY: The pointer is in-bounds of an allocation both before and after offsetting the | |
163 | // pointer, since it originates from a previous call to `Arc::into_raw` on an `Arc` that is | |
164 | // still valid. | |
165 | let ptr = unsafe { ptr.byte_sub(val_offset) }; | |
166 | ||
167 | // SAFETY: The pointer can't be null since you can't have an `ArcInner<T>` value at the null | |
168 | // address. | |
169 | unsafe { NonNull::new_unchecked(ptr.cast_mut()) } | |
170 | } | |
171 | } | |
172 | ||
53528772 WAF |
173 | // This is to allow [`Arc`] (and variants) to be used as the type of `self`. |
174 | impl<T: ?Sized> core::ops::Receiver for Arc<T> {} | |
175 | ||
f75cb6fc WAF |
176 | // This is to allow coercion from `Arc<T>` to `Arc<U>` if `T` can be converted to the |
177 | // dynamically-sized type (DST) `U`. | |
178 | impl<T: ?Sized + Unsize<U>, U: ?Sized> core::ops::CoerceUnsized<Arc<U>> for Arc<T> {} | |
179 | ||
0748424a WAF |
180 | // This is to allow `Arc<U>` to be dispatched on when `Arc<T>` can be coerced into `Arc<U>`. |
181 | impl<T: ?Sized + Unsize<U>, U: ?Sized> core::ops::DispatchFromDyn<Arc<U>> for Arc<T> {} | |
182 | ||
9dc04365 WAF |
183 | // SAFETY: It is safe to send `Arc<T>` to another thread when the underlying `T` is `Sync` because |
184 | // it effectively means sharing `&T` (which is safe because `T` is `Sync`); additionally, it needs | |
f8110cd1 AR |
185 | // `T` to be `Send` because any thread that has an `Arc<T>` may ultimately access `T` using a |
186 | // mutable reference when the reference count reaches zero and `T` is dropped. | |
9dc04365 WAF |
187 | unsafe impl<T: ?Sized + Sync + Send> Send for Arc<T> {} |
188 | ||
d701e061 AR |
189 | // SAFETY: It is safe to send `&Arc<T>` to another thread when the underlying `T` is `Sync` |
190 | // because it effectively means sharing `&T` (which is safe because `T` is `Sync`); additionally, | |
191 | // it needs `T` to be `Send` because any thread that has a `&Arc<T>` may clone it and get an | |
192 | // `Arc<T>` on that thread, so the thread may ultimately access `T` using a mutable reference when | |
193 | // the reference count reaches zero and `T` is dropped. | |
9dc04365 WAF |
194 | unsafe impl<T: ?Sized + Sync + Send> Sync for Arc<T> {} |
195 | ||
196 | impl<T> Arc<T> { | |
197 | /// Constructs a new reference counted instance of `T`. | |
cc41670e | 198 | pub fn new(contents: T, flags: Flags) -> Result<Self, AllocError> { |
9dc04365 WAF |
199 | // INVARIANT: The refcount is initialised to a non-zero value. |
200 | let value = ArcInner { | |
201 | // SAFETY: There are no safety requirements for this FFI call. | |
202 | refcount: Opaque::new(unsafe { bindings::REFCOUNT_INIT(1) }), | |
203 | data: contents, | |
204 | }; | |
205 | ||
cc41670e | 206 | let inner = <Box<_> as BoxExt<_>>::new(value, flags)?; |
9dc04365 WAF |
207 | |
208 | // SAFETY: We just created `inner` with a reference count of 1, which is owned by the new | |
209 | // `Arc` object. | |
210 | Ok(unsafe { Self::from_inner(Box::leak(inner).into()) }) | |
211 | } | |
92c4a1e7 BL |
212 | |
213 | /// Use the given initializer to in-place initialize a `T`. | |
214 | /// | |
215 | /// If `T: !Unpin` it will not be able to move afterwards. | |
216 | #[inline] | |
c34aa00d | 217 | pub fn pin_init<E>(init: impl PinInit<T, E>, flags: Flags) -> error::Result<Self> |
92c4a1e7 BL |
218 | where |
219 | Error: From<E>, | |
220 | { | |
c34aa00d | 221 | UniqueArc::pin_init(init, flags).map(|u| u.into()) |
92c4a1e7 BL |
222 | } |
223 | ||
224 | /// Use the given initializer to in-place initialize a `T`. | |
225 | /// | |
eed7a146 | 226 | /// This is equivalent to [`Arc<T>::pin_init`], since an [`Arc`] is always pinned. |
92c4a1e7 | 227 | #[inline] |
c34aa00d | 228 | pub fn init<E>(init: impl Init<T, E>, flags: Flags) -> error::Result<Self> |
92c4a1e7 BL |
229 | where |
230 | Error: From<E>, | |
231 | { | |
c34aa00d | 232 | UniqueArc::init(init, flags).map(|u| u.into()) |
92c4a1e7 | 233 | } |
9dc04365 WAF |
234 | } |
235 | ||
236 | impl<T: ?Sized> Arc<T> { | |
237 | /// Constructs a new [`Arc`] from an existing [`ArcInner`]. | |
238 | /// | |
239 | /// # Safety | |
240 | /// | |
241 | /// The caller must ensure that `inner` points to a valid location and has a non-zero reference | |
242 | /// count, one of which will be owned by the new [`Arc`] instance. | |
243 | unsafe fn from_inner(inner: NonNull<ArcInner<T>>) -> Self { | |
244 | // INVARIANT: By the safety requirements, the invariants hold. | |
245 | Arc { | |
246 | ptr: inner, | |
247 | _p: PhantomData, | |
248 | } | |
249 | } | |
17f67160 | 250 | |
a8321776 WAF |
251 | /// Convert the [`Arc`] into a raw pointer. |
252 | /// | |
253 | /// The raw pointer has ownership of the refcount that this Arc object owned. | |
254 | pub fn into_raw(self) -> *const T { | |
255 | let ptr = self.ptr.as_ptr(); | |
256 | core::mem::forget(self); | |
257 | // SAFETY: The pointer is valid. | |
258 | unsafe { core::ptr::addr_of!((*ptr).data) } | |
259 | } | |
260 | ||
261 | /// Recreates an [`Arc`] instance previously deconstructed via [`Arc::into_raw`]. | |
262 | /// | |
263 | /// # Safety | |
264 | /// | |
265 | /// `ptr` must have been returned by a previous call to [`Arc::into_raw`]. Additionally, it | |
266 | /// must not be called more than once for each previous call to [`Arc::into_raw`]. | |
267 | pub unsafe fn from_raw(ptr: *const T) -> Self { | |
51f6af86 AR |
268 | // SAFETY: The caller promises that this pointer originates from a call to `into_raw` on an |
269 | // `Arc` that is still valid. | |
270 | let ptr = unsafe { ArcInner::container_of(ptr) }; | |
a8321776 WAF |
271 | |
272 | // SAFETY: By the safety requirements we know that `ptr` came from `Arc::into_raw`, so the | |
273 | // reference count held then will be owned by the new `Arc` object. | |
51f6af86 | 274 | unsafe { Self::from_inner(ptr) } |
a8321776 WAF |
275 | } |
276 | ||
17f67160 WAF |
277 | /// Returns an [`ArcBorrow`] from the given [`Arc`]. |
278 | /// | |
279 | /// This is useful when the argument of a function call is an [`ArcBorrow`] (e.g., in a method | |
280 | /// receiver), but we have an [`Arc`] instead. Getting an [`ArcBorrow`] is free when optimised. | |
281 | #[inline] | |
282 | pub fn as_arc_borrow(&self) -> ArcBorrow<'_, T> { | |
283 | // SAFETY: The constraint that the lifetime of the shared reference must outlive that of | |
284 | // the returned `ArcBorrow` ensures that the object remains alive and that no mutable | |
285 | // reference can be created. | |
286 | unsafe { ArcBorrow::new(self.ptr) } | |
287 | } | |
bd780aea AR |
288 | |
289 | /// Compare whether two [`Arc`] pointers reference the same underlying object. | |
290 | pub fn ptr_eq(this: &Self, other: &Self) -> bool { | |
291 | core::ptr::eq(this.ptr.as_ptr(), other.ptr.as_ptr()) | |
292 | } | |
a0a4e170 AR |
293 | |
294 | /// Converts this [`Arc`] into a [`UniqueArc`], or destroys it if it is not unique. | |
295 | /// | |
296 | /// When this destroys the `Arc`, it does so while properly avoiding races. This means that | |
297 | /// this method will never call the destructor of the value. | |
298 | /// | |
299 | /// # Examples | |
300 | /// | |
301 | /// ``` | |
302 | /// use kernel::sync::{Arc, UniqueArc}; | |
303 | /// | |
304 | /// let arc = Arc::new(42, GFP_KERNEL)?; | |
305 | /// let unique_arc = arc.into_unique_or_drop(); | |
306 | /// | |
307 | /// // The above conversion should succeed since refcount of `arc` is 1. | |
308 | /// assert!(unique_arc.is_some()); | |
309 | /// | |
310 | /// assert_eq!(*(unique_arc.unwrap()), 42); | |
311 | /// | |
312 | /// # Ok::<(), Error>(()) | |
313 | /// ``` | |
314 | /// | |
315 | /// ``` | |
316 | /// use kernel::sync::{Arc, UniqueArc}; | |
317 | /// | |
318 | /// let arc = Arc::new(42, GFP_KERNEL)?; | |
319 | /// let another = arc.clone(); | |
320 | /// | |
321 | /// let unique_arc = arc.into_unique_or_drop(); | |
322 | /// | |
323 | /// // The above conversion should fail since refcount of `arc` is >1. | |
324 | /// assert!(unique_arc.is_none()); | |
325 | /// | |
326 | /// # Ok::<(), Error>(()) | |
327 | /// ``` | |
328 | pub fn into_unique_or_drop(self) -> Option<Pin<UniqueArc<T>>> { | |
329 | // We will manually manage the refcount in this method, so we disable the destructor. | |
330 | let me = ManuallyDrop::new(self); | |
331 | // SAFETY: We own a refcount, so the pointer is still valid. | |
332 | let refcount = unsafe { me.ptr.as_ref() }.refcount.get(); | |
333 | ||
334 | // If the refcount reaches a non-zero value, then we have destroyed this `Arc` and will | |
335 | // return without further touching the `Arc`. If the refcount reaches zero, then there are | |
336 | // no other arcs, and we can create a `UniqueArc`. | |
337 | // | |
338 | // SAFETY: We own a refcount, so the pointer is not dangling. | |
339 | let is_zero = unsafe { bindings::refcount_dec_and_test(refcount) }; | |
340 | if is_zero { | |
341 | // SAFETY: We have exclusive access to the arc, so we can perform unsynchronized | |
342 | // accesses to the refcount. | |
343 | unsafe { core::ptr::write(refcount, bindings::REFCOUNT_INIT(1)) }; | |
344 | ||
345 | // INVARIANT: We own the only refcount to this arc, so we may create a `UniqueArc`. We | |
346 | // must pin the `UniqueArc` because the values was previously in an `Arc`, and they pin | |
347 | // their values. | |
348 | Some(Pin::from(UniqueArc { | |
349 | inner: ManuallyDrop::into_inner(me), | |
350 | })) | |
351 | } else { | |
352 | None | |
353 | } | |
354 | } | |
9dc04365 WAF |
355 | } |
356 | ||
0c7ae432 WAF |
357 | impl<T: 'static> ForeignOwnable for Arc<T> { |
358 | type Borrowed<'a> = ArcBorrow<'a, T>; | |
359 | ||
360 | fn into_foreign(self) -> *const core::ffi::c_void { | |
361 | ManuallyDrop::new(self).ptr.as_ptr() as _ | |
362 | } | |
363 | ||
364 | unsafe fn borrow<'a>(ptr: *const core::ffi::c_void) -> ArcBorrow<'a, T> { | |
365 | // SAFETY: By the safety requirement of this function, we know that `ptr` came from | |
366 | // a previous call to `Arc::into_foreign`. | |
367 | let inner = NonNull::new(ptr as *mut ArcInner<T>).unwrap(); | |
368 | ||
369 | // SAFETY: The safety requirements of `from_foreign` ensure that the object remains alive | |
1d24eb2d | 370 | // for the lifetime of the returned value. |
0c7ae432 WAF |
371 | unsafe { ArcBorrow::new(inner) } |
372 | } | |
373 | ||
374 | unsafe fn from_foreign(ptr: *const core::ffi::c_void) -> Self { | |
375 | // SAFETY: By the safety requirement of this function, we know that `ptr` came from | |
376 | // a previous call to `Arc::into_foreign`, which guarantees that `ptr` is valid and | |
377 | // holds a reference count increment that is transferrable to us. | |
378 | unsafe { Self::from_inner(NonNull::new(ptr as _).unwrap()) } | |
379 | } | |
380 | } | |
381 | ||
9dc04365 WAF |
382 | impl<T: ?Sized> Deref for Arc<T> { |
383 | type Target = T; | |
384 | ||
385 | fn deref(&self) -> &Self::Target { | |
386 | // SAFETY: By the type invariant, there is necessarily a reference to the object, so it is | |
387 | // safe to dereference it. | |
388 | unsafe { &self.ptr.as_ref().data } | |
389 | } | |
390 | } | |
391 | ||
47329ba1 AR |
392 | impl<T: ?Sized> AsRef<T> for Arc<T> { |
393 | fn as_ref(&self) -> &T { | |
394 | self.deref() | |
395 | } | |
396 | } | |
397 | ||
9dc04365 WAF |
398 | impl<T: ?Sized> Clone for Arc<T> { |
399 | fn clone(&self) -> Self { | |
400 | // INVARIANT: C `refcount_inc` saturates the refcount, so it cannot overflow to zero. | |
401 | // SAFETY: By the type invariant, there is necessarily a reference to the object, so it is | |
402 | // safe to increment the refcount. | |
403 | unsafe { bindings::refcount_inc(self.ptr.as_ref().refcount.get()) }; | |
404 | ||
405 | // SAFETY: We just incremented the refcount. This increment is now owned by the new `Arc`. | |
406 | unsafe { Self::from_inner(self.ptr) } | |
407 | } | |
408 | } | |
409 | ||
410 | impl<T: ?Sized> Drop for Arc<T> { | |
411 | fn drop(&mut self) { | |
412 | // SAFETY: By the type invariant, there is necessarily a reference to the object. We cannot | |
413 | // touch `refcount` after it's decremented to a non-zero value because another thread/CPU | |
414 | // may concurrently decrement it to zero and free it. It is ok to have a raw pointer to | |
415 | // freed/invalid memory as long as it is never dereferenced. | |
416 | let refcount = unsafe { self.ptr.as_ref() }.refcount.get(); | |
417 | ||
418 | // INVARIANT: If the refcount reaches zero, there are no other instances of `Arc`, and | |
419 | // this instance is being dropped, so the broken invariant is not observable. | |
420 | // SAFETY: Also by the type invariant, we are allowed to decrement the refcount. | |
421 | let is_zero = unsafe { bindings::refcount_dec_and_test(refcount) }; | |
422 | if is_zero { | |
423 | // The count reached zero, we must free the memory. | |
424 | // | |
425 | // SAFETY: The pointer was initialised from the result of `Box::leak`. | |
828176d0 | 426 | unsafe { drop(Box::from_raw(self.ptr.as_ptr())) }; |
9dc04365 WAF |
427 | } |
428 | } | |
429 | } | |
17f67160 | 430 | |
70e42ebb WAF |
431 | impl<T: ?Sized> From<UniqueArc<T>> for Arc<T> { |
432 | fn from(item: UniqueArc<T>) -> Self { | |
433 | item.inner | |
434 | } | |
435 | } | |
436 | ||
437 | impl<T: ?Sized> From<Pin<UniqueArc<T>>> for Arc<T> { | |
438 | fn from(item: Pin<UniqueArc<T>>) -> Self { | |
439 | // SAFETY: The type invariants of `Arc` guarantee that the data is pinned. | |
440 | unsafe { Pin::into_inner_unchecked(item).inner } | |
441 | } | |
442 | } | |
443 | ||
17f67160 WAF |
444 | /// A borrowed reference to an [`Arc`] instance. |
445 | /// | |
446 | /// For cases when one doesn't ever need to increment the refcount on the allocation, it is simpler | |
4c799d1d | 447 | /// to use just `&T`, which we can trivially get from an [`Arc<T>`] instance. |
17f67160 WAF |
448 | /// |
449 | /// However, when one may need to increment the refcount, it is preferable to use an `ArcBorrow<T>` | |
450 | /// over `&Arc<T>` because the latter results in a double-indirection: a pointer (shared reference) | |
4c799d1d VO |
451 | /// to a pointer ([`Arc<T>`]) to the object (`T`). An [`ArcBorrow`] eliminates this double |
452 | /// indirection while still allowing one to increment the refcount and getting an [`Arc<T>`] when/if | |
17f67160 WAF |
453 | /// needed. |
454 | /// | |
455 | /// # Invariants | |
456 | /// | |
457 | /// There are no mutable references to the underlying [`Arc`], and it remains valid for the | |
458 | /// lifetime of the [`ArcBorrow`] instance. | |
459 | /// | |
460 | /// # Example | |
461 | /// | |
462 | /// ``` | |
bfa7dff0 | 463 | /// use kernel::sync::{Arc, ArcBorrow}; |
17f67160 WAF |
464 | /// |
465 | /// struct Example; | |
466 | /// | |
467 | /// fn do_something(e: ArcBorrow<'_, Example>) -> Arc<Example> { | |
468 | /// e.into() | |
469 | /// } | |
470 | /// | |
cc41670e | 471 | /// let obj = Arc::new(Example, GFP_KERNEL)?; |
17f67160 WAF |
472 | /// let cloned = do_something(obj.as_arc_borrow()); |
473 | /// | |
474 | /// // Assert that both `obj` and `cloned` point to the same underlying object. | |
475 | /// assert!(core::ptr::eq(&*obj, &*cloned)); | |
bfa7dff0 | 476 | /// # Ok::<(), Error>(()) |
17f67160 | 477 | /// ``` |
92a655ae WAF |
478 | /// |
479 | /// Using `ArcBorrow<T>` as the type of `self`: | |
480 | /// | |
481 | /// ``` | |
bfa7dff0 | 482 | /// use kernel::sync::{Arc, ArcBorrow}; |
92a655ae WAF |
483 | /// |
484 | /// struct Example { | |
485 | /// a: u32, | |
486 | /// b: u32, | |
487 | /// } | |
488 | /// | |
489 | /// impl Example { | |
490 | /// fn use_reference(self: ArcBorrow<'_, Self>) { | |
491 | /// // ... | |
492 | /// } | |
493 | /// } | |
494 | /// | |
cc41670e | 495 | /// let obj = Arc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?; |
92a655ae | 496 | /// obj.as_arc_borrow().use_reference(); |
bfa7dff0 | 497 | /// # Ok::<(), Error>(()) |
92a655ae | 498 | /// ``` |
17f67160 WAF |
499 | pub struct ArcBorrow<'a, T: ?Sized + 'a> { |
500 | inner: NonNull<ArcInner<T>>, | |
501 | _p: PhantomData<&'a ()>, | |
502 | } | |
503 | ||
92a655ae WAF |
504 | // This is to allow [`ArcBorrow`] (and variants) to be used as the type of `self`. |
505 | impl<T: ?Sized> core::ops::Receiver for ArcBorrow<'_, T> {} | |
506 | ||
0748424a WAF |
507 | // This is to allow `ArcBorrow<U>` to be dispatched on when `ArcBorrow<T>` can be coerced into |
508 | // `ArcBorrow<U>`. | |
509 | impl<T: ?Sized + Unsize<U>, U: ?Sized> core::ops::DispatchFromDyn<ArcBorrow<'_, U>> | |
510 | for ArcBorrow<'_, T> | |
511 | { | |
512 | } | |
513 | ||
17f67160 WAF |
514 | impl<T: ?Sized> Clone for ArcBorrow<'_, T> { |
515 | fn clone(&self) -> Self { | |
516 | *self | |
517 | } | |
518 | } | |
519 | ||
520 | impl<T: ?Sized> Copy for ArcBorrow<'_, T> {} | |
521 | ||
522 | impl<T: ?Sized> ArcBorrow<'_, T> { | |
523 | /// Creates a new [`ArcBorrow`] instance. | |
524 | /// | |
525 | /// # Safety | |
526 | /// | |
527 | /// Callers must ensure the following for the lifetime of the returned [`ArcBorrow`] instance: | |
528 | /// 1. That `inner` remains valid; | |
529 | /// 2. That no mutable references to `inner` are created. | |
530 | unsafe fn new(inner: NonNull<ArcInner<T>>) -> Self { | |
531 | // INVARIANT: The safety requirements guarantee the invariants. | |
532 | Self { | |
533 | inner, | |
534 | _p: PhantomData, | |
535 | } | |
536 | } | |
51f6af86 AR |
537 | |
538 | /// Creates an [`ArcBorrow`] to an [`Arc`] that has previously been deconstructed with | |
539 | /// [`Arc::into_raw`]. | |
540 | /// | |
541 | /// # Safety | |
542 | /// | |
543 | /// * The provided pointer must originate from a call to [`Arc::into_raw`]. | |
544 | /// * For the duration of the lifetime annotated on this `ArcBorrow`, the reference count must | |
545 | /// not hit zero. | |
546 | /// * For the duration of the lifetime annotated on this `ArcBorrow`, there must not be a | |
547 | /// [`UniqueArc`] reference to this value. | |
548 | pub unsafe fn from_raw(ptr: *const T) -> Self { | |
549 | // SAFETY: The caller promises that this pointer originates from a call to `into_raw` on an | |
550 | // `Arc` that is still valid. | |
551 | let ptr = unsafe { ArcInner::container_of(ptr) }; | |
552 | ||
553 | // SAFETY: The caller promises that the value remains valid since the reference count must | |
554 | // not hit zero, and no mutable reference will be created since that would involve a | |
555 | // `UniqueArc`. | |
556 | unsafe { Self::new(ptr) } | |
557 | } | |
17f67160 WAF |
558 | } |
559 | ||
560 | impl<T: ?Sized> From<ArcBorrow<'_, T>> for Arc<T> { | |
561 | fn from(b: ArcBorrow<'_, T>) -> Self { | |
562 | // SAFETY: The existence of `b` guarantees that the refcount is non-zero. `ManuallyDrop` | |
563 | // guarantees that `drop` isn't called, so it's ok that the temporary `Arc` doesn't own the | |
564 | // increment. | |
565 | ManuallyDrop::new(unsafe { Arc::from_inner(b.inner) }) | |
566 | .deref() | |
567 | .clone() | |
568 | } | |
569 | } | |
570 | ||
571 | impl<T: ?Sized> Deref for ArcBorrow<'_, T> { | |
572 | type Target = T; | |
573 | ||
574 | fn deref(&self) -> &Self::Target { | |
575 | // SAFETY: By the type invariant, the underlying object is still alive with no mutable | |
576 | // references to it, so it is safe to create a shared reference. | |
577 | unsafe { &self.inner.as_ref().data } | |
578 | } | |
579 | } | |
70e42ebb WAF |
580 | |
581 | /// A refcounted object that is known to have a refcount of 1. | |
582 | /// | |
583 | /// It is mutable and can be converted to an [`Arc`] so that it can be shared. | |
584 | /// | |
585 | /// # Invariants | |
586 | /// | |
587 | /// `inner` always has a reference count of 1. | |
588 | /// | |
589 | /// # Examples | |
590 | /// | |
591 | /// In the following example, we make changes to the inner object before turning it into an | |
592 | /// `Arc<Test>` object (after which point, it cannot be mutated directly). Note that `x.into()` | |
593 | /// cannot fail. | |
594 | /// | |
595 | /// ``` | |
596 | /// use kernel::sync::{Arc, UniqueArc}; | |
597 | /// | |
598 | /// struct Example { | |
599 | /// a: u32, | |
600 | /// b: u32, | |
601 | /// } | |
602 | /// | |
603 | /// fn test() -> Result<Arc<Example>> { | |
cc41670e | 604 | /// let mut x = UniqueArc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?; |
70e42ebb WAF |
605 | /// x.a += 1; |
606 | /// x.b += 1; | |
607 | /// Ok(x.into()) | |
608 | /// } | |
609 | /// | |
610 | /// # test().unwrap(); | |
611 | /// ``` | |
612 | /// | |
ebf2b8a7 | 613 | /// In the following example we first allocate memory for a refcounted `Example` but we don't |
70e42ebb WAF |
614 | /// initialise it on allocation. We do initialise it later with a call to [`UniqueArc::write`], |
615 | /// followed by a conversion to `Arc<Example>`. This is particularly useful when allocation happens | |
616 | /// in one context (e.g., sleepable) and initialisation in another (e.g., atomic): | |
617 | /// | |
618 | /// ``` | |
619 | /// use kernel::sync::{Arc, UniqueArc}; | |
620 | /// | |
621 | /// struct Example { | |
622 | /// a: u32, | |
623 | /// b: u32, | |
624 | /// } | |
625 | /// | |
626 | /// fn test() -> Result<Arc<Example>> { | |
cc41670e | 627 | /// let x = UniqueArc::new_uninit(GFP_KERNEL)?; |
70e42ebb WAF |
628 | /// Ok(x.write(Example { a: 10, b: 20 }).into()) |
629 | /// } | |
630 | /// | |
631 | /// # test().unwrap(); | |
632 | /// ``` | |
633 | /// | |
634 | /// In the last example below, the caller gets a pinned instance of `Example` while converting to | |
635 | /// `Arc<Example>`; this is useful in scenarios where one needs a pinned reference during | |
636 | /// initialisation, for example, when initialising fields that are wrapped in locks. | |
637 | /// | |
638 | /// ``` | |
639 | /// use kernel::sync::{Arc, UniqueArc}; | |
640 | /// | |
641 | /// struct Example { | |
642 | /// a: u32, | |
643 | /// b: u32, | |
644 | /// } | |
645 | /// | |
646 | /// fn test() -> Result<Arc<Example>> { | |
cc41670e | 647 | /// let mut pinned = Pin::from(UniqueArc::new(Example { a: 10, b: 20 }, GFP_KERNEL)?); |
70e42ebb WAF |
648 | /// // We can modify `pinned` because it is `Unpin`. |
649 | /// pinned.as_mut().a += 1; | |
650 | /// Ok(pinned.into()) | |
651 | /// } | |
652 | /// | |
653 | /// # test().unwrap(); | |
654 | /// ``` | |
655 | pub struct UniqueArc<T: ?Sized> { | |
656 | inner: Arc<T>, | |
657 | } | |
658 | ||
659 | impl<T> UniqueArc<T> { | |
660 | /// Tries to allocate a new [`UniqueArc`] instance. | |
cc41670e | 661 | pub fn new(value: T, flags: Flags) -> Result<Self, AllocError> { |
70e42ebb | 662 | Ok(Self { |
ebf2b8a7 | 663 | // INVARIANT: The newly-created object has a refcount of 1. |
cc41670e | 664 | inner: Arc::new(value, flags)?, |
70e42ebb WAF |
665 | }) |
666 | } | |
667 | ||
668 | /// Tries to allocate a new [`UniqueArc`] instance whose contents are not initialised yet. | |
c34aa00d | 669 | pub fn new_uninit(flags: Flags) -> Result<UniqueArc<MaybeUninit<T>>, AllocError> { |
701608bd | 670 | // INVARIANT: The refcount is initialised to a non-zero value. |
c34aa00d WAF |
671 | let inner = Box::try_init::<AllocError>( |
672 | try_init!(ArcInner { | |
673 | // SAFETY: There are no safety requirements for this FFI call. | |
674 | refcount: Opaque::new(unsafe { bindings::REFCOUNT_INIT(1) }), | |
675 | data <- init::uninit::<T, AllocError>(), | |
676 | }? AllocError), | |
677 | flags, | |
678 | )?; | |
701608bd | 679 | Ok(UniqueArc { |
ebf2b8a7 | 680 | // INVARIANT: The newly-created object has a refcount of 1. |
701608bd BL |
681 | // SAFETY: The pointer from the `Box` is valid. |
682 | inner: unsafe { Arc::from_inner(Box::leak(inner).into()) }, | |
70e42ebb WAF |
683 | }) |
684 | } | |
685 | } | |
686 | ||
687 | impl<T> UniqueArc<MaybeUninit<T>> { | |
688 | /// Converts a `UniqueArc<MaybeUninit<T>>` into a `UniqueArc<T>` by writing a value into it. | |
689 | pub fn write(mut self, value: T) -> UniqueArc<T> { | |
690 | self.deref_mut().write(value); | |
31d95c2f AL |
691 | // SAFETY: We just wrote the value to be initialized. |
692 | unsafe { self.assume_init() } | |
693 | } | |
694 | ||
695 | /// Unsafely assume that `self` is initialized. | |
696 | /// | |
697 | /// # Safety | |
698 | /// | |
699 | /// The caller guarantees that the value behind this pointer has been initialized. It is | |
700 | /// *immediate* UB to call this when the value is not initialized. | |
701 | pub unsafe fn assume_init(self) -> UniqueArc<T> { | |
70e42ebb WAF |
702 | let inner = ManuallyDrop::new(self).inner.ptr; |
703 | UniqueArc { | |
704 | // SAFETY: The new `Arc` is taking over `ptr` from `self.inner` (which won't be | |
705 | // dropped). The types are compatible because `MaybeUninit<T>` is compatible with `T`. | |
706 | inner: unsafe { Arc::from_inner(inner.cast()) }, | |
707 | } | |
708 | } | |
1944caa8 BL |
709 | |
710 | /// Initialize `self` using the given initializer. | |
711 | pub fn init_with<E>(mut self, init: impl Init<T, E>) -> core::result::Result<UniqueArc<T>, E> { | |
712 | // SAFETY: The supplied pointer is valid for initialization. | |
713 | match unsafe { init.__init(self.as_mut_ptr()) } { | |
714 | // SAFETY: Initialization completed successfully. | |
715 | Ok(()) => Ok(unsafe { self.assume_init() }), | |
716 | Err(err) => Err(err), | |
717 | } | |
718 | } | |
719 | ||
720 | /// Pin-initialize `self` using the given pin-initializer. | |
721 | pub fn pin_init_with<E>( | |
722 | mut self, | |
723 | init: impl PinInit<T, E>, | |
724 | ) -> core::result::Result<Pin<UniqueArc<T>>, E> { | |
725 | // SAFETY: The supplied pointer is valid for initialization and we will later pin the value | |
726 | // to ensure it does not move. | |
727 | match unsafe { init.__pinned_init(self.as_mut_ptr()) } { | |
728 | // SAFETY: Initialization completed successfully. | |
729 | Ok(()) => Ok(unsafe { self.assume_init() }.into()), | |
730 | Err(err) => Err(err), | |
731 | } | |
732 | } | |
70e42ebb WAF |
733 | } |
734 | ||
735 | impl<T: ?Sized> From<UniqueArc<T>> for Pin<UniqueArc<T>> { | |
736 | fn from(obj: UniqueArc<T>) -> Self { | |
737 | // SAFETY: It is not possible to move/replace `T` inside a `Pin<UniqueArc<T>>` (unless `T` | |
738 | // is `Unpin`), so it is ok to convert it to `Pin<UniqueArc<T>>`. | |
739 | unsafe { Pin::new_unchecked(obj) } | |
740 | } | |
741 | } | |
742 | ||
743 | impl<T: ?Sized> Deref for UniqueArc<T> { | |
744 | type Target = T; | |
745 | ||
746 | fn deref(&self) -> &Self::Target { | |
747 | self.inner.deref() | |
748 | } | |
749 | } | |
750 | ||
751 | impl<T: ?Sized> DerefMut for UniqueArc<T> { | |
752 | fn deref_mut(&mut self) -> &mut Self::Target { | |
753 | // SAFETY: By the `Arc` type invariant, there is necessarily a reference to the object, so | |
754 | // it is safe to dereference it. Additionally, we know there is only one reference when | |
755 | // it's inside a `UniqueArc`, so it is safe to get a mutable reference. | |
756 | unsafe { &mut self.inner.ptr.as_mut().data } | |
757 | } | |
758 | } | |
00140a83 BF |
759 | |
760 | impl<T: fmt::Display + ?Sized> fmt::Display for UniqueArc<T> { | |
761 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { | |
762 | fmt::Display::fmt(self.deref(), f) | |
763 | } | |
764 | } | |
765 | ||
766 | impl<T: fmt::Display + ?Sized> fmt::Display for Arc<T> { | |
767 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { | |
768 | fmt::Display::fmt(self.deref(), f) | |
769 | } | |
770 | } | |
771 | ||
772 | impl<T: fmt::Debug + ?Sized> fmt::Debug for UniqueArc<T> { | |
773 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { | |
774 | fmt::Debug::fmt(self.deref(), f) | |
775 | } | |
776 | } | |
777 | ||
778 | impl<T: fmt::Debug + ?Sized> fmt::Debug for Arc<T> { | |
779 | fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { | |
780 | fmt::Debug::fmt(self.deref(), f) | |
781 | } | |
782 | } |