Commit | Line | Data |
---|---|---|
d4d791d4 AR |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | ||
3 | //! Work queues. | |
4 | //! | |
7324b889 AR |
5 | //! This file has two components: The raw work item API, and the safe work item API. |
6 | //! | |
7 | //! One pattern that is used in both APIs is the `ID` const generic, which exists to allow a single | |
8 | //! type to define multiple `work_struct` fields. This is done by choosing an id for each field, | |
9 | //! and using that id to specify which field you wish to use. (The actual value doesn't matter, as | |
10 | //! long as you use different values for different fields of the same struct.) Since these IDs are | |
11 | //! generic, they are used only at compile-time, so they shouldn't exist in the final binary. | |
12 | //! | |
13 | //! # The raw API | |
14 | //! | |
15 | //! The raw API consists of the `RawWorkItem` trait, where the work item needs to provide an | |
16 | //! arbitrary function that knows how to enqueue the work item. It should usually not be used | |
17 | //! directly, but if you want to, you can use it without using the pieces from the safe API. | |
18 | //! | |
19 | //! # The safe API | |
20 | //! | |
21 | //! The safe API is used via the `Work` struct and `WorkItem` traits. Furthermore, it also includes | |
22 | //! a trait called `WorkItemPointer`, which is usually not used directly by the user. | |
23 | //! | |
24 | //! * The `Work` struct is the Rust wrapper for the C `work_struct` type. | |
25 | //! * The `WorkItem` trait is implemented for structs that can be enqueued to a workqueue. | |
26 | //! * The `WorkItemPointer` trait is implemented for the pointer type that points at a something | |
27 | //! that implements `WorkItem`. | |
28 | //! | |
15b286d1 AR |
29 | //! ## Example |
30 | //! | |
31 | //! This example defines a struct that holds an integer and can be scheduled on the workqueue. When | |
32 | //! the struct is executed, it will print the integer. Since there is only one `work_struct` field, | |
33 | //! we do not need to specify ids for the fields. | |
34 | //! | |
35 | //! ``` | |
36 | //! use kernel::prelude::*; | |
37 | //! use kernel::sync::Arc; | |
38 | //! use kernel::workqueue::{self, Work, WorkItem}; | |
39 | //! use kernel::{impl_has_work, new_work}; | |
40 | //! | |
41 | //! #[pin_data] | |
42 | //! struct MyStruct { | |
43 | //! value: i32, | |
44 | //! #[pin] | |
45 | //! work: Work<MyStruct>, | |
46 | //! } | |
47 | //! | |
48 | //! impl_has_work! { | |
49 | //! impl HasWork<Self> for MyStruct { self.work } | |
50 | //! } | |
51 | //! | |
52 | //! impl MyStruct { | |
53 | //! fn new(value: i32) -> Result<Arc<Self>> { | |
54 | //! Arc::pin_init(pin_init!(MyStruct { | |
55 | //! value, | |
56 | //! work <- new_work!("MyStruct::work"), | |
57 | //! })) | |
58 | //! } | |
59 | //! } | |
60 | //! | |
61 | //! impl WorkItem for MyStruct { | |
62 | //! type Pointer = Arc<MyStruct>; | |
63 | //! | |
64 | //! fn run(this: Arc<MyStruct>) { | |
65 | //! pr_info!("The value is: {}", this.value); | |
66 | //! } | |
67 | //! } | |
68 | //! | |
69 | //! /// This method will enqueue the struct for execution on the system workqueue, where its value | |
70 | //! /// will be printed. | |
71 | //! fn print_later(val: Arc<MyStruct>) { | |
72 | //! let _ = workqueue::system().enqueue(val); | |
73 | //! } | |
74 | //! ``` | |
75 | //! | |
76 | //! The following example shows how multiple `work_struct` fields can be used: | |
77 | //! | |
78 | //! ``` | |
79 | //! use kernel::prelude::*; | |
80 | //! use kernel::sync::Arc; | |
81 | //! use kernel::workqueue::{self, Work, WorkItem}; | |
82 | //! use kernel::{impl_has_work, new_work}; | |
83 | //! | |
84 | //! #[pin_data] | |
85 | //! struct MyStruct { | |
86 | //! value_1: i32, | |
87 | //! value_2: i32, | |
88 | //! #[pin] | |
89 | //! work_1: Work<MyStruct, 1>, | |
90 | //! #[pin] | |
91 | //! work_2: Work<MyStruct, 2>, | |
92 | //! } | |
93 | //! | |
94 | //! impl_has_work! { | |
95 | //! impl HasWork<Self, 1> for MyStruct { self.work_1 } | |
96 | //! impl HasWork<Self, 2> for MyStruct { self.work_2 } | |
97 | //! } | |
98 | //! | |
99 | //! impl MyStruct { | |
100 | //! fn new(value_1: i32, value_2: i32) -> Result<Arc<Self>> { | |
101 | //! Arc::pin_init(pin_init!(MyStruct { | |
102 | //! value_1, | |
103 | //! value_2, | |
104 | //! work_1 <- new_work!("MyStruct::work_1"), | |
105 | //! work_2 <- new_work!("MyStruct::work_2"), | |
106 | //! })) | |
107 | //! } | |
108 | //! } | |
109 | //! | |
110 | //! impl WorkItem<1> for MyStruct { | |
111 | //! type Pointer = Arc<MyStruct>; | |
112 | //! | |
113 | //! fn run(this: Arc<MyStruct>) { | |
114 | //! pr_info!("The value is: {}", this.value_1); | |
115 | //! } | |
116 | //! } | |
117 | //! | |
118 | //! impl WorkItem<2> for MyStruct { | |
119 | //! type Pointer = Arc<MyStruct>; | |
120 | //! | |
121 | //! fn run(this: Arc<MyStruct>) { | |
122 | //! pr_info!("The second value is: {}", this.value_2); | |
123 | //! } | |
124 | //! } | |
125 | //! | |
126 | //! fn print_1_later(val: Arc<MyStruct>) { | |
127 | //! let _ = workqueue::system().enqueue::<Arc<MyStruct>, 1>(val); | |
128 | //! } | |
129 | //! | |
130 | //! fn print_2_later(val: Arc<MyStruct>) { | |
131 | //! let _ = workqueue::system().enqueue::<Arc<MyStruct>, 2>(val); | |
132 | //! } | |
133 | //! ``` | |
134 | //! | |
d4d791d4 AR |
135 | //! C header: [`include/linux/workqueue.h`](../../../../include/linux/workqueue.h) |
136 | ||
47f0dbe8 | 137 | use crate::{bindings, prelude::*, sync::Arc, sync::LockClassKey, types::Opaque}; |
115c95e9 | 138 | use alloc::alloc::AllocError; |
47f0dbe8 | 139 | use alloc::boxed::Box; |
7324b889 | 140 | use core::marker::PhantomData; |
47f0dbe8 | 141 | use core::pin::Pin; |
7324b889 AR |
142 | |
143 | /// Creates a [`Work`] initialiser with the given name and a newly-created lock class. | |
144 | #[macro_export] | |
145 | macro_rules! new_work { | |
146 | ($($name:literal)?) => { | |
147 | $crate::workqueue::Work::new($crate::optional_name!($($name)?), $crate::static_lock_class!()) | |
148 | }; | |
149 | } | |
d4d791d4 AR |
150 | |
151 | /// A kernel work queue. | |
152 | /// | |
153 | /// Wraps the kernel's C `struct workqueue_struct`. | |
154 | /// | |
155 | /// It allows work items to be queued to run on thread pools managed by the kernel. Several are | |
156 | /// always available, for example, `system`, `system_highpri`, `system_long`, etc. | |
157 | #[repr(transparent)] | |
158 | pub struct Queue(Opaque<bindings::workqueue_struct>); | |
159 | ||
160 | // SAFETY: Accesses to workqueues used by [`Queue`] are thread-safe. | |
161 | unsafe impl Send for Queue {} | |
162 | // SAFETY: Accesses to workqueues used by [`Queue`] are thread-safe. | |
163 | unsafe impl Sync for Queue {} | |
164 | ||
165 | impl Queue { | |
166 | /// Use the provided `struct workqueue_struct` with Rust. | |
167 | /// | |
168 | /// # Safety | |
169 | /// | |
170 | /// The caller must ensure that the provided raw pointer is not dangling, that it points at a | |
171 | /// valid workqueue, and that it remains valid until the end of 'a. | |
172 | pub unsafe fn from_raw<'a>(ptr: *const bindings::workqueue_struct) -> &'a Queue { | |
173 | // SAFETY: The `Queue` type is `#[repr(transparent)]`, so the pointer cast is valid. The | |
174 | // caller promises that the pointer is not dangling. | |
175 | unsafe { &*(ptr as *const Queue) } | |
176 | } | |
177 | ||
178 | /// Enqueues a work item. | |
179 | /// | |
180 | /// This may fail if the work item is already enqueued in a workqueue. | |
181 | /// | |
182 | /// The work item will be submitted using `WORK_CPU_UNBOUND`. | |
183 | pub fn enqueue<W, const ID: u64>(&self, w: W) -> W::EnqueueOutput | |
184 | where | |
185 | W: RawWorkItem<ID> + Send + 'static, | |
186 | { | |
187 | let queue_ptr = self.0.get(); | |
188 | ||
189 | // SAFETY: We only return `false` if the `work_struct` is already in a workqueue. The other | |
190 | // `__enqueue` requirements are not relevant since `W` is `Send` and static. | |
191 | // | |
192 | // The call to `bindings::queue_work_on` will dereference the provided raw pointer, which | |
193 | // is ok because `__enqueue` guarantees that the pointer is valid for the duration of this | |
194 | // closure. | |
195 | // | |
196 | // Furthermore, if the C workqueue code accesses the pointer after this call to | |
197 | // `__enqueue`, then the work item was successfully enqueued, and `bindings::queue_work_on` | |
198 | // will have returned true. In this case, `__enqueue` promises that the raw pointer will | |
199 | // stay valid until we call the function pointer in the `work_struct`, so the access is ok. | |
200 | unsafe { | |
201 | w.__enqueue(move |work_ptr| { | |
202 | bindings::queue_work_on(bindings::WORK_CPU_UNBOUND as _, queue_ptr, work_ptr) | |
203 | }) | |
204 | } | |
205 | } | |
115c95e9 AR |
206 | |
207 | /// Tries to spawn the given function or closure as a work item. | |
208 | /// | |
209 | /// This method can fail because it allocates memory to store the work item. | |
210 | pub fn try_spawn<T: 'static + Send + FnOnce()>(&self, func: T) -> Result<(), AllocError> { | |
211 | let init = pin_init!(ClosureWork { | |
212 | work <- new_work!("Queue::try_spawn"), | |
213 | func: Some(func), | |
214 | }); | |
215 | ||
216 | self.enqueue(Box::pin_init(init).map_err(|_| AllocError)?); | |
217 | Ok(()) | |
218 | } | |
219 | } | |
220 | ||
221 | /// A helper type used in `try_spawn`. | |
222 | #[pin_data] | |
223 | struct ClosureWork<T> { | |
224 | #[pin] | |
225 | work: Work<ClosureWork<T>>, | |
226 | func: Option<T>, | |
227 | } | |
228 | ||
229 | impl<T> ClosureWork<T> { | |
230 | fn project(self: Pin<&mut Self>) -> &mut Option<T> { | |
231 | // SAFETY: The `func` field is not structurally pinned. | |
232 | unsafe { &mut self.get_unchecked_mut().func } | |
233 | } | |
234 | } | |
235 | ||
236 | impl<T: FnOnce()> WorkItem for ClosureWork<T> { | |
237 | type Pointer = Pin<Box<Self>>; | |
238 | ||
239 | fn run(mut this: Pin<Box<Self>>) { | |
240 | if let Some(func) = this.as_mut().project().take() { | |
241 | (func)() | |
242 | } | |
243 | } | |
d4d791d4 AR |
244 | } |
245 | ||
246 | /// A raw work item. | |
247 | /// | |
248 | /// This is the low-level trait that is designed for being as general as possible. | |
249 | /// | |
250 | /// The `ID` parameter to this trait exists so that a single type can provide multiple | |
251 | /// implementations of this trait. For example, if a struct has multiple `work_struct` fields, then | |
252 | /// you will implement this trait once for each field, using a different id for each field. The | |
253 | /// actual value of the id is not important as long as you use different ids for different fields | |
254 | /// of the same struct. (Fields of different structs need not use different ids.) | |
255 | /// | |
256 | /// Note that the id is used only to select the right method to call during compilation. It wont be | |
257 | /// part of the final executable. | |
258 | /// | |
259 | /// # Safety | |
260 | /// | |
261 | /// Implementers must ensure that any pointers passed to a `queue_work_on` closure by `__enqueue` | |
262 | /// remain valid for the duration specified in the guarantees section of the documentation for | |
263 | /// `__enqueue`. | |
264 | pub unsafe trait RawWorkItem<const ID: u64> { | |
265 | /// The return type of [`Queue::enqueue`]. | |
266 | type EnqueueOutput; | |
267 | ||
268 | /// Enqueues this work item on a queue using the provided `queue_work_on` method. | |
269 | /// | |
270 | /// # Guarantees | |
271 | /// | |
272 | /// If this method calls the provided closure, then the raw pointer is guaranteed to point at a | |
273 | /// valid `work_struct` for the duration of the call to the closure. If the closure returns | |
274 | /// true, then it is further guaranteed that the pointer remains valid until someone calls the | |
275 | /// function pointer stored in the `work_struct`. | |
276 | /// | |
277 | /// # Safety | |
278 | /// | |
279 | /// The provided closure may only return `false` if the `work_struct` is already in a workqueue. | |
280 | /// | |
281 | /// If the work item type is annotated with any lifetimes, then you must not call the function | |
282 | /// pointer after any such lifetime expires. (Never calling the function pointer is okay.) | |
283 | /// | |
284 | /// If the work item type is not [`Send`], then the function pointer must be called on the same | |
285 | /// thread as the call to `__enqueue`. | |
286 | unsafe fn __enqueue<F>(self, queue_work_on: F) -> Self::EnqueueOutput | |
287 | where | |
288 | F: FnOnce(*mut bindings::work_struct) -> bool; | |
289 | } | |
03394130 | 290 | |
7324b889 AR |
291 | /// Defines the method that should be called directly when a work item is executed. |
292 | /// | |
293 | /// This trait is implemented by `Pin<Box<T>>` and `Arc<T>`, and is mainly intended to be | |
294 | /// implemented for smart pointer types. For your own structs, you would implement [`WorkItem`] | |
295 | /// instead. The `run` method on this trait will usually just perform the appropriate | |
296 | /// `container_of` translation and then call into the `run` method from the [`WorkItem`] trait. | |
297 | /// | |
298 | /// This trait is used when the `work_struct` field is defined using the [`Work`] helper. | |
299 | /// | |
300 | /// # Safety | |
301 | /// | |
302 | /// Implementers must ensure that [`__enqueue`] uses a `work_struct` initialized with the [`run`] | |
303 | /// method of this trait as the function pointer. | |
304 | /// | |
305 | /// [`__enqueue`]: RawWorkItem::__enqueue | |
306 | /// [`run`]: WorkItemPointer::run | |
307 | pub unsafe trait WorkItemPointer<const ID: u64>: RawWorkItem<ID> { | |
308 | /// Run this work item. | |
309 | /// | |
310 | /// # Safety | |
311 | /// | |
312 | /// The provided `work_struct` pointer must originate from a previous call to `__enqueue` where | |
313 | /// the `queue_work_on` closure returned true, and the pointer must still be valid. | |
314 | unsafe extern "C" fn run(ptr: *mut bindings::work_struct); | |
315 | } | |
316 | ||
317 | /// Defines the method that should be called when this work item is executed. | |
318 | /// | |
319 | /// This trait is used when the `work_struct` field is defined using the [`Work`] helper. | |
320 | pub trait WorkItem<const ID: u64 = 0> { | |
321 | /// The pointer type that this struct is wrapped in. This will typically be `Arc<Self>` or | |
322 | /// `Pin<Box<Self>>`. | |
323 | type Pointer: WorkItemPointer<ID>; | |
324 | ||
325 | /// The method that should be called when this work item is executed. | |
326 | fn run(this: Self::Pointer); | |
327 | } | |
328 | ||
329 | /// Links for a work item. | |
330 | /// | |
331 | /// This struct contains a function pointer to the `run` function from the [`WorkItemPointer`] | |
332 | /// trait, and defines the linked list pointers necessary to enqueue a work item in a workqueue. | |
333 | /// | |
334 | /// Wraps the kernel's C `struct work_struct`. | |
335 | /// | |
336 | /// This is a helper type used to associate a `work_struct` with the [`WorkItem`] that uses it. | |
337 | #[repr(transparent)] | |
338 | pub struct Work<T: ?Sized, const ID: u64 = 0> { | |
339 | work: Opaque<bindings::work_struct>, | |
340 | _inner: PhantomData<T>, | |
341 | } | |
342 | ||
343 | // SAFETY: Kernel work items are usable from any thread. | |
344 | // | |
345 | // We do not need to constrain `T` since the work item does not actually contain a `T`. | |
346 | unsafe impl<T: ?Sized, const ID: u64> Send for Work<T, ID> {} | |
347 | // SAFETY: Kernel work items are usable from any thread. | |
348 | // | |
349 | // We do not need to constrain `T` since the work item does not actually contain a `T`. | |
350 | unsafe impl<T: ?Sized, const ID: u64> Sync for Work<T, ID> {} | |
351 | ||
352 | impl<T: ?Sized, const ID: u64> Work<T, ID> { | |
353 | /// Creates a new instance of [`Work`]. | |
354 | #[inline] | |
355 | #[allow(clippy::new_ret_no_self)] | |
356 | pub fn new(name: &'static CStr, key: &'static LockClassKey) -> impl PinInit<Self> | |
357 | where | |
358 | T: WorkItem<ID>, | |
359 | { | |
360 | // SAFETY: The `WorkItemPointer` implementation promises that `run` can be used as the work | |
361 | // item function. | |
362 | unsafe { | |
363 | kernel::init::pin_init_from_closure(move |slot| { | |
364 | let slot = Self::raw_get(slot); | |
365 | bindings::init_work_with_key( | |
366 | slot, | |
367 | Some(T::Pointer::run), | |
368 | false, | |
369 | name.as_char_ptr(), | |
370 | key.as_ptr(), | |
371 | ); | |
372 | Ok(()) | |
373 | }) | |
374 | } | |
375 | } | |
376 | ||
377 | /// Get a pointer to the inner `work_struct`. | |
378 | /// | |
379 | /// # Safety | |
380 | /// | |
381 | /// The provided pointer must not be dangling and must be properly aligned. (But the memory | |
382 | /// need not be initialized.) | |
383 | #[inline] | |
384 | pub unsafe fn raw_get(ptr: *const Self) -> *mut bindings::work_struct { | |
385 | // SAFETY: The caller promises that the pointer is aligned and not dangling. | |
386 | // | |
387 | // A pointer cast would also be ok due to `#[repr(transparent)]`. We use `addr_of!` so that | |
388 | // the compiler does not complain that the `work` field is unused. | |
389 | unsafe { Opaque::raw_get(core::ptr::addr_of!((*ptr).work)) } | |
390 | } | |
391 | } | |
392 | ||
393 | /// Declares that a type has a [`Work<T, ID>`] field. | |
394 | /// | |
395 | /// The intended way of using this trait is via the [`impl_has_work!`] macro. You can use the macro | |
396 | /// like this: | |
397 | /// | |
398 | /// ```no_run | |
399 | /// use kernel::impl_has_work; | |
400 | /// use kernel::prelude::*; | |
401 | /// use kernel::workqueue::Work; | |
402 | /// | |
403 | /// struct MyWorkItem { | |
404 | /// work_field: Work<MyWorkItem, 1>, | |
405 | /// } | |
406 | /// | |
407 | /// impl_has_work! { | |
408 | /// impl HasWork<MyWorkItem, 1> for MyWorkItem { self.work_field } | |
409 | /// } | |
410 | /// ``` | |
411 | /// | |
412 | /// Note that since the `Work` type is annotated with an id, you can have several `work_struct` | |
413 | /// fields by using a different id for each one. | |
414 | /// | |
415 | /// # Safety | |
416 | /// | |
417 | /// The [`OFFSET`] constant must be the offset of a field in Self of type [`Work<T, ID>`]. The methods on | |
418 | /// this trait must have exactly the behavior that the definitions given below have. | |
419 | /// | |
420 | /// [`Work<T, ID>`]: Work | |
421 | /// [`impl_has_work!`]: crate::impl_has_work | |
422 | /// [`OFFSET`]: HasWork::OFFSET | |
423 | pub unsafe trait HasWork<T, const ID: u64 = 0> { | |
424 | /// The offset of the [`Work<T, ID>`] field. | |
425 | /// | |
426 | /// [`Work<T, ID>`]: Work | |
427 | const OFFSET: usize; | |
428 | ||
429 | /// Returns the offset of the [`Work<T, ID>`] field. | |
430 | /// | |
431 | /// This method exists because the [`OFFSET`] constant cannot be accessed if the type is not Sized. | |
432 | /// | |
433 | /// [`Work<T, ID>`]: Work | |
434 | /// [`OFFSET`]: HasWork::OFFSET | |
435 | #[inline] | |
436 | fn get_work_offset(&self) -> usize { | |
437 | Self::OFFSET | |
438 | } | |
439 | ||
440 | /// Returns a pointer to the [`Work<T, ID>`] field. | |
441 | /// | |
442 | /// # Safety | |
443 | /// | |
444 | /// The provided pointer must point at a valid struct of type `Self`. | |
445 | /// | |
446 | /// [`Work<T, ID>`]: Work | |
447 | #[inline] | |
448 | unsafe fn raw_get_work(ptr: *mut Self) -> *mut Work<T, ID> { | |
449 | // SAFETY: The caller promises that the pointer is valid. | |
450 | unsafe { (ptr as *mut u8).add(Self::OFFSET) as *mut Work<T, ID> } | |
451 | } | |
452 | ||
453 | /// Returns a pointer to the struct containing the [`Work<T, ID>`] field. | |
454 | /// | |
455 | /// # Safety | |
456 | /// | |
457 | /// The pointer must point at a [`Work<T, ID>`] field in a struct of type `Self`. | |
458 | /// | |
459 | /// [`Work<T, ID>`]: Work | |
460 | #[inline] | |
461 | unsafe fn work_container_of(ptr: *mut Work<T, ID>) -> *mut Self | |
462 | where | |
463 | Self: Sized, | |
464 | { | |
465 | // SAFETY: The caller promises that the pointer points at a field of the right type in the | |
466 | // right kind of struct. | |
467 | unsafe { (ptr as *mut u8).sub(Self::OFFSET) as *mut Self } | |
468 | } | |
469 | } | |
470 | ||
471 | /// Used to safely implement the [`HasWork<T, ID>`] trait. | |
472 | /// | |
473 | /// # Examples | |
474 | /// | |
475 | /// ``` | |
476 | /// use kernel::impl_has_work; | |
477 | /// use kernel::sync::Arc; | |
478 | /// use kernel::workqueue::{self, Work}; | |
479 | /// | |
480 | /// struct MyStruct { | |
481 | /// work_field: Work<MyStruct, 17>, | |
482 | /// } | |
483 | /// | |
484 | /// impl_has_work! { | |
485 | /// impl HasWork<MyStruct, 17> for MyStruct { self.work_field } | |
486 | /// } | |
487 | /// ``` | |
488 | /// | |
489 | /// [`HasWork<T, ID>`]: HasWork | |
490 | #[macro_export] | |
491 | macro_rules! impl_has_work { | |
492 | ($(impl$(<$($implarg:ident),*>)? | |
493 | HasWork<$work_type:ty $(, $id:tt)?> | |
494 | for $self:ident $(<$($selfarg:ident),*>)? | |
495 | { self.$field:ident } | |
496 | )*) => {$( | |
497 | // SAFETY: The implementation of `raw_get_work` only compiles if the field has the right | |
498 | // type. | |
499 | unsafe impl$(<$($implarg),*>)? $crate::workqueue::HasWork<$work_type $(, $id)?> for $self $(<$($selfarg),*>)? { | |
500 | const OFFSET: usize = ::core::mem::offset_of!(Self, $field) as usize; | |
501 | ||
502 | #[inline] | |
503 | unsafe fn raw_get_work(ptr: *mut Self) -> *mut $crate::workqueue::Work<$work_type $(, $id)?> { | |
504 | // SAFETY: The caller promises that the pointer is not dangling. | |
505 | unsafe { | |
506 | ::core::ptr::addr_of_mut!((*ptr).$field) | |
507 | } | |
508 | } | |
509 | } | |
510 | )*}; | |
511 | } | |
512 | ||
115c95e9 AR |
513 | impl_has_work! { |
514 | impl<T> HasWork<Self> for ClosureWork<T> { self.work } | |
515 | } | |
516 | ||
47f0dbe8 AR |
517 | unsafe impl<T, const ID: u64> WorkItemPointer<ID> for Arc<T> |
518 | where | |
519 | T: WorkItem<ID, Pointer = Self>, | |
520 | T: HasWork<T, ID>, | |
521 | { | |
522 | unsafe extern "C" fn run(ptr: *mut bindings::work_struct) { | |
523 | // SAFETY: The `__enqueue` method always uses a `work_struct` stored in a `Work<T, ID>`. | |
524 | let ptr = ptr as *mut Work<T, ID>; | |
525 | // SAFETY: This computes the pointer that `__enqueue` got from `Arc::into_raw`. | |
526 | let ptr = unsafe { T::work_container_of(ptr) }; | |
527 | // SAFETY: This pointer comes from `Arc::into_raw` and we've been given back ownership. | |
528 | let arc = unsafe { Arc::from_raw(ptr) }; | |
529 | ||
530 | T::run(arc) | |
531 | } | |
532 | } | |
533 | ||
534 | unsafe impl<T, const ID: u64> RawWorkItem<ID> for Arc<T> | |
535 | where | |
536 | T: WorkItem<ID, Pointer = Self>, | |
537 | T: HasWork<T, ID>, | |
538 | { | |
539 | type EnqueueOutput = Result<(), Self>; | |
540 | ||
541 | unsafe fn __enqueue<F>(self, queue_work_on: F) -> Self::EnqueueOutput | |
542 | where | |
543 | F: FnOnce(*mut bindings::work_struct) -> bool, | |
544 | { | |
545 | // Casting between const and mut is not a problem as long as the pointer is a raw pointer. | |
546 | let ptr = Arc::into_raw(self).cast_mut(); | |
547 | ||
548 | // SAFETY: Pointers into an `Arc` point at a valid value. | |
549 | let work_ptr = unsafe { T::raw_get_work(ptr) }; | |
550 | // SAFETY: `raw_get_work` returns a pointer to a valid value. | |
551 | let work_ptr = unsafe { Work::raw_get(work_ptr) }; | |
552 | ||
553 | if queue_work_on(work_ptr) { | |
554 | Ok(()) | |
555 | } else { | |
556 | // SAFETY: The work queue has not taken ownership of the pointer. | |
557 | Err(unsafe { Arc::from_raw(ptr) }) | |
558 | } | |
559 | } | |
560 | } | |
561 | ||
562 | unsafe impl<T, const ID: u64> WorkItemPointer<ID> for Pin<Box<T>> | |
563 | where | |
564 | T: WorkItem<ID, Pointer = Self>, | |
565 | T: HasWork<T, ID>, | |
566 | { | |
567 | unsafe extern "C" fn run(ptr: *mut bindings::work_struct) { | |
568 | // SAFETY: The `__enqueue` method always uses a `work_struct` stored in a `Work<T, ID>`. | |
569 | let ptr = ptr as *mut Work<T, ID>; | |
570 | // SAFETY: This computes the pointer that `__enqueue` got from `Arc::into_raw`. | |
571 | let ptr = unsafe { T::work_container_of(ptr) }; | |
572 | // SAFETY: This pointer comes from `Arc::into_raw` and we've been given back ownership. | |
573 | let boxed = unsafe { Box::from_raw(ptr) }; | |
574 | // SAFETY: The box was already pinned when it was enqueued. | |
575 | let pinned = unsafe { Pin::new_unchecked(boxed) }; | |
576 | ||
577 | T::run(pinned) | |
578 | } | |
579 | } | |
580 | ||
581 | unsafe impl<T, const ID: u64> RawWorkItem<ID> for Pin<Box<T>> | |
582 | where | |
583 | T: WorkItem<ID, Pointer = Self>, | |
584 | T: HasWork<T, ID>, | |
585 | { | |
586 | type EnqueueOutput = (); | |
587 | ||
588 | unsafe fn __enqueue<F>(self, queue_work_on: F) -> Self::EnqueueOutput | |
589 | where | |
590 | F: FnOnce(*mut bindings::work_struct) -> bool, | |
591 | { | |
592 | // SAFETY: We're not going to move `self` or any of its fields, so its okay to temporarily | |
593 | // remove the `Pin` wrapper. | |
594 | let boxed = unsafe { Pin::into_inner_unchecked(self) }; | |
595 | let ptr = Box::into_raw(boxed); | |
596 | ||
597 | // SAFETY: Pointers into a `Box` point at a valid value. | |
598 | let work_ptr = unsafe { T::raw_get_work(ptr) }; | |
599 | // SAFETY: `raw_get_work` returns a pointer to a valid value. | |
600 | let work_ptr = unsafe { Work::raw_get(work_ptr) }; | |
601 | ||
602 | if !queue_work_on(work_ptr) { | |
603 | // SAFETY: This method requires exclusive ownership of the box, so it cannot be in a | |
604 | // workqueue. | |
605 | unsafe { ::core::hint::unreachable_unchecked() } | |
606 | } | |
607 | } | |
608 | } | |
609 | ||
03394130 WAF |
610 | /// Returns the system work queue (`system_wq`). |
611 | /// | |
612 | /// It is the one used by `schedule[_delayed]_work[_on]()`. Multi-CPU multi-threaded. There are | |
613 | /// users which expect relatively short queue flush time. | |
614 | /// | |
615 | /// Callers shouldn't queue work items which can run for too long. | |
616 | pub fn system() -> &'static Queue { | |
617 | // SAFETY: `system_wq` is a C global, always available. | |
618 | unsafe { Queue::from_raw(bindings::system_wq) } | |
619 | } | |
620 | ||
621 | /// Returns the system high-priority work queue (`system_highpri_wq`). | |
622 | /// | |
623 | /// It is similar to the one returned by [`system`] but for work items which require higher | |
624 | /// scheduling priority. | |
625 | pub fn system_highpri() -> &'static Queue { | |
626 | // SAFETY: `system_highpri_wq` is a C global, always available. | |
627 | unsafe { Queue::from_raw(bindings::system_highpri_wq) } | |
628 | } | |
629 | ||
630 | /// Returns the system work queue for potentially long-running work items (`system_long_wq`). | |
631 | /// | |
632 | /// It is similar to the one returned by [`system`] but may host long running work items. Queue | |
633 | /// flushing might take relatively long. | |
634 | pub fn system_long() -> &'static Queue { | |
635 | // SAFETY: `system_long_wq` is a C global, always available. | |
636 | unsafe { Queue::from_raw(bindings::system_long_wq) } | |
637 | } | |
638 | ||
639 | /// Returns the system unbound work queue (`system_unbound_wq`). | |
640 | /// | |
641 | /// Workers are not bound to any specific CPU, not concurrency managed, and all queued work items | |
642 | /// are executed immediately as long as `max_active` limit is not reached and resources are | |
643 | /// available. | |
644 | pub fn system_unbound() -> &'static Queue { | |
645 | // SAFETY: `system_unbound_wq` is a C global, always available. | |
646 | unsafe { Queue::from_raw(bindings::system_unbound_wq) } | |
647 | } | |
648 | ||
649 | /// Returns the system freezable work queue (`system_freezable_wq`). | |
650 | /// | |
651 | /// It is equivalent to the one returned by [`system`] except that it's freezable. | |
652 | /// | |
653 | /// A freezable workqueue participates in the freeze phase of the system suspend operations. Work | |
654 | /// items on the workqueue are drained and no new work item starts execution until thawed. | |
655 | pub fn system_freezable() -> &'static Queue { | |
656 | // SAFETY: `system_freezable_wq` is a C global, always available. | |
657 | unsafe { Queue::from_raw(bindings::system_freezable_wq) } | |
658 | } | |
659 | ||
660 | /// Returns the system power-efficient work queue (`system_power_efficient_wq`). | |
661 | /// | |
662 | /// It is inclined towards saving power and is converted to "unbound" variants if the | |
663 | /// `workqueue.power_efficient` kernel parameter is specified; otherwise, it is similar to the one | |
664 | /// returned by [`system`]. | |
665 | pub fn system_power_efficient() -> &'static Queue { | |
666 | // SAFETY: `system_power_efficient_wq` is a C global, always available. | |
667 | unsafe { Queue::from_raw(bindings::system_power_efficient_wq) } | |
668 | } | |
669 | ||
670 | /// Returns the system freezable power-efficient work queue (`system_freezable_power_efficient_wq`). | |
671 | /// | |
672 | /// It is similar to the one returned by [`system_power_efficient`] except that is freezable. | |
673 | /// | |
674 | /// A freezable workqueue participates in the freeze phase of the system suspend operations. Work | |
675 | /// items on the workqueue are drained and no new work item starts execution until thawed. | |
676 | pub fn system_freezable_power_efficient() -> &'static Queue { | |
677 | // SAFETY: `system_freezable_power_efficient_wq` is a C global, always available. | |
678 | unsafe { Queue::from_raw(bindings::system_freezable_power_efficient_wq) } | |
679 | } |