Commit | Line | Data |
---|---|---|
d4d791d4 AR |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | ||
3 | //! Work queues. | |
4 | //! | |
7324b889 AR |
5 | //! This file has two components: The raw work item API, and the safe work item API. |
6 | //! | |
7 | //! One pattern that is used in both APIs is the `ID` const generic, which exists to allow a single | |
8 | //! type to define multiple `work_struct` fields. This is done by choosing an id for each field, | |
9 | //! and using that id to specify which field you wish to use. (The actual value doesn't matter, as | |
10 | //! long as you use different values for different fields of the same struct.) Since these IDs are | |
11 | //! generic, they are used only at compile-time, so they shouldn't exist in the final binary. | |
12 | //! | |
13 | //! # The raw API | |
14 | //! | |
4c799d1d | 15 | //! The raw API consists of the [`RawWorkItem`] trait, where the work item needs to provide an |
7324b889 AR |
16 | //! arbitrary function that knows how to enqueue the work item. It should usually not be used |
17 | //! directly, but if you want to, you can use it without using the pieces from the safe API. | |
18 | //! | |
19 | //! # The safe API | |
20 | //! | |
4c799d1d VO |
21 | //! The safe API is used via the [`Work`] struct and [`WorkItem`] traits. Furthermore, it also |
22 | //! includes a trait called [`WorkItemPointer`], which is usually not used directly by the user. | |
7324b889 | 23 | //! |
4c799d1d VO |
24 | //! * The [`Work`] struct is the Rust wrapper for the C `work_struct` type. |
25 | //! * The [`WorkItem`] trait is implemented for structs that can be enqueued to a workqueue. | |
26 | //! * The [`WorkItemPointer`] trait is implemented for the pointer type that points at a something | |
27 | //! that implements [`WorkItem`]. | |
7324b889 | 28 | //! |
15b286d1 AR |
29 | //! ## Example |
30 | //! | |
31 | //! This example defines a struct that holds an integer and can be scheduled on the workqueue. When | |
32 | //! the struct is executed, it will print the integer. Since there is only one `work_struct` field, | |
33 | //! we do not need to specify ids for the fields. | |
34 | //! | |
35 | //! ``` | |
15b286d1 | 36 | //! use kernel::sync::Arc; |
e283ee23 | 37 | //! use kernel::workqueue::{self, impl_has_work, new_work, Work, WorkItem}; |
15b286d1 AR |
38 | //! |
39 | //! #[pin_data] | |
40 | //! struct MyStruct { | |
41 | //! value: i32, | |
42 | //! #[pin] | |
43 | //! work: Work<MyStruct>, | |
44 | //! } | |
45 | //! | |
46 | //! impl_has_work! { | |
47 | //! impl HasWork<Self> for MyStruct { self.work } | |
48 | //! } | |
49 | //! | |
50 | //! impl MyStruct { | |
51 | //! fn new(value: i32) -> Result<Arc<Self>> { | |
52 | //! Arc::pin_init(pin_init!(MyStruct { | |
53 | //! value, | |
54 | //! work <- new_work!("MyStruct::work"), | |
c34aa00d | 55 | //! }), GFP_KERNEL) |
15b286d1 AR |
56 | //! } |
57 | //! } | |
58 | //! | |
59 | //! impl WorkItem for MyStruct { | |
60 | //! type Pointer = Arc<MyStruct>; | |
61 | //! | |
62 | //! fn run(this: Arc<MyStruct>) { | |
63 | //! pr_info!("The value is: {}", this.value); | |
64 | //! } | |
65 | //! } | |
66 | //! | |
67 | //! /// This method will enqueue the struct for execution on the system workqueue, where its value | |
68 | //! /// will be printed. | |
69 | //! fn print_later(val: Arc<MyStruct>) { | |
70 | //! let _ = workqueue::system().enqueue(val); | |
71 | //! } | |
72 | //! ``` | |
73 | //! | |
74 | //! The following example shows how multiple `work_struct` fields can be used: | |
75 | //! | |
76 | //! ``` | |
15b286d1 | 77 | //! use kernel::sync::Arc; |
e283ee23 | 78 | //! use kernel::workqueue::{self, impl_has_work, new_work, Work, WorkItem}; |
15b286d1 AR |
79 | //! |
80 | //! #[pin_data] | |
81 | //! struct MyStruct { | |
82 | //! value_1: i32, | |
83 | //! value_2: i32, | |
84 | //! #[pin] | |
85 | //! work_1: Work<MyStruct, 1>, | |
86 | //! #[pin] | |
87 | //! work_2: Work<MyStruct, 2>, | |
88 | //! } | |
89 | //! | |
90 | //! impl_has_work! { | |
91 | //! impl HasWork<Self, 1> for MyStruct { self.work_1 } | |
92 | //! impl HasWork<Self, 2> for MyStruct { self.work_2 } | |
93 | //! } | |
94 | //! | |
95 | //! impl MyStruct { | |
96 | //! fn new(value_1: i32, value_2: i32) -> Result<Arc<Self>> { | |
97 | //! Arc::pin_init(pin_init!(MyStruct { | |
98 | //! value_1, | |
99 | //! value_2, | |
100 | //! work_1 <- new_work!("MyStruct::work_1"), | |
101 | //! work_2 <- new_work!("MyStruct::work_2"), | |
c34aa00d | 102 | //! }), GFP_KERNEL) |
15b286d1 AR |
103 | //! } |
104 | //! } | |
105 | //! | |
106 | //! impl WorkItem<1> for MyStruct { | |
107 | //! type Pointer = Arc<MyStruct>; | |
108 | //! | |
109 | //! fn run(this: Arc<MyStruct>) { | |
110 | //! pr_info!("The value is: {}", this.value_1); | |
111 | //! } | |
112 | //! } | |
113 | //! | |
114 | //! impl WorkItem<2> for MyStruct { | |
115 | //! type Pointer = Arc<MyStruct>; | |
116 | //! | |
117 | //! fn run(this: Arc<MyStruct>) { | |
118 | //! pr_info!("The second value is: {}", this.value_2); | |
119 | //! } | |
120 | //! } | |
121 | //! | |
122 | //! fn print_1_later(val: Arc<MyStruct>) { | |
123 | //! let _ = workqueue::system().enqueue::<Arc<MyStruct>, 1>(val); | |
124 | //! } | |
125 | //! | |
126 | //! fn print_2_later(val: Arc<MyStruct>) { | |
127 | //! let _ = workqueue::system().enqueue::<Arc<MyStruct>, 2>(val); | |
128 | //! } | |
129 | //! ``` | |
130 | //! | |
bc2e7d5c | 131 | //! C header: [`include/linux/workqueue.h`](srctree/include/linux/workqueue.h) |
d4d791d4 | 132 | |
2c109285 | 133 | use crate::alloc::{AllocError, Flags}; |
00280272 | 134 | use crate::{prelude::*, sync::Arc, sync::LockClassKey, types::Opaque}; |
7324b889 AR |
135 | use core::marker::PhantomData; |
136 | ||
137 | /// Creates a [`Work`] initialiser with the given name and a newly-created lock class. | |
138 | #[macro_export] | |
139 | macro_rules! new_work { | |
140 | ($($name:literal)?) => { | |
141 | $crate::workqueue::Work::new($crate::optional_name!($($name)?), $crate::static_lock_class!()) | |
142 | }; | |
143 | } | |
e283ee23 | 144 | pub use new_work; |
d4d791d4 AR |
145 | |
146 | /// A kernel work queue. | |
147 | /// | |
148 | /// Wraps the kernel's C `struct workqueue_struct`. | |
149 | /// | |
150 | /// It allows work items to be queued to run on thread pools managed by the kernel. Several are | |
151 | /// always available, for example, `system`, `system_highpri`, `system_long`, etc. | |
152 | #[repr(transparent)] | |
153 | pub struct Queue(Opaque<bindings::workqueue_struct>); | |
154 | ||
155 | // SAFETY: Accesses to workqueues used by [`Queue`] are thread-safe. | |
156 | unsafe impl Send for Queue {} | |
157 | // SAFETY: Accesses to workqueues used by [`Queue`] are thread-safe. | |
158 | unsafe impl Sync for Queue {} | |
159 | ||
160 | impl Queue { | |
161 | /// Use the provided `struct workqueue_struct` with Rust. | |
162 | /// | |
163 | /// # Safety | |
164 | /// | |
165 | /// The caller must ensure that the provided raw pointer is not dangling, that it points at a | |
af8b18d7 | 166 | /// valid workqueue, and that it remains valid until the end of `'a`. |
d4d791d4 AR |
167 | pub unsafe fn from_raw<'a>(ptr: *const bindings::workqueue_struct) -> &'a Queue { |
168 | // SAFETY: The `Queue` type is `#[repr(transparent)]`, so the pointer cast is valid. The | |
169 | // caller promises that the pointer is not dangling. | |
170 | unsafe { &*(ptr as *const Queue) } | |
171 | } | |
172 | ||
173 | /// Enqueues a work item. | |
174 | /// | |
175 | /// This may fail if the work item is already enqueued in a workqueue. | |
176 | /// | |
177 | /// The work item will be submitted using `WORK_CPU_UNBOUND`. | |
178 | pub fn enqueue<W, const ID: u64>(&self, w: W) -> W::EnqueueOutput | |
179 | where | |
180 | W: RawWorkItem<ID> + Send + 'static, | |
181 | { | |
182 | let queue_ptr = self.0.get(); | |
183 | ||
184 | // SAFETY: We only return `false` if the `work_struct` is already in a workqueue. The other | |
185 | // `__enqueue` requirements are not relevant since `W` is `Send` and static. | |
186 | // | |
187 | // The call to `bindings::queue_work_on` will dereference the provided raw pointer, which | |
188 | // is ok because `__enqueue` guarantees that the pointer is valid for the duration of this | |
189 | // closure. | |
190 | // | |
191 | // Furthermore, if the C workqueue code accesses the pointer after this call to | |
192 | // `__enqueue`, then the work item was successfully enqueued, and `bindings::queue_work_on` | |
193 | // will have returned true. In this case, `__enqueue` promises that the raw pointer will | |
194 | // stay valid until we call the function pointer in the `work_struct`, so the access is ok. | |
195 | unsafe { | |
196 | w.__enqueue(move |work_ptr| { | |
3e0bc285 MO |
197 | bindings::queue_work_on( |
198 | bindings::wq_misc_consts_WORK_CPU_UNBOUND as _, | |
199 | queue_ptr, | |
200 | work_ptr, | |
201 | ) | |
d4d791d4 AR |
202 | }) |
203 | } | |
204 | } | |
115c95e9 AR |
205 | |
206 | /// Tries to spawn the given function or closure as a work item. | |
207 | /// | |
208 | /// This method can fail because it allocates memory to store the work item. | |
c34aa00d WAF |
209 | pub fn try_spawn<T: 'static + Send + FnOnce()>( |
210 | &self, | |
211 | flags: Flags, | |
212 | func: T, | |
213 | ) -> Result<(), AllocError> { | |
115c95e9 AR |
214 | let init = pin_init!(ClosureWork { |
215 | work <- new_work!("Queue::try_spawn"), | |
216 | func: Some(func), | |
217 | }); | |
218 | ||
c34aa00d | 219 | self.enqueue(Box::pin_init(init, flags).map_err(|_| AllocError)?); |
115c95e9 AR |
220 | Ok(()) |
221 | } | |
222 | } | |
223 | ||
4c799d1d VO |
224 | /// A helper type used in [`try_spawn`]. |
225 | /// | |
226 | /// [`try_spawn`]: Queue::try_spawn | |
115c95e9 AR |
227 | #[pin_data] |
228 | struct ClosureWork<T> { | |
229 | #[pin] | |
230 | work: Work<ClosureWork<T>>, | |
231 | func: Option<T>, | |
232 | } | |
233 | ||
234 | impl<T> ClosureWork<T> { | |
235 | fn project(self: Pin<&mut Self>) -> &mut Option<T> { | |
236 | // SAFETY: The `func` field is not structurally pinned. | |
237 | unsafe { &mut self.get_unchecked_mut().func } | |
238 | } | |
239 | } | |
240 | ||
241 | impl<T: FnOnce()> WorkItem for ClosureWork<T> { | |
242 | type Pointer = Pin<Box<Self>>; | |
243 | ||
244 | fn run(mut this: Pin<Box<Self>>) { | |
245 | if let Some(func) = this.as_mut().project().take() { | |
246 | (func)() | |
247 | } | |
248 | } | |
d4d791d4 AR |
249 | } |
250 | ||
251 | /// A raw work item. | |
252 | /// | |
253 | /// This is the low-level trait that is designed for being as general as possible. | |
254 | /// | |
255 | /// The `ID` parameter to this trait exists so that a single type can provide multiple | |
256 | /// implementations of this trait. For example, if a struct has multiple `work_struct` fields, then | |
257 | /// you will implement this trait once for each field, using a different id for each field. The | |
258 | /// actual value of the id is not important as long as you use different ids for different fields | |
259 | /// of the same struct. (Fields of different structs need not use different ids.) | |
260 | /// | |
b6cda913 | 261 | /// Note that the id is used only to select the right method to call during compilation. It won't be |
d4d791d4 AR |
262 | /// part of the final executable. |
263 | /// | |
264 | /// # Safety | |
265 | /// | |
4c799d1d | 266 | /// Implementers must ensure that any pointers passed to a `queue_work_on` closure by [`__enqueue`] |
d4d791d4 | 267 | /// remain valid for the duration specified in the guarantees section of the documentation for |
4c799d1d VO |
268 | /// [`__enqueue`]. |
269 | /// | |
270 | /// [`__enqueue`]: RawWorkItem::__enqueue | |
d4d791d4 AR |
271 | pub unsafe trait RawWorkItem<const ID: u64> { |
272 | /// The return type of [`Queue::enqueue`]. | |
273 | type EnqueueOutput; | |
274 | ||
275 | /// Enqueues this work item on a queue using the provided `queue_work_on` method. | |
276 | /// | |
277 | /// # Guarantees | |
278 | /// | |
279 | /// If this method calls the provided closure, then the raw pointer is guaranteed to point at a | |
280 | /// valid `work_struct` for the duration of the call to the closure. If the closure returns | |
281 | /// true, then it is further guaranteed that the pointer remains valid until someone calls the | |
282 | /// function pointer stored in the `work_struct`. | |
283 | /// | |
284 | /// # Safety | |
285 | /// | |
286 | /// The provided closure may only return `false` if the `work_struct` is already in a workqueue. | |
287 | /// | |
288 | /// If the work item type is annotated with any lifetimes, then you must not call the function | |
289 | /// pointer after any such lifetime expires. (Never calling the function pointer is okay.) | |
290 | /// | |
291 | /// If the work item type is not [`Send`], then the function pointer must be called on the same | |
292 | /// thread as the call to `__enqueue`. | |
293 | unsafe fn __enqueue<F>(self, queue_work_on: F) -> Self::EnqueueOutput | |
294 | where | |
295 | F: FnOnce(*mut bindings::work_struct) -> bool; | |
296 | } | |
03394130 | 297 | |
7324b889 AR |
298 | /// Defines the method that should be called directly when a work item is executed. |
299 | /// | |
4c799d1d | 300 | /// This trait is implemented by `Pin<Box<T>>` and [`Arc<T>`], and is mainly intended to be |
7324b889 | 301 | /// implemented for smart pointer types. For your own structs, you would implement [`WorkItem`] |
4c799d1d VO |
302 | /// instead. The [`run`] method on this trait will usually just perform the appropriate |
303 | /// `container_of` translation and then call into the [`run`][WorkItem::run] method from the | |
304 | /// [`WorkItem`] trait. | |
7324b889 AR |
305 | /// |
306 | /// This trait is used when the `work_struct` field is defined using the [`Work`] helper. | |
307 | /// | |
308 | /// # Safety | |
309 | /// | |
310 | /// Implementers must ensure that [`__enqueue`] uses a `work_struct` initialized with the [`run`] | |
311 | /// method of this trait as the function pointer. | |
312 | /// | |
313 | /// [`__enqueue`]: RawWorkItem::__enqueue | |
314 | /// [`run`]: WorkItemPointer::run | |
315 | pub unsafe trait WorkItemPointer<const ID: u64>: RawWorkItem<ID> { | |
316 | /// Run this work item. | |
317 | /// | |
318 | /// # Safety | |
319 | /// | |
4c799d1d VO |
320 | /// The provided `work_struct` pointer must originate from a previous call to [`__enqueue`] |
321 | /// where the `queue_work_on` closure returned true, and the pointer must still be valid. | |
322 | /// | |
323 | /// [`__enqueue`]: RawWorkItem::__enqueue | |
7324b889 AR |
324 | unsafe extern "C" fn run(ptr: *mut bindings::work_struct); |
325 | } | |
326 | ||
327 | /// Defines the method that should be called when this work item is executed. | |
328 | /// | |
329 | /// This trait is used when the `work_struct` field is defined using the [`Work`] helper. | |
330 | pub trait WorkItem<const ID: u64 = 0> { | |
331 | /// The pointer type that this struct is wrapped in. This will typically be `Arc<Self>` or | |
332 | /// `Pin<Box<Self>>`. | |
333 | type Pointer: WorkItemPointer<ID>; | |
334 | ||
335 | /// The method that should be called when this work item is executed. | |
336 | fn run(this: Self::Pointer); | |
337 | } | |
338 | ||
339 | /// Links for a work item. | |
340 | /// | |
4c799d1d | 341 | /// This struct contains a function pointer to the [`run`] function from the [`WorkItemPointer`] |
7324b889 AR |
342 | /// trait, and defines the linked list pointers necessary to enqueue a work item in a workqueue. |
343 | /// | |
344 | /// Wraps the kernel's C `struct work_struct`. | |
345 | /// | |
346 | /// This is a helper type used to associate a `work_struct` with the [`WorkItem`] that uses it. | |
4c799d1d VO |
347 | /// |
348 | /// [`run`]: WorkItemPointer::run | |
8db31d3f | 349 | #[pin_data] |
7324b889 AR |
350 | #[repr(transparent)] |
351 | pub struct Work<T: ?Sized, const ID: u64 = 0> { | |
8db31d3f | 352 | #[pin] |
7324b889 AR |
353 | work: Opaque<bindings::work_struct>, |
354 | _inner: PhantomData<T>, | |
355 | } | |
356 | ||
357 | // SAFETY: Kernel work items are usable from any thread. | |
358 | // | |
359 | // We do not need to constrain `T` since the work item does not actually contain a `T`. | |
360 | unsafe impl<T: ?Sized, const ID: u64> Send for Work<T, ID> {} | |
361 | // SAFETY: Kernel work items are usable from any thread. | |
362 | // | |
363 | // We do not need to constrain `T` since the work item does not actually contain a `T`. | |
364 | unsafe impl<T: ?Sized, const ID: u64> Sync for Work<T, ID> {} | |
365 | ||
366 | impl<T: ?Sized, const ID: u64> Work<T, ID> { | |
367 | /// Creates a new instance of [`Work`]. | |
368 | #[inline] | |
369 | #[allow(clippy::new_ret_no_self)] | |
370 | pub fn new(name: &'static CStr, key: &'static LockClassKey) -> impl PinInit<Self> | |
371 | where | |
372 | T: WorkItem<ID>, | |
373 | { | |
8db31d3f BL |
374 | pin_init!(Self { |
375 | work <- Opaque::ffi_init(|slot| { | |
376 | // SAFETY: The `WorkItemPointer` implementation promises that `run` can be used as | |
377 | // the work item function. | |
378 | unsafe { | |
379 | bindings::init_work_with_key( | |
380 | slot, | |
381 | Some(T::Pointer::run), | |
382 | false, | |
383 | name.as_char_ptr(), | |
384 | key.as_ptr(), | |
385 | ) | |
386 | } | |
387 | }), | |
388 | _inner: PhantomData, | |
389 | }) | |
7324b889 AR |
390 | } |
391 | ||
392 | /// Get a pointer to the inner `work_struct`. | |
393 | /// | |
394 | /// # Safety | |
395 | /// | |
396 | /// The provided pointer must not be dangling and must be properly aligned. (But the memory | |
397 | /// need not be initialized.) | |
398 | #[inline] | |
399 | pub unsafe fn raw_get(ptr: *const Self) -> *mut bindings::work_struct { | |
400 | // SAFETY: The caller promises that the pointer is aligned and not dangling. | |
401 | // | |
402 | // A pointer cast would also be ok due to `#[repr(transparent)]`. We use `addr_of!` so that | |
403 | // the compiler does not complain that the `work` field is unused. | |
404 | unsafe { Opaque::raw_get(core::ptr::addr_of!((*ptr).work)) } | |
405 | } | |
406 | } | |
407 | ||
408 | /// Declares that a type has a [`Work<T, ID>`] field. | |
409 | /// | |
410 | /// The intended way of using this trait is via the [`impl_has_work!`] macro. You can use the macro | |
411 | /// like this: | |
412 | /// | |
413 | /// ```no_run | |
e283ee23 | 414 | /// use kernel::workqueue::{impl_has_work, Work}; |
7324b889 AR |
415 | /// |
416 | /// struct MyWorkItem { | |
417 | /// work_field: Work<MyWorkItem, 1>, | |
418 | /// } | |
419 | /// | |
420 | /// impl_has_work! { | |
421 | /// impl HasWork<MyWorkItem, 1> for MyWorkItem { self.work_field } | |
422 | /// } | |
423 | /// ``` | |
424 | /// | |
4c799d1d | 425 | /// Note that since the [`Work`] type is annotated with an id, you can have several `work_struct` |
7324b889 AR |
426 | /// fields by using a different id for each one. |
427 | /// | |
428 | /// # Safety | |
429 | /// | |
af8b18d7 VO |
430 | /// The [`OFFSET`] constant must be the offset of a field in `Self` of type [`Work<T, ID>`]. The |
431 | /// methods on this trait must have exactly the behavior that the definitions given below have. | |
7324b889 | 432 | /// |
7324b889 AR |
433 | /// [`impl_has_work!`]: crate::impl_has_work |
434 | /// [`OFFSET`]: HasWork::OFFSET | |
435 | pub unsafe trait HasWork<T, const ID: u64 = 0> { | |
436 | /// The offset of the [`Work<T, ID>`] field. | |
7324b889 AR |
437 | const OFFSET: usize; |
438 | ||
439 | /// Returns the offset of the [`Work<T, ID>`] field. | |
440 | /// | |
af8b18d7 | 441 | /// This method exists because the [`OFFSET`] constant cannot be accessed if the type is not |
4c799d1d | 442 | /// [`Sized`]. |
7324b889 | 443 | /// |
7324b889 AR |
444 | /// [`OFFSET`]: HasWork::OFFSET |
445 | #[inline] | |
446 | fn get_work_offset(&self) -> usize { | |
447 | Self::OFFSET | |
448 | } | |
449 | ||
450 | /// Returns a pointer to the [`Work<T, ID>`] field. | |
451 | /// | |
452 | /// # Safety | |
453 | /// | |
454 | /// The provided pointer must point at a valid struct of type `Self`. | |
7324b889 AR |
455 | #[inline] |
456 | unsafe fn raw_get_work(ptr: *mut Self) -> *mut Work<T, ID> { | |
457 | // SAFETY: The caller promises that the pointer is valid. | |
458 | unsafe { (ptr as *mut u8).add(Self::OFFSET) as *mut Work<T, ID> } | |
459 | } | |
460 | ||
461 | /// Returns a pointer to the struct containing the [`Work<T, ID>`] field. | |
462 | /// | |
463 | /// # Safety | |
464 | /// | |
465 | /// The pointer must point at a [`Work<T, ID>`] field in a struct of type `Self`. | |
7324b889 AR |
466 | #[inline] |
467 | unsafe fn work_container_of(ptr: *mut Work<T, ID>) -> *mut Self | |
468 | where | |
469 | Self: Sized, | |
470 | { | |
471 | // SAFETY: The caller promises that the pointer points at a field of the right type in the | |
472 | // right kind of struct. | |
473 | unsafe { (ptr as *mut u8).sub(Self::OFFSET) as *mut Self } | |
474 | } | |
475 | } | |
476 | ||
477 | /// Used to safely implement the [`HasWork<T, ID>`] trait. | |
478 | /// | |
479 | /// # Examples | |
480 | /// | |
481 | /// ``` | |
7324b889 | 482 | /// use kernel::sync::Arc; |
e283ee23 | 483 | /// use kernel::workqueue::{self, impl_has_work, Work}; |
7324b889 AR |
484 | /// |
485 | /// struct MyStruct { | |
486 | /// work_field: Work<MyStruct, 17>, | |
487 | /// } | |
488 | /// | |
489 | /// impl_has_work! { | |
490 | /// impl HasWork<MyStruct, 17> for MyStruct { self.work_field } | |
491 | /// } | |
492 | /// ``` | |
7324b889 AR |
493 | #[macro_export] |
494 | macro_rules! impl_has_work { | |
495 | ($(impl$(<$($implarg:ident),*>)? | |
496 | HasWork<$work_type:ty $(, $id:tt)?> | |
497 | for $self:ident $(<$($selfarg:ident),*>)? | |
498 | { self.$field:ident } | |
499 | )*) => {$( | |
500 | // SAFETY: The implementation of `raw_get_work` only compiles if the field has the right | |
501 | // type. | |
502 | unsafe impl$(<$($implarg),*>)? $crate::workqueue::HasWork<$work_type $(, $id)?> for $self $(<$($selfarg),*>)? { | |
503 | const OFFSET: usize = ::core::mem::offset_of!(Self, $field) as usize; | |
504 | ||
505 | #[inline] | |
506 | unsafe fn raw_get_work(ptr: *mut Self) -> *mut $crate::workqueue::Work<$work_type $(, $id)?> { | |
507 | // SAFETY: The caller promises that the pointer is not dangling. | |
508 | unsafe { | |
509 | ::core::ptr::addr_of_mut!((*ptr).$field) | |
510 | } | |
511 | } | |
512 | } | |
513 | )*}; | |
514 | } | |
e283ee23 | 515 | pub use impl_has_work; |
7324b889 | 516 | |
115c95e9 AR |
517 | impl_has_work! { |
518 | impl<T> HasWork<Self> for ClosureWork<T> { self.work } | |
519 | } | |
520 | ||
47f0dbe8 AR |
521 | unsafe impl<T, const ID: u64> WorkItemPointer<ID> for Arc<T> |
522 | where | |
523 | T: WorkItem<ID, Pointer = Self>, | |
524 | T: HasWork<T, ID>, | |
525 | { | |
526 | unsafe extern "C" fn run(ptr: *mut bindings::work_struct) { | |
527 | // SAFETY: The `__enqueue` method always uses a `work_struct` stored in a `Work<T, ID>`. | |
528 | let ptr = ptr as *mut Work<T, ID>; | |
529 | // SAFETY: This computes the pointer that `__enqueue` got from `Arc::into_raw`. | |
530 | let ptr = unsafe { T::work_container_of(ptr) }; | |
531 | // SAFETY: This pointer comes from `Arc::into_raw` and we've been given back ownership. | |
532 | let arc = unsafe { Arc::from_raw(ptr) }; | |
533 | ||
534 | T::run(arc) | |
535 | } | |
536 | } | |
537 | ||
538 | unsafe impl<T, const ID: u64> RawWorkItem<ID> for Arc<T> | |
539 | where | |
540 | T: WorkItem<ID, Pointer = Self>, | |
541 | T: HasWork<T, ID>, | |
542 | { | |
543 | type EnqueueOutput = Result<(), Self>; | |
544 | ||
545 | unsafe fn __enqueue<F>(self, queue_work_on: F) -> Self::EnqueueOutput | |
546 | where | |
547 | F: FnOnce(*mut bindings::work_struct) -> bool, | |
548 | { | |
549 | // Casting between const and mut is not a problem as long as the pointer is a raw pointer. | |
550 | let ptr = Arc::into_raw(self).cast_mut(); | |
551 | ||
552 | // SAFETY: Pointers into an `Arc` point at a valid value. | |
553 | let work_ptr = unsafe { T::raw_get_work(ptr) }; | |
554 | // SAFETY: `raw_get_work` returns a pointer to a valid value. | |
555 | let work_ptr = unsafe { Work::raw_get(work_ptr) }; | |
556 | ||
557 | if queue_work_on(work_ptr) { | |
558 | Ok(()) | |
559 | } else { | |
560 | // SAFETY: The work queue has not taken ownership of the pointer. | |
561 | Err(unsafe { Arc::from_raw(ptr) }) | |
562 | } | |
563 | } | |
564 | } | |
565 | ||
566 | unsafe impl<T, const ID: u64> WorkItemPointer<ID> for Pin<Box<T>> | |
567 | where | |
568 | T: WorkItem<ID, Pointer = Self>, | |
569 | T: HasWork<T, ID>, | |
570 | { | |
571 | unsafe extern "C" fn run(ptr: *mut bindings::work_struct) { | |
572 | // SAFETY: The `__enqueue` method always uses a `work_struct` stored in a `Work<T, ID>`. | |
573 | let ptr = ptr as *mut Work<T, ID>; | |
574 | // SAFETY: This computes the pointer that `__enqueue` got from `Arc::into_raw`. | |
575 | let ptr = unsafe { T::work_container_of(ptr) }; | |
576 | // SAFETY: This pointer comes from `Arc::into_raw` and we've been given back ownership. | |
577 | let boxed = unsafe { Box::from_raw(ptr) }; | |
578 | // SAFETY: The box was already pinned when it was enqueued. | |
579 | let pinned = unsafe { Pin::new_unchecked(boxed) }; | |
580 | ||
581 | T::run(pinned) | |
582 | } | |
583 | } | |
584 | ||
585 | unsafe impl<T, const ID: u64> RawWorkItem<ID> for Pin<Box<T>> | |
586 | where | |
587 | T: WorkItem<ID, Pointer = Self>, | |
588 | T: HasWork<T, ID>, | |
589 | { | |
590 | type EnqueueOutput = (); | |
591 | ||
592 | unsafe fn __enqueue<F>(self, queue_work_on: F) -> Self::EnqueueOutput | |
593 | where | |
594 | F: FnOnce(*mut bindings::work_struct) -> bool, | |
595 | { | |
596 | // SAFETY: We're not going to move `self` or any of its fields, so its okay to temporarily | |
597 | // remove the `Pin` wrapper. | |
598 | let boxed = unsafe { Pin::into_inner_unchecked(self) }; | |
599 | let ptr = Box::into_raw(boxed); | |
600 | ||
601 | // SAFETY: Pointers into a `Box` point at a valid value. | |
602 | let work_ptr = unsafe { T::raw_get_work(ptr) }; | |
603 | // SAFETY: `raw_get_work` returns a pointer to a valid value. | |
604 | let work_ptr = unsafe { Work::raw_get(work_ptr) }; | |
605 | ||
606 | if !queue_work_on(work_ptr) { | |
607 | // SAFETY: This method requires exclusive ownership of the box, so it cannot be in a | |
608 | // workqueue. | |
609 | unsafe { ::core::hint::unreachable_unchecked() } | |
610 | } | |
611 | } | |
612 | } | |
613 | ||
03394130 WAF |
614 | /// Returns the system work queue (`system_wq`). |
615 | /// | |
616 | /// It is the one used by `schedule[_delayed]_work[_on]()`. Multi-CPU multi-threaded. There are | |
617 | /// users which expect relatively short queue flush time. | |
618 | /// | |
619 | /// Callers shouldn't queue work items which can run for too long. | |
620 | pub fn system() -> &'static Queue { | |
621 | // SAFETY: `system_wq` is a C global, always available. | |
622 | unsafe { Queue::from_raw(bindings::system_wq) } | |
623 | } | |
624 | ||
625 | /// Returns the system high-priority work queue (`system_highpri_wq`). | |
626 | /// | |
627 | /// It is similar to the one returned by [`system`] but for work items which require higher | |
628 | /// scheduling priority. | |
629 | pub fn system_highpri() -> &'static Queue { | |
630 | // SAFETY: `system_highpri_wq` is a C global, always available. | |
631 | unsafe { Queue::from_raw(bindings::system_highpri_wq) } | |
632 | } | |
633 | ||
634 | /// Returns the system work queue for potentially long-running work items (`system_long_wq`). | |
635 | /// | |
636 | /// It is similar to the one returned by [`system`] but may host long running work items. Queue | |
637 | /// flushing might take relatively long. | |
638 | pub fn system_long() -> &'static Queue { | |
639 | // SAFETY: `system_long_wq` is a C global, always available. | |
640 | unsafe { Queue::from_raw(bindings::system_long_wq) } | |
641 | } | |
642 | ||
643 | /// Returns the system unbound work queue (`system_unbound_wq`). | |
644 | /// | |
645 | /// Workers are not bound to any specific CPU, not concurrency managed, and all queued work items | |
646 | /// are executed immediately as long as `max_active` limit is not reached and resources are | |
647 | /// available. | |
648 | pub fn system_unbound() -> &'static Queue { | |
649 | // SAFETY: `system_unbound_wq` is a C global, always available. | |
650 | unsafe { Queue::from_raw(bindings::system_unbound_wq) } | |
651 | } | |
652 | ||
653 | /// Returns the system freezable work queue (`system_freezable_wq`). | |
654 | /// | |
655 | /// It is equivalent to the one returned by [`system`] except that it's freezable. | |
656 | /// | |
657 | /// A freezable workqueue participates in the freeze phase of the system suspend operations. Work | |
658 | /// items on the workqueue are drained and no new work item starts execution until thawed. | |
659 | pub fn system_freezable() -> &'static Queue { | |
660 | // SAFETY: `system_freezable_wq` is a C global, always available. | |
661 | unsafe { Queue::from_raw(bindings::system_freezable_wq) } | |
662 | } | |
663 | ||
664 | /// Returns the system power-efficient work queue (`system_power_efficient_wq`). | |
665 | /// | |
666 | /// It is inclined towards saving power and is converted to "unbound" variants if the | |
667 | /// `workqueue.power_efficient` kernel parameter is specified; otherwise, it is similar to the one | |
668 | /// returned by [`system`]. | |
669 | pub fn system_power_efficient() -> &'static Queue { | |
670 | // SAFETY: `system_power_efficient_wq` is a C global, always available. | |
671 | unsafe { Queue::from_raw(bindings::system_power_efficient_wq) } | |
672 | } | |
673 | ||
674 | /// Returns the system freezable power-efficient work queue (`system_freezable_power_efficient_wq`). | |
675 | /// | |
676 | /// It is similar to the one returned by [`system_power_efficient`] except that is freezable. | |
677 | /// | |
678 | /// A freezable workqueue participates in the freeze phase of the system suspend operations. Work | |
679 | /// items on the workqueue are drained and no new work item starts execution until thawed. | |
680 | pub fn system_freezable_power_efficient() -> &'static Queue { | |
681 | // SAFETY: `system_freezable_power_efficient_wq` is a C global, always available. | |
682 | unsafe { Queue::from_raw(bindings::system_freezable_power_efficient_wq) } | |
683 | } |