Commit | Line | Data |
---|---|---|
b2441318 | 1 | // SPDX-License-Identifier: GPL-2.0 |
e73f8959 ON |
2 | #include <linux/spinlock.h> |
3 | #include <linux/task_work.h> | |
03248add | 4 | #include <linux/resume_user_mode.h> |
e73f8959 | 5 | |
9da33de6 ON |
6 | static struct callback_head work_exited; /* all we need is ->next == NULL */ |
7 | ||
892f6668 ON |
8 | /** |
9 | * task_work_add - ask the @task to execute @work->func() | |
10 | * @task: the task which should run the callback | |
11 | * @work: the callback to run | |
91989c70 | 12 | * @notify: how to notify the targeted task |
892f6668 | 13 | * |
91989c70 | 14 | * Queue @work for task_work_run() below and notify the @task if @notify |
e788be95 JA |
15 | * is @TWA_RESUME, @TWA_SIGNAL, or @TWA_SIGNAL_NO_IPI. |
16 | * | |
17 | * @TWA_SIGNAL works like signals, in that the it will interrupt the targeted | |
18 | * task and run the task_work, regardless of whether the task is currently | |
19 | * running in the kernel or userspace. | |
20 | * @TWA_SIGNAL_NO_IPI works like @TWA_SIGNAL, except it doesn't send a | |
21 | * reschedule IPI to force the targeted task to reschedule and run task_work. | |
22 | * This can be advantageous if there's no strict requirement that the | |
23 | * task_work be run as soon as possible, just whenever the task enters the | |
24 | * kernel anyway. | |
25 | * @TWA_RESUME work is run only when the task exits the kernel and returns to | |
26 | * user mode, or before entering guest mode. | |
27 | * | |
28 | * Fails if the @task is exiting/exited and thus it can't process this @work. | |
29 | * Otherwise @work->func() will be called when the @task goes through one of | |
30 | * the aforementioned transitions, or exits. | |
892f6668 | 31 | * |
91989c70 JA |
32 | * If the targeted task is exiting, then an error is returned and the work item |
33 | * is not queued. It's up to the caller to arrange for an alternative mechanism | |
34 | * in that case. | |
892f6668 | 35 | * |
91989c70 JA |
36 | * Note: there is no ordering guarantee on works queued here. The task_work |
37 | * list is LIFO. | |
c8219906 | 38 | * |
892f6668 ON |
39 | * RETURNS: |
40 | * 0 if succeeds or -ESRCH. | |
41 | */ | |
91989c70 JA |
42 | int task_work_add(struct task_struct *task, struct callback_head *work, |
43 | enum task_work_notify_mode notify) | |
e73f8959 | 44 | { |
ac3d0da8 | 45 | struct callback_head *head; |
9da33de6 | 46 | |
23f61f0f WW |
47 | /* record the work call stack in order to print it in KASAN reports */ |
48 | kasan_record_aux_stack(work); | |
49 | ||
ac3d0da8 | 50 | do { |
61e96496 | 51 | head = READ_ONCE(task->task_works); |
9da33de6 ON |
52 | if (unlikely(head == &work_exited)) |
53 | return -ESRCH; | |
ac3d0da8 ON |
54 | work->next = head; |
55 | } while (cmpxchg(&task->task_works, head, work) != head); | |
e73f8959 | 56 | |
e91b4816 | 57 | switch (notify) { |
91989c70 JA |
58 | case TWA_NONE: |
59 | break; | |
e91b4816 | 60 | case TWA_RESUME: |
e73f8959 | 61 | set_notify_resume(task); |
e91b4816 ON |
62 | break; |
63 | case TWA_SIGNAL: | |
03941ccf | 64 | set_notify_signal(task); |
e91b4816 | 65 | break; |
e788be95 JA |
66 | case TWA_SIGNAL_NO_IPI: |
67 | __set_notify_signal(task); | |
68 | break; | |
91989c70 JA |
69 | default: |
70 | WARN_ON_ONCE(1); | |
71 | break; | |
e91b4816 ON |
72 | } |
73 | ||
ed3e694d | 74 | return 0; |
e73f8959 ON |
75 | } |
76 | ||
892f6668 | 77 | /** |
c7aab1a7 | 78 | * task_work_cancel_match - cancel a pending work added by task_work_add() |
892f6668 | 79 | * @task: the task which should execute the work |
c7aab1a7 | 80 | * @match: match function to call |
892f6668 ON |
81 | * |
82 | * RETURNS: | |
83 | * The found work or NULL if not found. | |
84 | */ | |
67d12145 | 85 | struct callback_head * |
c7aab1a7 JA |
86 | task_work_cancel_match(struct task_struct *task, |
87 | bool (*match)(struct callback_head *, void *data), | |
88 | void *data) | |
e73f8959 | 89 | { |
ac3d0da8 | 90 | struct callback_head **pprev = &task->task_works; |
205e550a | 91 | struct callback_head *work; |
e73f8959 | 92 | unsigned long flags; |
61e96496 | 93 | |
7f62d40d | 94 | if (likely(!task_work_pending(task))) |
61e96496 | 95 | return NULL; |
ac3d0da8 ON |
96 | /* |
97 | * If cmpxchg() fails we continue without updating pprev. | |
98 | * Either we raced with task_work_add() which added the | |
99 | * new entry before this work, we will find it again. Or | |
9da33de6 | 100 | * we raced with task_work_run(), *pprev == NULL/exited. |
ac3d0da8 | 101 | */ |
e73f8959 | 102 | raw_spin_lock_irqsave(&task->pi_lock, flags); |
506458ef | 103 | while ((work = READ_ONCE(*pprev))) { |
c7aab1a7 | 104 | if (!match(work, data)) |
ac3d0da8 ON |
105 | pprev = &work->next; |
106 | else if (cmpxchg(pprev, work, work->next) == work) | |
107 | break; | |
e73f8959 | 108 | } |
e73f8959 | 109 | raw_spin_unlock_irqrestore(&task->pi_lock, flags); |
ac3d0da8 ON |
110 | |
111 | return work; | |
e73f8959 ON |
112 | } |
113 | ||
c7aab1a7 JA |
114 | static bool task_work_func_match(struct callback_head *cb, void *data) |
115 | { | |
116 | return cb->func == data; | |
117 | } | |
118 | ||
119 | /** | |
120 | * task_work_cancel - cancel a pending work added by task_work_add() | |
121 | * @task: the task which should execute the work | |
122 | * @func: identifies the work to remove | |
123 | * | |
124 | * Find the last queued pending work with ->func == @func and remove | |
125 | * it from queue. | |
126 | * | |
127 | * RETURNS: | |
128 | * The found work or NULL if not found. | |
129 | */ | |
130 | struct callback_head * | |
131 | task_work_cancel(struct task_struct *task, task_work_func_t func) | |
132 | { | |
133 | return task_work_cancel_match(task, task_work_func_match, func); | |
134 | } | |
135 | ||
892f6668 ON |
136 | /** |
137 | * task_work_run - execute the works added by task_work_add() | |
138 | * | |
139 | * Flush the pending works. Should be used by the core kernel code. | |
140 | * Called before the task returns to the user-mode or stops, or when | |
141 | * it exits. In the latter case task_work_add() can no longer add the | |
142 | * new work after task_work_run() returns. | |
143 | */ | |
e73f8959 ON |
144 | void task_work_run(void) |
145 | { | |
146 | struct task_struct *task = current; | |
ac3d0da8 | 147 | struct callback_head *work, *head, *next; |
e73f8959 | 148 | |
ac3d0da8 | 149 | for (;;) { |
9da33de6 ON |
150 | /* |
151 | * work->func() can do task_work_add(), do not set | |
152 | * work_exited unless the list is empty. | |
153 | */ | |
154 | do { | |
6fb61492 | 155 | head = NULL; |
61e96496 | 156 | work = READ_ONCE(task->task_works); |
6fb61492 ON |
157 | if (!work) { |
158 | if (task->flags & PF_EXITING) | |
159 | head = &work_exited; | |
160 | else | |
161 | break; | |
162 | } | |
9da33de6 ON |
163 | } while (cmpxchg(&task->task_works, work, head) != work); |
164 | ||
ac3d0da8 ON |
165 | if (!work) |
166 | break; | |
6fb61492 ON |
167 | /* |
168 | * Synchronize with task_work_cancel(). It can not remove | |
169 | * the first entry == work, cmpxchg(task_works) must fail. | |
170 | * But it can remove another entry from the ->next list. | |
171 | */ | |
172 | raw_spin_lock_irq(&task->pi_lock); | |
173 | raw_spin_unlock_irq(&task->pi_lock); | |
e73f8959 | 174 | |
ac3d0da8 ON |
175 | do { |
176 | next = work->next; | |
177 | work->func(work); | |
178 | work = next; | |
f341861f | 179 | cond_resched(); |
ac3d0da8 | 180 | } while (work); |
e73f8959 ON |
181 | } |
182 | } |