Commit | Line | Data |
---|---|---|
54ecbe6f MH |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | ||
3 | #define pr_fmt(fmt) "rethook: " fmt | |
4 | ||
5 | #include <linux/bug.h> | |
6 | #include <linux/kallsyms.h> | |
7 | #include <linux/kprobes.h> | |
8 | #include <linux/preempt.h> | |
9 | #include <linux/rethook.h> | |
10 | #include <linux/slab.h> | |
54ecbe6f MH |
11 | |
12 | /* Return hook list (shadow stack by list) */ | |
13 | ||
14 | /* | |
15 | * This function is called from delayed_put_task_struct() when a task is | |
16 | * dead and cleaned up to recycle any kretprobe instances associated with | |
17 | * this task. These left over instances represent probed functions that | |
18 | * have been called but will never return. | |
19 | */ | |
20 | void rethook_flush_task(struct task_struct *tk) | |
21 | { | |
22 | struct rethook_node *rhn; | |
23 | struct llist_node *node; | |
24 | ||
25 | node = __llist_del_all(&tk->rethooks); | |
26 | while (node) { | |
27 | rhn = container_of(node, struct rethook_node, llist); | |
28 | node = node->next; | |
29 | preempt_disable(); | |
30 | rethook_recycle(rhn); | |
31 | preempt_enable(); | |
32 | } | |
33 | } | |
34 | ||
35 | static void rethook_free_rcu(struct rcu_head *head) | |
36 | { | |
37 | struct rethook *rh = container_of(head, struct rethook, rcu); | |
4bbd9345 | 38 | objpool_fini(&rh->pool); |
54ecbe6f MH |
39 | } |
40 | ||
195b9cb5 MHG |
41 | /** |
42 | * rethook_stop() - Stop using a rethook. | |
43 | * @rh: the struct rethook to stop. | |
44 | * | |
45 | * Stop using a rethook to prepare for freeing it. If you want to wait for | |
46 | * all running rethook handler before calling rethook_free(), you need to | |
47 | * call this first and wait RCU, and call rethook_free(). | |
48 | */ | |
49 | void rethook_stop(struct rethook *rh) | |
50 | { | |
a1461f1f | 51 | rcu_assign_pointer(rh->handler, NULL); |
195b9cb5 MHG |
52 | } |
53 | ||
54ecbe6f MH |
54 | /** |
55 | * rethook_free() - Free struct rethook. | |
56 | * @rh: the struct rethook to be freed. | |
57 | * | |
58 | * Free the rethook. Before calling this function, user must ensure the | |
59 | * @rh::data is cleaned if needed (or, the handler can access it after | |
60 | * calling this function.) This function will set the @rh to be freed | |
61 | * after all rethook_node are freed (not soon). And the caller must | |
62 | * not touch @rh after calling this. | |
63 | */ | |
64 | void rethook_free(struct rethook *rh) | |
65 | { | |
a1461f1f | 66 | rethook_stop(rh); |
54ecbe6f MH |
67 | |
68 | call_rcu(&rh->rcu, rethook_free_rcu); | |
69 | } | |
70 | ||
4bbd9345 | 71 | static int rethook_init_node(void *nod, void *context) |
72 | { | |
73 | struct rethook_node *node = nod; | |
74 | ||
75 | node->rethook = context; | |
76 | return 0; | |
77 | } | |
78 | ||
79 | static int rethook_fini_pool(struct objpool_head *head, void *context) | |
80 | { | |
81 | kfree(context); | |
82 | return 0; | |
83 | } | |
84 | ||
a1461f1f MHG |
85 | static inline rethook_handler_t rethook_get_handler(struct rethook *rh) |
86 | { | |
87 | return (rethook_handler_t)rcu_dereference_check(rh->handler, | |
88 | rcu_read_lock_any_held()); | |
89 | } | |
90 | ||
54ecbe6f MH |
91 | /** |
92 | * rethook_alloc() - Allocate struct rethook. | |
93 | * @data: a data to pass the @handler when hooking the return. | |
4bbd9345 | 94 | * @handler: the return hook callback function, must NOT be NULL |
95 | * @size: node size: rethook node and additional data | |
96 | * @num: number of rethook nodes to be preallocated | |
54ecbe6f MH |
97 | * |
98 | * Allocate and initialize a new rethook with @data and @handler. | |
4bbd9345 | 99 | * Return pointer of new rethook, or error codes for failures. |
100 | * | |
54ecbe6f MH |
101 | * Note that @handler == NULL means this rethook is going to be freed. |
102 | */ | |
4bbd9345 | 103 | struct rethook *rethook_alloc(void *data, rethook_handler_t handler, |
104 | int size, int num) | |
54ecbe6f | 105 | { |
4bbd9345 | 106 | struct rethook *rh; |
54ecbe6f | 107 | |
4bbd9345 | 108 | if (!handler || num <= 0 || size < sizeof(struct rethook_node)) |
109 | return ERR_PTR(-EINVAL); | |
110 | ||
111 | rh = kzalloc(sizeof(struct rethook), GFP_KERNEL); | |
112 | if (!rh) | |
113 | return ERR_PTR(-ENOMEM); | |
54ecbe6f MH |
114 | |
115 | rh->data = data; | |
a1461f1f | 116 | rcu_assign_pointer(rh->handler, handler); |
54ecbe6f | 117 | |
4bbd9345 | 118 | /* initialize the objpool for rethook nodes */ |
119 | if (objpool_init(&rh->pool, num, size, GFP_KERNEL, rh, | |
120 | rethook_init_node, rethook_fini_pool)) { | |
121 | kfree(rh); | |
122 | return ERR_PTR(-ENOMEM); | |
123 | } | |
54ecbe6f MH |
124 | return rh; |
125 | } | |
126 | ||
54ecbe6f MH |
127 | static void free_rethook_node_rcu(struct rcu_head *head) |
128 | { | |
129 | struct rethook_node *node = container_of(head, struct rethook_node, rcu); | |
4bbd9345 | 130 | struct rethook *rh = node->rethook; |
54ecbe6f | 131 | |
4bbd9345 | 132 | objpool_drop(node, &rh->pool); |
54ecbe6f MH |
133 | } |
134 | ||
135 | /** | |
136 | * rethook_recycle() - return the node to rethook. | |
137 | * @node: The struct rethook_node to be returned. | |
138 | * | |
139 | * Return back the @node to @node::rethook. If the @node::rethook is already | |
140 | * marked as freed, this will free the @node. | |
141 | */ | |
142 | void rethook_recycle(struct rethook_node *node) | |
143 | { | |
a1461f1f | 144 | rethook_handler_t handler; |
54ecbe6f | 145 | |
a1461f1f MHG |
146 | handler = rethook_get_handler(node->rethook); |
147 | if (likely(handler)) | |
4bbd9345 | 148 | objpool_push(node, &node->rethook->pool); |
54ecbe6f MH |
149 | else |
150 | call_rcu(&node->rcu, free_rethook_node_rcu); | |
151 | } | |
152 | NOKPROBE_SYMBOL(rethook_recycle); | |
153 | ||
154 | /** | |
155 | * rethook_try_get() - get an unused rethook node. | |
156 | * @rh: The struct rethook which pools the nodes. | |
157 | * | |
158 | * Get an unused rethook node from @rh. If the node pool is empty, this | |
159 | * will return NULL. Caller must disable preemption. | |
160 | */ | |
161 | struct rethook_node *rethook_try_get(struct rethook *rh) | |
162 | { | |
a1461f1f | 163 | rethook_handler_t handler = rethook_get_handler(rh); |
54ecbe6f MH |
164 | |
165 | /* Check whether @rh is going to be freed. */ | |
166 | if (unlikely(!handler)) | |
167 | return NULL; | |
168 | ||
c0f3bb40 MHG |
169 | /* |
170 | * This expects the caller will set up a rethook on a function entry. | |
171 | * When the function returns, the rethook will eventually be reclaimed | |
172 | * or released in the rethook_recycle() with call_rcu(). | |
173 | * This means the caller must be run in the RCU-availabe context. | |
174 | */ | |
175 | if (unlikely(!rcu_is_watching())) | |
176 | return NULL; | |
177 | ||
4bbd9345 | 178 | return (struct rethook_node *)objpool_pop(&rh->pool); |
54ecbe6f MH |
179 | } |
180 | NOKPROBE_SYMBOL(rethook_try_get); | |
181 | ||
182 | /** | |
183 | * rethook_hook() - Hook the current function return. | |
184 | * @node: The struct rethook node to hook the function return. | |
185 | * @regs: The struct pt_regs for the function entry. | |
186 | * @mcount: True if this is called from mcount(ftrace) context. | |
187 | * | |
188 | * Hook the current running function return. This must be called when the | |
189 | * function entry (or at least @regs must be the registers of the function | |
190 | * entry.) @mcount is used for identifying the context. If this is called | |
191 | * from ftrace (mcount) callback, @mcount must be set true. If this is called | |
192 | * from the real function entry (e.g. kprobes) @mcount must be set false. | |
193 | * This is because the way to hook the function return depends on the context. | |
194 | */ | |
195 | void rethook_hook(struct rethook_node *node, struct pt_regs *regs, bool mcount) | |
196 | { | |
197 | arch_rethook_prepare(node, regs, mcount); | |
198 | __llist_add(&node->llist, ¤t->rethooks); | |
199 | } | |
200 | NOKPROBE_SYMBOL(rethook_hook); | |
201 | ||
202 | /* This assumes the 'tsk' is the current task or is not running. */ | |
203 | static unsigned long __rethook_find_ret_addr(struct task_struct *tsk, | |
204 | struct llist_node **cur) | |
205 | { | |
206 | struct rethook_node *rh = NULL; | |
207 | struct llist_node *node = *cur; | |
208 | ||
209 | if (!node) | |
210 | node = tsk->rethooks.first; | |
211 | else | |
212 | node = node->next; | |
213 | ||
214 | while (node) { | |
215 | rh = container_of(node, struct rethook_node, llist); | |
216 | if (rh->ret_addr != (unsigned long)arch_rethook_trampoline) { | |
217 | *cur = node; | |
218 | return rh->ret_addr; | |
219 | } | |
220 | node = node->next; | |
221 | } | |
222 | return 0; | |
223 | } | |
224 | NOKPROBE_SYMBOL(__rethook_find_ret_addr); | |
225 | ||
226 | /** | |
227 | * rethook_find_ret_addr -- Find correct return address modified by rethook | |
228 | * @tsk: Target task | |
229 | * @frame: A frame pointer | |
230 | * @cur: a storage of the loop cursor llist_node pointer for next call | |
231 | * | |
232 | * Find the correct return address modified by a rethook on @tsk in unsigned | |
233 | * long type. | |
234 | * The @tsk must be 'current' or a task which is not running. @frame is a hint | |
235 | * to get the currect return address - which is compared with the | |
236 | * rethook::frame field. The @cur is a loop cursor for searching the | |
237 | * kretprobe return addresses on the @tsk. The '*@cur' should be NULL at the | |
238 | * first call, but '@cur' itself must NOT NULL. | |
239 | * | |
240 | * Returns found address value or zero if not found. | |
241 | */ | |
242 | unsigned long rethook_find_ret_addr(struct task_struct *tsk, unsigned long frame, | |
243 | struct llist_node **cur) | |
244 | { | |
245 | struct rethook_node *rhn = NULL; | |
246 | unsigned long ret; | |
247 | ||
248 | if (WARN_ON_ONCE(!cur)) | |
249 | return 0; | |
250 | ||
251 | if (WARN_ON_ONCE(tsk != current && task_is_running(tsk))) | |
252 | return 0; | |
253 | ||
254 | do { | |
255 | ret = __rethook_find_ret_addr(tsk, cur); | |
256 | if (!ret) | |
257 | break; | |
258 | rhn = container_of(*cur, struct rethook_node, llist); | |
259 | } while (rhn->frame != frame); | |
260 | ||
261 | return ret; | |
262 | } | |
263 | NOKPROBE_SYMBOL(rethook_find_ret_addr); | |
264 | ||
265 | void __weak arch_rethook_fixup_return(struct pt_regs *regs, | |
266 | unsigned long correct_ret_addr) | |
267 | { | |
268 | /* | |
269 | * Do nothing by default. If the architecture which uses a | |
270 | * frame pointer to record real return address on the stack, | |
271 | * it should fill this function to fixup the return address | |
272 | * so that stacktrace works from the rethook handler. | |
273 | */ | |
274 | } | |
275 | ||
276 | /* This function will be called from each arch-defined trampoline. */ | |
277 | unsigned long rethook_trampoline_handler(struct pt_regs *regs, | |
278 | unsigned long frame) | |
279 | { | |
280 | struct llist_node *first, *node = NULL; | |
281 | unsigned long correct_ret_addr; | |
282 | rethook_handler_t handler; | |
283 | struct rethook_node *rhn; | |
284 | ||
285 | correct_ret_addr = __rethook_find_ret_addr(current, &node); | |
286 | if (!correct_ret_addr) { | |
287 | pr_err("rethook: Return address not found! Maybe there is a bug in the kernel\n"); | |
288 | BUG_ON(1); | |
289 | } | |
290 | ||
291 | instruction_pointer_set(regs, correct_ret_addr); | |
292 | ||
293 | /* | |
294 | * These loops must be protected from rethook_free_rcu() because those | |
295 | * are accessing 'rhn->rethook'. | |
296 | */ | |
be243bac | 297 | preempt_disable_notrace(); |
54ecbe6f MH |
298 | |
299 | /* | |
300 | * Run the handler on the shadow stack. Do not unlink the list here because | |
301 | * stackdump inside the handlers needs to decode it. | |
302 | */ | |
303 | first = current->rethooks.first; | |
304 | while (first) { | |
305 | rhn = container_of(first, struct rethook_node, llist); | |
306 | if (WARN_ON_ONCE(rhn->frame != frame)) | |
307 | break; | |
a1461f1f | 308 | handler = rethook_get_handler(rhn->rethook); |
54ecbe6f | 309 | if (handler) |
cb16330d MHG |
310 | handler(rhn, rhn->rethook->data, |
311 | correct_ret_addr, regs); | |
54ecbe6f MH |
312 | |
313 | if (first == node) | |
314 | break; | |
315 | first = first->next; | |
316 | } | |
317 | ||
318 | /* Fixup registers for returning to correct address. */ | |
319 | arch_rethook_fixup_return(regs, correct_ret_addr); | |
320 | ||
321 | /* Unlink used shadow stack */ | |
322 | first = current->rethooks.first; | |
323 | current->rethooks.first = node->next; | |
324 | node->next = NULL; | |
325 | ||
326 | while (first) { | |
327 | rhn = container_of(first, struct rethook_node, llist); | |
328 | first = first->next; | |
329 | rethook_recycle(rhn); | |
330 | } | |
be243bac | 331 | preempt_enable_notrace(); |
54ecbe6f MH |
332 | |
333 | return correct_ret_addr; | |
334 | } | |
335 | NOKPROBE_SYMBOL(rethook_trampoline_handler); |