Commit | Line | Data |
---|---|---|
9c92ab61 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
1867a23b | 2 | /* |
e912c881 | 3 | * Sync File validation framework |
1867a23b GP |
4 | * |
5 | * Copyright (C) 2012 Google, Inc. | |
1867a23b GP |
6 | */ |
7 | ||
8 | #include <linux/file.h> | |
9 | #include <linux/fs.h> | |
10 | #include <linux/uaccess.h> | |
aff9da10 | 11 | #include <linux/slab.h> |
1867a23b GP |
12 | #include <linux/sync_file.h> |
13 | ||
1fe82e2e | 14 | #include "sync_debug.h" |
1867a23b | 15 | |
aff9da10 | 16 | #define CREATE_TRACE_POINTS |
a04f915e | 17 | #include "sync_trace.h" |
aff9da10 | 18 | |
fc0c9a03 GP |
19 | /* |
20 | * SW SYNC validation framework | |
21 | * | |
22 | * A sync object driver that uses a 32bit counter to coordinate | |
23 | * synchronization. Useful when there is no hardware primitive backing | |
24 | * the synchronization. | |
25 | * | |
26 | * To start the framework just open: | |
27 | * | |
28 | * <debugfs>/sync/sw_sync | |
29 | * | |
30 | * That will create a sync timeline, all fences created under this timeline | |
31 | * file descriptor will belong to the this timeline. | |
32 | * | |
33 | * The 'sw_sync' file can be opened many times as to create different | |
34 | * timelines. | |
35 | * | |
36 | * Fences can be created with SW_SYNC_IOC_CREATE_FENCE ioctl with struct | |
e56a0fbd | 37 | * sw_sync_create_fence_data as parameter. |
fc0c9a03 GP |
38 | * |
39 | * To increment the timeline counter, SW_SYNC_IOC_INC ioctl should be used | |
40 | * with the increment as u32. This will update the last signaled value | |
41 | * from the timeline and signal any fence that has a seqno smaller or equal | |
42 | * to it. | |
43 | * | |
e56a0fbd | 44 | * struct sw_sync_create_fence_data |
fc0c9a03 GP |
45 | * @value: the seqno to initialise the fence with |
46 | * @name: the name of the new sync point | |
47 | * @fence: return the fd of the new sync_file with the created fence | |
48 | */ | |
6f65aa89 GP |
49 | struct sw_sync_create_fence_data { |
50 | __u32 value; | |
51 | char name[32]; | |
52 | __s32 fence; /* fd of new fence */ | |
53 | }; | |
54 | ||
55 | #define SW_SYNC_IOC_MAGIC 'W' | |
56 | ||
57 | #define SW_SYNC_IOC_CREATE_FENCE _IOWR(SW_SYNC_IOC_MAGIC, 0,\ | |
58 | struct sw_sync_create_fence_data) | |
fc0c9a03 | 59 | |
6f65aa89 GP |
60 | #define SW_SYNC_IOC_INC _IOW(SW_SYNC_IOC_MAGIC, 1, __u32) |
61 | ||
f54d1867 | 62 | static const struct dma_fence_ops timeline_fence_ops; |
aff9da10 | 63 | |
f54d1867 | 64 | static inline struct sync_pt *dma_fence_to_sync_pt(struct dma_fence *fence) |
aff9da10 GP |
65 | { |
66 | if (fence->ops != &timeline_fence_ops) | |
67 | return NULL; | |
68 | return container_of(fence, struct sync_pt, base); | |
69 | } | |
70 | ||
71 | /** | |
72 | * sync_timeline_create() - creates a sync object | |
aff9da10 GP |
73 | * @name: sync_timeline name |
74 | * | |
75 | * Creates a new sync_timeline. Returns the sync_timeline object or NULL in | |
76 | * case of error. | |
77 | */ | |
74881588 | 78 | static struct sync_timeline *sync_timeline_create(const char *name) |
aff9da10 GP |
79 | { |
80 | struct sync_timeline *obj; | |
81 | ||
82 | obj = kzalloc(sizeof(*obj), GFP_KERNEL); | |
83 | if (!obj) | |
84 | return NULL; | |
85 | ||
86 | kref_init(&obj->kref); | |
f54d1867 | 87 | obj->context = dma_fence_context_alloc(1); |
aff9da10 | 88 | strlcpy(obj->name, name, sizeof(obj->name)); |
aff9da10 | 89 | |
f1e8c671 | 90 | obj->pt_tree = RB_ROOT; |
d3862e44 CW |
91 | INIT_LIST_HEAD(&obj->pt_list); |
92 | spin_lock_init(&obj->lock); | |
aff9da10 GP |
93 | |
94 | sync_timeline_debug_add(obj); | |
95 | ||
96 | return obj; | |
97 | } | |
98 | ||
99 | static void sync_timeline_free(struct kref *kref) | |
100 | { | |
101 | struct sync_timeline *obj = | |
102 | container_of(kref, struct sync_timeline, kref); | |
103 | ||
104 | sync_timeline_debug_remove(obj); | |
105 | ||
106 | kfree(obj); | |
107 | } | |
108 | ||
109 | static void sync_timeline_get(struct sync_timeline *obj) | |
110 | { | |
111 | kref_get(&obj->kref); | |
112 | } | |
113 | ||
114 | static void sync_timeline_put(struct sync_timeline *obj) | |
115 | { | |
116 | kref_put(&obj->kref, sync_timeline_free); | |
117 | } | |
118 | ||
150b6a9d GP |
119 | static const char *timeline_fence_get_driver_name(struct dma_fence *fence) |
120 | { | |
121 | return "sw_sync"; | |
122 | } | |
123 | ||
124 | static const char *timeline_fence_get_timeline_name(struct dma_fence *fence) | |
125 | { | |
126 | struct sync_timeline *parent = dma_fence_parent(fence); | |
127 | ||
128 | return parent->name; | |
129 | } | |
130 | ||
131 | static void timeline_fence_release(struct dma_fence *fence) | |
132 | { | |
133 | struct sync_pt *pt = dma_fence_to_sync_pt(fence); | |
134 | struct sync_timeline *parent = dma_fence_parent(fence); | |
d3c6dd1f | 135 | unsigned long flags; |
150b6a9d | 136 | |
d3c6dd1f | 137 | spin_lock_irqsave(fence->lock, flags); |
150b6a9d | 138 | if (!list_empty(&pt->link)) { |
d3c6dd1f CW |
139 | list_del(&pt->link); |
140 | rb_erase(&pt->node, &parent->pt_tree); | |
150b6a9d | 141 | } |
d3c6dd1f | 142 | spin_unlock_irqrestore(fence->lock, flags); |
150b6a9d GP |
143 | |
144 | sync_timeline_put(parent); | |
145 | dma_fence_free(fence); | |
146 | } | |
147 | ||
148 | static bool timeline_fence_signaled(struct dma_fence *fence) | |
149 | { | |
150 | struct sync_timeline *parent = dma_fence_parent(fence); | |
151 | ||
5e498abf | 152 | return !__dma_fence_is_later(fence->seqno, parent->value, fence->ops); |
150b6a9d GP |
153 | } |
154 | ||
155 | static bool timeline_fence_enable_signaling(struct dma_fence *fence) | |
156 | { | |
157 | return true; | |
158 | } | |
159 | ||
160 | static void timeline_fence_value_str(struct dma_fence *fence, | |
161 | char *str, int size) | |
162 | { | |
b312d8ca | 163 | snprintf(str, size, "%lld", fence->seqno); |
150b6a9d GP |
164 | } |
165 | ||
166 | static void timeline_fence_timeline_value_str(struct dma_fence *fence, | |
167 | char *str, int size) | |
168 | { | |
169 | struct sync_timeline *parent = dma_fence_parent(fence); | |
170 | ||
171 | snprintf(str, size, "%d", parent->value); | |
172 | } | |
173 | ||
174 | static const struct dma_fence_ops timeline_fence_ops = { | |
175 | .get_driver_name = timeline_fence_get_driver_name, | |
176 | .get_timeline_name = timeline_fence_get_timeline_name, | |
177 | .enable_signaling = timeline_fence_enable_signaling, | |
178 | .signaled = timeline_fence_signaled, | |
150b6a9d GP |
179 | .release = timeline_fence_release, |
180 | .fence_value_str = timeline_fence_value_str, | |
181 | .timeline_value_str = timeline_fence_timeline_value_str, | |
182 | }; | |
183 | ||
aff9da10 GP |
184 | /** |
185 | * sync_timeline_signal() - signal a status change on a sync_timeline | |
186 | * @obj: sync_timeline to signal | |
187 | * @inc: num to increment on timeline->value | |
188 | * | |
189 | * A sync implementation should call this any time one of it's fences | |
190 | * has signaled or has an error condition. | |
191 | */ | |
192 | static void sync_timeline_signal(struct sync_timeline *obj, unsigned int inc) | |
193 | { | |
aff9da10 GP |
194 | struct sync_pt *pt, *next; |
195 | ||
196 | trace_sync_timeline(obj); | |
197 | ||
d3862e44 | 198 | spin_lock_irq(&obj->lock); |
aff9da10 GP |
199 | |
200 | obj->value += inc; | |
201 | ||
f1e8c671 | 202 | list_for_each_entry_safe(pt, next, &obj->pt_list, link) { |
3792b7c1 | 203 | if (!timeline_fence_signaled(&pt->base)) |
f1e8c671 CW |
204 | break; |
205 | ||
206 | list_del_init(&pt->link); | |
207 | rb_erase(&pt->node, &obj->pt_tree); | |
3792b7c1 GP |
208 | |
209 | /* | |
210 | * A signal callback may release the last reference to this | |
211 | * fence, causing it to be freed. That operation has to be | |
212 | * last to avoid a use after free inside this loop, and must | |
213 | * be after we remove the fence from the timeline in order to | |
214 | * prevent deadlocking on timeline->lock inside | |
215 | * timeline_fence_release(). | |
216 | */ | |
217 | dma_fence_signal_locked(&pt->base); | |
f1e8c671 | 218 | } |
aff9da10 | 219 | |
d3862e44 | 220 | spin_unlock_irq(&obj->lock); |
aff9da10 GP |
221 | } |
222 | ||
223 | /** | |
224 | * sync_pt_create() - creates a sync pt | |
b88132b4 CW |
225 | * @obj: parent sync_timeline |
226 | * @value: value of the fence | |
aff9da10 | 227 | * |
b88132b4 | 228 | * Creates a new sync_pt (fence) as a child of @parent. @size bytes will be |
aff9da10 GP |
229 | * allocated allowing for implementation specific data to be kept after |
230 | * the generic sync_timeline struct. Returns the sync_pt object or | |
231 | * NULL in case of error. | |
232 | */ | |
3b52ce44 CW |
233 | static struct sync_pt *sync_pt_create(struct sync_timeline *obj, |
234 | unsigned int value) | |
aff9da10 | 235 | { |
aff9da10 GP |
236 | struct sync_pt *pt; |
237 | ||
3b52ce44 | 238 | pt = kzalloc(sizeof(*pt), GFP_KERNEL); |
aff9da10 GP |
239 | if (!pt) |
240 | return NULL; | |
241 | ||
aff9da10 | 242 | sync_timeline_get(obj); |
d3862e44 | 243 | dma_fence_init(&pt->base, &timeline_fence_ops, &obj->lock, |
f54d1867 | 244 | obj->context, value); |
d3862e44 | 245 | INIT_LIST_HEAD(&pt->link); |
a6aa8fca | 246 | |
d3862e44 | 247 | spin_lock_irq(&obj->lock); |
f1e8c671 CW |
248 | if (!dma_fence_is_signaled_locked(&pt->base)) { |
249 | struct rb_node **p = &obj->pt_tree.rb_node; | |
250 | struct rb_node *parent = NULL; | |
251 | ||
252 | while (*p) { | |
253 | struct sync_pt *other; | |
254 | int cmp; | |
255 | ||
256 | parent = *p; | |
257 | other = rb_entry(parent, typeof(*pt), node); | |
258 | cmp = value - other->base.seqno; | |
259 | if (cmp > 0) { | |
260 | p = &parent->rb_right; | |
261 | } else if (cmp < 0) { | |
262 | p = &parent->rb_left; | |
263 | } else { | |
264 | if (dma_fence_get_rcu(&other->base)) { | |
d3c6dd1f CW |
265 | sync_timeline_put(obj); |
266 | kfree(pt); | |
f1e8c671 CW |
267 | pt = other; |
268 | goto unlock; | |
269 | } | |
270 | p = &parent->rb_left; | |
271 | } | |
272 | } | |
273 | rb_link_node(&pt->node, parent, p); | |
274 | rb_insert_color(&pt->node, &obj->pt_tree); | |
275 | ||
276 | parent = rb_next(&pt->node); | |
277 | list_add_tail(&pt->link, | |
278 | parent ? &rb_entry(parent, typeof(*pt), node)->link : &obj->pt_list); | |
279 | } | |
280 | unlock: | |
d3862e44 | 281 | spin_unlock_irq(&obj->lock); |
a6aa8fca | 282 | |
aff9da10 GP |
283 | return pt; |
284 | } | |
285 | ||
1867a23b GP |
286 | /* |
287 | * *WARNING* | |
288 | * | |
289 | * improper use of this can result in deadlocking kernel drivers from userspace. | |
290 | */ | |
291 | ||
292 | /* opening sw_sync create a new sync obj */ | |
293 | static int sw_sync_debugfs_open(struct inode *inode, struct file *file) | |
294 | { | |
295 | struct sync_timeline *obj; | |
296 | char task_comm[TASK_COMM_LEN]; | |
297 | ||
298 | get_task_comm(task_comm, current); | |
299 | ||
b9bc2b7b | 300 | obj = sync_timeline_create(task_comm); |
1867a23b GP |
301 | if (!obj) |
302 | return -ENOMEM; | |
303 | ||
304 | file->private_data = obj; | |
305 | ||
306 | return 0; | |
307 | } | |
308 | ||
309 | static int sw_sync_debugfs_release(struct inode *inode, struct file *file) | |
310 | { | |
311 | struct sync_timeline *obj = file->private_data; | |
ea4d5a27 DB |
312 | struct sync_pt *pt, *next; |
313 | ||
314 | spin_lock_irq(&obj->lock); | |
315 | ||
316 | list_for_each_entry_safe(pt, next, &obj->pt_list, link) { | |
317 | dma_fence_set_error(&pt->base, -ENOENT); | |
318 | dma_fence_signal_locked(&pt->base); | |
319 | } | |
1867a23b | 320 | |
ea4d5a27 | 321 | spin_unlock_irq(&obj->lock); |
71110232 GP |
322 | |
323 | sync_timeline_put(obj); | |
1867a23b GP |
324 | return 0; |
325 | } | |
326 | ||
327 | static long sw_sync_ioctl_create_fence(struct sync_timeline *obj, | |
328 | unsigned long arg) | |
329 | { | |
330 | int fd = get_unused_fd_flags(O_CLOEXEC); | |
331 | int err; | |
332 | struct sync_pt *pt; | |
333 | struct sync_file *sync_file; | |
334 | struct sw_sync_create_fence_data data; | |
335 | ||
336 | if (fd < 0) | |
337 | return fd; | |
338 | ||
339 | if (copy_from_user(&data, (void __user *)arg, sizeof(data))) { | |
340 | err = -EFAULT; | |
341 | goto err; | |
342 | } | |
343 | ||
3b52ce44 | 344 | pt = sync_pt_create(obj, data.value); |
1867a23b GP |
345 | if (!pt) { |
346 | err = -ENOMEM; | |
347 | goto err; | |
348 | } | |
349 | ||
350 | sync_file = sync_file_create(&pt->base); | |
4592bfcd | 351 | dma_fence_put(&pt->base); |
1867a23b | 352 | if (!sync_file) { |
1867a23b GP |
353 | err = -ENOMEM; |
354 | goto err; | |
355 | } | |
356 | ||
357 | data.fence = fd; | |
358 | if (copy_to_user((void __user *)arg, &data, sizeof(data))) { | |
359 | fput(sync_file->file); | |
360 | err = -EFAULT; | |
361 | goto err; | |
362 | } | |
363 | ||
364 | fd_install(fd, sync_file->file); | |
365 | ||
366 | return 0; | |
367 | ||
368 | err: | |
369 | put_unused_fd(fd); | |
370 | return err; | |
371 | } | |
372 | ||
373 | static long sw_sync_ioctl_inc(struct sync_timeline *obj, unsigned long arg) | |
374 | { | |
375 | u32 value; | |
376 | ||
377 | if (copy_from_user(&value, (void __user *)arg, sizeof(value))) | |
378 | return -EFAULT; | |
379 | ||
8f66d3aa CW |
380 | while (value > INT_MAX) { |
381 | sync_timeline_signal(obj, INT_MAX); | |
382 | value -= INT_MAX; | |
383 | } | |
384 | ||
1867a23b GP |
385 | sync_timeline_signal(obj, value); |
386 | ||
387 | return 0; | |
388 | } | |
389 | ||
390 | static long sw_sync_ioctl(struct file *file, unsigned int cmd, | |
391 | unsigned long arg) | |
392 | { | |
393 | struct sync_timeline *obj = file->private_data; | |
394 | ||
395 | switch (cmd) { | |
396 | case SW_SYNC_IOC_CREATE_FENCE: | |
397 | return sw_sync_ioctl_create_fence(obj, arg); | |
398 | ||
399 | case SW_SYNC_IOC_INC: | |
400 | return sw_sync_ioctl_inc(obj, arg); | |
401 | ||
402 | default: | |
403 | return -ENOTTY; | |
404 | } | |
405 | } | |
406 | ||
407 | const struct file_operations sw_sync_debugfs_fops = { | |
408 | .open = sw_sync_debugfs_open, | |
409 | .release = sw_sync_debugfs_release, | |
410 | .unlocked_ioctl = sw_sync_ioctl, | |
1832f2d8 | 411 | .compat_ioctl = compat_ptr_ioctl, |
1867a23b | 412 | }; |