Commit | Line | Data |
---|---|---|
9952f691 | 1 | // SPDX-License-Identifier: GPL-2.0-only |
7ede0b0b TB |
2 | /* |
3 | * Tegra host1x Interrupt Management | |
4 | * | |
5 | * Copyright (c) 2010-2013, NVIDIA Corporation. | |
7ede0b0b TB |
6 | */ |
7 | ||
8 | #include <linux/clk.h> | |
9 | #include <linux/interrupt.h> | |
10 | #include <linux/slab.h> | |
11 | #include <linux/irq.h> | |
12 | ||
6579324a TB |
13 | #include <trace/events/host1x.h> |
14 | #include "channel.h" | |
7ede0b0b | 15 | #include "dev.h" |
687db220 | 16 | #include "fence.h" |
7ede0b0b TB |
17 | #include "intr.h" |
18 | ||
19 | /* Wait list management */ | |
20 | ||
21 | enum waitlist_state { | |
22 | WLS_PENDING, | |
23 | WLS_REMOVED, | |
24 | WLS_CANCELLED, | |
25 | WLS_HANDLED | |
26 | }; | |
27 | ||
28 | static void waiter_release(struct kref *kref) | |
29 | { | |
30 | kfree(container_of(kref, struct host1x_waitlist, refcount)); | |
31 | } | |
32 | ||
33 | /* | |
34 | * add a waiter to a waiter queue, sorted by threshold | |
35 | * returns true if it was added at the head of the queue | |
36 | */ | |
37 | static bool add_waiter_to_queue(struct host1x_waitlist *waiter, | |
38 | struct list_head *queue) | |
39 | { | |
40 | struct host1x_waitlist *pos; | |
41 | u32 thresh = waiter->thresh; | |
42 | ||
43 | list_for_each_entry_reverse(pos, queue, list) | |
44 | if ((s32)(pos->thresh - thresh) <= 0) { | |
45 | list_add(&waiter->list, &pos->list); | |
46 | return false; | |
47 | } | |
48 | ||
49 | list_add(&waiter->list, queue); | |
50 | return true; | |
51 | } | |
52 | ||
53 | /* | |
54 | * run through a waiter queue for a single sync point ID | |
55 | * and gather all completed waiters into lists by actions | |
56 | */ | |
57 | static void remove_completed_waiters(struct list_head *head, u32 sync, | |
58 | struct list_head completed[HOST1X_INTR_ACTION_COUNT]) | |
59 | { | |
60 | struct list_head *dest; | |
6579324a | 61 | struct host1x_waitlist *waiter, *next, *prev; |
7ede0b0b TB |
62 | |
63 | list_for_each_entry_safe(waiter, next, head, list) { | |
64 | if ((s32)(waiter->thresh - sync) > 0) | |
65 | break; | |
66 | ||
67 | dest = completed + waiter->action; | |
68 | ||
6579324a TB |
69 | /* consolidate submit cleanups */ |
70 | if (waiter->action == HOST1X_INTR_ACTION_SUBMIT_COMPLETE && | |
71 | !list_empty(dest)) { | |
72 | prev = list_entry(dest->prev, | |
73 | struct host1x_waitlist, list); | |
74 | if (prev->data == waiter->data) { | |
75 | prev->count++; | |
76 | dest = NULL; | |
77 | } | |
78 | } | |
79 | ||
7ede0b0b TB |
80 | /* PENDING->REMOVED or CANCELLED->HANDLED */ |
81 | if (atomic_inc_return(&waiter->state) == WLS_HANDLED || !dest) { | |
82 | list_del(&waiter->list); | |
83 | kref_put(&waiter->refcount, waiter_release); | |
84 | } else | |
85 | list_move_tail(&waiter->list, dest); | |
86 | } | |
87 | } | |
88 | ||
89 | static void reset_threshold_interrupt(struct host1x *host, | |
90 | struct list_head *head, | |
91 | unsigned int id) | |
92 | { | |
93 | u32 thresh = | |
94 | list_first_entry(head, struct host1x_waitlist, list)->thresh; | |
95 | ||
96 | host1x_hw_intr_set_syncpt_threshold(host, id, thresh); | |
97 | host1x_hw_intr_enable_syncpt_intr(host, id); | |
98 | } | |
99 | ||
6579324a TB |
100 | static void action_submit_complete(struct host1x_waitlist *waiter) |
101 | { | |
102 | struct host1x_channel *channel = waiter->data; | |
103 | ||
104 | host1x_cdma_update(&channel->cdma); | |
105 | ||
106 | /* Add nr_completed to trace */ | |
107 | trace_host1x_channel_submit_complete(dev_name(channel->dev), | |
108 | waiter->count, waiter->thresh); | |
6579324a TB |
109 | } |
110 | ||
7ede0b0b TB |
111 | static void action_wakeup(struct host1x_waitlist *waiter) |
112 | { | |
113 | wait_queue_head_t *wq = waiter->data; | |
6df633d0 | 114 | |
7ede0b0b TB |
115 | wake_up(wq); |
116 | } | |
117 | ||
118 | static void action_wakeup_interruptible(struct host1x_waitlist *waiter) | |
119 | { | |
120 | wait_queue_head_t *wq = waiter->data; | |
6df633d0 | 121 | |
7ede0b0b TB |
122 | wake_up_interruptible(wq); |
123 | } | |
124 | ||
687db220 MP |
125 | static void action_signal_fence(struct host1x_waitlist *waiter) |
126 | { | |
127 | struct host1x_syncpt_fence *f = waiter->data; | |
128 | ||
129 | host1x_fence_signal(f); | |
130 | } | |
131 | ||
7ede0b0b TB |
132 | typedef void (*action_handler)(struct host1x_waitlist *waiter); |
133 | ||
9f2e57ca | 134 | static const action_handler action_handlers[HOST1X_INTR_ACTION_COUNT] = { |
6579324a | 135 | action_submit_complete, |
7ede0b0b TB |
136 | action_wakeup, |
137 | action_wakeup_interruptible, | |
687db220 | 138 | action_signal_fence, |
7ede0b0b TB |
139 | }; |
140 | ||
141 | static void run_handlers(struct list_head completed[HOST1X_INTR_ACTION_COUNT]) | |
142 | { | |
143 | struct list_head *head = completed; | |
d4ad3ad9 | 144 | unsigned int i; |
7ede0b0b TB |
145 | |
146 | for (i = 0; i < HOST1X_INTR_ACTION_COUNT; ++i, ++head) { | |
147 | action_handler handler = action_handlers[i]; | |
148 | struct host1x_waitlist *waiter, *next; | |
149 | ||
150 | list_for_each_entry_safe(waiter, next, head, list) { | |
151 | list_del(&waiter->list); | |
152 | handler(waiter); | |
153 | WARN_ON(atomic_xchg(&waiter->state, WLS_HANDLED) != | |
154 | WLS_REMOVED); | |
155 | kref_put(&waiter->refcount, waiter_release); | |
156 | } | |
157 | } | |
158 | } | |
159 | ||
160 | /* | |
161 | * Remove & handle all waiters that have completed for the given syncpt | |
162 | */ | |
163 | static int process_wait_list(struct host1x *host, | |
164 | struct host1x_syncpt *syncpt, | |
165 | u32 threshold) | |
166 | { | |
167 | struct list_head completed[HOST1X_INTR_ACTION_COUNT]; | |
168 | unsigned int i; | |
169 | int empty; | |
170 | ||
171 | for (i = 0; i < HOST1X_INTR_ACTION_COUNT; ++i) | |
172 | INIT_LIST_HEAD(completed + i); | |
173 | ||
174 | spin_lock(&syncpt->intr.lock); | |
175 | ||
176 | remove_completed_waiters(&syncpt->intr.wait_head, threshold, | |
177 | completed); | |
178 | ||
179 | empty = list_empty(&syncpt->intr.wait_head); | |
180 | if (empty) | |
181 | host1x_hw_intr_disable_syncpt_intr(host, syncpt->id); | |
182 | else | |
183 | reset_threshold_interrupt(host, &syncpt->intr.wait_head, | |
184 | syncpt->id); | |
185 | ||
186 | spin_unlock(&syncpt->intr.lock); | |
187 | ||
188 | run_handlers(completed); | |
189 | ||
190 | return empty; | |
191 | } | |
192 | ||
193 | /* | |
194 | * Sync point threshold interrupt service thread function | |
195 | * Handles sync point threshold triggers, in thread context | |
196 | */ | |
197 | ||
198 | static void syncpt_thresh_work(struct work_struct *work) | |
199 | { | |
200 | struct host1x_syncpt_intr *syncpt_intr = | |
201 | container_of(work, struct host1x_syncpt_intr, work); | |
202 | struct host1x_syncpt *syncpt = | |
203 | container_of(syncpt_intr, struct host1x_syncpt, intr); | |
204 | unsigned int id = syncpt->id; | |
205 | struct host1x *host = syncpt->host; | |
206 | ||
207 | (void)process_wait_list(host, syncpt, | |
208 | host1x_syncpt_load(host->syncpt + id)); | |
209 | } | |
210 | ||
ac330f45 TR |
211 | int host1x_intr_add_action(struct host1x *host, struct host1x_syncpt *syncpt, |
212 | u32 thresh, enum host1x_intr_action action, | |
213 | void *data, struct host1x_waitlist *waiter, | |
214 | void **ref) | |
7ede0b0b | 215 | { |
7ede0b0b TB |
216 | int queue_was_empty; |
217 | ||
218 | if (waiter == NULL) { | |
219 | pr_warn("%s: NULL waiter\n", __func__); | |
220 | return -EINVAL; | |
221 | } | |
222 | ||
223 | /* initialize a new waiter */ | |
224 | INIT_LIST_HEAD(&waiter->list); | |
225 | kref_init(&waiter->refcount); | |
226 | if (ref) | |
227 | kref_get(&waiter->refcount); | |
228 | waiter->thresh = thresh; | |
229 | waiter->action = action; | |
230 | atomic_set(&waiter->state, WLS_PENDING); | |
231 | waiter->data = data; | |
232 | waiter->count = 1; | |
233 | ||
7ede0b0b TB |
234 | spin_lock(&syncpt->intr.lock); |
235 | ||
236 | queue_was_empty = list_empty(&syncpt->intr.wait_head); | |
237 | ||
238 | if (add_waiter_to_queue(waiter, &syncpt->intr.wait_head)) { | |
239 | /* added at head of list - new threshold value */ | |
ac330f45 | 240 | host1x_hw_intr_set_syncpt_threshold(host, syncpt->id, thresh); |
7ede0b0b TB |
241 | |
242 | /* added as first waiter - enable interrupt */ | |
243 | if (queue_was_empty) | |
ac330f45 | 244 | host1x_hw_intr_enable_syncpt_intr(host, syncpt->id); |
7ede0b0b TB |
245 | } |
246 | ||
7ede0b0b TB |
247 | if (ref) |
248 | *ref = waiter; | |
5a8d95d2 MP |
249 | |
250 | spin_unlock(&syncpt->intr.lock); | |
251 | ||
7ede0b0b TB |
252 | return 0; |
253 | } | |
254 | ||
ecfb888a MP |
255 | void host1x_intr_put_ref(struct host1x *host, unsigned int id, void *ref, |
256 | bool flush) | |
7ede0b0b TB |
257 | { |
258 | struct host1x_waitlist *waiter = ref; | |
259 | struct host1x_syncpt *syncpt; | |
260 | ||
ecfb888a | 261 | atomic_cmpxchg(&waiter->state, WLS_PENDING, WLS_CANCELLED); |
7ede0b0b TB |
262 | |
263 | syncpt = host->syncpt + id; | |
ecfb888a MP |
264 | |
265 | spin_lock(&syncpt->intr.lock); | |
266 | if (atomic_cmpxchg(&waiter->state, WLS_CANCELLED, WLS_HANDLED) == | |
267 | WLS_CANCELLED) { | |
268 | list_del(&waiter->list); | |
269 | kref_put(&waiter->refcount, waiter_release); | |
270 | } | |
271 | spin_unlock(&syncpt->intr.lock); | |
272 | ||
273 | if (flush) { | |
274 | /* Wait until any concurrently executing handler has finished. */ | |
275 | while (atomic_read(&waiter->state) != WLS_HANDLED) | |
276 | schedule(); | |
277 | } | |
7ede0b0b TB |
278 | |
279 | kref_put(&waiter->refcount, waiter_release); | |
280 | } | |
281 | ||
282 | int host1x_intr_init(struct host1x *host, unsigned int irq_sync) | |
283 | { | |
284 | unsigned int id; | |
285 | u32 nb_pts = host1x_syncpt_nb_pts(host); | |
286 | ||
287 | mutex_init(&host->intr_mutex); | |
288 | host->intr_syncpt_irq = irq_sync; | |
7ede0b0b TB |
289 | |
290 | for (id = 0; id < nb_pts; ++id) { | |
291 | struct host1x_syncpt *syncpt = host->syncpt + id; | |
292 | ||
293 | spin_lock_init(&syncpt->intr.lock); | |
294 | INIT_LIST_HEAD(&syncpt->intr.wait_head); | |
295 | snprintf(syncpt->intr.thresh_irq_name, | |
296 | sizeof(syncpt->intr.thresh_irq_name), | |
5c0d8d38 | 297 | "host1x_sp_%02u", id); |
7ede0b0b TB |
298 | } |
299 | ||
7ede0b0b TB |
300 | return 0; |
301 | } | |
302 | ||
303 | void host1x_intr_deinit(struct host1x *host) | |
304 | { | |
7ede0b0b TB |
305 | } |
306 | ||
307 | void host1x_intr_start(struct host1x *host) | |
308 | { | |
309 | u32 hz = clk_get_rate(host->clk); | |
310 | int err; | |
311 | ||
312 | mutex_lock(&host->intr_mutex); | |
313 | err = host1x_hw_intr_init_host_sync(host, DIV_ROUND_UP(hz, 1000000), | |
314 | syncpt_thresh_work); | |
315 | if (err) { | |
316 | mutex_unlock(&host->intr_mutex); | |
317 | return; | |
318 | } | |
319 | mutex_unlock(&host->intr_mutex); | |
320 | } | |
321 | ||
322 | void host1x_intr_stop(struct host1x *host) | |
323 | { | |
324 | unsigned int id; | |
325 | struct host1x_syncpt *syncpt = host->syncpt; | |
326 | u32 nb_pts = host1x_syncpt_nb_pts(host); | |
327 | ||
328 | mutex_lock(&host->intr_mutex); | |
329 | ||
330 | host1x_hw_intr_disable_all_syncpt_intrs(host); | |
331 | ||
332 | for (id = 0; id < nb_pts; ++id) { | |
333 | struct host1x_waitlist *waiter, *next; | |
334 | ||
335 | list_for_each_entry_safe(waiter, next, | |
336 | &syncpt[id].intr.wait_head, list) { | |
337 | if (atomic_cmpxchg(&waiter->state, | |
338 | WLS_CANCELLED, WLS_HANDLED) == WLS_CANCELLED) { | |
339 | list_del(&waiter->list); | |
340 | kref_put(&waiter->refcount, waiter_release); | |
341 | } | |
342 | } | |
343 | ||
344 | if (!list_empty(&syncpt[id].intr.wait_head)) { | |
345 | /* output diagnostics */ | |
346 | mutex_unlock(&host->intr_mutex); | |
5c0d8d38 | 347 | pr_warn("%s cannot stop syncpt intr id=%u\n", |
7ede0b0b TB |
348 | __func__, id); |
349 | return; | |
350 | } | |
351 | } | |
352 | ||
353 | host1x_hw_intr_free_syncpt_irq(host); | |
354 | ||
355 | mutex_unlock(&host->intr_mutex); | |
356 | } |