Commit | Line | Data |
---|---|---|
caab277b | 1 | // SPDX-License-Identifier: GPL-2.0-only |
7198e6b0 RC |
2 | /* |
3 | * Copyright (C) 2013 Red Hat | |
4 | * Author: Rob Clark <robdclark@gmail.com> | |
7198e6b0 RC |
5 | */ |
6 | ||
cfebe3fd RC |
7 | #include "drm/drm_drv.h" |
8 | ||
7198e6b0 RC |
9 | #include "msm_gpu.h" |
10 | #include "msm_gem.h" | |
871d812a | 11 | #include "msm_mmu.h" |
fde5de6c | 12 | #include "msm_fence.h" |
4241db42 | 13 | #include "msm_gpu_trace.h" |
c2052a4e | 14 | #include "adreno/adreno_gpu.h" |
7198e6b0 | 15 | |
c0fec7f5 | 16 | #include <generated/utsrelease.h> |
18bb8a6c | 17 | #include <linux/string_helpers.h> |
c0fec7f5 | 18 | #include <linux/devcoredump.h> |
1f6cca40 | 19 | #include <linux/reset.h> |
70082a52 | 20 | #include <linux/sched/task.h> |
7198e6b0 RC |
21 | |
22 | /* | |
23 | * Power Management: | |
24 | */ | |
25 | ||
7198e6b0 RC |
26 | static int enable_pwrrail(struct msm_gpu *gpu) |
27 | { | |
28 | struct drm_device *dev = gpu->dev; | |
29 | int ret = 0; | |
30 | ||
31 | if (gpu->gpu_reg) { | |
32 | ret = regulator_enable(gpu->gpu_reg); | |
33 | if (ret) { | |
6a41da17 | 34 | DRM_DEV_ERROR(dev->dev, "failed to enable 'gpu_reg': %d\n", ret); |
7198e6b0 RC |
35 | return ret; |
36 | } | |
37 | } | |
38 | ||
39 | if (gpu->gpu_cx) { | |
40 | ret = regulator_enable(gpu->gpu_cx); | |
41 | if (ret) { | |
6a41da17 | 42 | DRM_DEV_ERROR(dev->dev, "failed to enable 'gpu_cx': %d\n", ret); |
7198e6b0 RC |
43 | return ret; |
44 | } | |
45 | } | |
46 | ||
47 | return 0; | |
48 | } | |
49 | ||
50 | static int disable_pwrrail(struct msm_gpu *gpu) | |
51 | { | |
52 | if (gpu->gpu_cx) | |
53 | regulator_disable(gpu->gpu_cx); | |
54 | if (gpu->gpu_reg) | |
55 | regulator_disable(gpu->gpu_reg); | |
56 | return 0; | |
57 | } | |
58 | ||
59 | static int enable_clk(struct msm_gpu *gpu) | |
60 | { | |
98db803f JC |
61 | if (gpu->core_clk && gpu->fast_rate) |
62 | clk_set_rate(gpu->core_clk, gpu->fast_rate); | |
7198e6b0 | 63 | |
b5f103ab | 64 | /* Set the RBBM timer rate to 19.2Mhz */ |
98db803f JC |
65 | if (gpu->rbbmtimer_clk) |
66 | clk_set_rate(gpu->rbbmtimer_clk, 19200000); | |
b5f103ab | 67 | |
8e54eea5 | 68 | return clk_bulk_prepare_enable(gpu->nr_clocks, gpu->grp_clks); |
7198e6b0 RC |
69 | } |
70 | ||
71 | static int disable_clk(struct msm_gpu *gpu) | |
72 | { | |
8e54eea5 | 73 | clk_bulk_disable_unprepare(gpu->nr_clocks, gpu->grp_clks); |
7198e6b0 | 74 | |
bf5af4ae JC |
75 | /* |
76 | * Set the clock to a deliberately low rate. On older targets the clock | |
77 | * speed had to be non zero to avoid problems. On newer targets this | |
78 | * will be rounded down to zero anyway so it all works out. | |
79 | */ | |
98db803f JC |
80 | if (gpu->core_clk) |
81 | clk_set_rate(gpu->core_clk, 27000000); | |
89d777a5 | 82 | |
98db803f JC |
83 | if (gpu->rbbmtimer_clk) |
84 | clk_set_rate(gpu->rbbmtimer_clk, 0); | |
b5f103ab | 85 | |
7198e6b0 RC |
86 | return 0; |
87 | } | |
88 | ||
89 | static int enable_axi(struct msm_gpu *gpu) | |
90 | { | |
dd29bd41 | 91 | return clk_prepare_enable(gpu->ebi1_clk); |
7198e6b0 RC |
92 | } |
93 | ||
94 | static int disable_axi(struct msm_gpu *gpu) | |
95 | { | |
dd29bd41 | 96 | clk_disable_unprepare(gpu->ebi1_clk); |
7198e6b0 RC |
97 | return 0; |
98 | } | |
99 | ||
100 | int msm_gpu_pm_resume(struct msm_gpu *gpu) | |
101 | { | |
102 | int ret; | |
103 | ||
eeb75474 | 104 | DBG("%s", gpu->name); |
ec1cb6e4 | 105 | trace_msm_gpu_resume(0); |
7198e6b0 RC |
106 | |
107 | ret = enable_pwrrail(gpu); | |
108 | if (ret) | |
109 | return ret; | |
110 | ||
111 | ret = enable_clk(gpu); | |
112 | if (ret) | |
113 | return ret; | |
114 | ||
115 | ret = enable_axi(gpu); | |
116 | if (ret) | |
117 | return ret; | |
118 | ||
af5b4fff | 119 | msm_devfreq_resume(gpu); |
f91c14ab | 120 | |
eeb75474 RC |
121 | gpu->needs_hw_init = true; |
122 | ||
7198e6b0 RC |
123 | return 0; |
124 | } | |
125 | ||
126 | int msm_gpu_pm_suspend(struct msm_gpu *gpu) | |
127 | { | |
128 | int ret; | |
129 | ||
eeb75474 | 130 | DBG("%s", gpu->name); |
ec1cb6e4 | 131 | trace_msm_gpu_suspend(0); |
7198e6b0 | 132 | |
af5b4fff | 133 | msm_devfreq_suspend(gpu); |
f91c14ab | 134 | |
7198e6b0 RC |
135 | ret = disable_axi(gpu); |
136 | if (ret) | |
137 | return ret; | |
138 | ||
139 | ret = disable_clk(gpu); | |
140 | if (ret) | |
141 | return ret; | |
142 | ||
143 | ret = disable_pwrrail(gpu); | |
144 | if (ret) | |
145 | return ret; | |
146 | ||
3ab1c5cc RC |
147 | gpu->suspend_count++; |
148 | ||
7198e6b0 RC |
149 | return 0; |
150 | } | |
151 | ||
cfebe3fd RC |
152 | void msm_gpu_show_fdinfo(struct msm_gpu *gpu, struct msm_file_private *ctx, |
153 | struct drm_printer *p) | |
154 | { | |
155 | drm_printf(p, "drm-driver:\t%s\n", gpu->dev->driver->name); | |
156 | drm_printf(p, "drm-client-id:\t%u\n", ctx->seqno); | |
157 | drm_printf(p, "drm-engine-gpu:\t%llu ns\n", ctx->elapsed_ns); | |
158 | drm_printf(p, "drm-cycles-gpu:\t%llu\n", ctx->cycles); | |
159 | drm_printf(p, "drm-maxfreq-gpu:\t%u Hz\n", gpu->fast_rate); | |
160 | } | |
161 | ||
eeb75474 | 162 | int msm_gpu_hw_init(struct msm_gpu *gpu) |
37d77c3a | 163 | { |
eeb75474 | 164 | int ret; |
37d77c3a | 165 | |
c28e2f2b | 166 | WARN_ON(!mutex_is_locked(&gpu->lock)); |
cb1e3818 | 167 | |
eeb75474 RC |
168 | if (!gpu->needs_hw_init) |
169 | return 0; | |
37d77c3a | 170 | |
eeb75474 RC |
171 | disable_irq(gpu->irq); |
172 | ret = gpu->funcs->hw_init(gpu); | |
173 | if (!ret) | |
174 | gpu->needs_hw_init = false; | |
175 | enable_irq(gpu->irq); | |
37d77c3a | 176 | |
eeb75474 | 177 | return ret; |
37d77c3a RC |
178 | } |
179 | ||
c0fec7f5 JC |
180 | #ifdef CONFIG_DEV_COREDUMP |
181 | static ssize_t msm_gpu_devcoredump_read(char *buffer, loff_t offset, | |
182 | size_t count, void *data, size_t datalen) | |
183 | { | |
184 | struct msm_gpu *gpu = data; | |
185 | struct drm_print_iterator iter; | |
186 | struct drm_printer p; | |
187 | struct msm_gpu_state *state; | |
188 | ||
189 | state = msm_gpu_crashstate_get(gpu); | |
190 | if (!state) | |
191 | return 0; | |
192 | ||
193 | iter.data = buffer; | |
194 | iter.offset = 0; | |
195 | iter.start = offset; | |
196 | iter.remain = count; | |
197 | ||
198 | p = drm_coredump_printer(&iter); | |
199 | ||
200 | drm_printf(&p, "---\n"); | |
201 | drm_printf(&p, "kernel: " UTS_RELEASE "\n"); | |
202 | drm_printf(&p, "module: " KBUILD_MODNAME "\n"); | |
3530a17f AB |
203 | drm_printf(&p, "time: %lld.%09ld\n", |
204 | state->time.tv_sec, state->time.tv_nsec); | |
c0fec7f5 JC |
205 | if (state->comm) |
206 | drm_printf(&p, "comm: %s\n", state->comm); | |
207 | if (state->cmd) | |
208 | drm_printf(&p, "cmdline: %s\n", state->cmd); | |
209 | ||
210 | gpu->funcs->show(gpu, state, &p); | |
211 | ||
212 | msm_gpu_crashstate_put(gpu); | |
213 | ||
214 | return count - iter.remain; | |
215 | } | |
216 | ||
217 | static void msm_gpu_devcoredump_free(void *data) | |
218 | { | |
219 | struct msm_gpu *gpu = data; | |
220 | ||
221 | msm_gpu_crashstate_put(gpu); | |
222 | } | |
223 | ||
cdb95931 | 224 | static void msm_gpu_crashstate_get_bo(struct msm_gpu_state *state, |
cc66a42c | 225 | struct msm_gem_object *obj, u64 iova, bool full) |
cdb95931 JC |
226 | { |
227 | struct msm_gpu_state_bo *state_bo = &state->bos[state->nr_bos]; | |
228 | ||
229 | /* Don't record write only objects */ | |
cdb95931 JC |
230 | state_bo->size = obj->base.size; |
231 | state_bo->iova = iova; | |
232 | ||
18514c38 RC |
233 | BUILD_BUG_ON(sizeof(state_bo->name) != sizeof(obj->name)); |
234 | ||
235 | memcpy(state_bo->name, obj->name, sizeof(state_bo->name)); | |
236 | ||
cc66a42c | 237 | if (full) { |
cdb95931 JC |
238 | void *ptr; |
239 | ||
240 | state_bo->data = kvmalloc(obj->base.size, GFP_KERNEL); | |
241 | if (!state_bo->data) | |
896a248a | 242 | goto out; |
cdb95931 | 243 | |
6c0e3ea2 | 244 | msm_gem_lock(&obj->base); |
cdb95931 | 245 | ptr = msm_gem_get_vaddr_active(&obj->base); |
6c0e3ea2 | 246 | msm_gem_unlock(&obj->base); |
cdb95931 JC |
247 | if (IS_ERR(ptr)) { |
248 | kvfree(state_bo->data); | |
896a248a JC |
249 | state_bo->data = NULL; |
250 | goto out; | |
cdb95931 JC |
251 | } |
252 | ||
253 | memcpy(state_bo->data, ptr, obj->base.size); | |
254 | msm_gem_put_vaddr(&obj->base); | |
255 | } | |
896a248a | 256 | out: |
cdb95931 JC |
257 | state->nr_bos++; |
258 | } | |
259 | ||
260 | static void msm_gpu_crashstate_capture(struct msm_gpu *gpu, | |
261 | struct msm_gem_submit *submit, char *comm, char *cmd) | |
c0fec7f5 JC |
262 | { |
263 | struct msm_gpu_state *state; | |
264 | ||
4f3a31a8 SM |
265 | /* Check if the target supports capturing crash state */ |
266 | if (!gpu->funcs->gpu_state_get) | |
267 | return; | |
268 | ||
c0fec7f5 JC |
269 | /* Only save one crash state at a time */ |
270 | if (gpu->crashstate) | |
271 | return; | |
272 | ||
273 | state = gpu->funcs->gpu_state_get(gpu); | |
274 | if (IS_ERR_OR_NULL(state)) | |
275 | return; | |
276 | ||
277 | /* Fill in the additional crash state information */ | |
278 | state->comm = kstrdup(comm, GFP_KERNEL); | |
279 | state->cmd = kstrdup(cmd, GFP_KERNEL); | |
e25e92e0 | 280 | state->fault_info = gpu->fault_info; |
c0fec7f5 | 281 | |
cdb95931 | 282 | if (submit) { |
cc66a42c RC |
283 | int i; |
284 | ||
285 | state->bos = kcalloc(submit->nr_bos, | |
cdb95931 JC |
286 | sizeof(struct msm_gpu_state_bo), GFP_KERNEL); |
287 | ||
b220c154 | 288 | for (i = 0; state->bos && i < submit->nr_bos; i++) { |
cc66a42c RC |
289 | msm_gpu_crashstate_get_bo(state, submit->bos[i].obj, |
290 | submit->bos[i].iova, | |
291 | should_dump(submit, i)); | |
896a248a | 292 | } |
cdb95931 JC |
293 | } |
294 | ||
c0fec7f5 JC |
295 | /* Set the active crash state to be dumped on failure */ |
296 | gpu->crashstate = state; | |
297 | ||
298 | /* FIXME: Release the crashstate if this errors out? */ | |
299 | dev_coredumpm(gpu->dev->dev, THIS_MODULE, gpu, 0, GFP_KERNEL, | |
300 | msm_gpu_devcoredump_read, msm_gpu_devcoredump_free); | |
301 | } | |
302 | #else | |
6969019f AR |
303 | static void msm_gpu_crashstate_capture(struct msm_gpu *gpu, |
304 | struct msm_gem_submit *submit, char *comm, char *cmd) | |
c0fec7f5 JC |
305 | { |
306 | } | |
307 | #endif | |
308 | ||
bd6f82d8 RC |
309 | /* |
310 | * Hangcheck detection for locked gpu: | |
311 | */ | |
312 | ||
18bb8a6c RC |
313 | static struct msm_gem_submit * |
314 | find_submit(struct msm_ringbuffer *ring, uint32_t fence) | |
315 | { | |
316 | struct msm_gem_submit *submit; | |
298287f6 | 317 | unsigned long flags; |
18bb8a6c | 318 | |
298287f6 | 319 | spin_lock_irqsave(&ring->submit_lock, flags); |
77d20529 RC |
320 | list_for_each_entry(submit, &ring->submits, node) { |
321 | if (submit->seqno == fence) { | |
298287f6 | 322 | spin_unlock_irqrestore(&ring->submit_lock, flags); |
18bb8a6c | 323 | return submit; |
77d20529 RC |
324 | } |
325 | } | |
298287f6 | 326 | spin_unlock_irqrestore(&ring->submit_lock, flags); |
18bb8a6c RC |
327 | |
328 | return NULL; | |
329 | } | |
330 | ||
b6295f9a | 331 | static void retire_submits(struct msm_gpu *gpu); |
1a370be9 | 332 | |
39ba0c0d RC |
333 | static void get_comm_cmdline(struct msm_gem_submit *submit, char **comm, char **cmd) |
334 | { | |
d4726d77 | 335 | struct msm_file_private *ctx = submit->queue->ctx; |
39ba0c0d RC |
336 | struct task_struct *task; |
337 | ||
a66f1efc RC |
338 | WARN_ON(!mutex_is_locked(&submit->gpu->lock)); |
339 | ||
d4726d77 RC |
340 | /* Note that kstrdup will return NULL if argument is NULL: */ |
341 | *comm = kstrdup(ctx->comm, GFP_KERNEL); | |
342 | *cmd = kstrdup(ctx->cmdline, GFP_KERNEL); | |
343 | ||
39ba0c0d RC |
344 | task = get_pid_task(submit->pid, PIDTYPE_PID); |
345 | if (!task) | |
346 | return; | |
347 | ||
d4726d77 RC |
348 | if (!*comm) |
349 | *comm = kstrdup(task->comm, GFP_KERNEL); | |
350 | ||
351 | if (!*cmd) | |
352 | *cmd = kstrdup_quotable_cmdline(task, GFP_KERNEL); | |
39ba0c0d RC |
353 | |
354 | put_task_struct(task); | |
355 | } | |
356 | ||
7e688294 | 357 | static void recover_worker(struct kthread_work *work) |
bd6f82d8 RC |
358 | { |
359 | struct msm_gpu *gpu = container_of(work, struct msm_gpu, recover_work); | |
360 | struct drm_device *dev = gpu->dev; | |
96169f4e | 361 | struct msm_drm_private *priv = dev->dev_private; |
4816b626 | 362 | struct msm_gem_submit *submit; |
f97decac | 363 | struct msm_ringbuffer *cur_ring = gpu->funcs->active_ring(gpu); |
65a3c274 | 364 | char *comm = NULL, *cmd = NULL; |
f97decac JC |
365 | int i; |
366 | ||
c28e2f2b | 367 | mutex_lock(&gpu->lock); |
1a370be9 | 368 | |
6a41da17 | 369 | DRM_DEV_ERROR(dev->dev, "%s: hangcheck recover!\n", gpu->name); |
f97decac | 370 | |
96169f4e | 371 | submit = find_submit(cur_ring, cur_ring->memptrs->fence + 1); |
18bb8a6c | 372 | if (submit) { |
b0fb6604 | 373 | /* Increment the fault counts */ |
b0fb6604 | 374 | submit->queue->faults++; |
36a1d1bd LW |
375 | if (submit->aspace) |
376 | submit->aspace->faults++; | |
48dc4241 | 377 | |
39ba0c0d | 378 | get_comm_cmdline(submit, &comm, &cmd); |
18bb8a6c | 379 | |
65a3c274 | 380 | if (comm && cmd) { |
6a41da17 | 381 | DRM_DEV_ERROR(dev->dev, "%s: offending task: %s (%s)\n", |
65a3c274 | 382 | gpu->name, comm, cmd); |
96169f4e RC |
383 | |
384 | msm_rd_dump_submit(priv->hangrd, submit, | |
65a3c274 | 385 | "offending task: %s (%s)", comm, cmd); |
6c0e3ea2 | 386 | } else { |
96169f4e | 387 | msm_rd_dump_submit(priv->hangrd, submit, NULL); |
6c0e3ea2 | 388 | } |
bc211258 RC |
389 | } else { |
390 | /* | |
391 | * We couldn't attribute this fault to any particular context, | |
392 | * so increment the global fault count instead. | |
393 | */ | |
394 | gpu->global_faults++; | |
96169f4e RC |
395 | } |
396 | ||
c0fec7f5 JC |
397 | /* Record the crash state */ |
398 | pm_runtime_get_sync(&gpu->pdev->dev); | |
cdb95931 | 399 | msm_gpu_crashstate_capture(gpu, submit, comm, cmd); |
c0fec7f5 | 400 | |
65a3c274 JC |
401 | kfree(cmd); |
402 | kfree(comm); | |
96169f4e RC |
403 | |
404 | /* | |
405 | * Update all the rings with the latest and greatest fence.. this | |
406 | * needs to happen after msm_rd_dump_submit() to ensure that the | |
407 | * bo's referenced by the offending submit are still around. | |
408 | */ | |
7ddae82e | 409 | for (i = 0; i < gpu->nr_rings; i++) { |
96169f4e RC |
410 | struct msm_ringbuffer *ring = gpu->rb[i]; |
411 | ||
412 | uint32_t fence = ring->memptrs->fence; | |
18bb8a6c | 413 | |
96169f4e RC |
414 | /* |
415 | * For the current (faulting?) ring/submit advance the fence by | |
416 | * one more to clear the faulting submit | |
417 | */ | |
418 | if (ring == cur_ring) | |
c8af219d | 419 | ring->memptrs->fence = ++fence; |
96169f4e | 420 | |
3c7a5221 | 421 | msm_update_fence(ring->fctx, fence); |
4816b626 RC |
422 | } |
423 | ||
424 | if (msm_gpu_active(gpu)) { | |
1a370be9 | 425 | /* retire completed submits, plus the one that hung: */ |
b6295f9a | 426 | retire_submits(gpu); |
1a370be9 | 427 | |
37d77c3a | 428 | gpu->funcs->recover(gpu); |
1a370be9 | 429 | |
f97decac JC |
430 | /* |
431 | * Replay all remaining submits starting with highest priority | |
432 | * ring | |
433 | */ | |
b1fc2839 | 434 | for (i = 0; i < gpu->nr_rings; i++) { |
f97decac | 435 | struct msm_ringbuffer *ring = gpu->rb[i]; |
298287f6 | 436 | unsigned long flags; |
f97decac | 437 | |
298287f6 | 438 | spin_lock_irqsave(&ring->submit_lock, flags); |
f97decac | 439 | list_for_each_entry(submit, &ring->submits, node) |
15eb9ad0 | 440 | gpu->funcs->submit(gpu, submit); |
298287f6 | 441 | spin_unlock_irqrestore(&ring->submit_lock, flags); |
1a370be9 | 442 | } |
37d77c3a | 443 | } |
4816b626 | 444 | |
f350bfb9 | 445 | pm_runtime_put(&gpu->pdev->dev); |
06097e37 | 446 | |
c28e2f2b | 447 | mutex_unlock(&gpu->lock); |
bd6f82d8 RC |
448 | |
449 | msm_gpu_retire(gpu); | |
450 | } | |
451 | ||
e25e92e0 RC |
452 | static void fault_worker(struct kthread_work *work) |
453 | { | |
454 | struct msm_gpu *gpu = container_of(work, struct msm_gpu, fault_work); | |
e25e92e0 RC |
455 | struct msm_gem_submit *submit; |
456 | struct msm_ringbuffer *cur_ring = gpu->funcs->active_ring(gpu); | |
457 | char *comm = NULL, *cmd = NULL; | |
458 | ||
c28e2f2b | 459 | mutex_lock(&gpu->lock); |
e25e92e0 RC |
460 | |
461 | submit = find_submit(cur_ring, cur_ring->memptrs->fence + 1); | |
462 | if (submit && submit->fault_dumped) | |
463 | goto resume_smmu; | |
464 | ||
465 | if (submit) { | |
39ba0c0d | 466 | get_comm_cmdline(submit, &comm, &cmd); |
e25e92e0 RC |
467 | |
468 | /* | |
469 | * When we get GPU iova faults, we can get 1000s of them, | |
470 | * but we really only want to log the first one. | |
471 | */ | |
472 | submit->fault_dumped = true; | |
473 | } | |
474 | ||
475 | /* Record the crash state */ | |
476 | pm_runtime_get_sync(&gpu->pdev->dev); | |
477 | msm_gpu_crashstate_capture(gpu, submit, comm, cmd); | |
478 | pm_runtime_put_sync(&gpu->pdev->dev); | |
479 | ||
480 | kfree(cmd); | |
481 | kfree(comm); | |
482 | ||
483 | resume_smmu: | |
484 | memset(&gpu->fault_info, 0, sizeof(gpu->fault_info)); | |
485 | gpu->aspace->mmu->funcs->resume_translation(gpu->aspace->mmu); | |
486 | ||
c28e2f2b | 487 | mutex_unlock(&gpu->lock); |
e25e92e0 RC |
488 | } |
489 | ||
bd6f82d8 RC |
490 | static void hangcheck_timer_reset(struct msm_gpu *gpu) |
491 | { | |
1d2fa58e | 492 | struct msm_drm_private *priv = gpu->dev->dev_private; |
bd6f82d8 | 493 | mod_timer(&gpu->hangcheck_timer, |
1d2fa58e | 494 | round_jiffies_up(jiffies + msecs_to_jiffies(priv->hangcheck_period))); |
bd6f82d8 RC |
495 | } |
496 | ||
d73b1d02 RC |
497 | static bool made_progress(struct msm_gpu *gpu, struct msm_ringbuffer *ring) |
498 | { | |
499 | if (ring->hangcheck_progress_retries >= DRM_MSM_HANGCHECK_PROGRESS_RETRIES) | |
500 | return false; | |
501 | ||
502 | if (!gpu->funcs->progress) | |
503 | return false; | |
504 | ||
505 | if (!gpu->funcs->progress(gpu, ring)) | |
506 | return false; | |
507 | ||
508 | ring->hangcheck_progress_retries++; | |
509 | return true; | |
510 | } | |
511 | ||
e99e88a9 | 512 | static void hangcheck_handler(struct timer_list *t) |
bd6f82d8 | 513 | { |
e99e88a9 | 514 | struct msm_gpu *gpu = from_timer(gpu, t, hangcheck_timer); |
6b8819c8 | 515 | struct drm_device *dev = gpu->dev; |
f97decac JC |
516 | struct msm_ringbuffer *ring = gpu->funcs->active_ring(gpu); |
517 | uint32_t fence = ring->memptrs->fence; | |
bd6f82d8 | 518 | |
f97decac | 519 | if (fence != ring->hangcheck_fence) { |
bd6f82d8 | 520 | /* some progress has been made.. ya! */ |
f97decac | 521 | ring->hangcheck_fence = fence; |
d73b1d02 RC |
522 | ring->hangcheck_progress_retries = 0; |
523 | } else if (fence_before(fence, ring->fctx->last_fence) && | |
524 | !made_progress(gpu, ring)) { | |
bd6f82d8 | 525 | /* no progress and not done.. hung! */ |
f97decac | 526 | ring->hangcheck_fence = fence; |
d73b1d02 | 527 | ring->hangcheck_progress_retries = 0; |
6a41da17 | 528 | DRM_DEV_ERROR(dev->dev, "%s: hangcheck detected gpu lockup rb %d!\n", |
f97decac | 529 | gpu->name, ring->id); |
6a41da17 | 530 | DRM_DEV_ERROR(dev->dev, "%s: completed fence: %u\n", |
26791c48 | 531 | gpu->name, fence); |
6a41da17 | 532 | DRM_DEV_ERROR(dev->dev, "%s: submitted fence: %u\n", |
f9d5355f | 533 | gpu->name, ring->fctx->last_fence); |
f97decac | 534 | |
7e688294 | 535 | kthread_queue_work(gpu->worker, &gpu->recover_work); |
bd6f82d8 RC |
536 | } |
537 | ||
538 | /* if still more pending work, reset the hangcheck timer: */ | |
f9d5355f | 539 | if (fence_after(ring->fctx->last_fence, ring->hangcheck_fence)) |
bd6f82d8 | 540 | hangcheck_timer_reset(gpu); |
6b8819c8 RC |
541 | |
542 | /* workaround for missing irq: */ | |
298287f6 | 543 | msm_gpu_retire(gpu); |
bd6f82d8 RC |
544 | } |
545 | ||
70c70f09 RC |
546 | /* |
547 | * Performance Counters: | |
548 | */ | |
549 | ||
550 | /* called under perf_lock */ | |
551 | static int update_hw_cntrs(struct msm_gpu *gpu, uint32_t ncntrs, uint32_t *cntrs) | |
552 | { | |
553 | uint32_t current_cntrs[ARRAY_SIZE(gpu->last_cntrs)]; | |
554 | int i, n = min(ncntrs, gpu->num_perfcntrs); | |
555 | ||
556 | /* read current values: */ | |
557 | for (i = 0; i < gpu->num_perfcntrs; i++) | |
558 | current_cntrs[i] = gpu_read(gpu, gpu->perfcntrs[i].sample_reg); | |
559 | ||
560 | /* update cntrs: */ | |
561 | for (i = 0; i < n; i++) | |
562 | cntrs[i] = current_cntrs[i] - gpu->last_cntrs[i]; | |
563 | ||
564 | /* save current values: */ | |
565 | for (i = 0; i < gpu->num_perfcntrs; i++) | |
566 | gpu->last_cntrs[i] = current_cntrs[i]; | |
567 | ||
568 | return n; | |
569 | } | |
570 | ||
571 | static void update_sw_cntrs(struct msm_gpu *gpu) | |
572 | { | |
573 | ktime_t time; | |
574 | uint32_t elapsed; | |
575 | unsigned long flags; | |
576 | ||
577 | spin_lock_irqsave(&gpu->perf_lock, flags); | |
578 | if (!gpu->perfcntr_active) | |
579 | goto out; | |
580 | ||
581 | time = ktime_get(); | |
582 | elapsed = ktime_to_us(ktime_sub(time, gpu->last_sample.time)); | |
583 | ||
584 | gpu->totaltime += elapsed; | |
585 | if (gpu->last_sample.active) | |
586 | gpu->activetime += elapsed; | |
587 | ||
588 | gpu->last_sample.active = msm_gpu_active(gpu); | |
589 | gpu->last_sample.time = time; | |
590 | ||
591 | out: | |
592 | spin_unlock_irqrestore(&gpu->perf_lock, flags); | |
593 | } | |
594 | ||
595 | void msm_gpu_perfcntr_start(struct msm_gpu *gpu) | |
596 | { | |
597 | unsigned long flags; | |
598 | ||
eeb75474 RC |
599 | pm_runtime_get_sync(&gpu->pdev->dev); |
600 | ||
70c70f09 RC |
601 | spin_lock_irqsave(&gpu->perf_lock, flags); |
602 | /* we could dynamically enable/disable perfcntr registers too.. */ | |
603 | gpu->last_sample.active = msm_gpu_active(gpu); | |
604 | gpu->last_sample.time = ktime_get(); | |
605 | gpu->activetime = gpu->totaltime = 0; | |
606 | gpu->perfcntr_active = true; | |
607 | update_hw_cntrs(gpu, 0, NULL); | |
608 | spin_unlock_irqrestore(&gpu->perf_lock, flags); | |
609 | } | |
610 | ||
611 | void msm_gpu_perfcntr_stop(struct msm_gpu *gpu) | |
612 | { | |
613 | gpu->perfcntr_active = false; | |
eeb75474 | 614 | pm_runtime_put_sync(&gpu->pdev->dev); |
70c70f09 RC |
615 | } |
616 | ||
617 | /* returns -errno or # of cntrs sampled */ | |
618 | int msm_gpu_perfcntr_sample(struct msm_gpu *gpu, uint32_t *activetime, | |
619 | uint32_t *totaltime, uint32_t ncntrs, uint32_t *cntrs) | |
620 | { | |
621 | unsigned long flags; | |
622 | int ret; | |
623 | ||
624 | spin_lock_irqsave(&gpu->perf_lock, flags); | |
625 | ||
626 | if (!gpu->perfcntr_active) { | |
627 | ret = -EINVAL; | |
628 | goto out; | |
629 | } | |
630 | ||
631 | *activetime = gpu->activetime; | |
632 | *totaltime = gpu->totaltime; | |
633 | ||
634 | gpu->activetime = gpu->totaltime = 0; | |
635 | ||
636 | ret = update_hw_cntrs(gpu, ncntrs, cntrs); | |
637 | ||
638 | out: | |
639 | spin_unlock_irqrestore(&gpu->perf_lock, flags); | |
640 | ||
641 | return ret; | |
642 | } | |
643 | ||
7198e6b0 RC |
644 | /* |
645 | * Cmdstream submission/retirement: | |
646 | */ | |
647 | ||
4241db42 JC |
648 | static void retire_submit(struct msm_gpu *gpu, struct msm_ringbuffer *ring, |
649 | struct msm_gem_submit *submit) | |
7d12a279 | 650 | { |
4241db42 JC |
651 | int index = submit->seqno % MSM_GPU_SUBMIT_STATS_COUNT; |
652 | volatile struct msm_gpu_submit_stats *stats; | |
cfebe3fd | 653 | u64 elapsed, clock = 0, cycles; |
298287f6 | 654 | unsigned long flags; |
7d12a279 | 655 | |
4241db42 JC |
656 | stats = &ring->memptrs->stats[index]; |
657 | /* Convert 19.2Mhz alwayson ticks to nanoseconds for elapsed time */ | |
658 | elapsed = (stats->alwayson_end - stats->alwayson_start) * 10000; | |
659 | do_div(elapsed, 192); | |
660 | ||
cfebe3fd RC |
661 | cycles = stats->cpcycles_end - stats->cpcycles_start; |
662 | ||
4241db42 JC |
663 | /* Calculate the clock frequency from the number of CP cycles */ |
664 | if (elapsed) { | |
cfebe3fd | 665 | clock = cycles * 1000; |
4241db42 JC |
666 | do_div(clock, elapsed); |
667 | } | |
668 | ||
cfebe3fd RC |
669 | submit->queue->ctx->elapsed_ns += elapsed; |
670 | submit->queue->ctx->cycles += cycles; | |
671 | ||
4241db42 JC |
672 | trace_msm_gpu_submit_retired(submit, elapsed, clock, |
673 | stats->alwayson_start, stats->alwayson_end); | |
674 | ||
be40596b | 675 | msm_submit_retire(submit); |
7d12a279 | 676 | |
eeb75474 | 677 | pm_runtime_mark_last_busy(&gpu->pdev->dev); |
964d2f97 | 678 | |
298287f6 | 679 | spin_lock_irqsave(&ring->submit_lock, flags); |
964d2f97 | 680 | list_del(&submit->node); |
298287f6 | 681 | spin_unlock_irqrestore(&ring->submit_lock, flags); |
964d2f97 | 682 | |
9bc95570 RC |
683 | /* Update devfreq on transition from active->idle: */ |
684 | mutex_lock(&gpu->active_lock); | |
685 | gpu->active_submits--; | |
686 | WARN_ON(gpu->active_submits < 0); | |
5b26f37d | 687 | if (!gpu->active_submits) { |
9bc95570 | 688 | msm_devfreq_idle(gpu); |
5b26f37d AO |
689 | pm_runtime_put_autosuspend(&gpu->pdev->dev); |
690 | } | |
9bc95570 | 691 | |
5b26f37d | 692 | mutex_unlock(&gpu->active_lock); |
49e47761 | 693 | |
964d2f97 | 694 | msm_gem_submit_put(submit); |
7d12a279 RC |
695 | } |
696 | ||
b6295f9a | 697 | static void retire_submits(struct msm_gpu *gpu) |
1a370be9 | 698 | { |
f97decac | 699 | int i; |
1a370be9 | 700 | |
f97decac | 701 | /* Retire the commits starting with highest priority */ |
b1fc2839 | 702 | for (i = 0; i < gpu->nr_rings; i++) { |
f97decac | 703 | struct msm_ringbuffer *ring = gpu->rb[i]; |
1a370be9 | 704 | |
77d20529 RC |
705 | while (true) { |
706 | struct msm_gem_submit *submit = NULL; | |
298287f6 | 707 | unsigned long flags; |
77d20529 | 708 | |
298287f6 | 709 | spin_lock_irqsave(&ring->submit_lock, flags); |
77d20529 RC |
710 | submit = list_first_entry_or_null(&ring->submits, |
711 | struct msm_gem_submit, node); | |
298287f6 | 712 | spin_unlock_irqrestore(&ring->submit_lock, flags); |
77d20529 RC |
713 | |
714 | /* | |
715 | * If no submit, we are done. If submit->fence hasn't | |
716 | * been signalled, then later submits are not signalled | |
717 | * either, so we are also done. | |
718 | */ | |
1d8a5ca4 | 719 | if (submit && dma_fence_is_signaled(submit->hw_fence)) { |
4241db42 | 720 | retire_submit(gpu, ring, submit); |
77d20529 RC |
721 | } else { |
722 | break; | |
723 | } | |
1a370be9 RC |
724 | } |
725 | } | |
167a668a RC |
726 | |
727 | wake_up_all(&gpu->retire_event); | |
1a370be9 RC |
728 | } |
729 | ||
7e688294 | 730 | static void retire_worker(struct kthread_work *work) |
7198e6b0 RC |
731 | { |
732 | struct msm_gpu *gpu = container_of(work, struct msm_gpu, retire_work); | |
edd4fc63 | 733 | |
b6295f9a | 734 | retire_submits(gpu); |
7198e6b0 RC |
735 | } |
736 | ||
737 | /* call from irq handler to schedule work to retire bo's */ | |
738 | void msm_gpu_retire(struct msm_gpu *gpu) | |
739 | { | |
298287f6 RC |
740 | int i; |
741 | ||
742 | for (i = 0; i < gpu->nr_rings; i++) | |
3c7a5221 | 743 | msm_update_fence(gpu->rb[i]->fctx, gpu->rb[i]->memptrs->fence); |
298287f6 | 744 | |
7e688294 | 745 | kthread_queue_work(gpu->worker, &gpu->retire_work); |
70c70f09 | 746 | update_sw_cntrs(gpu); |
7198e6b0 RC |
747 | } |
748 | ||
749 | /* add bo's to gpu's ring, and kick gpu: */ | |
15eb9ad0 | 750 | void msm_gpu_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit) |
7198e6b0 RC |
751 | { |
752 | struct drm_device *dev = gpu->dev; | |
753 | struct msm_drm_private *priv = dev->dev_private; | |
f97decac | 754 | struct msm_ringbuffer *ring = submit->ring; |
298287f6 | 755 | unsigned long flags; |
7198e6b0 | 756 | |
c28e2f2b | 757 | WARN_ON(!mutex_is_locked(&gpu->lock)); |
1a370be9 | 758 | |
eeb75474 RC |
759 | pm_runtime_get_sync(&gpu->pdev->dev); |
760 | ||
761 | msm_gpu_hw_init(gpu); | |
37d77c3a | 762 | |
f9d5355f | 763 | submit->seqno = submit->hw_fence->seqno; |
f97decac | 764 | |
998b9a58 | 765 | msm_rd_dump_submit(priv->rd, submit, NULL); |
a7d3c950 | 766 | |
70c70f09 RC |
767 | update_sw_cntrs(gpu); |
768 | ||
964d2f97 RC |
769 | /* |
770 | * ring->submits holds a ref to the submit, to deal with the case | |
771 | * that a submit completes before msm_ioctl_gem_submit() returns. | |
772 | */ | |
773 | msm_gem_submit_get(submit); | |
774 | ||
298287f6 | 775 | spin_lock_irqsave(&ring->submit_lock, flags); |
964d2f97 | 776 | list_add_tail(&submit->node, &ring->submits); |
298287f6 | 777 | spin_unlock_irqrestore(&ring->submit_lock, flags); |
964d2f97 | 778 | |
9bc95570 RC |
779 | /* Update devfreq on transition from idle->active: */ |
780 | mutex_lock(&gpu->active_lock); | |
5b26f37d AO |
781 | if (!gpu->active_submits) { |
782 | pm_runtime_get(&gpu->pdev->dev); | |
9bc95570 | 783 | msm_devfreq_active(gpu); |
5b26f37d | 784 | } |
9bc95570 RC |
785 | gpu->active_submits++; |
786 | mutex_unlock(&gpu->active_lock); | |
787 | ||
15eb9ad0 | 788 | gpu->funcs->submit(gpu, submit); |
1d054c9b | 789 | gpu->cur_ctx_seqno = submit->queue->ctx->seqno; |
1a370be9 | 790 | |
5b26f37d | 791 | pm_runtime_put(&gpu->pdev->dev); |
bd6f82d8 | 792 | hangcheck_timer_reset(gpu); |
7198e6b0 RC |
793 | } |
794 | ||
795 | /* | |
796 | * Init/Cleanup: | |
797 | */ | |
798 | ||
799 | static irqreturn_t irq_handler(int irq, void *data) | |
800 | { | |
801 | struct msm_gpu *gpu = data; | |
802 | return gpu->funcs->irq(gpu); | |
803 | } | |
804 | ||
98db803f JC |
805 | static int get_clocks(struct platform_device *pdev, struct msm_gpu *gpu) |
806 | { | |
8e3e791d | 807 | int ret = devm_clk_bulk_get_all(&pdev->dev, &gpu->grp_clks); |
98db803f | 808 | |
8e54eea5 | 809 | if (ret < 1) { |
98db803f | 810 | gpu->nr_clocks = 0; |
8e54eea5 | 811 | return ret; |
9d20a0e6 | 812 | } |
98db803f | 813 | |
8e54eea5 | 814 | gpu->nr_clocks = ret; |
98db803f | 815 | |
8e54eea5 JC |
816 | gpu->core_clk = msm_clk_bulk_get_clock(gpu->grp_clks, |
817 | gpu->nr_clocks, "core"); | |
98db803f | 818 | |
8e54eea5 JC |
819 | gpu->rbbmtimer_clk = msm_clk_bulk_get_clock(gpu->grp_clks, |
820 | gpu->nr_clocks, "rbbmtimer"); | |
98db803f JC |
821 | |
822 | return 0; | |
823 | } | |
7198e6b0 | 824 | |
933415e2 JC |
825 | /* Return a new address space for a msm_drm_private instance */ |
826 | struct msm_gem_address_space * | |
25faf2f2 | 827 | msm_gpu_create_private_address_space(struct msm_gpu *gpu, struct task_struct *task) |
933415e2 JC |
828 | { |
829 | struct msm_gem_address_space *aspace = NULL; | |
933415e2 JC |
830 | if (!gpu) |
831 | return NULL; | |
832 | ||
833 | /* | |
834 | * If the target doesn't support private address spaces then return | |
835 | * the global one | |
836 | */ | |
25faf2f2 | 837 | if (gpu->funcs->create_private_address_space) { |
933415e2 | 838 | aspace = gpu->funcs->create_private_address_space(gpu); |
25faf2f2 RC |
839 | if (!IS_ERR(aspace)) |
840 | aspace->pid = get_pid(task_pid(task)); | |
841 | } | |
933415e2 JC |
842 | |
843 | if (IS_ERR_OR_NULL(aspace)) | |
844 | aspace = msm_gem_address_space_get(gpu->aspace); | |
845 | ||
846 | return aspace; | |
847 | } | |
848 | ||
7198e6b0 RC |
849 | int msm_gpu_init(struct drm_device *drm, struct platform_device *pdev, |
850 | struct msm_gpu *gpu, const struct msm_gpu_funcs *funcs, | |
5770fc7a | 851 | const char *name, struct msm_gpu_config *config) |
7198e6b0 | 852 | { |
d73b1d02 | 853 | struct msm_drm_private *priv = drm->dev_private; |
f97decac JC |
854 | int i, ret, nr_rings = config->nr_rings; |
855 | void *memptrs; | |
856 | uint64_t memptrs_iova; | |
7198e6b0 | 857 | |
70c70f09 RC |
858 | if (WARN_ON(gpu->num_perfcntrs > ARRAY_SIZE(gpu->last_cntrs))) |
859 | gpu->num_perfcntrs = ARRAY_SIZE(gpu->last_cntrs); | |
860 | ||
7198e6b0 RC |
861 | gpu->dev = drm; |
862 | gpu->funcs = funcs; | |
863 | gpu->name = name; | |
864 | ||
0737ab95 | 865 | gpu->worker = kthread_create_worker(0, "gpu-worker"); |
7e688294 RC |
866 | if (IS_ERR(gpu->worker)) { |
867 | ret = PTR_ERR(gpu->worker); | |
868 | gpu->worker = NULL; | |
869 | goto fail; | |
870 | } | |
bd6f82d8 | 871 | |
7e688294 RC |
872 | sched_set_fifo_low(gpu->worker->task); |
873 | ||
9bc95570 | 874 | mutex_init(&gpu->active_lock); |
c28e2f2b | 875 | mutex_init(&gpu->lock); |
167a668a | 876 | init_waitqueue_head(&gpu->retire_event); |
7e688294 RC |
877 | kthread_init_work(&gpu->retire_work, retire_worker); |
878 | kthread_init_work(&gpu->recover_work, recover_worker); | |
e25e92e0 | 879 | kthread_init_work(&gpu->fault_work, fault_worker); |
1a370be9 | 880 | |
d73b1d02 RC |
881 | priv->hangcheck_period = DRM_MSM_HANGCHECK_DEFAULT_PERIOD; |
882 | ||
883 | /* | |
884 | * If progress detection is supported, halve the hangcheck timer | |
885 | * duration, as it takes two iterations of the hangcheck handler | |
886 | * to detect a hang. | |
887 | */ | |
888 | if (funcs->progress) | |
889 | priv->hangcheck_period /= 2; | |
890 | ||
e99e88a9 | 891 | timer_setup(&gpu->hangcheck_timer, hangcheck_handler, 0); |
7198e6b0 | 892 | |
70c70f09 RC |
893 | spin_lock_init(&gpu->perf_lock); |
894 | ||
7198e6b0 RC |
895 | |
896 | /* Map registers: */ | |
c0e745d7 | 897 | gpu->mmio = msm_ioremap(pdev, config->ioname); |
7198e6b0 RC |
898 | if (IS_ERR(gpu->mmio)) { |
899 | ret = PTR_ERR(gpu->mmio); | |
900 | goto fail; | |
901 | } | |
902 | ||
903 | /* Get Interrupt: */ | |
878411ae | 904 | gpu->irq = platform_get_irq(pdev, 0); |
7198e6b0 RC |
905 | if (gpu->irq < 0) { |
906 | ret = gpu->irq; | |
6a41da17 | 907 | DRM_DEV_ERROR(drm->dev, "failed to get irq: %d\n", ret); |
7198e6b0 RC |
908 | goto fail; |
909 | } | |
910 | ||
911 | ret = devm_request_irq(&pdev->dev, gpu->irq, irq_handler, | |
0737ab95 | 912 | IRQF_TRIGGER_HIGH, "gpu-irq", gpu); |
7198e6b0 | 913 | if (ret) { |
6a41da17 | 914 | DRM_DEV_ERROR(drm->dev, "failed to request IRQ%u: %d\n", gpu->irq, ret); |
7198e6b0 RC |
915 | goto fail; |
916 | } | |
917 | ||
98db803f JC |
918 | ret = get_clocks(pdev, gpu); |
919 | if (ret) | |
920 | goto fail; | |
7198e6b0 | 921 | |
720c3bb8 | 922 | gpu->ebi1_clk = msm_clk_get(pdev, "bus"); |
7198e6b0 RC |
923 | DBG("ebi1_clk: %p", gpu->ebi1_clk); |
924 | if (IS_ERR(gpu->ebi1_clk)) | |
925 | gpu->ebi1_clk = NULL; | |
926 | ||
927 | /* Acquire regulators: */ | |
928 | gpu->gpu_reg = devm_regulator_get(&pdev->dev, "vdd"); | |
929 | DBG("gpu_reg: %p", gpu->gpu_reg); | |
930 | if (IS_ERR(gpu->gpu_reg)) | |
931 | gpu->gpu_reg = NULL; | |
932 | ||
933 | gpu->gpu_cx = devm_regulator_get(&pdev->dev, "vddcx"); | |
934 | DBG("gpu_cx: %p", gpu->gpu_cx); | |
935 | if (IS_ERR(gpu->gpu_cx)) | |
936 | gpu->gpu_cx = NULL; | |
937 | ||
1f6cca40 AO |
938 | gpu->cx_collapse = devm_reset_control_get_optional_exclusive(&pdev->dev, |
939 | "cx_collapse"); | |
940 | ||
1267a4df | 941 | gpu->pdev = pdev; |
9cba4056 | 942 | platform_set_drvdata(pdev, &gpu->adreno_smmu); |
1267a4df | 943 | |
f91c14ab JC |
944 | msm_devfreq_init(gpu); |
945 | ||
ccac7ce3 JC |
946 | |
947 | gpu->aspace = gpu->funcs->create_address_space(gpu, pdev); | |
1267a4df JC |
948 | |
949 | if (gpu->aspace == NULL) | |
6a41da17 | 950 | DRM_DEV_INFO(drm->dev, "%s: no IOMMU, fallback to VRAM carveout!\n", name); |
1267a4df JC |
951 | else if (IS_ERR(gpu->aspace)) { |
952 | ret = PTR_ERR(gpu->aspace); | |
953 | goto fail; | |
7198e6b0 | 954 | } |
a1ad3523 | 955 | |
546ec7b4 JC |
956 | memptrs = msm_gem_kernel_new(drm, |
957 | sizeof(struct msm_rbmemptrs) * nr_rings, | |
8b5de735 | 958 | check_apriv(gpu, MSM_BO_WC), gpu->aspace, &gpu->memptrs_bo, |
f97decac | 959 | &memptrs_iova); |
cd414f3d | 960 | |
f97decac JC |
961 | if (IS_ERR(memptrs)) { |
962 | ret = PTR_ERR(memptrs); | |
6a41da17 | 963 | DRM_DEV_ERROR(drm->dev, "could not allocate memptrs: %d\n", ret); |
cd414f3d JC |
964 | goto fail; |
965 | } | |
966 | ||
0815d774 JC |
967 | msm_gem_object_set_name(gpu->memptrs_bo, "memptrs"); |
968 | ||
f97decac | 969 | if (nr_rings > ARRAY_SIZE(gpu->rb)) { |
39ae0d3e | 970 | DRM_DEV_INFO_ONCE(drm->dev, "Only creating %zu ringbuffers\n", |
f97decac JC |
971 | ARRAY_SIZE(gpu->rb)); |
972 | nr_rings = ARRAY_SIZE(gpu->rb); | |
7198e6b0 RC |
973 | } |
974 | ||
f97decac JC |
975 | /* Create ringbuffer(s): */ |
976 | for (i = 0; i < nr_rings; i++) { | |
977 | gpu->rb[i] = msm_ringbuffer_new(gpu, i, memptrs, memptrs_iova); | |
978 | ||
979 | if (IS_ERR(gpu->rb[i])) { | |
980 | ret = PTR_ERR(gpu->rb[i]); | |
6a41da17 | 981 | DRM_DEV_ERROR(drm->dev, |
f97decac JC |
982 | "could not create ringbuffer %d: %d\n", i, ret); |
983 | goto fail; | |
984 | } | |
985 | ||
986 | memptrs += sizeof(struct msm_rbmemptrs); | |
987 | memptrs_iova += sizeof(struct msm_rbmemptrs); | |
988 | } | |
989 | ||
990 | gpu->nr_rings = nr_rings; | |
991 | ||
90f45c42 RC |
992 | refcount_set(&gpu->sysprof_active, 1); |
993 | ||
7198e6b0 RC |
994 | return 0; |
995 | ||
996 | fail: | |
f97decac JC |
997 | for (i = 0; i < ARRAY_SIZE(gpu->rb); i++) { |
998 | msm_ringbuffer_destroy(gpu->rb[i]); | |
999 | gpu->rb[i] = NULL; | |
1000 | } | |
1001 | ||
030af2b0 | 1002 | msm_gem_kernel_put(gpu->memptrs_bo, gpu->aspace); |
cd414f3d | 1003 | |
1267a4df | 1004 | platform_set_drvdata(pdev, NULL); |
7198e6b0 RC |
1005 | return ret; |
1006 | } | |
1007 | ||
1008 | void msm_gpu_cleanup(struct msm_gpu *gpu) | |
1009 | { | |
f97decac JC |
1010 | int i; |
1011 | ||
7198e6b0 RC |
1012 | DBG("%s", gpu->name); |
1013 | ||
f97decac JC |
1014 | for (i = 0; i < ARRAY_SIZE(gpu->rb); i++) { |
1015 | msm_ringbuffer_destroy(gpu->rb[i]); | |
1016 | gpu->rb[i] = NULL; | |
7198e6b0 | 1017 | } |
cd414f3d | 1018 | |
030af2b0 | 1019 | msm_gem_kernel_put(gpu->memptrs_bo, gpu->aspace); |
cd414f3d JC |
1020 | |
1021 | if (!IS_ERR_OR_NULL(gpu->aspace)) { | |
53bf7f7a | 1022 | gpu->aspace->mmu->funcs->detach(gpu->aspace->mmu); |
1267a4df JC |
1023 | msm_gem_address_space_put(gpu->aspace); |
1024 | } | |
7e688294 RC |
1025 | |
1026 | if (gpu->worker) { | |
1027 | kthread_destroy_worker(gpu->worker); | |
1028 | } | |
ec793cf0 | 1029 | |
af5b4fff | 1030 | msm_devfreq_cleanup(gpu); |
76efc245 AO |
1031 | |
1032 | platform_set_drvdata(gpu->pdev, NULL); | |
7198e6b0 | 1033 | } |