Commit | Line | Data |
---|---|---|
c1b69ed0 CZ |
1 | /* |
2 | * Copyright 2015 Advanced Micro Devices, Inc. | |
3 | * | |
4 | * Permission is hereby granted, free of charge, to any person obtaining a | |
5 | * copy of this software and associated documentation files (the "Software"), | |
6 | * to deal in the Software without restriction, including without limitation | |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
8 | * and/or sell copies of the Software, and to permit persons to whom the | |
9 | * Software is furnished to do so, subject to the following conditions: | |
10 | * | |
11 | * The above copyright notice and this permission notice shall be included in | |
12 | * all copies or substantial portions of the Software. | |
13 | * | |
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
20 | * OTHER DEALINGS IN THE SOFTWARE. | |
21 | * | |
22 | * | |
23 | */ | |
24 | #include <linux/kthread.h> | |
25 | #include <linux/wait.h> | |
26 | #include <linux/sched.h> | |
27 | #include <drm/drmP.h> | |
28 | #include "amdgpu.h" | |
7034decf | 29 | #include "amdgpu_trace.h" |
c1b69ed0 | 30 | |
1b1f42d8 | 31 | static void amdgpu_job_timedout(struct drm_sched_job *s_job) |
0de2479c | 32 | { |
3320b8d2 CK |
33 | struct amdgpu_ring *ring = to_amdgpu_ring(s_job->sched); |
34 | struct amdgpu_job *job = to_amdgpu_job(s_job); | |
0e51a772 | 35 | |
0de2479c | 36 | DRM_ERROR("ring %s timeout, last signaled seq=%u, last emitted seq=%u\n", |
3320b8d2 CK |
37 | job->base.sched->name, atomic_read(&ring->fence_drv.last_seq), |
38 | ring->fence_drv.sync_seq); | |
4fbf87e2 | 39 | |
5f152b5e | 40 | amdgpu_device_gpu_recover(job->adev, job, false); |
0de2479c ML |
41 | } |
42 | ||
50838c8c | 43 | int amdgpu_job_alloc(struct amdgpu_device *adev, unsigned num_ibs, |
c5637837 | 44 | struct amdgpu_job **job, struct amdgpu_vm *vm) |
50838c8c CK |
45 | { |
46 | size_t size = sizeof(struct amdgpu_job); | |
47 | ||
48 | if (num_ibs == 0) | |
49 | return -EINVAL; | |
50 | ||
51 | size += sizeof(struct amdgpu_ib) * num_ibs; | |
52 | ||
53 | *job = kzalloc(size, GFP_KERNEL); | |
54 | if (!*job) | |
55 | return -ENOMEM; | |
56 | ||
57 | (*job)->adev = adev; | |
c5637837 | 58 | (*job)->vm = vm; |
50838c8c CK |
59 | (*job)->ibs = (void *)&(*job)[1]; |
60 | (*job)->num_ibs = num_ibs; | |
50838c8c | 61 | |
e86f9cee | 62 | amdgpu_sync_create(&(*job)->sync); |
df83d1eb | 63 | amdgpu_sync_create(&(*job)->sched_sync); |
c70b78a7 | 64 | (*job)->vram_lost_counter = atomic_read(&adev->vram_lost_counter); |
e86f9cee | 65 | |
50838c8c CK |
66 | return 0; |
67 | } | |
68 | ||
d71518b5 CK |
69 | int amdgpu_job_alloc_with_ib(struct amdgpu_device *adev, unsigned size, |
70 | struct amdgpu_job **job) | |
71 | { | |
72 | int r; | |
73 | ||
c5637837 | 74 | r = amdgpu_job_alloc(adev, 1, job, NULL); |
d71518b5 CK |
75 | if (r) |
76 | return r; | |
77 | ||
78 | r = amdgpu_ib_get(adev, NULL, size, &(*job)->ibs[0]); | |
79 | if (r) | |
80 | kfree(*job); | |
df264f9e CK |
81 | else |
82 | (*job)->vm_pd_addr = adev->gart.table_addr; | |
d71518b5 CK |
83 | |
84 | return r; | |
85 | } | |
86 | ||
a5fb4ec2 | 87 | void amdgpu_job_free_resources(struct amdgpu_job *job) |
50838c8c | 88 | { |
f54d1867 | 89 | struct dma_fence *f; |
1ab0d211 CK |
90 | unsigned i; |
91 | ||
676d8c24 | 92 | /* use sched fence if available */ |
6fc13675 | 93 | f = job->base.s_fence ? &job->base.s_fence->finished : job->fence; |
50838c8c CK |
94 | |
95 | for (i = 0; i < job->num_ibs; ++i) | |
1ab0d211 | 96 | amdgpu_ib_free(job->adev, &job->ibs[i], f); |
d71518b5 CK |
97 | } |
98 | ||
1b1f42d8 | 99 | static void amdgpu_job_free_cb(struct drm_sched_job *s_job) |
b6723c8d | 100 | { |
3320b8d2 CK |
101 | struct amdgpu_ring *ring = to_amdgpu_ring(s_job->sched); |
102 | struct amdgpu_job *job = to_amdgpu_job(s_job); | |
c5f74f78 | 103 | |
3320b8d2 | 104 | amdgpu_ring_priority_put(ring, s_job->s_priority); |
f54d1867 | 105 | dma_fence_put(job->fence); |
a79a5bdc | 106 | amdgpu_sync_free(&job->sync); |
df83d1eb | 107 | amdgpu_sync_free(&job->sched_sync); |
b6723c8d ML |
108 | kfree(job); |
109 | } | |
110 | ||
1e24e31f CK |
111 | void amdgpu_job_free(struct amdgpu_job *job) |
112 | { | |
113 | amdgpu_job_free_resources(job); | |
a79a5bdc | 114 | |
f54d1867 | 115 | dma_fence_put(job->fence); |
a79a5bdc | 116 | amdgpu_sync_free(&job->sync); |
df83d1eb | 117 | amdgpu_sync_free(&job->sched_sync); |
1e24e31f CK |
118 | kfree(job); |
119 | } | |
120 | ||
0e28b10f CK |
121 | int amdgpu_job_submit(struct amdgpu_job *job, struct drm_sched_entity *entity, |
122 | void *owner, struct dma_fence **f) | |
d71518b5 | 123 | { |
3320b8d2 | 124 | struct amdgpu_ring *ring = to_amdgpu_ring(entity->sched); |
e686941a | 125 | int r; |
d71518b5 | 126 | |
e686941a ML |
127 | if (!f) |
128 | return -EINVAL; | |
129 | ||
0e28b10f | 130 | r = drm_sched_job_init(&job->base, entity->sched, entity, owner); |
e686941a ML |
131 | if (r) |
132 | return r; | |
d71518b5 CK |
133 | |
134 | job->owner = owner; | |
f54d1867 | 135 | *f = dma_fence_get(&job->base.s_fence->finished); |
a5fb4ec2 | 136 | amdgpu_job_free_resources(job); |
3320b8d2 | 137 | amdgpu_ring_priority_get(ring, job->base.s_priority); |
1b1f42d8 | 138 | drm_sched_entity_push_job(&job->base, entity); |
d71518b5 CK |
139 | |
140 | return 0; | |
50838c8c CK |
141 | } |
142 | ||
ee913fd9 CK |
143 | int amdgpu_job_submit_direct(struct amdgpu_job *job, struct amdgpu_ring *ring, |
144 | struct dma_fence **fence) | |
145 | { | |
146 | int r; | |
147 | ||
148 | job->base.sched = &ring->sched; | |
149 | r = amdgpu_ib_schedule(ring, job->num_ibs, job->ibs, NULL, fence); | |
150 | job->fence = dma_fence_get(*fence); | |
151 | if (r) | |
152 | return r; | |
153 | ||
154 | amdgpu_job_free(job); | |
155 | return 0; | |
156 | } | |
157 | ||
1b1f42d8 LS |
158 | static struct dma_fence *amdgpu_job_dependency(struct drm_sched_job *sched_job, |
159 | struct drm_sched_entity *s_entity) | |
e61235db | 160 | { |
3320b8d2 | 161 | struct amdgpu_ring *ring = to_amdgpu_ring(s_entity->sched); |
a6db8a33 | 162 | struct amdgpu_job *job = to_amdgpu_job(sched_job); |
c5637837 | 163 | struct amdgpu_vm *vm = job->vm; |
cebb52b7 | 164 | bool explicit = false; |
df83d1eb | 165 | int r; |
cebb52b7 AG |
166 | struct dma_fence *fence = amdgpu_sync_get_fence(&job->sync, &explicit); |
167 | ||
168 | if (fence && explicit) { | |
1b1f42d8 | 169 | if (drm_sched_dependency_optimized(fence, s_entity)) { |
cebb52b7 AG |
170 | r = amdgpu_sync_fence(job->adev, &job->sched_sync, fence, false); |
171 | if (r) | |
172 | DRM_ERROR("Error adding fence to sync (%d)\n", r); | |
173 | } | |
a340c7bc | 174 | } |
cebb52b7 | 175 | |
c4f46f22 | 176 | while (fence == NULL && vm && !job->vmid) { |
620f774f CK |
177 | r = amdgpu_vmid_grab(vm, ring, &job->sync, |
178 | &job->base.s_fence->finished, | |
179 | job); | |
94dd0a4a | 180 | if (r) |
8d0a7cea | 181 | DRM_ERROR("Error getting VM ID (%d)\n", r); |
8d0a7cea | 182 | |
cebb52b7 | 183 | fence = amdgpu_sync_get_fence(&job->sync, NULL); |
8d0a7cea CK |
184 | } |
185 | ||
186 | return fence; | |
e61235db CK |
187 | } |
188 | ||
1b1f42d8 | 189 | static struct dma_fence *amdgpu_job_run(struct drm_sched_job *sched_job) |
c1b69ed0 | 190 | { |
3320b8d2 | 191 | struct amdgpu_ring *ring = to_amdgpu_ring(sched_job->sched); |
48f05f29 | 192 | struct dma_fence *fence = NULL, *finished; |
14e47f93 | 193 | struct amdgpu_device *adev; |
4c7eb91c | 194 | struct amdgpu_job *job; |
bd755d08 | 195 | int r; |
c1b69ed0 | 196 | |
4c7eb91c | 197 | if (!sched_job) { |
4cef9267 | 198 | DRM_ERROR("job is null\n"); |
6f0e54a9 | 199 | return NULL; |
4cef9267 | 200 | } |
a6db8a33 | 201 | job = to_amdgpu_job(sched_job); |
48f05f29 | 202 | finished = &job->base.s_fence->finished; |
14e47f93 | 203 | adev = job->adev; |
e86f9cee | 204 | |
1fbb2e92 | 205 | BUG_ON(amdgpu_sync_peek_fence(&job->sync, NULL)); |
e86f9cee | 206 | |
7034decf | 207 | trace_amdgpu_sched_run_job(job); |
48f05f29 ML |
208 | |
209 | if (job->vram_lost_counter != atomic_read(&adev->vram_lost_counter)) | |
210 | dma_fence_set_error(finished, -ECANCELED);/* skip IB as well if VRAM lost */ | |
211 | ||
212 | if (finished->error < 0) { | |
213 | DRM_INFO("Skip scheduling IBs!\n"); | |
14e47f93 | 214 | } else { |
3320b8d2 | 215 | r = amdgpu_ib_schedule(ring, job->num_ibs, job->ibs, job, |
14e47f93 | 216 | &fence); |
15d73ce6 CZ |
217 | if (r) |
218 | DRM_ERROR("Error scheduling IBs (%d)\n", r); | |
219 | } | |
c7c5fbcd | 220 | /* if gpu reset, hw fence will be replaced here */ |
f54d1867 CW |
221 | dma_fence_put(job->fence); |
222 | job->fence = dma_fence_get(fence); | |
b2ff0e8a | 223 | |
22a77cf6 | 224 | amdgpu_job_free_resources(job); |
ec72b800 | 225 | return fence; |
c1b69ed0 CZ |
226 | } |
227 | ||
1b1f42d8 | 228 | const struct drm_sched_backend_ops amdgpu_sched_ops = { |
0856cab1 CK |
229 | .dependency = amdgpu_job_dependency, |
230 | .run_job = amdgpu_job_run, | |
0e51a772 | 231 | .timedout_job = amdgpu_job_timedout, |
c5f74f78 | 232 | .free_job = amdgpu_job_free_cb |
c1b69ed0 | 233 | }; |