Commit | Line | Data |
---|---|---|
9a65a38c BS |
1 | /* |
2 | * Copyright 2012 Red Hat Inc. | |
3 | * | |
4 | * Permission is hereby granted, free of charge, to any person obtaining a | |
5 | * copy of this software and associated documentation files (the "Software"), | |
6 | * to deal in the Software without restriction, including without limitation | |
7 | * the rights to use, copy, modify, merge, publish, distribute, sublicense, | |
8 | * and/or sell copies of the Software, and to permit persons to whom the | |
9 | * Software is furnished to do so, subject to the following conditions: | |
10 | * | |
11 | * The above copyright notice and this permission notice shall be included in | |
12 | * all copies or substantial portions of the Software. | |
13 | * | |
14 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | |
15 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | |
16 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL | |
17 | * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR | |
18 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, | |
19 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR | |
20 | * OTHER DEALINGS IN THE SOFTWARE. | |
21 | * | |
22 | * Authors: Ben Skeggs | |
23 | */ | |
24 | #include "changk104.h" | |
25 | ||
26 | #include <core/client.h> | |
13de7f46 | 27 | #include <core/gpuobj.h> |
9a65a38c BS |
28 | #include <subdev/fb.h> |
29 | #include <subdev/mmu.h> | |
30 | #include <subdev/timer.h> | |
31 | ||
32 | #include <nvif/class.h> | |
8ed1730c | 33 | #include <nvif/cla06f.h> |
9a65a38c BS |
34 | #include <nvif/unpack.h> |
35 | ||
9a65a38c | 36 | static int |
8f0649b5 | 37 | gk104_fifo_gpfifo_kick(struct gk104_fifo_chan *chan) |
9a65a38c | 38 | { |
8f0649b5 | 39 | struct gk104_fifo *fifo = chan->fifo; |
9a65a38c BS |
40 | struct nvkm_subdev *subdev = &fifo->base.engine.subdev; |
41 | struct nvkm_device *device = subdev->device; | |
8f0649b5 | 42 | struct nvkm_client *client = chan->base.object.client; |
9a65a38c BS |
43 | |
44 | nvkm_wr32(device, 0x002634, chan->base.chid); | |
45 | if (nvkm_msec(device, 2000, | |
46 | if (!(nvkm_rd32(device, 0x002634) & 0x00100000)) | |
47 | break; | |
48 | ) < 0) { | |
49 | nvkm_error(subdev, "channel %d [%s] kick timeout\n", | |
8f0649b5 | 50 | chan->base.chid, client->name); |
9a65a38c BS |
51 | return -EBUSY; |
52 | } | |
53 | ||
54 | return 0; | |
55 | } | |
56 | ||
8f0649b5 BS |
57 | static u32 |
58 | gk104_fifo_gpfifo_engine_addr(struct nvkm_engine *engine) | |
9a65a38c | 59 | { |
8f0649b5 | 60 | switch (engine->subdev.index) { |
68f3f702 BS |
61 | case NVKM_ENGINE_SW : |
62 | case NVKM_ENGINE_CE0 : | |
63 | case NVKM_ENGINE_CE1 : | |
64 | case NVKM_ENGINE_CE2 : return 0x0000; | |
65 | case NVKM_ENGINE_GR : return 0x0210; | |
a8b005fd | 66 | case NVKM_ENGINE_SEC : return 0x0220; |
68f3f702 BS |
67 | case NVKM_ENGINE_MSPDEC: return 0x0250; |
68 | case NVKM_ENGINE_MSPPP : return 0x0260; | |
69 | case NVKM_ENGINE_MSVLD : return 0x0270; | |
4a3f63f8 | 70 | case NVKM_ENGINE_VIC : return 0x0280; |
5d7fa4de | 71 | case NVKM_ENGINE_MSENC : return 0x0290; |
608fd040 | 72 | case NVKM_ENGINE_NVDEC : return 0x02100270; |
9e4fff32 BS |
73 | case NVKM_ENGINE_NVENC0: return 0x02100290; |
74 | case NVKM_ENGINE_NVENC1: return 0x0210; | |
9a65a38c | 75 | default: |
8f0649b5 BS |
76 | WARN_ON(1); |
77 | return 0; | |
9a65a38c | 78 | } |
8f0649b5 | 79 | } |
9a65a38c | 80 | |
8f0649b5 BS |
81 | static int |
82 | gk104_fifo_gpfifo_engine_fini(struct nvkm_fifo_chan *base, | |
83 | struct nvkm_engine *engine, bool suspend) | |
84 | { | |
8f0649b5 BS |
85 | struct gk104_fifo_chan *chan = gk104_fifo_chan(base); |
86 | struct nvkm_gpuobj *inst = chan->base.inst; | |
9e4fff32 | 87 | u32 offset = gk104_fifo_gpfifo_engine_addr(engine); |
8f0649b5 BS |
88 | int ret; |
89 | ||
90 | ret = gk104_fifo_gpfifo_kick(chan); | |
9a65a38c BS |
91 | if (ret && suspend) |
92 | return ret; | |
93 | ||
8f0649b5 BS |
94 | if (offset) { |
95 | nvkm_kmap(inst); | |
9e4fff32 BS |
96 | nvkm_wo32(inst, (offset & 0xffff) + 0x00, 0x00000000); |
97 | nvkm_wo32(inst, (offset & 0xffff) + 0x04, 0x00000000); | |
98 | if ((offset >>= 16)) { | |
99 | nvkm_wo32(inst, offset + 0x00, 0x00000000); | |
100 | nvkm_wo32(inst, offset + 0x04, 0x00000000); | |
101 | } | |
8f0649b5 BS |
102 | nvkm_done(inst); |
103 | } | |
104 | ||
105 | return ret; | |
106 | } | |
107 | ||
108 | static int | |
109 | gk104_fifo_gpfifo_engine_init(struct nvkm_fifo_chan *base, | |
110 | struct nvkm_engine *engine) | |
111 | { | |
8f0649b5 BS |
112 | struct gk104_fifo_chan *chan = gk104_fifo_chan(base); |
113 | struct nvkm_gpuobj *inst = chan->base.inst; | |
9e4fff32 | 114 | u32 offset = gk104_fifo_gpfifo_engine_addr(engine); |
8f0649b5 BS |
115 | |
116 | if (offset) { | |
9e4fff32 BS |
117 | u64 addr = chan->engn[engine->subdev.index].vma.offset; |
118 | u32 datalo = lower_32_bits(addr) | 0x00000004; | |
119 | u32 datahi = upper_32_bits(addr); | |
8f0649b5 | 120 | nvkm_kmap(inst); |
9e4fff32 BS |
121 | nvkm_wo32(inst, (offset & 0xffff) + 0x00, datalo); |
122 | nvkm_wo32(inst, (offset & 0xffff) + 0x04, datahi); | |
123 | if ((offset >>= 16)) { | |
124 | nvkm_wo32(inst, offset + 0x00, datalo); | |
125 | nvkm_wo32(inst, offset + 0x04, datahi); | |
126 | } | |
8f0649b5 | 127 | nvkm_done(inst); |
9a65a38c BS |
128 | } |
129 | ||
130 | return 0; | |
131 | } | |
132 | ||
8f0649b5 BS |
133 | static void |
134 | gk104_fifo_gpfifo_engine_dtor(struct nvkm_fifo_chan *base, | |
135 | struct nvkm_engine *engine) | |
136 | { | |
137 | struct gk104_fifo_chan *chan = gk104_fifo_chan(base); | |
138 | nvkm_gpuobj_unmap(&chan->engn[engine->subdev.index].vma); | |
139 | nvkm_gpuobj_del(&chan->engn[engine->subdev.index].inst); | |
140 | } | |
141 | ||
9a65a38c | 142 | static int |
8f0649b5 BS |
143 | gk104_fifo_gpfifo_engine_ctor(struct nvkm_fifo_chan *base, |
144 | struct nvkm_engine *engine, | |
145 | struct nvkm_object *object) | |
9a65a38c | 146 | { |
8f0649b5 BS |
147 | struct gk104_fifo_chan *chan = gk104_fifo_chan(base); |
148 | int engn = engine->subdev.index; | |
9a65a38c BS |
149 | int ret; |
150 | ||
8f0649b5 | 151 | if (!gk104_fifo_gpfifo_engine_addr(engine)) |
9a65a38c | 152 | return 0; |
9a65a38c | 153 | |
8f0649b5 BS |
154 | ret = nvkm_object_bind(object, NULL, 0, &chan->engn[engn].inst); |
155 | if (ret) | |
156 | return ret; | |
157 | ||
158 | return nvkm_gpuobj_map(chan->engn[engn].inst, chan->vm, | |
159 | NV_MEM_ACCESS_RW, &chan->engn[engn].vma); | |
9a65a38c BS |
160 | } |
161 | ||
8f0649b5 BS |
162 | static void |
163 | gk104_fifo_gpfifo_fini(struct nvkm_fifo_chan *base) | |
9a65a38c | 164 | { |
8f0649b5 BS |
165 | struct gk104_fifo_chan *chan = gk104_fifo_chan(base); |
166 | struct gk104_fifo *fifo = chan->fifo; | |
9a65a38c | 167 | struct nvkm_device *device = fifo->base.engine.subdev.device; |
8f0649b5 | 168 | u32 coff = chan->base.chid * 8; |
9a65a38c | 169 | |
8f0649b5 | 170 | if (!list_empty(&chan->head)) { |
386ffd5e | 171 | gk104_fifo_runlist_remove(fifo, chan); |
8f0649b5 | 172 | nvkm_mask(device, 0x800004 + coff, 0x00000800, 0x00000800); |
2bf1833e | 173 | gk104_fifo_gpfifo_kick(chan); |
69aa40e2 | 174 | gk104_fifo_runlist_commit(fifo, chan->runl); |
9a65a38c BS |
175 | } |
176 | ||
8f0649b5 | 177 | nvkm_wr32(device, 0x800000 + coff, 0x00000000); |
9a65a38c BS |
178 | } |
179 | ||
8f0649b5 BS |
180 | static void |
181 | gk104_fifo_gpfifo_init(struct nvkm_fifo_chan *base) | |
9a65a38c | 182 | { |
8f0649b5 BS |
183 | struct gk104_fifo_chan *chan = gk104_fifo_chan(base); |
184 | struct gk104_fifo *fifo = chan->fifo; | |
9a65a38c | 185 | struct nvkm_device *device = fifo->base.engine.subdev.device; |
8f0649b5 BS |
186 | u32 addr = chan->base.inst->addr >> 12; |
187 | u32 coff = chan->base.chid * 8; | |
9a65a38c | 188 | |
69aa40e2 | 189 | nvkm_mask(device, 0x800004 + coff, 0x000f0000, chan->runl << 16); |
8f0649b5 | 190 | nvkm_wr32(device, 0x800000 + coff, 0x80000000 | addr); |
9a65a38c | 191 | |
8f0649b5 | 192 | if (list_empty(&chan->head) && !chan->killed) { |
386ffd5e | 193 | gk104_fifo_runlist_insert(fifo, chan); |
8f0649b5 | 194 | nvkm_mask(device, 0x800004 + coff, 0x00000400, 0x00000400); |
69aa40e2 | 195 | gk104_fifo_runlist_commit(fifo, chan->runl); |
8f0649b5 | 196 | nvkm_mask(device, 0x800004 + coff, 0x00000400, 0x00000400); |
9a65a38c | 197 | } |
8f0649b5 | 198 | } |
9a65a38c | 199 | |
8f0649b5 BS |
200 | static void * |
201 | gk104_fifo_gpfifo_dtor(struct nvkm_fifo_chan *base) | |
202 | { | |
203 | struct gk104_fifo_chan *chan = gk104_fifo_chan(base); | |
204 | nvkm_vm_ref(NULL, &chan->vm, chan->pgd); | |
205 | nvkm_gpuobj_del(&chan->pgd); | |
206 | return chan; | |
9a65a38c BS |
207 | } |
208 | ||
8f0649b5 BS |
209 | static const struct nvkm_fifo_chan_func |
210 | gk104_fifo_gpfifo_func = { | |
211 | .dtor = gk104_fifo_gpfifo_dtor, | |
212 | .init = gk104_fifo_gpfifo_init, | |
213 | .fini = gk104_fifo_gpfifo_fini, | |
214 | .ntfy = g84_fifo_chan_ntfy, | |
215 | .engine_ctor = gk104_fifo_gpfifo_engine_ctor, | |
216 | .engine_dtor = gk104_fifo_gpfifo_engine_dtor, | |
217 | .engine_init = gk104_fifo_gpfifo_engine_init, | |
218 | .engine_fini = gk104_fifo_gpfifo_engine_fini, | |
219 | }; | |
220 | ||
1f5ff7f5 BS |
221 | struct gk104_fifo_chan_func { |
222 | u32 engine; | |
223 | u64 subdev; | |
224 | }; | |
225 | ||
226 | static int | |
227 | gk104_fifo_gpfifo_new_(const struct gk104_fifo_chan_func *func, | |
228 | struct gk104_fifo *fifo, u32 *engmask, u16 *chid, | |
229 | u64 vm, u64 ioffset, u64 ilength, | |
230 | const struct nvkm_oclass *oclass, | |
231 | struct nvkm_object **pobject) | |
9a65a38c | 232 | { |
8f0649b5 | 233 | struct nvkm_device *device = fifo->base.engine.subdev.device; |
9a65a38c | 234 | struct gk104_fifo_chan *chan; |
1f5ff7f5 BS |
235 | int runlist = -1, ret = -ENOSYS, i, j; |
236 | u32 engines = 0, present = 0; | |
237 | u64 subdevs = 0; | |
238 | u64 usermem; | |
239 | ||
240 | /* Determine which downstream engines are present */ | |
241 | for (i = 0; i < fifo->engine_nr; i++) { | |
242 | struct nvkm_engine *engine = fifo->engine[i].engine; | |
243 | if (engine) { | |
244 | u64 submask = BIT_ULL(engine->subdev.index); | |
245 | for (j = 0; func[j].subdev; j++) { | |
246 | if (func[j].subdev & submask) { | |
247 | present |= func[j].engine; | |
248 | break; | |
249 | } | |
250 | } | |
251 | ||
252 | if (!func[j].subdev) | |
253 | continue; | |
254 | ||
255 | if (runlist < 0 && (*engmask & present)) | |
256 | runlist = fifo->engine[i].runl; | |
257 | if (runlist == fifo->engine[i].runl) { | |
258 | engines |= func[j].engine; | |
259 | subdevs |= func[j].subdev; | |
260 | } | |
261 | } | |
9a65a38c BS |
262 | } |
263 | ||
1f5ff7f5 BS |
264 | /* Just an engine mask query? All done here! */ |
265 | if (!*engmask) { | |
266 | *engmask = present; | |
8f0649b5 | 267 | return nvkm_object_new(oclass, NULL, 0, pobject); |
9a65a38c BS |
268 | } |
269 | ||
1f5ff7f5 BS |
270 | /* No runlist? No supported engines. */ |
271 | *engmask = present; | |
272 | if (runlist < 0) | |
9a65a38c | 273 | return -ENODEV; |
1f5ff7f5 | 274 | *engmask = engines; |
9a65a38c | 275 | |
1f5ff7f5 | 276 | /* Allocate the channel. */ |
8f0649b5 BS |
277 | if (!(chan = kzalloc(sizeof(*chan), GFP_KERNEL))) |
278 | return -ENOMEM; | |
279 | *pobject = &chan->base.object; | |
280 | chan->fifo = fifo; | |
1f5ff7f5 | 281 | chan->runl = runlist; |
8f0649b5 BS |
282 | INIT_LIST_HEAD(&chan->head); |
283 | ||
284 | ret = nvkm_fifo_chan_ctor(&gk104_fifo_gpfifo_func, &fifo->base, | |
1f5ff7f5 | 285 | 0x1000, 0x1000, true, vm, 0, subdevs, |
8f0649b5 BS |
286 | 1, fifo->user.bar.offset, 0x200, |
287 | oclass, &chan->base); | |
9a65a38c BS |
288 | if (ret) |
289 | return ret; | |
290 | ||
1f5ff7f5 | 291 | *chid = chan->base.chid; |
9a65a38c | 292 | |
1f5ff7f5 | 293 | /* Page directory. */ |
8f0649b5 BS |
294 | ret = nvkm_gpuobj_new(device, 0x10000, 0x1000, false, NULL, &chan->pgd); |
295 | if (ret) | |
296 | return ret; | |
297 | ||
298 | nvkm_kmap(chan->base.inst); | |
299 | nvkm_wo32(chan->base.inst, 0x0200, lower_32_bits(chan->pgd->addr)); | |
300 | nvkm_wo32(chan->base.inst, 0x0204, upper_32_bits(chan->pgd->addr)); | |
301 | nvkm_wo32(chan->base.inst, 0x0208, 0xffffffff); | |
302 | nvkm_wo32(chan->base.inst, 0x020c, 0x000000ff); | |
303 | nvkm_done(chan->base.inst); | |
9a65a38c | 304 | |
8f0649b5 BS |
305 | ret = nvkm_vm_ref(chan->base.vm, &chan->vm, chan->pgd); |
306 | if (ret) | |
307 | return ret; | |
308 | ||
1f5ff7f5 | 309 | /* Clear channel control registers. */ |
9a65a38c | 310 | usermem = chan->base.chid * 0x200; |
1f5ff7f5 | 311 | ilength = order_base_2(ilength / 8); |
9a65a38c BS |
312 | |
313 | nvkm_kmap(fifo->user.mem); | |
314 | for (i = 0; i < 0x200; i += 4) | |
315 | nvkm_wo32(fifo->user.mem, usermem + i, 0x00000000); | |
316 | nvkm_done(fifo->user.mem); | |
317 | usermem = nvkm_memory_addr(fifo->user.mem) + usermem; | |
318 | ||
8f0649b5 BS |
319 | /* RAMFC */ |
320 | nvkm_kmap(chan->base.inst); | |
321 | nvkm_wo32(chan->base.inst, 0x08, lower_32_bits(usermem)); | |
322 | nvkm_wo32(chan->base.inst, 0x0c, upper_32_bits(usermem)); | |
323 | nvkm_wo32(chan->base.inst, 0x10, 0x0000face); | |
324 | nvkm_wo32(chan->base.inst, 0x30, 0xfffff902); | |
325 | nvkm_wo32(chan->base.inst, 0x48, lower_32_bits(ioffset)); | |
326 | nvkm_wo32(chan->base.inst, 0x4c, upper_32_bits(ioffset) | | |
327 | (ilength << 16)); | |
328 | nvkm_wo32(chan->base.inst, 0x84, 0x20400000); | |
329 | nvkm_wo32(chan->base.inst, 0x94, 0x30000001); | |
330 | nvkm_wo32(chan->base.inst, 0x9c, 0x00000100); | |
331 | nvkm_wo32(chan->base.inst, 0xac, 0x0000001f); | |
332 | nvkm_wo32(chan->base.inst, 0xe8, chan->base.chid); | |
333 | nvkm_wo32(chan->base.inst, 0xb8, 0xf8000000); | |
334 | nvkm_wo32(chan->base.inst, 0xf8, 0x10003080); /* 0x002310 */ | |
335 | nvkm_wo32(chan->base.inst, 0xfc, 0x10000010); /* 0x002350 */ | |
336 | nvkm_done(chan->base.inst); | |
9a65a38c BS |
337 | return 0; |
338 | } | |
339 | ||
1f5ff7f5 BS |
340 | static const struct gk104_fifo_chan_func |
341 | gk104_fifo_gpfifo[] = { | |
342 | { NVA06F_V0_ENGINE_SW | NVA06F_V0_ENGINE_GR, | |
343 | BIT_ULL(NVKM_ENGINE_SW) | BIT_ULL(NVKM_ENGINE_GR) | |
344 | }, | |
a8b005fd | 345 | { NVA06F_V0_ENGINE_SEC , BIT_ULL(NVKM_ENGINE_SEC ) }, |
1f5ff7f5 BS |
346 | { NVA06F_V0_ENGINE_MSVLD , BIT_ULL(NVKM_ENGINE_MSVLD ) }, |
347 | { NVA06F_V0_ENGINE_MSPDEC, BIT_ULL(NVKM_ENGINE_MSPDEC) }, | |
348 | { NVA06F_V0_ENGINE_MSPPP , BIT_ULL(NVKM_ENGINE_MSPPP ) }, | |
349 | { NVA06F_V0_ENGINE_MSENC , BIT_ULL(NVKM_ENGINE_MSENC ) }, | |
4a3f63f8 | 350 | { NVA06F_V0_ENGINE_VIC , BIT_ULL(NVKM_ENGINE_VIC ) }, |
608fd040 | 351 | { NVA06F_V0_ENGINE_NVDEC , BIT_ULL(NVKM_ENGINE_NVDEC ) }, |
9e4fff32 BS |
352 | { NVA06F_V0_ENGINE_NVENC0, BIT_ULL(NVKM_ENGINE_NVENC0) }, |
353 | { NVA06F_V0_ENGINE_NVENC1, BIT_ULL(NVKM_ENGINE_NVENC1) }, | |
1f5ff7f5 BS |
354 | { NVA06F_V0_ENGINE_CE0 , BIT_ULL(NVKM_ENGINE_CE0 ) }, |
355 | { NVA06F_V0_ENGINE_CE1 , BIT_ULL(NVKM_ENGINE_CE1 ) }, | |
356 | { NVA06F_V0_ENGINE_CE2 , BIT_ULL(NVKM_ENGINE_CE2 ) }, | |
357 | {} | |
358 | }; | |
359 | ||
360 | int | |
361 | gk104_fifo_gpfifo_new(struct nvkm_fifo *base, const struct nvkm_oclass *oclass, | |
362 | void *data, u32 size, struct nvkm_object **pobject) | |
363 | { | |
364 | struct nvkm_object *parent = oclass->parent; | |
365 | union { | |
366 | struct kepler_channel_gpfifo_a_v0 v0; | |
367 | } *args = data; | |
368 | struct gk104_fifo *fifo = gk104_fifo(base); | |
369 | int ret = -ENOSYS; | |
370 | ||
371 | nvif_ioctl(parent, "create channel gpfifo size %d\n", size); | |
372 | if (!(ret = nvif_unpack(ret, &data, &size, args->v0, 0, 0, false))) { | |
373 | nvif_ioctl(parent, "create channel gpfifo vers %d vm %llx " | |
374 | "ioffset %016llx ilength %08x engine %08x\n", | |
375 | args->v0.version, args->v0.vm, args->v0.ioffset, | |
376 | args->v0.ilength, args->v0.engines); | |
377 | return gk104_fifo_gpfifo_new_(gk104_fifo_gpfifo, fifo, | |
378 | &args->v0.engines, | |
379 | &args->v0.chid, | |
380 | args->v0.vm, | |
381 | args->v0.ioffset, | |
382 | args->v0.ilength, | |
383 | oclass, pobject); | |
384 | ||
385 | } | |
386 | ||
387 | return ret; | |
388 | } | |
389 | ||
8f0649b5 BS |
390 | const struct nvkm_fifo_chan_oclass |
391 | gk104_fifo_gpfifo_oclass = { | |
392 | .base.oclass = KEPLER_CHANNEL_GPFIFO_A, | |
393 | .base.minver = 0, | |
394 | .base.maxver = 0, | |
395 | .ctor = gk104_fifo_gpfifo_new, | |
9a65a38c | 396 | }; |