projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'x86-asm-2024-03-11' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip
[linux-2.6-block.git] / drivers / gpu / drm / nouveau / nvkm / subdev / gsp / r535.c
CommitLineData
176fdcbd
BS		 1/*
2 * Copyright 2023 Red Hat Inc.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
13 *
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
21 */
22#include "priv.h"
23
24#include <core/pci.h>
25#include <subdev/timer.h>
830531e9 26#include <subdev/vfn.h>
2a77d015 27#include <engine/fifo/chan.h>
176fdcbd
BS		 28#include <engine/sec2.h>
29
30#include <nvfw/fw.h>
31
32#include <nvrm/nvtypes.h>
b5bad8c1
DA		 33#include <nvrm/535.113.01/common/sdk/nvidia/inc/class/cl0000.h>
34#include <nvrm/535.113.01/common/sdk/nvidia/inc/class/cl0005.h>
35#include <nvrm/535.113.01/common/sdk/nvidia/inc/class/cl0080.h>
36#include <nvrm/535.113.01/common/sdk/nvidia/inc/class/cl2080.h>
37#include <nvrm/535.113.01/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080event.h>
38#include <nvrm/535.113.01/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080gpu.h>
39#include <nvrm/535.113.01/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080internal.h>
40#include <nvrm/535.113.01/common/sdk/nvidia/inc/nvos.h>
41#include <nvrm/535.113.01/common/shared/msgq/inc/msgq/msgq_priv.h>
42#include <nvrm/535.113.01/common/uproc/os/common/include/libos_init_args.h>
43#include <nvrm/535.113.01/nvidia/arch/nvalloc/common/inc/gsp/gsp_fw_sr_meta.h>
44#include <nvrm/535.113.01/nvidia/arch/nvalloc/common/inc/gsp/gsp_fw_wpr_meta.h>
45#include <nvrm/535.113.01/nvidia/arch/nvalloc/common/inc/rmRiscvUcode.h>
46#include <nvrm/535.113.01/nvidia/arch/nvalloc/common/inc/rmgspseq.h>
47#include <nvrm/535.113.01/nvidia/generated/g_allclasses.h>
48#include <nvrm/535.113.01/nvidia/generated/g_os_nvoc.h>
49#include <nvrm/535.113.01/nvidia/generated/g_rpc-structures.h>
50#include <nvrm/535.113.01/nvidia/inc/kernel/gpu/gsp/gsp_fw_heap.h>
51#include <nvrm/535.113.01/nvidia/inc/kernel/gpu/gsp/gsp_init_args.h>
52#include <nvrm/535.113.01/nvidia/inc/kernel/gpu/gsp/gsp_static_config.h>
53#include <nvrm/535.113.01/nvidia/inc/kernel/gpu/intr/engine_idx.h>
54#include <nvrm/535.113.01/nvidia/kernel/inc/vgpu/rpc_global_enums.h>
176fdcbd
BS		 55
56#include <linux/acpi.h>
57
b5bad8c1
DA		 58#define GSP_MSG_MIN_SIZE GSP_PAGE_SIZE
59#define GSP_MSG_MAX_SIZE GSP_PAGE_MIN_SIZE * 16
60
176fdcbd 61struct r535_gsp_msg {
b5bad8c1
DA		 62 u8 auth_tag_buffer[16];
63 u8 aad_buffer[16];
176fdcbd
BS		 64 u32 checksum;
65 u32 sequence;
b5bad8c1
DA		 66 u32 elem_count;
67 u32 pad;
176fdcbd
BS		 68 u8 data[];
69};
70
b5bad8c1
DA		 71#define GSP_MSG_HDR_SIZE offsetof(struct r535_gsp_msg, data)
72
59f6a3d8
DA		 73static int
74r535_rpc_status_to_errno(uint32_t rpc_status)
75{
76 switch (rpc_status) {
77 case 0x55: /* NV_ERR_NOT_READY */
78 case 0x66: /* NV_ERR_TIMEOUT_RETRY */
79 return -EAGAIN;
80 case 0x51: /* NV_ERR_NO_MEMORY */
81 return -ENOMEM;
82 default:
83 return -EINVAL;
84 }
85}
86
176fdcbd
BS		 87static void *
88r535_gsp_msgq_wait(struct nvkm_gsp *gsp, u32 repc, u32 *prepc, int *ptime)
89{
90 struct r535_gsp_msg *mqe;
91 u32 size, rptr = *gsp->msgq.rptr;
92 int used;
93 u8 *msg;
94 u32 len;
95
b5bad8c1 96 size = DIV_ROUND_UP(GSP_MSG_HDR_SIZE + repc, GSP_PAGE_SIZE);
176fdcbd
BS		 97 if (WARN_ON(!size || size >= gsp->msgq.cnt))
98 return ERR_PTR(-EINVAL);
99
100 do {
101 u32 wptr = *gsp->msgq.wptr;
102
103 used = wptr + gsp->msgq.cnt - rptr;
104 if (used >= gsp->msgq.cnt)
105 used -= gsp->msgq.cnt;
106 if (used >= size)
107 break;
108
109 usleep_range(1, 2);
110 } while (--(*ptime));
111
112 if (WARN_ON(!*ptime))
113 return ERR_PTR(-ETIMEDOUT);
114
115 mqe = (void *)((u8 *)gsp->shm.msgq.ptr + 0x1000 + rptr * 0x1000);
116
117 if (prepc) {
118 *prepc = (used * GSP_PAGE_SIZE) - sizeof(*mqe);
119 return mqe->data;
120 }
121
122 msg = kvmalloc(repc, GFP_KERNEL);
123 if (!msg)
124 return ERR_PTR(-ENOMEM);
125
126 len = ((gsp->msgq.cnt - rptr) * GSP_PAGE_SIZE) - sizeof(*mqe);
127 len = min_t(u32, repc, len);
128 memcpy(msg, mqe->data, len);
129
130 rptr += DIV_ROUND_UP(len, GSP_PAGE_SIZE);
131 if (rptr == gsp->msgq.cnt)
132 rptr = 0;
133
134 repc -= len;
135
136 if (repc) {
137 mqe = (void *)((u8 *)gsp->shm.msgq.ptr + 0x1000 + 0 * 0x1000);
138 memcpy(msg + len, mqe, repc);
139
140 rptr += DIV_ROUND_UP(repc, GSP_PAGE_SIZE);
141 }
142
143 mb();
144 (*gsp->msgq.rptr) = rptr;
145 return msg;
146}
147
148static void *
149r535_gsp_msgq_recv(struct nvkm_gsp *gsp, u32 repc, int *ptime)
150{
151 return r535_gsp_msgq_wait(gsp, repc, NULL, ptime);
152}
153
154static int
155r535_gsp_cmdq_push(struct nvkm_gsp *gsp, void *argv)
156{
157 struct r535_gsp_msg *cmd = container_of(argv, typeof(*cmd), data);
158 struct r535_gsp_msg *cqe;
159 u32 argc = cmd->checksum;
160 u64 *ptr = (void *)cmd;
b5bad8c1 161 u64 *end;
176fdcbd
BS		 162 u64 csum = 0;
163 int free, time = 1000000;
164 u32 wptr, size;
165 u32 off = 0;
166
b5bad8c1
DA		 167 argc = ALIGN(GSP_MSG_HDR_SIZE + argc, GSP_PAGE_SIZE);
168
169 end = (u64 *)((char *)ptr + argc);
170 cmd->pad = 0;
176fdcbd
BS		 171 cmd->checksum = 0;
172 cmd->sequence = gsp->cmdq.seq++;
b5bad8c1
DA		 173 cmd->elem_count = DIV_ROUND_UP(argc, 0x1000);
174
176fdcbd
BS		 175 while (ptr < end)
176 csum ^= *ptr++;
177
178 cmd->checksum = upper_32_bits(csum) ^ lower_32_bits(csum);
179
176fdcbd
BS		 180 wptr = *gsp->cmdq.wptr;
181 do {
182 do {
183 free = *gsp->cmdq.rptr + gsp->cmdq.cnt - wptr - 1;
184 if (free >= gsp->cmdq.cnt)
185 free -= gsp->cmdq.cnt;
186 if (free >= 1)
187 break;
188
189 usleep_range(1, 2);
190 } while(--time);
191
192 if (WARN_ON(!time)) {
193 kvfree(cmd);
194 return -ETIMEDOUT;
195 }
196
197 cqe = (void *)((u8 *)gsp->shm.cmdq.ptr + 0x1000 + wptr * 0x1000);
198 size = min_t(u32, argc, (gsp->cmdq.cnt - wptr) * GSP_PAGE_SIZE);
199 memcpy(cqe, (u8 *)cmd + off, size);
200
201 wptr += DIV_ROUND_UP(size, 0x1000);
202 if (wptr == gsp->cmdq.cnt)
203 wptr = 0;
204
205 off += size;
206 argc -= size;
207 } while(argc);
208
209 nvkm_trace(&gsp->subdev, "cmdq: wptr %d\n", wptr);
210 wmb();
211 (*gsp->cmdq.wptr) = wptr;
212 mb();
213
214 nvkm_falcon_wr32(&gsp->falcon, 0xc00, 0x00000000);
215
216 kvfree(cmd);
217 return 0;
218}
219
220static void *
221r535_gsp_cmdq_get(struct nvkm_gsp *gsp, u32 argc)
222{
223 struct r535_gsp_msg *cmd;
b5bad8c1 224 u32 size = GSP_MSG_HDR_SIZE + argc;
176fdcbd 225
b5bad8c1
DA		 226 size = ALIGN(size, GSP_MSG_MIN_SIZE);
227 cmd = kvzalloc(size, GFP_KERNEL);
176fdcbd
BS		 228 if (!cmd)
229 return ERR_PTR(-ENOMEM);
230
231 cmd->checksum = argc;
232 return cmd->data;
233}
234
235struct nvfw_gsp_rpc {
236 u32 header_version;
237 u32 signature;
238 u32 length;
239 u32 function;
240 u32 rpc_result;
241 u32 rpc_result_private;
242 u32 sequence;
243 union {
244 u32 spare;
245 u32 cpuRmGfid;
246 };
247 u8 data[];
248};
249
250static void
251r535_gsp_msg_done(struct nvkm_gsp *gsp, struct nvfw_gsp_rpc *msg)
252{
253 kvfree(msg);
254}
255
256static void
257r535_gsp_msg_dump(struct nvkm_gsp *gsp, struct nvfw_gsp_rpc *msg, int lvl)
258{
259 if (gsp->subdev.debug >= lvl) {
260 nvkm_printk__(&gsp->subdev, lvl, info,
261 "msg fn:%d len:0x%x/0x%zx res:0x%x resp:0x%x\n",
262 msg->function, msg->length, msg->length - sizeof(*msg),
263 msg->rpc_result, msg->rpc_result_private);
264 print_hex_dump(KERN_INFO, "msg: ", DUMP_PREFIX_OFFSET, 16, 1,
265 msg->data, msg->length - sizeof(*msg), true);
266 }
267}
268
269static struct nvfw_gsp_rpc *
270r535_gsp_msg_recv(struct nvkm_gsp *gsp, int fn, u32 repc)
271{
272 struct nvkm_subdev *subdev = &gsp->subdev;
273 struct nvfw_gsp_rpc *msg;
274 int time = 4000000, i;
275 u32 size;
276
277retry:
278 msg = r535_gsp_msgq_wait(gsp, sizeof(*msg), &size, &time);
279 if (IS_ERR_OR_NULL(msg))
280 return msg;
281
282 msg = r535_gsp_msgq_recv(gsp, msg->length, &time);
283 if (IS_ERR_OR_NULL(msg))
284 return msg;
285
286 if (msg->rpc_result) {
287 r535_gsp_msg_dump(gsp, msg, NV_DBG_ERROR);
288 r535_gsp_msg_done(gsp, msg);
289 return ERR_PTR(-EINVAL);
290 }
291
292 r535_gsp_msg_dump(gsp, msg, NV_DBG_TRACE);
293
294 if (fn && msg->function == fn) {
295 if (repc) {
296 if (msg->length < sizeof(*msg) + repc) {
297 nvkm_error(subdev, "msg len %d < %zd\n",
298 msg->length, sizeof(*msg) + repc);
299 r535_gsp_msg_dump(gsp, msg, NV_DBG_ERROR);
300 r535_gsp_msg_done(gsp, msg);
301 return ERR_PTR(-EIO);
302 }
303
304 return msg;
305 }
306
307 r535_gsp_msg_done(gsp, msg);
308 return NULL;
309 }
310
311 for (i = 0; i < gsp->msgq.ntfy_nr; i++) {
312 struct nvkm_gsp_msgq_ntfy *ntfy = &gsp->msgq.ntfy[i];
313
314 if (ntfy->fn == msg->function) {
24ab185d
DA		 315 if (ntfy->func)
316 ntfy->func(ntfy->priv, ntfy->fn, msg->data, msg->length - sizeof(*msg));
176fdcbd
BS		 317 break;
318 }
319 }
320
321 if (i == gsp->msgq.ntfy_nr)
322 r535_gsp_msg_dump(gsp, msg, NV_DBG_WARN);
323
324 r535_gsp_msg_done(gsp, msg);
325 if (fn)
326 goto retry;
327
328 if (*gsp->msgq.rptr != *gsp->msgq.wptr)
329 goto retry;
330
331 return NULL;
332}
333
334static int
335r535_gsp_msg_ntfy_add(struct nvkm_gsp *gsp, u32 fn, nvkm_gsp_msg_ntfy_func func, void *priv)
336{
337 int ret = 0;
338
339 mutex_lock(&gsp->msgq.mutex);
340 if (WARN_ON(gsp->msgq.ntfy_nr >= ARRAY_SIZE(gsp->msgq.ntfy))) {
341 ret = -ENOSPC;
342 } else {
343 gsp->msgq.ntfy[gsp->msgq.ntfy_nr].fn = fn;
344 gsp->msgq.ntfy[gsp->msgq.ntfy_nr].func = func;
345 gsp->msgq.ntfy[gsp->msgq.ntfy_nr].priv = priv;
346 gsp->msgq.ntfy_nr++;
347 }
348 mutex_unlock(&gsp->msgq.mutex);
349 return ret;
350}
351
352static int
353r535_gsp_rpc_poll(struct nvkm_gsp *gsp, u32 fn)
354{
355 void *repv;
356
357 mutex_lock(&gsp->cmdq.mutex);
358 repv = r535_gsp_msg_recv(gsp, fn, 0);
359 mutex_unlock(&gsp->cmdq.mutex);
360 if (IS_ERR(repv))
361 return PTR_ERR(repv);
362
363 return 0;
364}
365
366static void *
367r535_gsp_rpc_send(struct nvkm_gsp *gsp, void *argv, bool wait, u32 repc)
368{
369 struct nvfw_gsp_rpc *rpc = container_of(argv, typeof(*rpc), data);
370 struct nvfw_gsp_rpc *msg;
371 u32 fn = rpc->function;
372 void *repv = NULL;
373 int ret;
374
375 if (gsp->subdev.debug >= NV_DBG_TRACE) {
376 nvkm_trace(&gsp->subdev, "rpc fn:%d len:0x%x/0x%zx\n", rpc->function,
377 rpc->length, rpc->length - sizeof(*rpc));
378 print_hex_dump(KERN_INFO, "rpc: ", DUMP_PREFIX_OFFSET, 16, 1,
379 rpc->data, rpc->length - sizeof(*rpc), true);
380 }
381
382 ret = r535_gsp_cmdq_push(gsp, rpc);
45b7955b 383 if (ret)
176fdcbd 384 return ERR_PTR(ret);
176fdcbd
BS		 385
386 if (wait) {
387 msg = r535_gsp_msg_recv(gsp, fn, repc);
388 if (!IS_ERR_OR_NULL(msg))
389 repv = msg->data;
390 else
391 repv = msg;
392 }
393
394 return repv;
395}
396
9e994444
BS		 397static void
398r535_gsp_event_dtor(struct nvkm_gsp_event *event)
399{
400 struct nvkm_gsp_device *device = event->device;
401 struct nvkm_gsp_client *client = device->object.client;
402 struct nvkm_gsp *gsp = client->gsp;
403
404 mutex_lock(&gsp->client_id.mutex);
405 if (event->func) {
406 list_del(&event->head);
407 event->func = NULL;
408 }
409 mutex_unlock(&gsp->client_id.mutex);
410
411 nvkm_gsp_rm_free(&event->object);
412 event->device = NULL;
413}
414
415static int
416r535_gsp_device_event_get(struct nvkm_gsp_event *event)
417{
418 struct nvkm_gsp_device *device = event->device;
419 NV2080_CTRL_EVENT_SET_NOTIFICATION_PARAMS *ctrl;
420
421 ctrl = nvkm_gsp_rm_ctrl_get(&device->subdevice,
422 NV2080_CTRL_CMD_EVENT_SET_NOTIFICATION, sizeof(*ctrl));
423 if (IS_ERR(ctrl))
424 return PTR_ERR(ctrl);
425
426 ctrl->event = event->id;
427 ctrl->action = NV2080_CTRL_EVENT_SET_NOTIFICATION_ACTION_REPEAT;
428 return nvkm_gsp_rm_ctrl_wr(&device->subdevice, ctrl);
429}
430
431static int
432r535_gsp_device_event_ctor(struct nvkm_gsp_device *device, u32 handle, u32 id,
433 nvkm_gsp_event_func func, struct nvkm_gsp_event *event)
434{
435 struct nvkm_gsp_client *client = device->object.client;
436 struct nvkm_gsp *gsp = client->gsp;
437 NV0005_ALLOC_PARAMETERS *args;
438 int ret;
439
440 args = nvkm_gsp_rm_alloc_get(&device->subdevice, handle,
441 NV01_EVENT_KERNEL_CALLBACK_EX, sizeof(*args),
442 &event->object);
443 if (IS_ERR(args))
444 return PTR_ERR(args);
445
446 args->hParentClient = client->object.handle;
447 args->hSrcResource = 0;
448 args->hClass = NV01_EVENT_KERNEL_CALLBACK_EX;
449 args->notifyIndex = NV01_EVENT_CLIENT_RM | id;
450 args->data = NULL;
451
452 ret = nvkm_gsp_rm_alloc_wr(&event->object, args);
453 if (ret)
454 return ret;
455
456 event->device = device;
457 event->id = id;
458
459 ret = r535_gsp_device_event_get(event);
460 if (ret) {
461 nvkm_gsp_event_dtor(event);
462 return ret;
463 }
464
465 mutex_lock(&gsp->client_id.mutex);
466 event->func = func;
467 list_add(&event->head, &client->events);
468 mutex_unlock(&gsp->client_id.mutex);
469 return 0;
470}
471
37e328a1
BS		 472static void
473r535_gsp_device_dtor(struct nvkm_gsp_device *device)
474{
475 nvkm_gsp_rm_free(&device->subdevice);
476 nvkm_gsp_rm_free(&device->object);
477}
478
479static int
480r535_gsp_subdevice_ctor(struct nvkm_gsp_device *device)
481{
482 NV2080_ALLOC_PARAMETERS *args;
483
484 return nvkm_gsp_rm_alloc(&device->object, 0x5d1d0000, NV20_SUBDEVICE_0, sizeof(*args),
485 &device->subdevice);
486}
487
488static int
489r535_gsp_device_ctor(struct nvkm_gsp_client *client, struct nvkm_gsp_device *device)
490{
491 NV0080_ALLOC_PARAMETERS *args;
492 int ret;
493
494 args = nvkm_gsp_rm_alloc_get(&client->object, 0xde1d0000, NV01_DEVICE_0, sizeof(*args),
495 &device->object);
496 if (IS_ERR(args))
497 return PTR_ERR(args);
498
499 args->hClientShare = client->object.handle;
500
501 ret = nvkm_gsp_rm_alloc_wr(&device->object, args);
502 if (ret)
503 return ret;
504
505 ret = r535_gsp_subdevice_ctor(device);
506 if (ret)
507 nvkm_gsp_rm_free(&device->object);
508
509 return ret;
510}
511
512static void
513r535_gsp_client_dtor(struct nvkm_gsp_client *client)
514{
515 struct nvkm_gsp *gsp = client->gsp;
516
517 nvkm_gsp_rm_free(&client->object);
518
519 mutex_lock(&gsp->client_id.mutex);
520 idr_remove(&gsp->client_id.idr, client->object.handle & 0xffff);
521 mutex_unlock(&gsp->client_id.mutex);
522
523 client->gsp = NULL;
524}
525
526static int
527r535_gsp_client_ctor(struct nvkm_gsp *gsp, struct nvkm_gsp_client *client)
528{
529 NV0000_ALLOC_PARAMETERS *args;
530 int ret;
531
532 mutex_lock(&gsp->client_id.mutex);
533 ret = idr_alloc(&gsp->client_id.idr, client, 0, 0xffff + 1, GFP_KERNEL);
534 mutex_unlock(&gsp->client_id.mutex);
535 if (ret < 0)
536 return ret;
537
538 client->gsp = gsp;
539 client->object.client = client;
9e994444 540 INIT_LIST_HEAD(&client->events);
37e328a1
BS		 541
542 args = nvkm_gsp_rm_alloc_get(&client->object, 0xc1d00000 | ret, NV01_ROOT, sizeof(*args),
543 &client->object);
544 if (IS_ERR(args)) {
545 r535_gsp_client_dtor(client);
546 return ret;
547 }
548
549 args->hClient = client->object.handle;
550 args->processID = ~0;
551
552 ret = nvkm_gsp_rm_alloc_wr(&client->object, args);
553 if (ret) {
554 r535_gsp_client_dtor(client);
555 return ret;
556 }
557
558 return 0;
559}
560
561static int
562r535_gsp_rpc_rm_free(struct nvkm_gsp_object *object)
563{
564 struct nvkm_gsp_client *client = object->client;
565 struct nvkm_gsp *gsp = client->gsp;
566 rpc_free_v03_00 *rpc;
567
568 nvkm_debug(&gsp->subdev, "cli:0x%08x obj:0x%08x free\n",
569 client->object.handle, object->handle);
570
571 rpc = nvkm_gsp_rpc_get(gsp, NV_VGPU_MSG_FUNCTION_FREE, sizeof(*rpc));
572 if (WARN_ON(IS_ERR_OR_NULL(rpc)))
573 return -EIO;
574
575 rpc->params.hRoot = client->object.handle;
576 rpc->params.hObjectParent = 0;
577 rpc->params.hObjectOld = object->handle;
578 return nvkm_gsp_rpc_wr(gsp, rpc, true);
579}
580
581static void
582r535_gsp_rpc_rm_alloc_done(struct nvkm_gsp_object *object, void *repv)
583{
584 rpc_gsp_rm_alloc_v03_00 *rpc = container_of(repv, typeof(*rpc), params);
585
586 nvkm_gsp_rpc_done(object->client->gsp, rpc);
587}
588
589static void *
590r535_gsp_rpc_rm_alloc_push(struct nvkm_gsp_object *object, void *argv, u32 repc)
591{
592 rpc_gsp_rm_alloc_v03_00 *rpc = container_of(argv, typeof(*rpc), params);
593 struct nvkm_gsp *gsp = object->client->gsp;
594 void *ret;
595
596 rpc = nvkm_gsp_rpc_push(gsp, rpc, true, sizeof(*rpc) + repc);
597 if (IS_ERR_OR_NULL(rpc))
598 return rpc;
599
600 if (rpc->status) {
59f6a3d8 601 ret = ERR_PTR(r535_rpc_status_to_errno(rpc->status));
4ae3a201 602 if (PTR_ERR(ret) != -EAGAIN)
59f6a3d8 603 nvkm_error(&gsp->subdev, "RM_ALLOC: 0x%x\n", rpc->status);
37e328a1
BS		 604 } else {
605 ret = repc ? rpc->params : NULL;
606 }
607
9c9dd22b 608 nvkm_gsp_rpc_done(gsp, rpc);
37e328a1
BS		 609
610 return ret;
611}
612
613static void *
614r535_gsp_rpc_rm_alloc_get(struct nvkm_gsp_object *object, u32 oclass, u32 argc)
615{
616 struct nvkm_gsp_client *client = object->client;
617 struct nvkm_gsp *gsp = client->gsp;
618 rpc_gsp_rm_alloc_v03_00 *rpc;
619
620 nvkm_debug(&gsp->subdev, "cli:0x%08x obj:0x%08x new obj:0x%08x cls:0x%08x argc:%d\n",
621 client->object.handle, object->parent->handle, object->handle, oclass, argc);
622
623 rpc = nvkm_gsp_rpc_get(gsp, NV_VGPU_MSG_FUNCTION_GSP_RM_ALLOC, sizeof(*rpc) + argc);
624 if (IS_ERR(rpc))
625 return rpc;
626
627 rpc->hClient = client->object.handle;
628 rpc->hParent = object->parent->handle;
629 rpc->hObject = object->handle;
630 rpc->hClass = oclass;
631 rpc->status = 0;
632 rpc->paramsSize = argc;
633 return rpc->params;
634}
635
4cf2c83e
BS		 636static void
637r535_gsp_rpc_rm_ctrl_done(struct nvkm_gsp_object *object, void *repv)
638{
639 rpc_gsp_rm_control_v03_00 *rpc = container_of(repv, typeof(*rpc), params);
640
4ae3a201
DA		 641 if (!repv)
642 return;
4cf2c83e
BS		 643 nvkm_gsp_rpc_done(object->client->gsp, rpc);
644}
645
4ae3a201
DA		 646static int
647r535_gsp_rpc_rm_ctrl_push(struct nvkm_gsp_object *object, void **argv, u32 repc)
4cf2c83e 648{
4ae3a201 649 rpc_gsp_rm_control_v03_00 *rpc = container_of((*argv), typeof(*rpc), params);
4cf2c83e 650 struct nvkm_gsp *gsp = object->client->gsp;
4ae3a201 651 int ret = 0;
4cf2c83e
BS		 652
653 rpc = nvkm_gsp_rpc_push(gsp, rpc, true, repc);
4ae3a201
DA		 654 if (IS_ERR_OR_NULL(rpc)) {
655 *argv = NULL;
656 return PTR_ERR(rpc);
657 }
4cf2c83e
BS		 658
659 if (rpc->status) {
4ae3a201 660 ret = r535_rpc_status_to_errno(rpc->status);
59f6a3d8
DA		 661 if (ret != -EAGAIN)
662 nvkm_error(&gsp->subdev, "cli:0x%08x obj:0x%08x ctrl cmd:0x%08x failed: 0x%08x\n",
663 object->client->object.handle, object->handle, rpc->cmd, rpc->status);
4cf2c83e
BS		 664 }
665
4ae3a201
DA		 666 if (repc)
667 *argv = rpc->params;
668 else
4cf2c83e
BS		 669 nvkm_gsp_rpc_done(gsp, rpc);
670
671 return ret;
672}
673
674static void *
675r535_gsp_rpc_rm_ctrl_get(struct nvkm_gsp_object *object, u32 cmd, u32 argc)
676{
677 struct nvkm_gsp_client *client = object->client;
678 struct nvkm_gsp *gsp = client->gsp;
679 rpc_gsp_rm_control_v03_00 *rpc;
680
681 nvkm_debug(&gsp->subdev, "cli:0x%08x obj:0x%08x ctrl cmd:0x%08x argc:%d\n",
682 client->object.handle, object->handle, cmd, argc);
683
684 rpc = nvkm_gsp_rpc_get(gsp, NV_VGPU_MSG_FUNCTION_GSP_RM_CONTROL, sizeof(*rpc) + argc);
685 if (IS_ERR(rpc))
686 return rpc;
687
688 rpc->hClient = client->object.handle;
689 rpc->hObject = object->handle;
690 rpc->cmd = cmd;
691 rpc->status = 0;
692 rpc->paramsSize = argc;
693 return rpc->params;
694}
695
176fdcbd
BS		 696static void
697r535_gsp_rpc_done(struct nvkm_gsp *gsp, void *repv)
698{
699 struct nvfw_gsp_rpc *rpc = container_of(repv, typeof(*rpc), data);
700
701 r535_gsp_msg_done(gsp, rpc);
702}
703
704static void *
705r535_gsp_rpc_get(struct nvkm_gsp *gsp, u32 fn, u32 argc)
706{
707 struct nvfw_gsp_rpc *rpc;
708
709 rpc = r535_gsp_cmdq_get(gsp, ALIGN(sizeof(*rpc) + argc, sizeof(u64)));
42bd415b
DC		 710 if (IS_ERR(rpc))
711 return ERR_CAST(rpc);
176fdcbd
BS		 712
713 rpc->header_version = 0x03000000;
5177e5fa 714 rpc->signature = ('C' << 24) | ('P' << 16) | ('R' << 8) | 'V';
176fdcbd
BS		 715 rpc->function = fn;
716 rpc->rpc_result = 0xffffffff;
717 rpc->rpc_result_private = 0xffffffff;
718 rpc->length = sizeof(*rpc) + argc;
719 return rpc->data;
720}
721
722static void *
723r535_gsp_rpc_push(struct nvkm_gsp *gsp, void *argv, bool wait, u32 repc)
724{
725 struct nvfw_gsp_rpc *rpc = container_of(argv, typeof(*rpc), data);
726 struct r535_gsp_msg *cmd = container_of((void *)rpc, typeof(*cmd), data);
727 const u32 max_msg_size = (16 * 0x1000) - sizeof(struct r535_gsp_msg);
728 const u32 max_rpc_size = max_msg_size - sizeof(*rpc);
729 u32 rpc_size = rpc->length - sizeof(*rpc);
730 void *repv;
731
732 mutex_lock(&gsp->cmdq.mutex);
733 if (rpc_size > max_rpc_size) {
734 const u32 fn = rpc->function;
735
736 /* Adjust length, and send initial RPC. */
737 rpc->length = sizeof(*rpc) + max_rpc_size;
738 cmd->checksum = rpc->length;
739
740 repv = r535_gsp_rpc_send(gsp, argv, false, 0);
741 if (IS_ERR(repv))
742 goto done;
743
744 argv += max_rpc_size;
745 rpc_size -= max_rpc_size;
746
747 /* Remaining chunks sent as CONTINUATION_RECORD RPCs. */
748 while (rpc_size) {
749 u32 size = min(rpc_size, max_rpc_size);
750 void *next;
751
752 next = r535_gsp_rpc_get(gsp, NV_VGPU_MSG_FUNCTION_CONTINUATION_RECORD, size);
753 if (IS_ERR(next)) {
754 repv = next;
755 goto done;
756 }
757
758 memcpy(next, argv, size);
759
760 repv = r535_gsp_rpc_send(gsp, next, false, 0);
761 if (IS_ERR(repv))
762 goto done;
763
764 argv += size;
765 rpc_size -= size;
766 }
767
768 /* Wait for reply. */
769 if (wait) {
770 rpc = r535_gsp_msg_recv(gsp, fn, repc);
771 if (!IS_ERR_OR_NULL(rpc))
772 repv = rpc->data;
773 else
774 repv = rpc;
775 } else {
776 repv = NULL;
777 }
778 } else {
779 repv = r535_gsp_rpc_send(gsp, argv, wait, repc);
780 }
781
782done:
783 mutex_unlock(&gsp->cmdq.mutex);
784 return repv;
785}
786
787const struct nvkm_gsp_rm
788r535_gsp_rm = {
789 .rpc_get = r535_gsp_rpc_get,
790 .rpc_push = r535_gsp_rpc_push,
791 .rpc_done = r535_gsp_rpc_done,
4cf2c83e
BS		 792
793 .rm_ctrl_get = r535_gsp_rpc_rm_ctrl_get,
794 .rm_ctrl_push = r535_gsp_rpc_rm_ctrl_push,
795 .rm_ctrl_done = r535_gsp_rpc_rm_ctrl_done,
37e328a1
BS		 796
797 .rm_alloc_get = r535_gsp_rpc_rm_alloc_get,
798 .rm_alloc_push = r535_gsp_rpc_rm_alloc_push,
799 .rm_alloc_done = r535_gsp_rpc_rm_alloc_done,
800
801 .rm_free = r535_gsp_rpc_rm_free,
802
803 .client_ctor = r535_gsp_client_ctor,
804 .client_dtor = r535_gsp_client_dtor,
805
806 .device_ctor = r535_gsp_device_ctor,
807 .device_dtor = r535_gsp_device_dtor,
9e994444
BS		 808
809 .event_ctor = r535_gsp_device_event_ctor,
810 .event_dtor = r535_gsp_event_dtor,
176fdcbd
BS		 811};
812
830531e9
BS		 813static void
814r535_gsp_msgq_work(struct work_struct *work)
815{
816 struct nvkm_gsp *gsp = container_of(work, typeof(*gsp), msgq.work);
817
818 mutex_lock(&gsp->cmdq.mutex);
819 if (*gsp->msgq.rptr != *gsp->msgq.wptr)
820 r535_gsp_msg_recv(gsp, 0, 0);
821 mutex_unlock(&gsp->cmdq.mutex);
822}
823
824static irqreturn_t
825r535_gsp_intr(struct nvkm_inth *inth)
826{
827 struct nvkm_gsp *gsp = container_of(inth, typeof(*gsp), subdev.inth);
828 struct nvkm_subdev *subdev = &gsp->subdev;
829 u32 intr = nvkm_falcon_rd32(&gsp->falcon, 0x0008);
830 u32 inte = nvkm_falcon_rd32(&gsp->falcon, gsp->falcon.func->addr2 +
831 gsp->falcon.func->riscv_irqmask);
832 u32 stat = intr & inte;
833
834 if (!stat) {
835 nvkm_debug(subdev, "inte %08x %08x\n", intr, inte);
836 return IRQ_NONE;
837 }
838
839 if (stat & 0x00000040) {
840 nvkm_falcon_wr32(&gsp->falcon, 0x004, 0x00000040);
841 schedule_work(&gsp->msgq.work);
842 stat &= ~0x00000040;
843 }
844
845 if (stat) {
846 nvkm_error(subdev, "intr %08x\n", stat);
847 nvkm_falcon_wr32(&gsp->falcon, 0x014, stat);
848 nvkm_falcon_wr32(&gsp->falcon, 0x004, stat);
849 }
850
851 nvkm_falcon_intr_retrigger(&gsp->falcon);
852 return IRQ_HANDLED;
853}
854
855static int
856r535_gsp_intr_get_table(struct nvkm_gsp *gsp)
857{
858 NV2080_CTRL_INTERNAL_INTR_GET_KERNEL_TABLE_PARAMS *ctrl;
859 int ret = 0;
860
861 ctrl = nvkm_gsp_rm_ctrl_get(&gsp->internal.device.subdevice,
862 NV2080_CTRL_CMD_INTERNAL_INTR_GET_KERNEL_TABLE, sizeof(*ctrl));
863 if (IS_ERR(ctrl))
864 return PTR_ERR(ctrl);
865
4ae3a201
DA		 866 ret = nvkm_gsp_rm_ctrl_push(&gsp->internal.device.subdevice, &ctrl, sizeof(*ctrl));
867 if (WARN_ON(ret)) {
868 nvkm_gsp_rm_ctrl_done(&gsp->internal.device.subdevice, ctrl);
869 return ret;
870 }
830531e9
BS		 871
872 for (unsigned i = 0; i < ctrl->tableLen; i++) {
873 enum nvkm_subdev_type type;
874 int inst;
875
876 nvkm_debug(&gsp->subdev,
877 "%2d: engineIdx %3d pmcIntrMask %08x stall %08x nonStall %08x\n", i,
878 ctrl->table[i].engineIdx, ctrl->table[i].pmcIntrMask,
879 ctrl->table[i].vectorStall, ctrl->table[i].vectorNonStall);
880
881 switch (ctrl->table[i].engineIdx) {
882 case MC_ENGINE_IDX_GSP:
883 type = NVKM_SUBDEV_GSP;
884 inst = 0;
885 break;
9e994444
BS		 886 case MC_ENGINE_IDX_DISP:
887 type = NVKM_ENGINE_DISP;
888 inst = 0;
889 break;
b5ce219a
BS		 890 case MC_ENGINE_IDX_CE0 ... MC_ENGINE_IDX_CE9:
891 type = NVKM_ENGINE_CE;
892 inst = ctrl->table[i].engineIdx - MC_ENGINE_IDX_CE0;
893 break;
361c3cd8
BS		 894 case MC_ENGINE_IDX_GR0:
895 type = NVKM_ENGINE_GR;
896 inst = 0;
897 break;
142cd602
BS		 898 case MC_ENGINE_IDX_NVDEC0 ... MC_ENGINE_IDX_NVDEC7:
899 type = NVKM_ENGINE_NVDEC;
900 inst = ctrl->table[i].engineIdx - MC_ENGINE_IDX_NVDEC0;
901 break;
08ab88f5
BS		 902 case MC_ENGINE_IDX_MSENC ... MC_ENGINE_IDX_MSENC2:
903 type = NVKM_ENGINE_NVENC;
904 inst = ctrl->table[i].engineIdx - MC_ENGINE_IDX_MSENC;
905 break;
ca968634
BS		 906 case MC_ENGINE_IDX_NVJPEG0 ... MC_ENGINE_IDX_NVJPEG7:
907 type = NVKM_ENGINE_NVJPG;
908 inst = ctrl->table[i].engineIdx - MC_ENGINE_IDX_NVJPEG0;
909 break;
015185cc
BS		 910 case MC_ENGINE_IDX_OFA0:
911 type = NVKM_ENGINE_OFA;
912 inst = 0;
913 break;
830531e9
BS		 914 default:
915 continue;
916 }
917
918 if (WARN_ON(gsp->intr_nr == ARRAY_SIZE(gsp->intr))) {
919 ret = -ENOSPC;
920 break;
921 }
922
923 gsp->intr[gsp->intr_nr].type = type;
924 gsp->intr[gsp->intr_nr].inst = inst;
925 gsp->intr[gsp->intr_nr].stall = ctrl->table[i].vectorStall;
926 gsp->intr[gsp->intr_nr].nonstall = ctrl->table[i].vectorNonStall;
927 gsp->intr_nr++;
928 }
929
930 nvkm_gsp_rm_ctrl_done(&gsp->internal.device.subdevice, ctrl);
931 return ret;
932}
933
4cf2c83e
BS		 934static int
935r535_gsp_rpc_get_gsp_static_info(struct nvkm_gsp *gsp)
936{
937 GspStaticConfigInfo *rpc;
5bf02571 938 int last_usable = -1;
4cf2c83e
BS		 939
940 rpc = nvkm_gsp_rpc_rd(gsp, NV_VGPU_MSG_FUNCTION_GET_GSP_STATIC_INFO, sizeof(*rpc));
941 if (IS_ERR(rpc))
942 return PTR_ERR(rpc);
943
944 gsp->internal.client.object.client = &gsp->internal.client;
945 gsp->internal.client.object.parent = NULL;
946 gsp->internal.client.object.handle = rpc->hInternalClient;
947 gsp->internal.client.gsp = gsp;
948
949 gsp->internal.device.object.client = &gsp->internal.client;
950 gsp->internal.device.object.parent = &gsp->internal.client.object;
951 gsp->internal.device.object.handle = rpc->hInternalDevice;
952
953 gsp->internal.device.subdevice.client = &gsp->internal.client;
954 gsp->internal.device.subdevice.parent = &gsp->internal.device.object;
955 gsp->internal.device.subdevice.handle = rpc->hInternalSubdevice;
956
5bf02571
BS		 957 gsp->bar.rm_bar1_pdb = rpc->bar1PdeBase;
958 gsp->bar.rm_bar2_pdb = rpc->bar2PdeBase;
959
960 for (int i = 0; i < rpc->fbRegionInfoParams.numFBRegions; i++) {
961 NV2080_CTRL_CMD_FB_GET_FB_REGION_FB_REGION_INFO *reg =
962 &rpc->fbRegionInfoParams.fbRegion[i];
963
964 nvkm_debug(&gsp->subdev, "fb region %d: "
965 "%016llx-%016llx rsvd:%016llx perf:%08x comp:%d iso:%d prot:%d\n", i,
966 reg->base, reg->limit, reg->reserved, reg->performance,
967 reg->supportCompressed, reg->supportISO, reg->bProtected);
968
969 if (!reg->reserved && !reg->bProtected) {
970 if (reg->supportCompressed && reg->supportISO &&
971 !WARN_ON_ONCE(gsp->fb.region_nr >= ARRAY_SIZE(gsp->fb.region))) {
972 const u64 size = (reg->limit + 1) - reg->base;
973
974 gsp->fb.region[gsp->fb.region_nr].addr = reg->base;
975 gsp->fb.region[gsp->fb.region_nr].size = size;
976 gsp->fb.region_nr++;
977 }
978
979 last_usable = i;
980 }
981 }
982
983 if (last_usable >= 0) {
984 u32 rsvd_base = rpc->fbRegionInfoParams.fbRegion[last_usable].limit + 1;
985
986 gsp->fb.rsvd_size = gsp->fb.heap.addr - rsvd_base;
987 }
988
361c3cd8
BS		 989 for (int gpc = 0; gpc < ARRAY_SIZE(rpc->tpcInfo); gpc++) {
990 if (rpc->gpcInfo.gpcMask & BIT(gpc)) {
991 gsp->gr.tpcs += hweight32(rpc->tpcInfo[gpc].tpcMask);
992 gsp->gr.gpcs++;
993 }
994 }
995
4cf2c83e
BS		 996 nvkm_gsp_rpc_done(gsp, rpc);
997 return 0;
998}
999
042b5f83
TT		 1000static void
1001nvkm_gsp_mem_dtor(struct nvkm_gsp *gsp, struct nvkm_gsp_mem *mem)
1002{
1003 if (mem->data) {
1004 /*
1005 * Poison the buffer to catch any unexpected access from
1006 * GSP-RM if the buffer was prematurely freed.
1007 */
1008 memset(mem->data, 0xFF, mem->size);
1009
1010 dma_free_coherent(gsp->subdev.device->dev, mem->size, mem->data, mem->addr);
1011 memset(mem, 0, sizeof(*mem));
1012 }
1013}
1014
1015static int
1016nvkm_gsp_mem_ctor(struct nvkm_gsp *gsp, size_t size, struct nvkm_gsp_mem *mem)
1017{
1018 mem->size = size;
1019 mem->data = dma_alloc_coherent(gsp->subdev.device->dev, size, &mem->addr, GFP_KERNEL);
1020 if (WARN_ON(!mem->data))
1021 return -ENOMEM;
1022
1023 return 0;
1024}
1025
4cf2c83e
BS		 1026static int
1027r535_gsp_postinit(struct nvkm_gsp *gsp)
1028{
830531e9 1029 struct nvkm_device *device = gsp->subdev.device;
4cf2c83e
BS		 1030 int ret;
1031
1032 ret = r535_gsp_rpc_get_gsp_static_info(gsp);
1033 if (WARN_ON(ret))
1034 return ret;
1035
830531e9
BS		 1036 INIT_WORK(&gsp->msgq.work, r535_gsp_msgq_work);
1037
1038 ret = r535_gsp_intr_get_table(gsp);
1039 if (WARN_ON(ret))
1040 return ret;
1041
1042 ret = nvkm_gsp_intr_stall(gsp, gsp->subdev.type, gsp->subdev.inst);
1043 if (WARN_ON(ret < 0))
1044 return ret;
1045
1046 ret = nvkm_inth_add(&device->vfn->intr, ret, NVKM_INTR_PRIO_NORMAL, &gsp->subdev,
1047 r535_gsp_intr, &gsp->subdev.inth);
1048 if (WARN_ON(ret))
1049 return ret;
1050
1051 nvkm_inth_allow(&gsp->subdev.inth);
1052 nvkm_wr32(device, 0x110004, 0x00000040);
042b5f83
TT		 1053
1054 /* Release the DMA buffers that were needed only for boot and init */
1055 nvkm_gsp_mem_dtor(gsp, &gsp->boot.fw);
1056 nvkm_gsp_mem_dtor(gsp, &gsp->libos);
042b5f83 1057
4cf2c83e
BS		 1058 return ret;
1059}
1060
176fdcbd
BS		 1061static int
1062r535_gsp_rpc_unloading_guest_driver(struct nvkm_gsp *gsp, bool suspend)
1063{
1064 rpc_unloading_guest_driver_v1F_07 *rpc;
1065
1066 rpc = nvkm_gsp_rpc_get(gsp, NV_VGPU_MSG_FUNCTION_UNLOADING_GUEST_DRIVER, sizeof(*rpc));
1067 if (IS_ERR(rpc))
1068 return PTR_ERR(rpc);
1069
1070 if (suspend) {
1071 rpc->bInPMTransition = 1;
1072 rpc->bGc6Entering = 0;
1073 rpc->newLevel = NV2080_CTRL_GPU_SET_POWER_STATE_GPU_LEVEL_3;
1074 } else {
1075 rpc->bInPMTransition = 0;
1076 rpc->bGc6Entering = 0;
1077 rpc->newLevel = NV2080_CTRL_GPU_SET_POWER_STATE_GPU_LEVEL_0;
1078 }
1079
1080 return nvkm_gsp_rpc_wr(gsp, rpc, true);
1081}
1082
8d55b0a9
DA		 1083/* dword only */
1084struct nv_gsp_registry_entries {
1085 const char *name;
1086 u32 value;
1087};
1088
1089static const struct nv_gsp_registry_entries r535_registry_entries[] = {
1090 { "RMSecBusResetEnable", 1 },
1091 { "RMForcePcieConfigSave", 1 },
1092};
1093#define NV_GSP_REG_NUM_ENTRIES ARRAY_SIZE(r535_registry_entries)
1094
176fdcbd
BS		 1095static int
1096r535_gsp_rpc_set_registry(struct nvkm_gsp *gsp)
1097{
1098 PACKED_REGISTRY_TABLE *rpc;
1099 char *strings;
8d55b0a9
DA		 1100 int str_offset;
1101 int i;
52fdb99c 1102 size_t rpc_size = struct_size(rpc, entries, NV_GSP_REG_NUM_ENTRIES);
8d55b0a9
DA		 1103
1104 /* add strings + null terminator */
1105 for (i = 0; i < NV_GSP_REG_NUM_ENTRIES; i++)
1106 rpc_size += strlen(r535_registry_entries[i].name) + 1;
176fdcbd 1107
8d55b0a9 1108 rpc = nvkm_gsp_rpc_get(gsp, NV_VGPU_MSG_FUNCTION_SET_REGISTRY, rpc_size);
176fdcbd
BS		 1109 if (IS_ERR(rpc))
1110 return PTR_ERR(rpc);
1111
8d55b0a9
DA		 1112 rpc->numEntries = NV_GSP_REG_NUM_ENTRIES;
1113
1114 str_offset = offsetof(typeof(*rpc), entries[NV_GSP_REG_NUM_ENTRIES]);
1115 strings = (char *)&rpc->entries[NV_GSP_REG_NUM_ENTRIES];
1116 for (i = 0; i < NV_GSP_REG_NUM_ENTRIES; i++) {
1117 int name_len = strlen(r535_registry_entries[i].name) + 1;
1118
1119 rpc->entries[i].nameOffset = str_offset;
1120 rpc->entries[i].type = 1;
1121 rpc->entries[i].data = r535_registry_entries[i].value;
1122 rpc->entries[i].length = 4;
1123 memcpy(strings, r535_registry_entries[i].name, name_len);
1124 strings += name_len;
1125 str_offset += name_len;
1126 }
61712c94 1127 rpc->size = str_offset;
176fdcbd
BS		 1128
1129 return nvkm_gsp_rpc_wr(gsp, rpc, false);
1130}
1131
1132#if defined(CONFIG_ACPI) && defined(CONFIG_X86)
1133static void
1134r535_gsp_acpi_caps(acpi_handle handle, CAPS_METHOD_DATA *caps)
1135{
1136 const guid_t NVOP_DSM_GUID =
1137 GUID_INIT(0xA486D8F8, 0x0BDA, 0x471B,
1138 0xA7, 0x2B, 0x60, 0x42, 0xA6, 0xB5, 0xBE, 0xE0);
1139 u64 NVOP_DSM_REV = 0x00000100;
1140 union acpi_object argv4 = {
1141 .buffer.type = ACPI_TYPE_BUFFER,
1142 .buffer.length = 4,
1143 .buffer.pointer = kmalloc(argv4.buffer.length, GFP_KERNEL),
1144 }, *obj;
1145
1146 caps->status = 0xffff;
1147
1148 if (!acpi_check_dsm(handle, &NVOP_DSM_GUID, NVOP_DSM_REV, BIT_ULL(0x1a)))
1149 return;
1150
1151 obj = acpi_evaluate_dsm(handle, &NVOP_DSM_GUID, NVOP_DSM_REV, 0x1a, &argv4);
1152 if (!obj)
1153 return;
1154
176fdcbd
BS		 1155 if (WARN_ON(obj->type != ACPI_TYPE_BUFFER) ||
1156 WARN_ON(obj->buffer.length != 4))
1157 return;
1158
1159 caps->status = 0;
1160 caps->optimusCaps = *(u32 *)obj->buffer.pointer;
176fdcbd
BS		 1161
1162 ACPI_FREE(obj);
1163
1164 kfree(argv4.buffer.pointer);
1165}
1166
1167static void
1168r535_gsp_acpi_jt(acpi_handle handle, JT_METHOD_DATA *jt)
1169{
1170 const guid_t JT_DSM_GUID =
1171 GUID_INIT(0xCBECA351L, 0x067B, 0x4924,
1172 0x9C, 0xBD, 0xB4, 0x6B, 0x00, 0xB8, 0x6F, 0x34);
1173 u64 JT_DSM_REV = 0x00000103;
1174 u32 caps;
1175 union acpi_object argv4 = {
1176 .buffer.type = ACPI_TYPE_BUFFER,
1177 .buffer.length = sizeof(caps),
1178 .buffer.pointer = kmalloc(argv4.buffer.length, GFP_KERNEL),
1179 }, *obj;
1180
1181 jt->status = 0xffff;
1182
1183 obj = acpi_evaluate_dsm(handle, &JT_DSM_GUID, JT_DSM_REV, 0x1, &argv4);
1184 if (!obj)
1185 return;
1186
176fdcbd
BS		 1187 if (WARN_ON(obj->type != ACPI_TYPE_BUFFER) ||
1188 WARN_ON(obj->buffer.length != 4))
1189 return;
1190
1191 jt->status = 0;
1192 jt->jtCaps = *(u32 *)obj->buffer.pointer;
1193 jt->jtRevId = (jt->jtCaps & 0xfff00000) >> 20;
1194 jt->bSBIOSCaps = 0;
176fdcbd
BS		 1195
1196 ACPI_FREE(obj);
1197
1198 kfree(argv4.buffer.pointer);
1199}
1200
1201static void
1202r535_gsp_acpi_mux_id(acpi_handle handle, u32 id, MUX_METHOD_DATA_ELEMENT *mode,
1203 MUX_METHOD_DATA_ELEMENT *part)
1204{
cf22fc28
LP		 1205 union acpi_object mux_arg = { ACPI_TYPE_INTEGER };
1206 struct acpi_object_list input = { 1, &mux_arg };
09f12bf9 1207 acpi_handle iter = NULL, handle_mux = NULL;
176fdcbd
BS		 1208 acpi_status status;
1209 unsigned long long value;
1210
1211 mode->status = 0xffff;
1212 part->status = 0xffff;
1213
1214 do {
1215 status = acpi_get_next_object(ACPI_TYPE_DEVICE, handle, iter, &iter);
1216 if (ACPI_FAILURE(status) || !iter)
1217 return;
1218
1219 status = acpi_evaluate_integer(iter, "_ADR", NULL, &value);
1220 if (ACPI_FAILURE(status) || value != id)
1221 continue;
1222
1223 handle_mux = iter;
1224 } while (!handle_mux);
1225
1226 if (!handle_mux)
1227 return;
1228
cf22fc28
LP		 1229 /* I -think- 0 means "acquire" according to nvidia's driver source */
1230 input.pointer->integer.type = ACPI_TYPE_INTEGER;
1231 input.pointer->integer.value = 0;
1232
1233 status = acpi_evaluate_integer(handle_mux, "MXDM", &input, &value);
176fdcbd
BS		 1234 if (ACPI_SUCCESS(status)) {
1235 mode->acpiId = id;
1236 mode->mode = value;
1237 mode->status = 0;
1238 }
1239
cf22fc28 1240 status = acpi_evaluate_integer(handle_mux, "MXDS", &input, &value);
176fdcbd
BS		 1241 if (ACPI_SUCCESS(status)) {
1242 part->acpiId = id;
1243 part->mode = value;
1244 part->status = 0;
1245 }
1246}
1247
1248static void
1249r535_gsp_acpi_mux(acpi_handle handle, DOD_METHOD_DATA *dod, MUX_METHOD_DATA *mux)
1250{
1251 mux->tableLen = dod->acpiIdListLen / sizeof(dod->acpiIdList[0]);
1252
1253 for (int i = 0; i < mux->tableLen; i++) {
1254 r535_gsp_acpi_mux_id(handle, dod->acpiIdList[i], &mux->acpiIdMuxModeTable[i],
1255 &mux->acpiIdMuxPartTable[i]);
1256 }
1257}
1258
1259static void
1260r535_gsp_acpi_dod(acpi_handle handle, DOD_METHOD_DATA *dod)
1261{
1262 acpi_status status;
1263 struct acpi_buffer output = { ACPI_ALLOCATE_BUFFER, NULL };
1264 union acpi_object *_DOD;
1265
1266 dod->status = 0xffff;
1267
1268 status = acpi_evaluate_object(handle, "_DOD", NULL, &output);
1269 if (ACPI_FAILURE(status))
1270 return;
1271
1272 _DOD = output.pointer;
1273
1274 if (WARN_ON(_DOD->type != ACPI_TYPE_PACKAGE) ||
1275 WARN_ON(_DOD->package.count > ARRAY_SIZE(dod->acpiIdList)))
1276 return;
1277
1278 for (int i = 0; i < _DOD->package.count; i++) {
1279 if (WARN_ON(_DOD->package.elements[i].type != ACPI_TYPE_INTEGER))
1280 return;
1281
1282 dod->acpiIdList[i] = _DOD->package.elements[i].integer.value;
1283 dod->acpiIdListLen += sizeof(dod->acpiIdList[0]);
1284 }
1285
176fdcbd 1286 dod->status = 0;
a9b9b42b 1287 kfree(output.pointer);
176fdcbd
BS		 1288}
1289#endif
1290
1291static void
1292r535_gsp_acpi_info(struct nvkm_gsp *gsp, ACPI_METHOD_DATA *acpi)
1293{
1294#if defined(CONFIG_ACPI) && defined(CONFIG_X86)
1295 acpi_handle handle = ACPI_HANDLE(gsp->subdev.device->dev);
1296
1297 if (!handle)
1298 return;
1299
1300 acpi->bValid = 1;
1301
1302 r535_gsp_acpi_dod(handle, &acpi->dodMethodData);
1303 if (acpi->dodMethodData.status == 0)
1304 r535_gsp_acpi_mux(handle, &acpi->dodMethodData, &acpi->muxMethodData);
1305
1306 r535_gsp_acpi_jt(handle, &acpi->jtMethodData);
1307 r535_gsp_acpi_caps(handle, &acpi->capsMethodData);
1308#endif
1309}
1310
1311static int
1312r535_gsp_rpc_set_system_info(struct nvkm_gsp *gsp)
1313{
1314 struct nvkm_device *device = gsp->subdev.device;
1315 struct nvkm_device_pci *pdev = container_of(device, typeof(*pdev), device);
1316 GspSystemInfo *info;
1317
1318 if (WARN_ON(device->type == NVKM_DEVICE_TEGRA))
1319 return -ENOSYS;
1320
1321 info = nvkm_gsp_rpc_get(gsp, NV_VGPU_MSG_FUNCTION_GSP_SET_SYSTEM_INFO, sizeof(*info));
1322 if (IS_ERR(info))
1323 return PTR_ERR(info);
1324
1325 info->gpuPhysAddr = device->func->resource_addr(device, 0);
1326 info->gpuPhysFbAddr = device->func->resource_addr(device, 1);
1327 info->gpuPhysInstAddr = device->func->resource_addr(device, 3);
1328 info->nvDomainBusDeviceFunc = pci_dev_id(pdev->pdev);
1329 info->maxUserVa = TASK_SIZE;
1330 info->pciConfigMirrorBase = 0x088000;
1331 info->pciConfigMirrorSize = 0x001000;
1332 r535_gsp_acpi_info(gsp, &info->acpiMethodData);
1333
1334 return nvkm_gsp_rpc_wr(gsp, info, false);
1335}
1336
1337static int
1338r535_gsp_msg_os_error_log(void *priv, u32 fn, void *repv, u32 repc)
1339{
1340 struct nvkm_gsp *gsp = priv;
1341 struct nvkm_subdev *subdev = &gsp->subdev;
1342 rpc_os_error_log_v17_00 *msg = repv;
1343
1344 if (WARN_ON(repc < sizeof(*msg)))
1345 return -EINVAL;
1346
1347 nvkm_error(subdev, "Xid:%d %s\n", msg->exceptType, msg->errString);
1348 return 0;
1349}
1350
2a77d015
BS		 1351static int
1352r535_gsp_msg_rc_triggered(void *priv, u32 fn, void *repv, u32 repc)
1353{
1354 rpc_rc_triggered_v17_02 *msg = repv;
1355 struct nvkm_gsp *gsp = priv;
1356 struct nvkm_subdev *subdev = &gsp->subdev;
1357 struct nvkm_chan *chan;
1358 unsigned long flags;
1359
1360 if (WARN_ON(repc < sizeof(*msg)))
1361 return -EINVAL;
1362
1363 nvkm_error(subdev, "rc engn:%08x chid:%d type:%d scope:%d part:%d\n",
1364 msg->nv2080EngineType, msg->chid, msg->exceptType, msg->scope,
1365 msg->partitionAttributionId);
1366
1367 chan = nvkm_chan_get_chid(&subdev->device->fifo->engine, msg->chid / 8, &flags);
1368 if (!chan) {
1369 nvkm_error(subdev, "rc chid:%d not found!\n", msg->chid);
1370 return 0;
1371 }
1372
1373 nvkm_chan_error(chan, false);
1374 nvkm_chan_put(&chan, flags);
1375 return 0;
1376}
1377
5bf02571
BS		 1378static int
1379r535_gsp_msg_mmu_fault_queued(void *priv, u32 fn, void *repv, u32 repc)
1380{
1381 struct nvkm_gsp *gsp = priv;
1382 struct nvkm_subdev *subdev = &gsp->subdev;
1383
1384 WARN_ON(repc != 0);
1385
1386 nvkm_error(subdev, "mmu fault queued\n");
1387 return 0;
1388}
1389
9e994444
BS		 1390static int
1391r535_gsp_msg_post_event(void *priv, u32 fn, void *repv, u32 repc)
1392{
1393 struct nvkm_gsp *gsp = priv;
1394 struct nvkm_gsp_client *client;
1395 struct nvkm_subdev *subdev = &gsp->subdev;
1396 rpc_post_event_v17_00 *msg = repv;
1397
1398 if (WARN_ON(repc < sizeof(*msg)))
1399 return -EINVAL;
1400 if (WARN_ON(repc != sizeof(*msg) + msg->eventDataSize))
1401 return -EINVAL;
1402
1403 nvkm_debug(subdev, "event: %08x %08x %d %08x %08x %d %d\n",
1404 msg->hClient, msg->hEvent, msg->notifyIndex, msg->data,
1405 msg->status, msg->eventDataSize, msg->bNotifyList);
1406
1407 mutex_lock(&gsp->client_id.mutex);
1408 client = idr_find(&gsp->client_id.idr, msg->hClient & 0xffff);
1409 if (client) {
1410 struct nvkm_gsp_event *event;
1411 bool handled = false;
1412
1413 list_for_each_entry(event, &client->events, head) {
1414 if (event->object.handle == msg->hEvent) {
1415 event->func(event, msg->eventData, msg->eventDataSize);
1416 handled = true;
1417 }
1418 }
1419
1420 if (!handled) {
1421 nvkm_error(subdev, "event: cid 0x%08x event 0x%08x not found!\n",
1422 msg->hClient, msg->hEvent);
1423 }
1424 } else {
1425 nvkm_error(subdev, "event: cid 0x%08x not found!\n", msg->hClient);
1426 }
1427 mutex_unlock(&gsp->client_id.mutex);
1428 return 0;
1429}
1430
88a2b4d3
TT		 1431/**
1432 * r535_gsp_msg_run_cpu_sequencer() -- process I/O commands from the GSP
1433 *
1434 * The GSP sequencer is a list of I/O commands that the GSP can send to
1435 * the driver to perform for various purposes. The most common usage is to
1436 * perform a special mid-initialization reset.
1437 */
176fdcbd
BS		 1438static int
1439r535_gsp_msg_run_cpu_sequencer(void *priv, u32 fn, void *repv, u32 repc)
1440{
1441 struct nvkm_gsp *gsp = priv;
1442 struct nvkm_subdev *subdev = &gsp->subdev;
1443 struct nvkm_device *device = subdev->device;
1444 rpc_run_cpu_sequencer_v17_00 *seq = repv;
1445 int ptr = 0, ret;
1446
1447 nvkm_debug(subdev, "seq: %08x %08x\n", seq->bufferSizeDWord, seq->cmdIndex);
1448
1449 while (ptr < seq->cmdIndex) {
1450 GSP_SEQUENCER_BUFFER_CMD *cmd = (void *)&seq->commandBuffer[ptr];
1451
1452 ptr += 1;
1453 ptr += GSP_SEQUENCER_PAYLOAD_SIZE_DWORDS(cmd->opCode);
1454
1455 switch (cmd->opCode) {
1456 case GSP_SEQ_BUF_OPCODE_REG_WRITE: {
1457 u32 addr = cmd->payload.regWrite.addr;
1458 u32 data = cmd->payload.regWrite.val;
1459
1460 nvkm_trace(subdev, "seq wr32 %06x %08x\n", addr, data);
1461 nvkm_wr32(device, addr, data);
1462 }
1463 break;
1464 case GSP_SEQ_BUF_OPCODE_REG_MODIFY: {
1465 u32 addr = cmd->payload.regModify.addr;
1466 u32 mask = cmd->payload.regModify.mask;
1467 u32 data = cmd->payload.regModify.val;
1468
1469 nvkm_trace(subdev, "seq mask %06x %08x %08x\n", addr, mask, data);
1470 nvkm_mask(device, addr, mask, data);
1471 }
1472 break;
1473 case GSP_SEQ_BUF_OPCODE_REG_POLL: {
1474 u32 addr = cmd->payload.regPoll.addr;
1475 u32 mask = cmd->payload.regPoll.mask;
1476 u32 data = cmd->payload.regPoll.val;
1477 u32 usec = cmd->payload.regPoll.timeout ?: 4000000;
1478 //u32 error = cmd->payload.regPoll.error;
1479
1480 nvkm_trace(subdev, "seq poll %06x %08x %08x %d\n", addr, mask, data, usec);
1481 nvkm_rd32(device, addr);
1482 nvkm_usec(device, usec,
1483 if ((nvkm_rd32(device, addr) & mask) == data)
1484 break;
1485 );
1486 }
1487 break;
1488 case GSP_SEQ_BUF_OPCODE_DELAY_US: {
1489 u32 usec = cmd->payload.delayUs.val;
1490
1491 nvkm_trace(subdev, "seq usec %d\n", usec);
1492 udelay(usec);
1493 }
1494 break;
1495 case GSP_SEQ_BUF_OPCODE_REG_STORE: {
1496 u32 addr = cmd->payload.regStore.addr;
1497 u32 slot = cmd->payload.regStore.index;
1498
1499 seq->regSaveArea[slot] = nvkm_rd32(device, addr);
1500 nvkm_trace(subdev, "seq save %08x -> %d: %08x\n", addr, slot,
1501 seq->regSaveArea[slot]);
1502 }
1503 break;
1504 case GSP_SEQ_BUF_OPCODE_CORE_RESET:
1505 nvkm_trace(subdev, "seq core reset\n");
1506 nvkm_falcon_reset(&gsp->falcon);
1507 nvkm_falcon_mask(&gsp->falcon, 0x624, 0x00000080, 0x00000080);
1508 nvkm_falcon_wr32(&gsp->falcon, 0x10c, 0x00000000);
1509 break;
1510 case GSP_SEQ_BUF_OPCODE_CORE_START:
1511 nvkm_trace(subdev, "seq core start\n");
1512 if (nvkm_falcon_rd32(&gsp->falcon, 0x100) & 0x00000040)
1513 nvkm_falcon_wr32(&gsp->falcon, 0x130, 0x00000002);
1514 else
1515 nvkm_falcon_wr32(&gsp->falcon, 0x100, 0x00000002);
1516 break;
1517 case GSP_SEQ_BUF_OPCODE_CORE_WAIT_FOR_HALT:
1518 nvkm_trace(subdev, "seq core wait halt\n");
1519 nvkm_msec(device, 2000,
1520 if (nvkm_falcon_rd32(&gsp->falcon, 0x100) & 0x00000010)
1521 break;
1522 );
1523 break;
1524 case GSP_SEQ_BUF_OPCODE_CORE_RESUME: {
1525 struct nvkm_sec2 *sec2 = device->sec2;
1526 u32 mbox0;
1527
1528 nvkm_trace(subdev, "seq core resume\n");
1529
1530 ret = gsp->func->reset(gsp);
1531 if (WARN_ON(ret))
1532 return ret;
1533
1534 nvkm_falcon_wr32(&gsp->falcon, 0x040, lower_32_bits(gsp->libos.addr));
1535 nvkm_falcon_wr32(&gsp->falcon, 0x044, upper_32_bits(gsp->libos.addr));
1536
1537 nvkm_falcon_start(&sec2->falcon);
1538
1539 if (nvkm_msec(device, 2000,
1540 if (nvkm_rd32(device, 0x1180f8) & 0x04000000)
1541 break;
1542 ) < 0)
1543 return -ETIMEDOUT;
1544
1545 mbox0 = nvkm_falcon_rd32(&sec2->falcon, 0x040);
1546 if (WARN_ON(mbox0)) {
1547 nvkm_error(&gsp->subdev, "seq core resume sec2: 0x%x\n", mbox0);
1548 return -EIO;
1549 }
1550
1551 nvkm_falcon_wr32(&gsp->falcon, 0x080, gsp->boot.app_version);
1552
1553 if (WARN_ON(!nvkm_falcon_riscv_active(&gsp->falcon)))
1554 return -EIO;
1555 }
1556 break;
1557 default:
1558 nvkm_error(subdev, "unknown sequencer opcode %08x\n", cmd->opCode);
1559 return -EINVAL;
1560 }
1561 }
1562
1563 return 0;
1564}
1565
176fdcbd
BS		 1566static int
1567r535_gsp_booter_unload(struct nvkm_gsp *gsp, u32 mbox0, u32 mbox1)
1568{
1569 struct nvkm_subdev *subdev = &gsp->subdev;
1570 struct nvkm_device *device = subdev->device;
1571 u32 wpr2_hi;
1572 int ret;
1573
1574 wpr2_hi = nvkm_rd32(device, 0x1fa828);
1575 if (!wpr2_hi) {
1576 nvkm_debug(subdev, "WPR2 not set - skipping booter unload\n");
1577 return 0;
1578 }
1579
1580 ret = nvkm_falcon_fw_boot(&gsp->booter.unload, &gsp->subdev, true, &mbox0, &mbox1, 0, 0);
1581 if (WARN_ON(ret))
1582 return ret;
1583
1584 wpr2_hi = nvkm_rd32(device, 0x1fa828);
1585 if (WARN_ON(wpr2_hi))
1586 return -EIO;
1587
1588 return 0;
1589}
1590
1591static int
1592r535_gsp_booter_load(struct nvkm_gsp *gsp, u32 mbox0, u32 mbox1)
1593{
1594 int ret;
1595
1596 ret = nvkm_falcon_fw_boot(&gsp->booter.load, &gsp->subdev, true, &mbox0, &mbox1, 0, 0);
1597 if (ret)
1598 return ret;
1599
1600 nvkm_falcon_wr32(&gsp->falcon, 0x080, gsp->boot.app_version);
1601
1602 if (WARN_ON(!nvkm_falcon_riscv_active(&gsp->falcon)))
1603 return -EIO;
1604
1605 return 0;
1606}
1607
1608static int
1609r535_gsp_wpr_meta_init(struct nvkm_gsp *gsp)
1610{
1611 GspFwWprMeta *meta;
1612 int ret;
1613
1614 ret = nvkm_gsp_mem_ctor(gsp, 0x1000, &gsp->wpr_meta);
1615 if (ret)
1616 return ret;
1617
1618 meta = gsp->wpr_meta.data;
1619
1620 meta->magic = GSP_FW_WPR_META_MAGIC;
1621 meta->revision = GSP_FW_WPR_META_REVISION;
1622
1623 meta->sysmemAddrOfRadix3Elf = gsp->radix3.mem[0].addr;
1624 meta->sizeOfRadix3Elf = gsp->fb.wpr2.elf.size;
1625
1626 meta->sysmemAddrOfBootloader = gsp->boot.fw.addr;
1627 meta->sizeOfBootloader = gsp->boot.fw.size;
1628 meta->bootloaderCodeOffset = gsp->boot.code_offset;
1629 meta->bootloaderDataOffset = gsp->boot.data_offset;
1630 meta->bootloaderManifestOffset = gsp->boot.manifest_offset;
1631
1632 meta->sysmemAddrOfSignature = gsp->sig.addr;
1633 meta->sizeOfSignature = gsp->sig.size;
1634
1635 meta->gspFwRsvdStart = gsp->fb.heap.addr;
1636 meta->nonWprHeapOffset = gsp->fb.heap.addr;
1637 meta->nonWprHeapSize = gsp->fb.heap.size;
1638 meta->gspFwWprStart = gsp->fb.wpr2.addr;
1639 meta->gspFwHeapOffset = gsp->fb.wpr2.heap.addr;
1640 meta->gspFwHeapSize = gsp->fb.wpr2.heap.size;
1641 meta->gspFwOffset = gsp->fb.wpr2.elf.addr;
1642 meta->bootBinOffset = gsp->fb.wpr2.boot.addr;
1643 meta->frtsOffset = gsp->fb.wpr2.frts.addr;
1644 meta->frtsSize = gsp->fb.wpr2.frts.size;
1645 meta->gspFwWprEnd = ALIGN_DOWN(gsp->fb.bios.vga_workspace.addr, 0x20000);
1646 meta->fbSize = gsp->fb.size;
1647 meta->vgaWorkspaceOffset = gsp->fb.bios.vga_workspace.addr;
1648 meta->vgaWorkspaceSize = gsp->fb.bios.vga_workspace.size;
1649 meta->bootCount = 0;
1650 meta->partitionRpcAddr = 0;
1651 meta->partitionRpcRequestOffset = 0;
1652 meta->partitionRpcReplyOffset = 0;
1653 meta->verified = 0;
1654 return 0;
1655}
1656
1657static int
1658r535_gsp_shared_init(struct nvkm_gsp *gsp)
1659{
1660 struct {
1661 msgqTxHeader tx;
1662 msgqRxHeader rx;
1663 } *cmdq, *msgq;
1664 int ret, i;
1665
1666 gsp->shm.cmdq.size = 0x40000;
1667 gsp->shm.msgq.size = 0x40000;
1668
1669 gsp->shm.ptes.nr = (gsp->shm.cmdq.size + gsp->shm.msgq.size) >> GSP_PAGE_SHIFT;
1670 gsp->shm.ptes.nr += DIV_ROUND_UP(gsp->shm.ptes.nr * sizeof(u64), GSP_PAGE_SIZE);
1671 gsp->shm.ptes.size = ALIGN(gsp->shm.ptes.nr * sizeof(u64), GSP_PAGE_SIZE);
1672
1673 ret = nvkm_gsp_mem_ctor(gsp, gsp->shm.ptes.size +
1674 gsp->shm.cmdq.size +
1675 gsp->shm.msgq.size,
1676 &gsp->shm.mem);
1677 if (ret)
1678 return ret;
1679
1680 gsp->shm.ptes.ptr = gsp->shm.mem.data;
1681 gsp->shm.cmdq.ptr = (u8 *)gsp->shm.ptes.ptr + gsp->shm.ptes.size;
1682 gsp->shm.msgq.ptr = (u8 *)gsp->shm.cmdq.ptr + gsp->shm.cmdq.size;
1683
1684 for (i = 0; i < gsp->shm.ptes.nr; i++)
1685 gsp->shm.ptes.ptr[i] = gsp->shm.mem.addr + (i << GSP_PAGE_SHIFT);
1686
1687 cmdq = gsp->shm.cmdq.ptr;
1688 cmdq->tx.version = 0;
1689 cmdq->tx.size = gsp->shm.cmdq.size;
1690 cmdq->tx.entryOff = GSP_PAGE_SIZE;
1691 cmdq->tx.msgSize = GSP_PAGE_SIZE;
1692 cmdq->tx.msgCount = (cmdq->tx.size - cmdq->tx.entryOff) / cmdq->tx.msgSize;
1693 cmdq->tx.writePtr = 0;
1694 cmdq->tx.flags = 1;
1695 cmdq->tx.rxHdrOff = offsetof(typeof(*cmdq), rx.readPtr);
1696
1697 msgq = gsp->shm.msgq.ptr;
1698
1699 gsp->cmdq.cnt = cmdq->tx.msgCount;
1700 gsp->cmdq.wptr = &cmdq->tx.writePtr;
1701 gsp->cmdq.rptr = &msgq->rx.readPtr;
1702 gsp->msgq.cnt = cmdq->tx.msgCount;
1703 gsp->msgq.wptr = &msgq->tx.writePtr;
1704 gsp->msgq.rptr = &cmdq->rx.readPtr;
1705 return 0;
1706}
1707
1708static int
1709r535_gsp_rmargs_init(struct nvkm_gsp *gsp, bool resume)
1710{
1711 GSP_ARGUMENTS_CACHED *args;
1712 int ret;
1713
1714 if (!resume) {
1715 ret = r535_gsp_shared_init(gsp);
1716 if (ret)
1717 return ret;
1718
1719 ret = nvkm_gsp_mem_ctor(gsp, 0x1000, &gsp->rmargs);
1720 if (ret)
1721 return ret;
1722 }
1723
1724 args = gsp->rmargs.data;
1725 args->messageQueueInitArguments.sharedMemPhysAddr = gsp->shm.mem.addr;
1726 args->messageQueueInitArguments.pageTableEntryCount = gsp->shm.ptes.nr;
1727 args->messageQueueInitArguments.cmdQueueOffset =
1728 (u8 *)gsp->shm.cmdq.ptr - (u8 *)gsp->shm.mem.data;
1729 args->messageQueueInitArguments.statQueueOffset =
1730 (u8 *)gsp->shm.msgq.ptr - (u8 *)gsp->shm.mem.data;
1731
1732 if (!resume) {
1733 args->srInitArguments.oldLevel = 0;
1734 args->srInitArguments.flags = 0;
1735 args->srInitArguments.bInPMTransition = 0;
1736 } else {
1737 args->srInitArguments.oldLevel = NV2080_CTRL_GPU_SET_POWER_STATE_GPU_LEVEL_3;
1738 args->srInitArguments.flags = 0;
1739 args->srInitArguments.bInPMTransition = 1;
1740 }
1741
1742 return 0;
1743}
1744
1745static inline u64
1746r535_gsp_libos_id8(const char *name)
1747{
1748 u64 id = 0;
1749
1750 for (int i = 0; i < sizeof(id) && *name; i++, name++)
1751 id = (id << 8) | *name;
1752
1753 return id;
1754}
1755
88a2b4d3
TT		 1756/**
1757 * create_pte_array() - creates a PTE array of a physically contiguous buffer
1758 * @ptes: pointer to the array
1759 * @addr: base address of physically contiguous buffer (GSP_PAGE_SIZE aligned)
1760 * @size: size of the buffer
1761 *
1762 * GSP-RM sometimes expects physically-contiguous buffers to have an array of
1763 * "PTEs" for each page in that buffer. Although in theory that allows for
1764 * the buffer to be physically discontiguous, GSP-RM does not currently
1765 * support that.
1766 *
1767 * In this case, the PTEs are DMA addresses of each page of the buffer. Since
1768 * the buffer is physically contiguous, calculating all the PTEs is simple
1769 * math.
1770 *
1771 * See memdescGetPhysAddrsForGpu()
1772 */
176fdcbd
BS		 1773static void create_pte_array(u64 *ptes, dma_addr_t addr, size_t size)
1774{
1775 unsigned int num_pages = DIV_ROUND_UP_ULL(size, GSP_PAGE_SIZE);
1776 unsigned int i;
1777
1778 for (i = 0; i < num_pages; i++)
1779 ptes[i] = (u64)addr + (i << GSP_PAGE_SHIFT);
1780}
1781
88a2b4d3
TT		 1782/**
1783 * r535_gsp_libos_init() -- create the libos arguments structure
1784 *
1785 * The logging buffers are byte queues that contain encoded printf-like
1786 * messages from GSP-RM. They need to be decoded by a special application
1787 * that can parse the buffers.
1788 *
1789 * The 'loginit' buffer contains logs from early GSP-RM init and
1790 * exception dumps. The 'logrm' buffer contains the subsequent logs. Both are
1791 * written to directly by GSP-RM and can be any multiple of GSP_PAGE_SIZE.
1792 *
1793 * The physical address map for the log buffer is stored in the buffer
1794 * itself, starting with offset 1. Offset 0 contains the "put" pointer.
1795 *
1796 * The GSP only understands 4K pages (GSP_PAGE_SIZE), so even if the kernel is
1797 * configured for a larger page size (e.g. 64K pages), we need to give
1798 * the GSP an array of 4K pages. Fortunately, since the buffer is
1799 * physically contiguous, it's simple math to calculate the addresses.
1800 *
1801 * The buffers must be a multiple of GSP_PAGE_SIZE. GSP-RM also currently
1802 * ignores the @kind field for LOGINIT, LOGINTR, and LOGRM, but expects the
1803 * buffers to be physically contiguous anyway.
1804 *
1805 * The memory allocated for the arguments must remain until the GSP sends the
1806 * init_done RPC.
1807 *
1808 * See _kgspInitLibosLoggingStructures (allocates memory for buffers)
1809 * See kgspSetupLibosInitArgs_IMPL (creates pLibosInitArgs[] array)
1810 */
176fdcbd
BS		 1811static int
1812r535_gsp_libos_init(struct nvkm_gsp *gsp)
1813{
1814 LibosMemoryRegionInitArgument *args;
1815 int ret;
1816
1817 ret = nvkm_gsp_mem_ctor(gsp, 0x1000, &gsp->libos);
1818 if (ret)
1819 return ret;
1820
1821 args = gsp->libos.data;
1822
1823 ret = nvkm_gsp_mem_ctor(gsp, 0x10000, &gsp->loginit);
1824 if (ret)
1825 return ret;
1826
1827 args[0].id8 = r535_gsp_libos_id8("LOGINIT");
1828 args[0].pa = gsp->loginit.addr;
1829 args[0].size = gsp->loginit.size;
1830 args[0].kind = LIBOS_MEMORY_REGION_CONTIGUOUS;
1831 args[0].loc = LIBOS_MEMORY_REGION_LOC_SYSMEM;
1832 create_pte_array(gsp->loginit.data + sizeof(u64), gsp->loginit.addr, gsp->loginit.size);
1833
1834 ret = nvkm_gsp_mem_ctor(gsp, 0x10000, &gsp->logintr);
1835 if (ret)
1836 return ret;
1837
1838 args[1].id8 = r535_gsp_libos_id8("LOGINTR");
1839 args[1].pa = gsp->logintr.addr;
1840 args[1].size = gsp->logintr.size;
1841 args[1].kind = LIBOS_MEMORY_REGION_CONTIGUOUS;
1842 args[1].loc = LIBOS_MEMORY_REGION_LOC_SYSMEM;
1843 create_pte_array(gsp->logintr.data + sizeof(u64), gsp->logintr.addr, gsp->logintr.size);
1844
1845 ret = nvkm_gsp_mem_ctor(gsp, 0x10000, &gsp->logrm);
1846 if (ret)
1847 return ret;
1848
1849 args[2].id8 = r535_gsp_libos_id8("LOGRM");
1850 args[2].pa = gsp->logrm.addr;
1851 args[2].size = gsp->logrm.size;
1852 args[2].kind = LIBOS_MEMORY_REGION_CONTIGUOUS;
1853 args[2].loc = LIBOS_MEMORY_REGION_LOC_SYSMEM;
1854 create_pte_array(gsp->logrm.data + sizeof(u64), gsp->logrm.addr, gsp->logrm.size);
1855
1856 ret = r535_gsp_rmargs_init(gsp, false);
1857 if (ret)
1858 return ret;
1859
1860 args[3].id8 = r535_gsp_libos_id8("RMARGS");
1861 args[3].pa = gsp->rmargs.addr;
1862 args[3].size = gsp->rmargs.size;
1863 args[3].kind = LIBOS_MEMORY_REGION_CONTIGUOUS;
1864 args[3].loc = LIBOS_MEMORY_REGION_LOC_SYSMEM;
1865 return 0;
1866}
1867
1868void
1869nvkm_gsp_sg_free(struct nvkm_device *device, struct sg_table *sgt)
1870{
1871 struct scatterlist *sgl;
1872 int i;
1873
1874 dma_unmap_sgtable(device->dev, sgt, DMA_BIDIRECTIONAL, 0);
1875
1876 for_each_sgtable_sg(sgt, sgl, i) {
1877 struct page *page = sg_page(sgl);
1878
1879 __free_page(page);
1880 }
1881
1882 sg_free_table(sgt);
1883}
1884
1885int
1886nvkm_gsp_sg(struct nvkm_device *device, u64 size, struct sg_table *sgt)
1887{
1888 const u64 pages = DIV_ROUND_UP(size, PAGE_SIZE);
1889 struct scatterlist *sgl;
1890 int ret, i;
1891
1892 ret = sg_alloc_table(sgt, pages, GFP_KERNEL);
1893 if (ret)
1894 return ret;
1895
1896 for_each_sgtable_sg(sgt, sgl, i) {
1897 struct page *page = alloc_page(GFP_KERNEL);
1898
1899 if (!page) {
1900 nvkm_gsp_sg_free(device, sgt);
1901 return -ENOMEM;
1902 }
1903
1904 sg_set_page(sgl, page, PAGE_SIZE, 0);
1905 }
1906
1907 ret = dma_map_sgtable(device->dev, sgt, DMA_BIDIRECTIONAL, 0);
1908 if (ret)
1909 nvkm_gsp_sg_free(device, sgt);
1910
1911 return ret;
1912}
1913
1914static void
1915nvkm_gsp_radix3_dtor(struct nvkm_gsp *gsp, struct nvkm_gsp_radix3 *rx3)
1916{
1917 for (int i = ARRAY_SIZE(rx3->mem) - 1; i >= 0; i--)
1918 nvkm_gsp_mem_dtor(gsp, &rx3->mem[i]);
1919}
1920
88a2b4d3
TT		 1921/**
1922 * nvkm_gsp_radix3_sg - build a radix3 table from a S/G list
1923 *
1924 * The GSP uses a three-level page table, called radix3, to map the firmware.
1925 * Each 64-bit "pointer" in the table is either the bus address of an entry in
1926 * the next table (for levels 0 and 1) or the bus address of the next page in
1927 * the GSP firmware image itself.
1928 *
1929 * Level 0 contains a single entry in one page that points to the first page
1930 * of level 1.
1931 *
1932 * Level 1, since it's also only one page in size, contains up to 512 entries,
1933 * one for each page in Level 2.
1934 *
1935 * Level 2 can be up to 512 pages in size, and each of those entries points to
1936 * the next page of the firmware image. Since there can be up to 512*512
1937 * pages, that limits the size of the firmware to 512*512*GSP_PAGE_SIZE = 1GB.
1938 *
1939 * Internally, the GSP has its window into system memory, but the base
1940 * physical address of the aperture is not 0. In fact, it varies depending on
1941 * the GPU architecture. Since the GPU is a PCI device, this window is
1942 * accessed via DMA and is therefore bound by IOMMU translation. The end
1943 * result is that GSP-RM must translate the bus addresses in the table to GSP
1944 * physical addresses. All this should happen transparently.
1945 *
1946 * Returns 0 on success, or negative error code
1947 *
1948 * See kgspCreateRadix3_IMPL
1949 */
176fdcbd 1950static int
34e659f3 1951nvkm_gsp_radix3_sg(struct nvkm_gsp *gsp, struct sg_table *sgt, u64 size,
176fdcbd
BS		 1952 struct nvkm_gsp_radix3 *rx3)
1953{
1954 u64 addr;
1955
1956 for (int i = ARRAY_SIZE(rx3->mem) - 1; i >= 0; i--) {
1957 u64 *ptes;
34e659f3
TT		 1958 size_t bufsize;
1959 int ret, idx;
176fdcbd 1960
34e659f3
TT		 1961 bufsize = ALIGN((size / GSP_PAGE_SIZE) * sizeof(u64), GSP_PAGE_SIZE);
1962 ret = nvkm_gsp_mem_ctor(gsp, bufsize, &rx3->mem[i]);
1963 if (ret)
1964 return ret;
176fdcbd
BS		 1965
1966 ptes = rx3->mem[i].data;
1967 if (i == 2) {
1968 struct scatterlist *sgl;
1969
1970 for_each_sgtable_dma_sg(sgt, sgl, idx) {
1971 for (int j = 0; j < sg_dma_len(sgl) / GSP_PAGE_SIZE; j++)
1972 *ptes++ = sg_dma_address(sgl) + (GSP_PAGE_SIZE * j);
1973 }
1974 } else {
1975 for (int j = 0; j < size / GSP_PAGE_SIZE; j++)
1976 *ptes++ = addr + GSP_PAGE_SIZE * j;
1977 }
1978
1979 size = rx3->mem[i].size;
1980 addr = rx3->mem[i].addr;
1981 }
1982
1983 return 0;
1984}
1985
1986int
1987r535_gsp_fini(struct nvkm_gsp *gsp, bool suspend)
1988{
1989 u32 mbox0 = 0xff, mbox1 = 0xff;
1990 int ret;
1991
1992 if (!gsp->running)
1993 return 0;
1994
1995 if (suspend) {
1996 GspFwWprMeta *meta = gsp->wpr_meta.data;
1997 u64 len = meta->gspFwWprEnd - meta->gspFwWprStart;
1998 GspFwSRMeta *sr;
1999
2000 ret = nvkm_gsp_sg(gsp->subdev.device, len, &gsp->sr.sgt);
2001 if (ret)
2002 return ret;
2003
34e659f3 2004 ret = nvkm_gsp_radix3_sg(gsp, &gsp->sr.sgt, len, &gsp->sr.radix3);
176fdcbd
BS		 2005 if (ret)
2006 return ret;
2007
2008 ret = nvkm_gsp_mem_ctor(gsp, sizeof(*sr), &gsp->sr.meta);
2009 if (ret)
2010 return ret;
2011
2012 sr = gsp->sr.meta.data;
2013 sr->magic = GSP_FW_SR_META_MAGIC;
2014 sr->revision = GSP_FW_SR_META_REVISION;
2015 sr->sysmemAddrOfSuspendResumeData = gsp->sr.radix3.mem[0].addr;
2016 sr->sizeOfSuspendResumeData = len;
2017
2018 mbox0 = lower_32_bits(gsp->sr.meta.addr);
2019 mbox1 = upper_32_bits(gsp->sr.meta.addr);
2020 }
2021
2022 ret = r535_gsp_rpc_unloading_guest_driver(gsp, suspend);
2023 if (WARN_ON(ret))
2024 return ret;
2025
2026 nvkm_msec(gsp->subdev.device, 2000,
2027 if (nvkm_falcon_rd32(&gsp->falcon, 0x040) & 0x80000000)
2028 break;
2029 );
2030
2031 nvkm_falcon_reset(&gsp->falcon);
2032
2033 ret = nvkm_gsp_fwsec_sb(gsp);
2034 WARN_ON(ret);
2035
2036 ret = r535_gsp_booter_unload(gsp, mbox0, mbox1);
2037 WARN_ON(ret);
2038
2039 gsp->running = false;
2040 return 0;
2041}
2042
2043int
2044r535_gsp_init(struct nvkm_gsp *gsp)
2045{
2046 u32 mbox0, mbox1;
2047 int ret;
2048
2049 if (!gsp->sr.meta.data) {
2050 mbox0 = lower_32_bits(gsp->wpr_meta.addr);
2051 mbox1 = upper_32_bits(gsp->wpr_meta.addr);
2052 } else {
2053 r535_gsp_rmargs_init(gsp, true);
2054
2055 mbox0 = lower_32_bits(gsp->sr.meta.addr);
2056 mbox1 = upper_32_bits(gsp->sr.meta.addr);
2057 }
2058
2059 /* Execute booter to handle (eventually...) booting GSP-RM. */
2060 ret = r535_gsp_booter_load(gsp, mbox0, mbox1);
2061 if (WARN_ON(ret))
2062 goto done;
2063
2064 ret = r535_gsp_rpc_poll(gsp, NV_VGPU_MSG_EVENT_GSP_INIT_DONE);
2065 if (ret)
2066 goto done;
2067
2068 gsp->running = true;
2069
2070done:
2071 if (gsp->sr.meta.data) {
2072 nvkm_gsp_mem_dtor(gsp, &gsp->sr.meta);
2073 nvkm_gsp_radix3_dtor(gsp, &gsp->sr.radix3);
2074 nvkm_gsp_sg_free(gsp->subdev.device, &gsp->sr.sgt);
4cf2c83e 2075 return ret;
176fdcbd
BS		 2076 }
2077
4cf2c83e
BS		 2078 if (ret == 0)
2079 ret = r535_gsp_postinit(gsp);
2080
176fdcbd
BS		 2081 return ret;
2082}
2083
2084static int
2085r535_gsp_rm_boot_ctor(struct nvkm_gsp *gsp)
2086{
2087 const struct firmware *fw = gsp->fws.bl;
2088 const struct nvfw_bin_hdr *hdr;
2089 RM_RISCV_UCODE_DESC *desc;
2090 int ret;
2091
2092 hdr = nvfw_bin_hdr(&gsp->subdev, fw->data);
2093 desc = (void *)fw->data + hdr->header_offset;
2094
2095 ret = nvkm_gsp_mem_ctor(gsp, hdr->data_size, &gsp->boot.fw);
2096 if (ret)
2097 return ret;
2098
2099 memcpy(gsp->boot.fw.data, fw->data + hdr->data_offset, hdr->data_size);
2100
2101 gsp->boot.code_offset = desc->monitorCodeOffset;
2102 gsp->boot.data_offset = desc->monitorDataOffset;
2103 gsp->boot.manifest_offset = desc->manifestOffset;
2104 gsp->boot.app_version = desc->appVersion;
2105 return 0;
2106}
2107
2108static const struct nvkm_firmware_func
2109r535_gsp_fw = {
2110 .type = NVKM_FIRMWARE_IMG_SGT,
2111};
2112
2113static int
2114r535_gsp_elf_section(struct nvkm_gsp *gsp, const char *name, const u8 **pdata, u64 *psize)
2115{
2116 const u8 *img = gsp->fws.rm->data;
2117 const struct elf64_hdr *ehdr = (const struct elf64_hdr *)img;
2118 const struct elf64_shdr *shdr = (const struct elf64_shdr *)&img[ehdr->e_shoff];
2119 const char *names = &img[shdr[ehdr->e_shstrndx].sh_offset];
2120
2121 for (int i = 0; i < ehdr->e_shnum; i++, shdr++) {
2122 if (!strcmp(&names[shdr->sh_name], name)) {
2123 *pdata = &img[shdr->sh_offset];
2124 *psize = shdr->sh_size;
2125 return 0;
2126 }
2127 }
2128
2129 nvkm_error(&gsp->subdev, "section '%s' not found\n", name);
2130 return -ENOENT;
2131}
2132
2133static void
2134r535_gsp_dtor_fws(struct nvkm_gsp *gsp)
2135{
2136 nvkm_firmware_put(gsp->fws.bl);
2137 gsp->fws.bl = NULL;
2138 nvkm_firmware_put(gsp->fws.booter.unload);
2139 gsp->fws.booter.unload = NULL;
2140 nvkm_firmware_put(gsp->fws.booter.load);
2141 gsp->fws.booter.load = NULL;
2142 nvkm_firmware_put(gsp->fws.rm);
2143 gsp->fws.rm = NULL;
2144}
2145
2146void
2147r535_gsp_dtor(struct nvkm_gsp *gsp)
2148{
37e328a1
BS		 2149 idr_destroy(&gsp->client_id.idr);
2150 mutex_destroy(&gsp->client_id.mutex);
2151
176fdcbd
BS		 2152 nvkm_gsp_radix3_dtor(gsp, &gsp->radix3);
2153 nvkm_gsp_mem_dtor(gsp, &gsp->sig);
2154 nvkm_firmware_dtor(&gsp->fw);
2155
2156 nvkm_falcon_fw_dtor(&gsp->booter.unload);
2157 nvkm_falcon_fw_dtor(&gsp->booter.load);
2158
2159 mutex_destroy(&gsp->msgq.mutex);
2160 mutex_destroy(&gsp->cmdq.mutex);
2161
2162 r535_gsp_dtor_fws(gsp);
042b5f83 2163
f6ecfdad
SP		 2164 nvkm_gsp_mem_dtor(gsp, &gsp->rmargs);
2165 nvkm_gsp_mem_dtor(gsp, &gsp->wpr_meta);
042b5f83
TT		 2166 nvkm_gsp_mem_dtor(gsp, &gsp->shm.mem);
2167 nvkm_gsp_mem_dtor(gsp, &gsp->loginit);
2168 nvkm_gsp_mem_dtor(gsp, &gsp->logintr);
2169 nvkm_gsp_mem_dtor(gsp, &gsp->logrm);
176fdcbd
BS		 2170}
2171
2172int
2173r535_gsp_oneinit(struct nvkm_gsp *gsp)
2174{
2175 struct nvkm_device *device = gsp->subdev.device;
2176 const u8 *data;
2177 u64 size;
2178 int ret;
2179
2180 mutex_init(&gsp->cmdq.mutex);
2181 mutex_init(&gsp->msgq.mutex);
2182
2183 ret = gsp->func->booter.ctor(gsp, "booter-load", gsp->fws.booter.load,
2184 &device->sec2->falcon, &gsp->booter.load);
2185 if (ret)
2186 return ret;
2187
2188 ret = gsp->func->booter.ctor(gsp, "booter-unload", gsp->fws.booter.unload,
2189 &device->sec2->falcon, &gsp->booter.unload);
2190 if (ret)
2191 return ret;
2192
2193 /* Load GSP firmware from ELF image into DMA-accessible memory. */
2194 ret = r535_gsp_elf_section(gsp, ".fwimage", &data, &size);
2195 if (ret)
2196 return ret;
2197
2198 ret = nvkm_firmware_ctor(&r535_gsp_fw, "gsp-rm", device, data, size, &gsp->fw);
2199 if (ret)
2200 return ret;
2201
2202 /* Load relevant signature from ELF image. */
2203 ret = r535_gsp_elf_section(gsp, gsp->func->sig_section, &data, &size);
2204 if (ret)
2205 return ret;
2206
2207 ret = nvkm_gsp_mem_ctor(gsp, ALIGN(size, 256), &gsp->sig);
2208 if (ret)
2209 return ret;
2210
2211 memcpy(gsp->sig.data, data, size);
2212
2213 /* Build radix3 page table for ELF image. */
34e659f3 2214 ret = nvkm_gsp_radix3_sg(gsp, &gsp->fw.mem.sgt, gsp->fw.len, &gsp->radix3);
176fdcbd
BS		 2215 if (ret)
2216 return ret;
2217
2218 r535_gsp_msg_ntfy_add(gsp, NV_VGPU_MSG_EVENT_GSP_RUN_CPU_SEQUENCER,
2219 r535_gsp_msg_run_cpu_sequencer, gsp);
9e994444 2220 r535_gsp_msg_ntfy_add(gsp, NV_VGPU_MSG_EVENT_POST_EVENT, r535_gsp_msg_post_event, gsp);
2a77d015
BS		 2221 r535_gsp_msg_ntfy_add(gsp, NV_VGPU_MSG_EVENT_RC_TRIGGERED,
2222 r535_gsp_msg_rc_triggered, gsp);
5bf02571
BS		 2223 r535_gsp_msg_ntfy_add(gsp, NV_VGPU_MSG_EVENT_MMU_FAULT_QUEUED,
2224 r535_gsp_msg_mmu_fault_queued, gsp);
176fdcbd 2225 r535_gsp_msg_ntfy_add(gsp, NV_VGPU_MSG_EVENT_OS_ERROR_LOG, r535_gsp_msg_os_error_log, gsp);
24ab185d
DA		 2226 r535_gsp_msg_ntfy_add(gsp, NV_VGPU_MSG_EVENT_PERF_BRIDGELESS_INFO_UPDATE, NULL, NULL);
2227 r535_gsp_msg_ntfy_add(gsp, NV_VGPU_MSG_EVENT_UCODE_LIBOS_PRINT, NULL, NULL);
2228 r535_gsp_msg_ntfy_add(gsp, NV_VGPU_MSG_EVENT_GSP_SEND_USER_SHARED_DATA, NULL, NULL);
176fdcbd
BS		 2229 ret = r535_gsp_rm_boot_ctor(gsp);
2230 if (ret)
2231 return ret;
2232
2233 /* Release FW images - we've copied them to DMA buffers now. */
2234 r535_gsp_dtor_fws(gsp);
2235
2236 /* Calculate FB layout. */
2237 gsp->fb.wpr2.frts.size = 0x100000;
2238 gsp->fb.wpr2.frts.addr = ALIGN_DOWN(gsp->fb.bios.addr, 0x20000) - gsp->fb.wpr2.frts.size;
2239
2240 gsp->fb.wpr2.boot.size = gsp->boot.fw.size;
2241 gsp->fb.wpr2.boot.addr = ALIGN_DOWN(gsp->fb.wpr2.frts.addr - gsp->fb.wpr2.boot.size, 0x1000);
2242
2243 gsp->fb.wpr2.elf.size = gsp->fw.len;
2244 gsp->fb.wpr2.elf.addr = ALIGN_DOWN(gsp->fb.wpr2.boot.addr - gsp->fb.wpr2.elf.size, 0x10000);
2245
2246 {
2247 u32 fb_size_gb = DIV_ROUND_UP_ULL(gsp->fb.size, 1 << 30);
2248
2249 gsp->fb.wpr2.heap.size =
2250 gsp->func->wpr_heap.os_carveout_size +
2251 gsp->func->wpr_heap.base_size +
2252 ALIGN(GSP_FW_HEAP_PARAM_SIZE_PER_GB_FB * fb_size_gb, 1 << 20) +
2253 ALIGN(GSP_FW_HEAP_PARAM_CLIENT_ALLOC_SIZE, 1 << 20);
2254
2255 gsp->fb.wpr2.heap.size = max(gsp->fb.wpr2.heap.size, gsp->func->wpr_heap.min_size);
2256 }
2257
2258 gsp->fb.wpr2.heap.addr = ALIGN_DOWN(gsp->fb.wpr2.elf.addr - gsp->fb.wpr2.heap.size, 0x100000);
2259 gsp->fb.wpr2.heap.size = ALIGN_DOWN(gsp->fb.wpr2.elf.addr - gsp->fb.wpr2.heap.addr, 0x100000);
2260
2261 gsp->fb.wpr2.addr = ALIGN_DOWN(gsp->fb.wpr2.heap.addr - sizeof(GspFwWprMeta), 0x100000);
2262 gsp->fb.wpr2.size = gsp->fb.wpr2.frts.addr + gsp->fb.wpr2.frts.size - gsp->fb.wpr2.addr;
2263
2264 gsp->fb.heap.size = 0x100000;
2265 gsp->fb.heap.addr = gsp->fb.wpr2.addr - gsp->fb.heap.size;
2266
2267 ret = nvkm_gsp_fwsec_frts(gsp);
2268 if (WARN_ON(ret))
2269 return ret;
2270
2271 ret = r535_gsp_libos_init(gsp);
2272 if (WARN_ON(ret))
2273 return ret;
2274
2275 ret = r535_gsp_wpr_meta_init(gsp);
2276 if (WARN_ON(ret))
2277 return ret;
2278
2279 ret = r535_gsp_rpc_set_system_info(gsp);
2280 if (WARN_ON(ret))
2281 return ret;
2282
2283 ret = r535_gsp_rpc_set_registry(gsp);
2284 if (WARN_ON(ret))
2285 return ret;
2286
2287 /* Reset GSP into RISC-V mode. */
2288 ret = gsp->func->reset(gsp);
2289 if (WARN_ON(ret))
2290 return ret;
2291
2292 nvkm_falcon_wr32(&gsp->falcon, 0x040, lower_32_bits(gsp->libos.addr));
2293 nvkm_falcon_wr32(&gsp->falcon, 0x044, upper_32_bits(gsp->libos.addr));
37e328a1
BS		 2294
2295 mutex_init(&gsp->client_id.mutex);
2296 idr_init(&gsp->client_id.idr);
176fdcbd
BS		 2297 return 0;
2298}
2299
2300static int
2301r535_gsp_load_fw(struct nvkm_gsp *gsp, const char *name, const char *ver,
2302 const struct firmware **pfw)
2303{
2304 char fwname[64];
2305
2306 snprintf(fwname, sizeof(fwname), "gsp/%s-%s", name, ver);
2307 return nvkm_firmware_get(&gsp->subdev, fwname, 0, pfw);
2308}
2309
2310int
2311r535_gsp_load(struct nvkm_gsp *gsp, int ver, const struct nvkm_gsp_fwif *fwif)
2312{
2313 struct nvkm_subdev *subdev = &gsp->subdev;
2314 int ret;
1d492944 2315 bool enable_gsp = fwif->enable;
176fdcbd 2316
1d492944
DA		 2317#if IS_ENABLED(CONFIG_DRM_NOUVEAU_GSP_DEFAULT)
2318 enable_gsp = true;
2319#endif
2320 if (!nvkm_boolopt(subdev->device->cfgopt, "NvGspRm", enable_gsp))
176fdcbd
BS		 2321 return -EINVAL;
2322
2323 if ((ret = r535_gsp_load_fw(gsp, "gsp", fwif->ver, &gsp->fws.rm)) ||
2324 (ret = r535_gsp_load_fw(gsp, "booter_load", fwif->ver, &gsp->fws.booter.load)) ||
2325 (ret = r535_gsp_load_fw(gsp, "booter_unload", fwif->ver, &gsp->fws.booter.unload)) ||
2326 (ret = r535_gsp_load_fw(gsp, "bootloader", fwif->ver, &gsp->fws.bl))) {
2327 r535_gsp_dtor_fws(gsp);
2328 return ret;
2329 }
2330
2331 return 0;
2332}
2333
2334#define NVKM_GSP_FIRMWARE(chip) \
b5bad8c1
DA		 2335MODULE_FIRMWARE("nvidia/"#chip"/gsp/booter_load-535.113.01.bin"); \
2336MODULE_FIRMWARE("nvidia/"#chip"/gsp/booter_unload-535.113.01.bin"); \
2337MODULE_FIRMWARE("nvidia/"#chip"/gsp/bootloader-535.113.01.bin"); \
2338MODULE_FIRMWARE("nvidia/"#chip"/gsp/gsp-535.113.01.bin")
176fdcbd
BS		 2339
2340NVKM_GSP_FIRMWARE(tu102);
2341NVKM_GSP_FIRMWARE(tu104);
2342NVKM_GSP_FIRMWARE(tu106);
2343
2344NVKM_GSP_FIRMWARE(tu116);
2345NVKM_GSP_FIRMWARE(tu117);
2346
2347NVKM_GSP_FIRMWARE(ga100);
2348
2349NVKM_GSP_FIRMWARE(ga102);
2350NVKM_GSP_FIRMWARE(ga103);
2351NVKM_GSP_FIRMWARE(ga104);
2352NVKM_GSP_FIRMWARE(ga106);
2353NVKM_GSP_FIRMWARE(ga107);
2354
2355NVKM_GSP_FIRMWARE(ad102);
2356NVKM_GSP_FIRMWARE(ad103);
2357NVKM_GSP_FIRMWARE(ad104);
2358NVKM_GSP_FIRMWARE(ad106);
2359NVKM_GSP_FIRMWARE(ad107);
Linux 6.x block layer and io_uring tree(s)