drivers/gpu/drm/nouveau/nvkm/subdev/gsp/r535.c

   1 /*
   2  * Copyright 2023 Red Hat Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  */
  22 #include "priv.h"
  23
  24 #include <core/pci.h>
  25 #include <subdev/timer.h>
  26 #include <subdev/vfn.h>
  27 #include <engine/fifo/chan.h>
  28 #include <engine/sec2.h>
  29
  30 #include <nvfw/fw.h>
  31
  32 #include <nvrm/nvtypes.h>
  33 #include <nvrm/535.113.01/common/sdk/nvidia/inc/class/cl0000.h>
  34 #include <nvrm/535.113.01/common/sdk/nvidia/inc/class/cl0005.h>
  35 #include <nvrm/535.113.01/common/sdk/nvidia/inc/class/cl0080.h>
  36 #include <nvrm/535.113.01/common/sdk/nvidia/inc/class/cl2080.h>
  37 #include <nvrm/535.113.01/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080event.h>
  38 #include <nvrm/535.113.01/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080gpu.h>
  39 #include <nvrm/535.113.01/common/sdk/nvidia/inc/ctrl/ctrl2080/ctrl2080internal.h>
  40 #include <nvrm/535.113.01/common/sdk/nvidia/inc/nvos.h>
  41 #include <nvrm/535.113.01/common/shared/msgq/inc/msgq/msgq_priv.h>
  42 #include <nvrm/535.113.01/common/uproc/os/common/include/libos_init_args.h>
  43 #include <nvrm/535.113.01/nvidia/arch/nvalloc/common/inc/gsp/gsp_fw_sr_meta.h>
  44 #include <nvrm/535.113.01/nvidia/arch/nvalloc/common/inc/gsp/gsp_fw_wpr_meta.h>
  45 #include <nvrm/535.113.01/nvidia/arch/nvalloc/common/inc/rmRiscvUcode.h>
  46 #include <nvrm/535.113.01/nvidia/arch/nvalloc/common/inc/rmgspseq.h>
  47 #include <nvrm/535.113.01/nvidia/generated/g_allclasses.h>
  48 #include <nvrm/535.113.01/nvidia/generated/g_os_nvoc.h>
  49 #include <nvrm/535.113.01/nvidia/generated/g_rpc-structures.h>
  50 #include <nvrm/535.113.01/nvidia/inc/kernel/gpu/gsp/gsp_fw_heap.h>
  51 #include <nvrm/535.113.01/nvidia/inc/kernel/gpu/gsp/gsp_init_args.h>
  52 #include <nvrm/535.113.01/nvidia/inc/kernel/gpu/gsp/gsp_static_config.h>
  53 #include <nvrm/535.113.01/nvidia/inc/kernel/gpu/intr/engine_idx.h>
  54 #include <nvrm/535.113.01/nvidia/kernel/inc/vgpu/rpc_global_enums.h>
  55
  56 #include <linux/acpi.h>
  57
  58 #define GSP_MSG_MIN_SIZE GSP_PAGE_SIZE
  59 #define GSP_MSG_MAX_SIZE GSP_PAGE_MIN_SIZE * 16
  60
  61 struct r535_gsp_msg {
  62         u8 auth_tag_buffer[16];
  63         u8 aad_buffer[16];
  64         u32 checksum;
  65         u32 sequence;
  66         u32 elem_count;
  67         u32 pad;
  68         u8  data[];
  69 };
  70
  71 #define GSP_MSG_HDR_SIZE offsetof(struct r535_gsp_msg, data)
  72
  73 static void *
  74 r535_gsp_msgq_wait(struct nvkm_gsp *gsp, u32 repc, u32 *prepc, int *ptime)
  75 {
  76         struct r535_gsp_msg *mqe;
  77         u32 size, rptr = *gsp->msgq.rptr;
  78         int used;
  79         u8 *msg;
  80         u32 len;
  81
  82         size = DIV_ROUND_UP(GSP_MSG_HDR_SIZE + repc, GSP_PAGE_SIZE);
  83         if (WARN_ON(!size || size >= gsp->msgq.cnt))
  84                 return ERR_PTR(-EINVAL);
  85
  86         do {
  87                 u32 wptr = *gsp->msgq.wptr;
  88
  89                 used = wptr + gsp->msgq.cnt - rptr;
  90                 if (used >= gsp->msgq.cnt)
  91                         used -= gsp->msgq.cnt;
  92                 if (used >= size)
  93                         break;
  94
  95                 usleep_range(1, 2);
  96         } while (--(*ptime));
  97
  98         if (WARN_ON(!*ptime))
  99                 return ERR_PTR(-ETIMEDOUT);
 100
 101         mqe = (void *)((u8 *)gsp->shm.msgq.ptr + 0x1000 + rptr * 0x1000);
 102
 103         if (prepc) {
 104                 *prepc = (used * GSP_PAGE_SIZE) - sizeof(*mqe);
 105                 return mqe->data;
 106         }
 107
 108         msg = kvmalloc(repc, GFP_KERNEL);
 109         if (!msg)
 110                 return ERR_PTR(-ENOMEM);
 111
 112         len = ((gsp->msgq.cnt - rptr) * GSP_PAGE_SIZE) - sizeof(*mqe);
 113         len = min_t(u32, repc, len);
 114         memcpy(msg, mqe->data, len);
 115
 116         rptr += DIV_ROUND_UP(len, GSP_PAGE_SIZE);
 117         if (rptr == gsp->msgq.cnt)
 118                 rptr = 0;
 119
 120         repc -= len;
 121
 122         if (repc) {
 123                 mqe = (void *)((u8 *)gsp->shm.msgq.ptr + 0x1000 + 0 * 0x1000);
 124                 memcpy(msg + len, mqe, repc);
 125
 126                 rptr += DIV_ROUND_UP(repc, GSP_PAGE_SIZE);
 127         }
 128
 129         mb();
 130         (*gsp->msgq.rptr) = rptr;
 131         return msg;
 132 }
 133
 134 static void *
 135 r535_gsp_msgq_recv(struct nvkm_gsp *gsp, u32 repc, int *ptime)
 136 {
 137         return r535_gsp_msgq_wait(gsp, repc, NULL, ptime);
 138 }
 139
 140 static int
 141 r535_gsp_cmdq_push(struct nvkm_gsp *gsp, void *argv)
 142 {
 143         struct r535_gsp_msg *cmd = container_of(argv, typeof(*cmd), data);
 144         struct r535_gsp_msg *cqe;
 145         u32 argc = cmd->checksum;
 146         u64 *ptr = (void *)cmd;
 147         u64 *end;
 148         u64 csum = 0;
 149         int free, time = 1000000;
 150         u32 wptr, size;
 151         u32 off = 0;
 152
 153         argc = ALIGN(GSP_MSG_HDR_SIZE + argc, GSP_PAGE_SIZE);
 154
 155         end = (u64 *)((char *)ptr + argc);
 156         cmd->pad = 0;
 157         cmd->checksum = 0;
 158         cmd->sequence = gsp->cmdq.seq++;
 159         cmd->elem_count = DIV_ROUND_UP(argc, 0x1000);
 160
 161         while (ptr < end)
 162                 csum ^= *ptr++;
 163
 164         cmd->checksum = upper_32_bits(csum) ^ lower_32_bits(csum);
 165
 166         wptr = *gsp->cmdq.wptr;
 167         do {
 168                 do {
 169                         free = *gsp->cmdq.rptr + gsp->cmdq.cnt - wptr - 1;
 170                         if (free >= gsp->cmdq.cnt)
 171                                 free -= gsp->cmdq.cnt;
 172                         if (free >= 1)
 173                                 break;
 174
 175                         usleep_range(1, 2);
 176                 } while(--time);
 177
 178                 if (WARN_ON(!time)) {
 179                         kvfree(cmd);
 180                         return -ETIMEDOUT;
 181                 }
 182
 183                 cqe = (void *)((u8 *)gsp->shm.cmdq.ptr + 0x1000 + wptr * 0x1000);
 184                 size = min_t(u32, argc, (gsp->cmdq.cnt - wptr) * GSP_PAGE_SIZE);
 185                 memcpy(cqe, (u8 *)cmd + off, size);
 186
 187                 wptr += DIV_ROUND_UP(size, 0x1000);
 188                 if (wptr == gsp->cmdq.cnt)
 189                         wptr = 0;
 190
 191                 off  += size;
 192                 argc -= size;
 193         } while(argc);
 194
 195         nvkm_trace(&gsp->subdev, "cmdq: wptr %d\n", wptr);
 196         wmb();
 197         (*gsp->cmdq.wptr) = wptr;
 198         mb();
 199
 200         nvkm_falcon_wr32(&gsp->falcon, 0xc00, 0x00000000);
 201
 202         kvfree(cmd);
 203         return 0;
 204 }
 205
 206 static void *
 207 r535_gsp_cmdq_get(struct nvkm_gsp *gsp, u32 argc)
 208 {
 209         struct r535_gsp_msg *cmd;
 210         u32 size = GSP_MSG_HDR_SIZE + argc;
 211
 212         size = ALIGN(size, GSP_MSG_MIN_SIZE);
 213         cmd = kvzalloc(size, GFP_KERNEL);
 214         if (!cmd)
 215                 return ERR_PTR(-ENOMEM);
 216
 217         cmd->checksum = argc;
 218         return cmd->data;
 219 }
 220
 221 struct nvfw_gsp_rpc {
 222         u32 header_version;
 223         u32 signature;
 224         u32 length;
 225         u32 function;
 226         u32 rpc_result;
 227         u32 rpc_result_private;
 228         u32 sequence;
 229         union {
 230                 u32 spare;
 231                 u32 cpuRmGfid;
 232         };
 233         u8  data[];
 234 };
 235
 236 static void
 237 r535_gsp_msg_done(struct nvkm_gsp *gsp, struct nvfw_gsp_rpc *msg)
 238 {
 239         kvfree(msg);
 240 }
 241
 242 static void
 243 r535_gsp_msg_dump(struct nvkm_gsp *gsp, struct nvfw_gsp_rpc *msg, int lvl)
 244 {
 245         if (gsp->subdev.debug >= lvl) {
 246                 nvkm_printk__(&gsp->subdev, lvl, info,
 247                               "msg fn:%d len:0x%x/0x%zx res:0x%x resp:0x%x\n",
 248                               msg->function, msg->length, msg->length - sizeof(*msg),
 249                               msg->rpc_result, msg->rpc_result_private);
 250                 print_hex_dump(KERN_INFO, "msg: ", DUMP_PREFIX_OFFSET, 16, 1,
 251                                msg->data, msg->length - sizeof(*msg), true);
 252         }
 253 }
 254
 255 static struct nvfw_gsp_rpc *
 256 r535_gsp_msg_recv(struct nvkm_gsp *gsp, int fn, u32 repc)
 257 {
 258         struct nvkm_subdev *subdev = &gsp->subdev;
 259         struct nvfw_gsp_rpc *msg;
 260         int time = 4000000, i;
 261         u32 size;
 262
 263 retry:
 264         msg = r535_gsp_msgq_wait(gsp, sizeof(*msg), &size, &time);
 265         if (IS_ERR_OR_NULL(msg))
 266                 return msg;
 267
 268         msg = r535_gsp_msgq_recv(gsp, msg->length, &time);
 269         if (IS_ERR_OR_NULL(msg))
 270                 return msg;
 271
 272         if (msg->rpc_result) {
 273                 r535_gsp_msg_dump(gsp, msg, NV_DBG_ERROR);
 274                 r535_gsp_msg_done(gsp, msg);
 275                 return ERR_PTR(-EINVAL);
 276         }
 277
 278         r535_gsp_msg_dump(gsp, msg, NV_DBG_TRACE);
 279
 280         if (fn && msg->function == fn) {
 281                 if (repc) {
 282                         if (msg->length < sizeof(*msg) + repc) {
 283                                 nvkm_error(subdev, "msg len %d < %zd\n",
 284                                            msg->length, sizeof(*msg) + repc);
 285                                 r535_gsp_msg_dump(gsp, msg, NV_DBG_ERROR);
 286                                 r535_gsp_msg_done(gsp, msg);
 287                                 return ERR_PTR(-EIO);
 288                         }
 289
 290                         return msg;
 291                 }
 292
 293                 r535_gsp_msg_done(gsp, msg);
 294                 return NULL;
 295         }
 296
 297         for (i = 0; i < gsp->msgq.ntfy_nr; i++) {
 298                 struct nvkm_gsp_msgq_ntfy *ntfy = &gsp->msgq.ntfy[i];
 299
 300                 if (ntfy->fn == msg->function) {
 301                         if (ntfy->func)
 302                                 ntfy->func(ntfy->priv, ntfy->fn, msg->data, msg->length - sizeof(*msg));
 303                         break;
 304                 }
 305         }
 306
 307         if (i == gsp->msgq.ntfy_nr)
 308                 r535_gsp_msg_dump(gsp, msg, NV_DBG_WARN);
 309
 310         r535_gsp_msg_done(gsp, msg);
 311         if (fn)
 312                 goto retry;
 313
 314         if (*gsp->msgq.rptr != *gsp->msgq.wptr)
 315                 goto retry;
 316
 317         return NULL;
 318 }
 319
 320 static int
 321 r535_gsp_msg_ntfy_add(struct nvkm_gsp *gsp, u32 fn, nvkm_gsp_msg_ntfy_func func, void *priv)
 322 {
 323         int ret = 0;
 324
 325         mutex_lock(&gsp->msgq.mutex);
 326         if (WARN_ON(gsp->msgq.ntfy_nr >= ARRAY_SIZE(gsp->msgq.ntfy))) {
 327                 ret = -ENOSPC;
 328         } else {
 329                 gsp->msgq.ntfy[gsp->msgq.ntfy_nr].fn = fn;
 330                 gsp->msgq.ntfy[gsp->msgq.ntfy_nr].func = func;
 331                 gsp->msgq.ntfy[gsp->msgq.ntfy_nr].priv = priv;
 332                 gsp->msgq.ntfy_nr++;
 333         }
 334         mutex_unlock(&gsp->msgq.mutex);
 335         return ret;
 336 }
 337
 338 static int
 339 r535_gsp_rpc_poll(struct nvkm_gsp *gsp, u32 fn)
 340 {
 341         void *repv;
 342
 343         mutex_lock(&gsp->cmdq.mutex);
 344         repv = r535_gsp_msg_recv(gsp, fn, 0);
 345         mutex_unlock(&gsp->cmdq.mutex);
 346         if (IS_ERR(repv))
 347                 return PTR_ERR(repv);
 348
 349         return 0;
 350 }
 351
 352 static void *
 353 r535_gsp_rpc_send(struct nvkm_gsp *gsp, void *argv, bool wait, u32 repc)
 354 {
 355         struct nvfw_gsp_rpc *rpc = container_of(argv, typeof(*rpc), data);
 356         struct nvfw_gsp_rpc *msg;
 357         u32 fn = rpc->function;
 358         void *repv = NULL;
 359         int ret;
 360
 361         if (gsp->subdev.debug >= NV_DBG_TRACE) {
 362                 nvkm_trace(&gsp->subdev, "rpc fn:%d len:0x%x/0x%zx\n", rpc->function,
 363                            rpc->length, rpc->length - sizeof(*rpc));
 364                 print_hex_dump(KERN_INFO, "rpc: ", DUMP_PREFIX_OFFSET, 16, 1,
 365                                rpc->data, rpc->length - sizeof(*rpc), true);
 366         }
 367
 368         ret = r535_gsp_cmdq_push(gsp, rpc);
 369         if (ret)
 370                 return ERR_PTR(ret);
 371
 372         if (wait) {
 373                 msg = r535_gsp_msg_recv(gsp, fn, repc);
 374                 if (!IS_ERR_OR_NULL(msg))
 375                         repv = msg->data;
 376                 else
 377                         repv = msg;
 378         }
 379
 380         return repv;
 381 }
 382
 383 static void
 384 r535_gsp_event_dtor(struct nvkm_gsp_event *event)
 385 {
 386         struct nvkm_gsp_device *device = event->device;
 387         struct nvkm_gsp_client *client = device->object.client;
 388         struct nvkm_gsp *gsp = client->gsp;
 389
 390         mutex_lock(&gsp->client_id.mutex);
 391         if (event->func) {
 392                 list_del(&event->head);
 393                 event->func = NULL;
 394         }
 395         mutex_unlock(&gsp->client_id.mutex);
 396
 397         nvkm_gsp_rm_free(&event->object);
 398         event->device = NULL;
 399 }
 400
 401 static int
 402 r535_gsp_device_event_get(struct nvkm_gsp_event *event)
 403 {
 404         struct nvkm_gsp_device *device = event->device;
 405         NV2080_CTRL_EVENT_SET_NOTIFICATION_PARAMS *ctrl;
 406
 407         ctrl = nvkm_gsp_rm_ctrl_get(&device->subdevice,
 408                                     NV2080_CTRL_CMD_EVENT_SET_NOTIFICATION, sizeof(*ctrl));
 409         if (IS_ERR(ctrl))
 410                 return PTR_ERR(ctrl);
 411
 412         ctrl->event = event->id;
 413         ctrl->action = NV2080_CTRL_EVENT_SET_NOTIFICATION_ACTION_REPEAT;
 414         return nvkm_gsp_rm_ctrl_wr(&device->subdevice, ctrl);
 415 }
 416
 417 static int
 418 r535_gsp_device_event_ctor(struct nvkm_gsp_device *device, u32 handle, u32 id,
 419                            nvkm_gsp_event_func func, struct nvkm_gsp_event *event)
 420 {
 421         struct nvkm_gsp_client *client = device->object.client;
 422         struct nvkm_gsp *gsp = client->gsp;
 423         NV0005_ALLOC_PARAMETERS *args;
 424         int ret;
 425
 426         args = nvkm_gsp_rm_alloc_get(&device->subdevice, handle,
 427                                      NV01_EVENT_KERNEL_CALLBACK_EX, sizeof(*args),
 428                                      &event->object);
 429         if (IS_ERR(args))
 430                 return PTR_ERR(args);
 431
 432         args->hParentClient = client->object.handle;
 433         args->hSrcResource = 0;
 434         args->hClass = NV01_EVENT_KERNEL_CALLBACK_EX;
 435         args->notifyIndex = NV01_EVENT_CLIENT_RM | id;
 436         args->data = NULL;
 437
 438         ret = nvkm_gsp_rm_alloc_wr(&event->object, args);
 439         if (ret)
 440                 return ret;
 441
 442         event->device = device;
 443         event->id = id;
 444
 445         ret = r535_gsp_device_event_get(event);
 446         if (ret) {
 447                 nvkm_gsp_event_dtor(event);
 448                 return ret;
 449         }
 450
 451         mutex_lock(&gsp->client_id.mutex);
 452         event->func = func;
 453         list_add(&event->head, &client->events);
 454         mutex_unlock(&gsp->client_id.mutex);
 455         return 0;
 456 }
 457
 458 static void
 459 r535_gsp_device_dtor(struct nvkm_gsp_device *device)
 460 {
 461         nvkm_gsp_rm_free(&device->subdevice);
 462         nvkm_gsp_rm_free(&device->object);
 463 }
 464
 465 static int
 466 r535_gsp_subdevice_ctor(struct nvkm_gsp_device *device)
 467 {
 468         NV2080_ALLOC_PARAMETERS *args;
 469
 470         return nvkm_gsp_rm_alloc(&device->object, 0x5d1d0000, NV20_SUBDEVICE_0, sizeof(*args),
 471                                  &device->subdevice);
 472 }
 473
 474 static int
 475 r535_gsp_device_ctor(struct nvkm_gsp_client *client, struct nvkm_gsp_device *device)
 476 {
 477         NV0080_ALLOC_PARAMETERS *args;
 478         int ret;
 479
 480         args = nvkm_gsp_rm_alloc_get(&client->object, 0xde1d0000, NV01_DEVICE_0, sizeof(*args),
 481                                      &device->object);
 482         if (IS_ERR(args))
 483                 return PTR_ERR(args);
 484
 485         args->hClientShare = client->object.handle;
 486
 487         ret = nvkm_gsp_rm_alloc_wr(&device->object, args);
 488         if (ret)
 489                 return ret;
 490
 491         ret = r535_gsp_subdevice_ctor(device);
 492         if (ret)
 493                 nvkm_gsp_rm_free(&device->object);
 494
 495         return ret;
 496 }
 497
 498 static void
 499 r535_gsp_client_dtor(struct nvkm_gsp_client *client)
 500 {
 501         struct nvkm_gsp *gsp = client->gsp;
 502
 503         nvkm_gsp_rm_free(&client->object);
 504
 505         mutex_lock(&gsp->client_id.mutex);
 506         idr_remove(&gsp->client_id.idr, client->object.handle & 0xffff);
 507         mutex_unlock(&gsp->client_id.mutex);
 508
 509         client->gsp = NULL;
 510 }
 511
 512 static int
 513 r535_gsp_client_ctor(struct nvkm_gsp *gsp, struct nvkm_gsp_client *client)
 514 {
 515         NV0000_ALLOC_PARAMETERS *args;
 516         int ret;
 517
 518         mutex_lock(&gsp->client_id.mutex);
 519         ret = idr_alloc(&gsp->client_id.idr, client, 0, 0xffff + 1, GFP_KERNEL);
 520         mutex_unlock(&gsp->client_id.mutex);
 521         if (ret < 0)
 522                 return ret;
 523
 524         client->gsp = gsp;
 525         client->object.client = client;
 526         INIT_LIST_HEAD(&client->events);
 527
 528         args = nvkm_gsp_rm_alloc_get(&client->object, 0xc1d00000 | ret, NV01_ROOT, sizeof(*args),
 529                                      &client->object);
 530         if (IS_ERR(args)) {
 531                 r535_gsp_client_dtor(client);
 532                 return ret;
 533         }
 534
 535         args->hClient = client->object.handle;
 536         args->processID = ~0;
 537
 538         ret = nvkm_gsp_rm_alloc_wr(&client->object, args);
 539         if (ret) {
 540                 r535_gsp_client_dtor(client);
 541                 return ret;
 542         }
 543
 544         return 0;
 545 }
 546
 547 static int
 548 r535_gsp_rpc_rm_free(struct nvkm_gsp_object *object)
 549 {
 550         struct nvkm_gsp_client *client = object->client;
 551         struct nvkm_gsp *gsp = client->gsp;
 552         rpc_free_v03_00 *rpc;
 553
 554         nvkm_debug(&gsp->subdev, "cli:0x%08x obj:0x%08x free\n",
 555                    client->object.handle, object->handle);
 556
 557         rpc = nvkm_gsp_rpc_get(gsp, NV_VGPU_MSG_FUNCTION_FREE, sizeof(*rpc));
 558         if (WARN_ON(IS_ERR_OR_NULL(rpc)))
 559                 return -EIO;
 560
 561         rpc->params.hRoot = client->object.handle;
 562         rpc->params.hObjectParent = 0;
 563         rpc->params.hObjectOld = object->handle;
 564         return nvkm_gsp_rpc_wr(gsp, rpc, true);
 565 }
 566
 567 static void
 568 r535_gsp_rpc_rm_alloc_done(struct nvkm_gsp_object *object, void *repv)
 569 {
 570         rpc_gsp_rm_alloc_v03_00 *rpc = container_of(repv, typeof(*rpc), params);
 571
 572         nvkm_gsp_rpc_done(object->client->gsp, rpc);
 573 }
 574
 575 static void *
 576 r535_gsp_rpc_rm_alloc_push(struct nvkm_gsp_object *object, void *argv, u32 repc)
 577 {
 578         rpc_gsp_rm_alloc_v03_00 *rpc = container_of(argv, typeof(*rpc), params);
 579         struct nvkm_gsp *gsp = object->client->gsp;
 580         void *ret;
 581
 582         rpc = nvkm_gsp_rpc_push(gsp, rpc, true, sizeof(*rpc) + repc);
 583         if (IS_ERR_OR_NULL(rpc))
 584                 return rpc;
 585
 586         if (rpc->status) {
 587                 nvkm_error(&gsp->subdev, "RM_ALLOC: 0x%x\n", rpc->status);
 588                 ret = ERR_PTR(-EINVAL);
 589         } else {
 590                 ret = repc ? rpc->params : NULL;
 591         }
 592
 593         if (IS_ERR_OR_NULL(ret))
 594                 nvkm_gsp_rpc_done(gsp, rpc);
 595
 596         return ret;
 597 }
 598
 599 static void *
 600 r535_gsp_rpc_rm_alloc_get(struct nvkm_gsp_object *object, u32 oclass, u32 argc)
 601 {
 602         struct nvkm_gsp_client *client = object->client;
 603         struct nvkm_gsp *gsp = client->gsp;
 604         rpc_gsp_rm_alloc_v03_00 *rpc;
 605
 606         nvkm_debug(&gsp->subdev, "cli:0x%08x obj:0x%08x new obj:0x%08x cls:0x%08x argc:%d\n",
 607                    client->object.handle, object->parent->handle, object->handle, oclass, argc);
 608
 609         rpc = nvkm_gsp_rpc_get(gsp, NV_VGPU_MSG_FUNCTION_GSP_RM_ALLOC, sizeof(*rpc) + argc);
 610         if (IS_ERR(rpc))
 611                 return rpc;
 612
 613         rpc->hClient = client->object.handle;
 614         rpc->hParent = object->parent->handle;
 615         rpc->hObject = object->handle;
 616         rpc->hClass = oclass;
 617         rpc->status = 0;
 618         rpc->paramsSize = argc;
 619         return rpc->params;
 620 }
 621
 622 static void
 623 r535_gsp_rpc_rm_ctrl_done(struct nvkm_gsp_object *object, void *repv)
 624 {
 625         rpc_gsp_rm_control_v03_00 *rpc = container_of(repv, typeof(*rpc), params);
 626
 627         nvkm_gsp_rpc_done(object->client->gsp, rpc);
 628 }
 629
 630 static void *
 631 r535_gsp_rpc_rm_ctrl_push(struct nvkm_gsp_object *object, void *argv, u32 repc)
 632 {
 633         rpc_gsp_rm_control_v03_00 *rpc = container_of(argv, typeof(*rpc), params);
 634         struct nvkm_gsp *gsp = object->client->gsp;
 635         void *ret;
 636
 637         rpc = nvkm_gsp_rpc_push(gsp, rpc, true, repc);
 638         if (IS_ERR_OR_NULL(rpc))
 639                 return rpc;
 640
 641         if (rpc->status) {
 642                 nvkm_error(&gsp->subdev, "cli:0x%08x obj:0x%08x ctrl cmd:0x%08x failed: 0x%08x\n",
 643                            object->client->object.handle, object->handle, rpc->cmd, rpc->status);
 644                 ret = ERR_PTR(-EINVAL);
 645         } else {
 646                 ret = repc ? rpc->params : NULL;
 647         }
 648
 649         if (IS_ERR_OR_NULL(ret))
 650                 nvkm_gsp_rpc_done(gsp, rpc);
 651
 652         return ret;
 653 }
 654
 655 static void *
 656 r535_gsp_rpc_rm_ctrl_get(struct nvkm_gsp_object *object, u32 cmd, u32 argc)
 657 {
 658         struct nvkm_gsp_client *client = object->client;
 659         struct nvkm_gsp *gsp = client->gsp;
 660         rpc_gsp_rm_control_v03_00 *rpc;
 661
 662         nvkm_debug(&gsp->subdev, "cli:0x%08x obj:0x%08x ctrl cmd:0x%08x argc:%d\n",
 663                    client->object.handle, object->handle, cmd, argc);
 664
 665         rpc = nvkm_gsp_rpc_get(gsp, NV_VGPU_MSG_FUNCTION_GSP_RM_CONTROL, sizeof(*rpc) + argc);
 666         if (IS_ERR(rpc))
 667                 return rpc;
 668
 669         rpc->hClient    = client->object.handle;
 670         rpc->hObject    = object->handle;
 671         rpc->cmd        = cmd;
 672         rpc->status     = 0;
 673         rpc->paramsSize = argc;
 674         return rpc->params;
 675 }
 676
 677 static void
 678 r535_gsp_rpc_done(struct nvkm_gsp *gsp, void *repv)
 679 {
 680         struct nvfw_gsp_rpc *rpc = container_of(repv, typeof(*rpc), data);
 681
 682         r535_gsp_msg_done(gsp, rpc);
 683 }
 684
 685 static void *
 686 r535_gsp_rpc_get(struct nvkm_gsp *gsp, u32 fn, u32 argc)
 687 {
 688         struct nvfw_gsp_rpc *rpc;
 689
 690         rpc = r535_gsp_cmdq_get(gsp, ALIGN(sizeof(*rpc) + argc, sizeof(u64)));
 691         if (IS_ERR(rpc))
 692                 return ERR_CAST(rpc);
 693
 694         rpc->header_version = 0x03000000;
 695         rpc->signature = ('C' << 24) | ('P' << 16) | ('R' << 8) | 'V';
 696         rpc->function = fn;
 697         rpc->rpc_result = 0xffffffff;
 698         rpc->rpc_result_private = 0xffffffff;
 699         rpc->length = sizeof(*rpc) + argc;
 700         return rpc->data;
 701 }
 702
 703 static void *
 704 r535_gsp_rpc_push(struct nvkm_gsp *gsp, void *argv, bool wait, u32 repc)
 705 {
 706         struct nvfw_gsp_rpc *rpc = container_of(argv, typeof(*rpc), data);
 707         struct r535_gsp_msg *cmd = container_of((void *)rpc, typeof(*cmd), data);
 708         const u32 max_msg_size = (16 * 0x1000) - sizeof(struct r535_gsp_msg);
 709         const u32 max_rpc_size = max_msg_size - sizeof(*rpc);
 710         u32 rpc_size = rpc->length - sizeof(*rpc);
 711         void *repv;
 712
 713         mutex_lock(&gsp->cmdq.mutex);
 714         if (rpc_size > max_rpc_size) {
 715                 const u32 fn = rpc->function;
 716
 717                 /* Adjust length, and send initial RPC. */
 718                 rpc->length = sizeof(*rpc) + max_rpc_size;
 719                 cmd->checksum = rpc->length;
 720
 721                 repv = r535_gsp_rpc_send(gsp, argv, false, 0);
 722                 if (IS_ERR(repv))
 723                         goto done;
 724
 725                 argv += max_rpc_size;
 726                 rpc_size -= max_rpc_size;
 727
 728                 /* Remaining chunks sent as CONTINUATION_RECORD RPCs. */
 729                 while (rpc_size) {
 730                         u32 size = min(rpc_size, max_rpc_size);
 731                         void *next;
 732
 733                         next = r535_gsp_rpc_get(gsp, NV_VGPU_MSG_FUNCTION_CONTINUATION_RECORD, size);
 734                         if (IS_ERR(next)) {
 735                                 repv = next;
 736                                 goto done;
 737                         }
 738
 739                         memcpy(next, argv, size);
 740
 741                         repv = r535_gsp_rpc_send(gsp, next, false, 0);
 742                         if (IS_ERR(repv))
 743                                 goto done;
 744
 745                         argv += size;
 746                         rpc_size -= size;
 747                 }
 748
 749                 /* Wait for reply. */
 750                 if (wait) {
 751                         rpc = r535_gsp_msg_recv(gsp, fn, repc);
 752                         if (!IS_ERR_OR_NULL(rpc))
 753                                 repv = rpc->data;
 754                         else
 755                                 repv = rpc;
 756                 } else {
 757                         repv = NULL;
 758                 }
 759         } else {
 760                 repv = r535_gsp_rpc_send(gsp, argv, wait, repc);
 761         }
 762
 763 done:
 764         mutex_unlock(&gsp->cmdq.mutex);
 765         return repv;
 766 }
 767
 768 const struct nvkm_gsp_rm
 769 r535_gsp_rm = {
 770         .rpc_get = r535_gsp_rpc_get,
 771         .rpc_push = r535_gsp_rpc_push,
 772         .rpc_done = r535_gsp_rpc_done,
 773
 774         .rm_ctrl_get = r535_gsp_rpc_rm_ctrl_get,
 775         .rm_ctrl_push = r535_gsp_rpc_rm_ctrl_push,
 776         .rm_ctrl_done = r535_gsp_rpc_rm_ctrl_done,
 777
 778         .rm_alloc_get = r535_gsp_rpc_rm_alloc_get,
 779         .rm_alloc_push = r535_gsp_rpc_rm_alloc_push,
 780         .rm_alloc_done = r535_gsp_rpc_rm_alloc_done,
 781
 782         .rm_free = r535_gsp_rpc_rm_free,
 783
 784         .client_ctor = r535_gsp_client_ctor,
 785         .client_dtor = r535_gsp_client_dtor,
 786
 787         .device_ctor = r535_gsp_device_ctor,
 788         .device_dtor = r535_gsp_device_dtor,
 789
 790         .event_ctor = r535_gsp_device_event_ctor,
 791         .event_dtor = r535_gsp_event_dtor,
 792 };
 793
 794 static void
 795 r535_gsp_msgq_work(struct work_struct *work)
 796 {
 797         struct nvkm_gsp *gsp = container_of(work, typeof(*gsp), msgq.work);
 798
 799         mutex_lock(&gsp->cmdq.mutex);
 800         if (*gsp->msgq.rptr != *gsp->msgq.wptr)
 801                 r535_gsp_msg_recv(gsp, 0, 0);
 802         mutex_unlock(&gsp->cmdq.mutex);
 803 }
 804
 805 static irqreturn_t
 806 r535_gsp_intr(struct nvkm_inth *inth)
 807 {
 808         struct nvkm_gsp *gsp = container_of(inth, typeof(*gsp), subdev.inth);
 809         struct nvkm_subdev *subdev = &gsp->subdev;
 810         u32 intr = nvkm_falcon_rd32(&gsp->falcon, 0x0008);
 811         u32 inte = nvkm_falcon_rd32(&gsp->falcon, gsp->falcon.func->addr2 +
 812                                                   gsp->falcon.func->riscv_irqmask);
 813         u32 stat = intr & inte;
 814
 815         if (!stat) {
 816                 nvkm_debug(subdev, "inte %08x %08x\n", intr, inte);
 817                 return IRQ_NONE;
 818         }
 819
 820         if (stat & 0x00000040) {
 821                 nvkm_falcon_wr32(&gsp->falcon, 0x004, 0x00000040);
 822                 schedule_work(&gsp->msgq.work);
 823                 stat &= ~0x00000040;
 824         }
 825
 826         if (stat) {
 827                 nvkm_error(subdev, "intr %08x\n", stat);
 828                 nvkm_falcon_wr32(&gsp->falcon, 0x014, stat);
 829                 nvkm_falcon_wr32(&gsp->falcon, 0x004, stat);
 830         }
 831
 832         nvkm_falcon_intr_retrigger(&gsp->falcon);
 833         return IRQ_HANDLED;
 834 }
 835
 836 static int
 837 r535_gsp_intr_get_table(struct nvkm_gsp *gsp)
 838 {
 839         NV2080_CTRL_INTERNAL_INTR_GET_KERNEL_TABLE_PARAMS *ctrl;
 840         int ret = 0;
 841
 842         ctrl = nvkm_gsp_rm_ctrl_get(&gsp->internal.device.subdevice,
 843                                     NV2080_CTRL_CMD_INTERNAL_INTR_GET_KERNEL_TABLE, sizeof(*ctrl));
 844         if (IS_ERR(ctrl))
 845                 return PTR_ERR(ctrl);
 846
 847         ctrl = nvkm_gsp_rm_ctrl_push(&gsp->internal.device.subdevice, ctrl, sizeof(*ctrl));
 848         if (WARN_ON(IS_ERR(ctrl)))
 849                 return PTR_ERR(ctrl);
 850
 851         for (unsigned i = 0; i < ctrl->tableLen; i++) {
 852                 enum nvkm_subdev_type type;
 853                 int inst;
 854
 855                 nvkm_debug(&gsp->subdev,
 856                            "%2d: engineIdx %3d pmcIntrMask %08x stall %08x nonStall %08x\n", i,
 857                            ctrl->table[i].engineIdx, ctrl->table[i].pmcIntrMask,
 858                            ctrl->table[i].vectorStall, ctrl->table[i].vectorNonStall);
 859
 860                 switch (ctrl->table[i].engineIdx) {
 861                 case MC_ENGINE_IDX_GSP:
 862                         type = NVKM_SUBDEV_GSP;
 863                         inst = 0;
 864                         break;
 865                 case MC_ENGINE_IDX_DISP:
 866                         type = NVKM_ENGINE_DISP;
 867                         inst = 0;
 868                         break;
 869                 case MC_ENGINE_IDX_CE0 ... MC_ENGINE_IDX_CE9:
 870                         type = NVKM_ENGINE_CE;
 871                         inst = ctrl->table[i].engineIdx - MC_ENGINE_IDX_CE0;
 872                         break;
 873                 case MC_ENGINE_IDX_GR0:
 874                         type = NVKM_ENGINE_GR;
 875                         inst = 0;
 876                         break;
 877                 case MC_ENGINE_IDX_NVDEC0 ... MC_ENGINE_IDX_NVDEC7:
 878                         type = NVKM_ENGINE_NVDEC;
 879                         inst = ctrl->table[i].engineIdx - MC_ENGINE_IDX_NVDEC0;
 880                         break;
 881                 case MC_ENGINE_IDX_MSENC ... MC_ENGINE_IDX_MSENC2:
 882                         type = NVKM_ENGINE_NVENC;
 883                         inst = ctrl->table[i].engineIdx - MC_ENGINE_IDX_MSENC;
 884                         break;
 885                 case MC_ENGINE_IDX_NVJPEG0 ... MC_ENGINE_IDX_NVJPEG7:
 886                         type = NVKM_ENGINE_NVJPG;
 887                         inst = ctrl->table[i].engineIdx - MC_ENGINE_IDX_NVJPEG0;
 888                         break;
 889                 case MC_ENGINE_IDX_OFA0:
 890                         type = NVKM_ENGINE_OFA;
 891                         inst = 0;
 892                         break;
 893                 default:
 894                         continue;
 895                 }
 896
 897                 if (WARN_ON(gsp->intr_nr == ARRAY_SIZE(gsp->intr))) {
 898                         ret = -ENOSPC;
 899                         break;
 900                 }
 901
 902                 gsp->intr[gsp->intr_nr].type = type;
 903                 gsp->intr[gsp->intr_nr].inst = inst;
 904                 gsp->intr[gsp->intr_nr].stall = ctrl->table[i].vectorStall;
 905                 gsp->intr[gsp->intr_nr].nonstall = ctrl->table[i].vectorNonStall;
 906                 gsp->intr_nr++;
 907         }
 908
 909         nvkm_gsp_rm_ctrl_done(&gsp->internal.device.subdevice, ctrl);
 910         return ret;
 911 }
 912
 913 static int
 914 r535_gsp_rpc_get_gsp_static_info(struct nvkm_gsp *gsp)
 915 {
 916         GspStaticConfigInfo *rpc;
 917         int last_usable = -1;
 918
 919         rpc = nvkm_gsp_rpc_rd(gsp, NV_VGPU_MSG_FUNCTION_GET_GSP_STATIC_INFO, sizeof(*rpc));
 920         if (IS_ERR(rpc))
 921                 return PTR_ERR(rpc);
 922
 923         gsp->internal.client.object.client = &gsp->internal.client;
 924         gsp->internal.client.object.parent = NULL;
 925         gsp->internal.client.object.handle = rpc->hInternalClient;
 926         gsp->internal.client.gsp = gsp;
 927
 928         gsp->internal.device.object.client = &gsp->internal.client;
 929         gsp->internal.device.object.parent = &gsp->internal.client.object;
 930         gsp->internal.device.object.handle = rpc->hInternalDevice;
 931
 932         gsp->internal.device.subdevice.client = &gsp->internal.client;
 933         gsp->internal.device.subdevice.parent = &gsp->internal.device.object;
 934         gsp->internal.device.subdevice.handle = rpc->hInternalSubdevice;
 935
 936         gsp->bar.rm_bar1_pdb = rpc->bar1PdeBase;
 937         gsp->bar.rm_bar2_pdb = rpc->bar2PdeBase;
 938
 939         for (int i = 0; i < rpc->fbRegionInfoParams.numFBRegions; i++) {
 940                 NV2080_CTRL_CMD_FB_GET_FB_REGION_FB_REGION_INFO *reg =
 941                         &rpc->fbRegionInfoParams.fbRegion[i];
 942
 943                 nvkm_debug(&gsp->subdev, "fb region %d: "
 944                            "%016llx-%016llx rsvd:%016llx perf:%08x comp:%d iso:%d prot:%d\n", i,
 945                            reg->base, reg->limit, reg->reserved, reg->performance,
 946                            reg->supportCompressed, reg->supportISO, reg->bProtected);
 947
 948                 if (!reg->reserved && !reg->bProtected) {
 949                         if (reg->supportCompressed && reg->supportISO &&
 950                             !WARN_ON_ONCE(gsp->fb.region_nr >= ARRAY_SIZE(gsp->fb.region))) {
 951                                         const u64 size = (reg->limit + 1) - reg->base;
 952
 953                                         gsp->fb.region[gsp->fb.region_nr].addr = reg->base;
 954                                         gsp->fb.region[gsp->fb.region_nr].size = size;
 955                                         gsp->fb.region_nr++;
 956                         }
 957
 958                         last_usable = i;
 959                 }
 960         }
 961
 962         if (last_usable >= 0) {
 963                 u32 rsvd_base = rpc->fbRegionInfoParams.fbRegion[last_usable].limit + 1;
 964
 965                 gsp->fb.rsvd_size = gsp->fb.heap.addr - rsvd_base;
 966         }
 967
 968         for (int gpc = 0; gpc < ARRAY_SIZE(rpc->tpcInfo); gpc++) {
 969                 if (rpc->gpcInfo.gpcMask & BIT(gpc)) {
 970                         gsp->gr.tpcs += hweight32(rpc->tpcInfo[gpc].tpcMask);
 971                         gsp->gr.gpcs++;
 972                 }
 973         }
 974
 975         nvkm_gsp_rpc_done(gsp, rpc);
 976         return 0;
 977 }
 978
 979 static int
 980 r535_gsp_postinit(struct nvkm_gsp *gsp)
 981 {
 982         struct nvkm_device *device = gsp->subdev.device;
 983         int ret;
 984
 985         ret = r535_gsp_rpc_get_gsp_static_info(gsp);
 986         if (WARN_ON(ret))
 987                 return ret;
 988
 989         INIT_WORK(&gsp->msgq.work, r535_gsp_msgq_work);
 990
 991         ret = r535_gsp_intr_get_table(gsp);
 992         if (WARN_ON(ret))
 993                 return ret;
 994
 995         ret = nvkm_gsp_intr_stall(gsp, gsp->subdev.type, gsp->subdev.inst);
 996         if (WARN_ON(ret < 0))
 997                 return ret;
 998
 999         ret = nvkm_inth_add(&device->vfn->intr, ret, NVKM_INTR_PRIO_NORMAL, &gsp->subdev,
1000                             r535_gsp_intr, &gsp->subdev.inth);
1001         if (WARN_ON(ret))
1002                 return ret;
1003
1004         nvkm_inth_allow(&gsp->subdev.inth);
1005         nvkm_wr32(device, 0x110004, 0x00000040);
1006         return ret;
1007 }
1008
1009 static int
1010 r535_gsp_rpc_unloading_guest_driver(struct nvkm_gsp *gsp, bool suspend)
1011 {
1012         rpc_unloading_guest_driver_v1F_07 *rpc;
1013
1014         rpc = nvkm_gsp_rpc_get(gsp, NV_VGPU_MSG_FUNCTION_UNLOADING_GUEST_DRIVER, sizeof(*rpc));
1015         if (IS_ERR(rpc))
1016                 return PTR_ERR(rpc);
1017
1018         if (suspend) {
1019                 rpc->bInPMTransition = 1;
1020                 rpc->bGc6Entering = 0;
1021                 rpc->newLevel = NV2080_CTRL_GPU_SET_POWER_STATE_GPU_LEVEL_3;
1022         } else {
1023                 rpc->bInPMTransition = 0;
1024                 rpc->bGc6Entering = 0;
1025                 rpc->newLevel = NV2080_CTRL_GPU_SET_POWER_STATE_GPU_LEVEL_0;
1026         }
1027
1028         return nvkm_gsp_rpc_wr(gsp, rpc, true);
1029 }
1030
1031 /* dword only */
1032 struct nv_gsp_registry_entries {
1033         const char *name;
1034         u32 value;
1035 };
1036
1037 static const struct nv_gsp_registry_entries r535_registry_entries[] = {
1038         { "RMSecBusResetEnable", 1 },
1039         { "RMForcePcieConfigSave", 1 },
1040 };
1041 #define NV_GSP_REG_NUM_ENTRIES ARRAY_SIZE(r535_registry_entries)
1042
1043 static int
1044 r535_gsp_rpc_set_registry(struct nvkm_gsp *gsp)
1045 {
1046         PACKED_REGISTRY_TABLE *rpc;
1047         char *strings;
1048         int str_offset;
1049         int i;
1050         size_t rpc_size = struct_size(rpc, entries, NV_GSP_REG_NUM_ENTRIES);
1051
1052         /* add strings + null terminator */
1053         for (i = 0; i < NV_GSP_REG_NUM_ENTRIES; i++)
1054                 rpc_size += strlen(r535_registry_entries[i].name) + 1;
1055
1056         rpc = nvkm_gsp_rpc_get(gsp, NV_VGPU_MSG_FUNCTION_SET_REGISTRY, rpc_size);
1057         if (IS_ERR(rpc))
1058                 return PTR_ERR(rpc);
1059
1060         rpc->size = sizeof(*rpc);
1061         rpc->numEntries = NV_GSP_REG_NUM_ENTRIES;
1062
1063         str_offset = offsetof(typeof(*rpc), entries[NV_GSP_REG_NUM_ENTRIES]);
1064         strings = (char *)&rpc->entries[NV_GSP_REG_NUM_ENTRIES];
1065         for (i = 0; i < NV_GSP_REG_NUM_ENTRIES; i++) {
1066                 int name_len = strlen(r535_registry_entries[i].name) + 1;
1067
1068                 rpc->entries[i].nameOffset = str_offset;
1069                 rpc->entries[i].type = 1;
1070                 rpc->entries[i].data = r535_registry_entries[i].value;
1071                 rpc->entries[i].length = 4;
1072                 memcpy(strings, r535_registry_entries[i].name, name_len);
1073                 strings += name_len;
1074                 str_offset += name_len;
1075         }
1076
1077         return nvkm_gsp_rpc_wr(gsp, rpc, false);
1078 }
1079
1080 #if defined(CONFIG_ACPI) && defined(CONFIG_X86)
1081 static void
1082 r535_gsp_acpi_caps(acpi_handle handle, CAPS_METHOD_DATA *caps)
1083 {
1084         const guid_t NVOP_DSM_GUID =
1085                 GUID_INIT(0xA486D8F8, 0x0BDA, 0x471B,
1086                           0xA7, 0x2B, 0x60, 0x42, 0xA6, 0xB5, 0xBE, 0xE0);
1087         u64 NVOP_DSM_REV = 0x00000100;
1088         union acpi_object argv4 = {
1089                 .buffer.type    = ACPI_TYPE_BUFFER,
1090                 .buffer.length  = 4,
1091                 .buffer.pointer = kmalloc(argv4.buffer.length, GFP_KERNEL),
1092         }, *obj;
1093
1094         caps->status = 0xffff;
1095
1096         if (!acpi_check_dsm(handle, &NVOP_DSM_GUID, NVOP_DSM_REV, BIT_ULL(0x1a)))
1097                 return;
1098
1099         obj = acpi_evaluate_dsm(handle, &NVOP_DSM_GUID, NVOP_DSM_REV, 0x1a, &argv4);
1100         if (!obj)
1101                 return;
1102
1103         printk(KERN_ERR "nvop: obj type %d\n", obj->type);
1104         printk(KERN_ERR "nvop: obj len %d\n", obj->buffer.length);
1105
1106         if (WARN_ON(obj->type != ACPI_TYPE_BUFFER) ||
1107             WARN_ON(obj->buffer.length != 4))
1108                 return;
1109
1110         caps->status = 0;
1111         caps->optimusCaps = *(u32 *)obj->buffer.pointer;
1112         printk(KERN_ERR "nvop: caps %08x\n", caps->optimusCaps);
1113
1114         ACPI_FREE(obj);
1115
1116         kfree(argv4.buffer.pointer);
1117 }
1118
1119 static void
1120 r535_gsp_acpi_jt(acpi_handle handle, JT_METHOD_DATA *jt)
1121 {
1122         const guid_t JT_DSM_GUID =
1123                 GUID_INIT(0xCBECA351L, 0x067B, 0x4924,
1124                           0x9C, 0xBD, 0xB4, 0x6B, 0x00, 0xB8, 0x6F, 0x34);
1125         u64 JT_DSM_REV = 0x00000103;
1126         u32 caps;
1127         union acpi_object argv4 = {
1128                 .buffer.type    = ACPI_TYPE_BUFFER,
1129                 .buffer.length  = sizeof(caps),
1130                 .buffer.pointer = kmalloc(argv4.buffer.length, GFP_KERNEL),
1131         }, *obj;
1132
1133         jt->status = 0xffff;
1134
1135         obj = acpi_evaluate_dsm(handle, &JT_DSM_GUID, JT_DSM_REV, 0x1, &argv4);
1136         if (!obj)
1137                 return;
1138
1139         printk(KERN_ERR "jt: obj type %d\n", obj->type);
1140         printk(KERN_ERR "jt: obj len %d\n", obj->buffer.length);
1141
1142         if (WARN_ON(obj->type != ACPI_TYPE_BUFFER) ||
1143             WARN_ON(obj->buffer.length != 4))
1144                 return;
1145
1146         jt->status = 0;
1147         jt->jtCaps = *(u32 *)obj->buffer.pointer;
1148         jt->jtRevId = (jt->jtCaps & 0xfff00000) >> 20;
1149         jt->bSBIOSCaps = 0;
1150         printk(KERN_ERR "jt: caps %08x rev:%04x\n", jt->jtCaps, jt->jtRevId);
1151
1152         ACPI_FREE(obj);
1153
1154         kfree(argv4.buffer.pointer);
1155 }
1156
1157 static void
1158 r535_gsp_acpi_mux_id(acpi_handle handle, u32 id, MUX_METHOD_DATA_ELEMENT *mode,
1159                                                  MUX_METHOD_DATA_ELEMENT *part)
1160 {
1161         acpi_handle iter = NULL, handle_mux = NULL;
1162         acpi_status status;
1163         unsigned long long value;
1164
1165         mode->status = 0xffff;
1166         part->status = 0xffff;
1167
1168         do {
1169                 status = acpi_get_next_object(ACPI_TYPE_DEVICE, handle, iter, &iter);
1170                 if (ACPI_FAILURE(status) || !iter)
1171                         return;
1172
1173                 status = acpi_evaluate_integer(iter, "_ADR", NULL, &value);
1174                 if (ACPI_FAILURE(status) || value != id)
1175                         continue;
1176
1177                 handle_mux = iter;
1178         } while (!handle_mux);
1179
1180         if (!handle_mux)
1181                 return;
1182
1183         status = acpi_evaluate_integer(handle_mux, "MXDM", NULL, &value);
1184         if (ACPI_SUCCESS(status)) {
1185                 mode->acpiId = id;
1186                 mode->mode   = value;
1187                 mode->status = 0;
1188         }
1189
1190         status = acpi_evaluate_integer(handle_mux, "MXDS", NULL, &value);
1191         if (ACPI_SUCCESS(status)) {
1192                 part->acpiId = id;
1193                 part->mode   = value;
1194                 part->status = 0;
1195         }
1196 }
1197
1198 static void
1199 r535_gsp_acpi_mux(acpi_handle handle, DOD_METHOD_DATA *dod, MUX_METHOD_DATA *mux)
1200 {
1201         mux->tableLen = dod->acpiIdListLen / sizeof(dod->acpiIdList[0]);
1202
1203         for (int i = 0; i < mux->tableLen; i++) {
1204                 r535_gsp_acpi_mux_id(handle, dod->acpiIdList[i], &mux->acpiIdMuxModeTable[i],
1205                                                                  &mux->acpiIdMuxPartTable[i]);
1206         }
1207 }
1208
1209 static void
1210 r535_gsp_acpi_dod(acpi_handle handle, DOD_METHOD_DATA *dod)
1211 {
1212         acpi_status status;
1213         struct acpi_buffer output = { ACPI_ALLOCATE_BUFFER, NULL };
1214         union acpi_object *_DOD;
1215
1216         dod->status = 0xffff;
1217
1218         status = acpi_evaluate_object(handle, "_DOD", NULL, &output);
1219         if (ACPI_FAILURE(status))
1220                 return;
1221
1222         _DOD = output.pointer;
1223
1224         if (WARN_ON(_DOD->type != ACPI_TYPE_PACKAGE) ||
1225             WARN_ON(_DOD->package.count > ARRAY_SIZE(dod->acpiIdList)))
1226                 return;
1227
1228         for (int i = 0; i < _DOD->package.count; i++) {
1229                 if (WARN_ON(_DOD->package.elements[i].type != ACPI_TYPE_INTEGER))
1230                         return;
1231
1232                 dod->acpiIdList[i] = _DOD->package.elements[i].integer.value;
1233                 dod->acpiIdListLen += sizeof(dod->acpiIdList[0]);
1234         }
1235
1236         printk(KERN_ERR "_DOD: ok! len:%d\n", dod->acpiIdListLen);
1237         dod->status = 0;
1238 }
1239 #endif
1240
1241 static void
1242 r535_gsp_acpi_info(struct nvkm_gsp *gsp, ACPI_METHOD_DATA *acpi)
1243 {
1244 #if defined(CONFIG_ACPI) && defined(CONFIG_X86)
1245         acpi_handle handle = ACPI_HANDLE(gsp->subdev.device->dev);
1246
1247         if (!handle)
1248                 return;
1249
1250         acpi->bValid = 1;
1251
1252         r535_gsp_acpi_dod(handle, &acpi->dodMethodData);
1253         if (acpi->dodMethodData.status == 0)
1254                 r535_gsp_acpi_mux(handle, &acpi->dodMethodData, &acpi->muxMethodData);
1255
1256         r535_gsp_acpi_jt(handle, &acpi->jtMethodData);
1257         r535_gsp_acpi_caps(handle, &acpi->capsMethodData);
1258 #endif
1259 }
1260
1261 static int
1262 r535_gsp_rpc_set_system_info(struct nvkm_gsp *gsp)
1263 {
1264         struct nvkm_device *device = gsp->subdev.device;
1265         struct nvkm_device_pci *pdev = container_of(device, typeof(*pdev), device);
1266         GspSystemInfo *info;
1267
1268         if (WARN_ON(device->type == NVKM_DEVICE_TEGRA))
1269                 return -ENOSYS;
1270
1271         info = nvkm_gsp_rpc_get(gsp, NV_VGPU_MSG_FUNCTION_GSP_SET_SYSTEM_INFO, sizeof(*info));
1272         if (IS_ERR(info))
1273                 return PTR_ERR(info);
1274
1275         info->gpuPhysAddr = device->func->resource_addr(device, 0);
1276         info->gpuPhysFbAddr = device->func->resource_addr(device, 1);
1277         info->gpuPhysInstAddr = device->func->resource_addr(device, 3);
1278         info->nvDomainBusDeviceFunc = pci_dev_id(pdev->pdev);
1279         info->maxUserVa = TASK_SIZE;
1280         info->pciConfigMirrorBase = 0x088000;
1281         info->pciConfigMirrorSize = 0x001000;
1282         r535_gsp_acpi_info(gsp, &info->acpiMethodData);
1283
1284         return nvkm_gsp_rpc_wr(gsp, info, false);
1285 }
1286
1287 static int
1288 r535_gsp_msg_os_error_log(void *priv, u32 fn, void *repv, u32 repc)
1289 {
1290         struct nvkm_gsp *gsp = priv;
1291         struct nvkm_subdev *subdev = &gsp->subdev;
1292         rpc_os_error_log_v17_00 *msg = repv;
1293
1294         if (WARN_ON(repc < sizeof(*msg)))
1295                 return -EINVAL;
1296
1297         nvkm_error(subdev, "Xid:%d %s\n", msg->exceptType, msg->errString);
1298         return 0;
1299 }
1300
1301 static int
1302 r535_gsp_msg_rc_triggered(void *priv, u32 fn, void *repv, u32 repc)
1303 {
1304         rpc_rc_triggered_v17_02 *msg = repv;
1305         struct nvkm_gsp *gsp = priv;
1306         struct nvkm_subdev *subdev = &gsp->subdev;
1307         struct nvkm_chan *chan;
1308         unsigned long flags;
1309
1310         if (WARN_ON(repc < sizeof(*msg)))
1311                 return -EINVAL;
1312
1313         nvkm_error(subdev, "rc engn:%08x chid:%d type:%d scope:%d part:%d\n",
1314                    msg->nv2080EngineType, msg->chid, msg->exceptType, msg->scope,
1315                    msg->partitionAttributionId);
1316
1317         chan = nvkm_chan_get_chid(&subdev->device->fifo->engine, msg->chid / 8, &flags);
1318         if (!chan) {
1319                 nvkm_error(subdev, "rc chid:%d not found!\n", msg->chid);
1320                 return 0;
1321         }
1322
1323         nvkm_chan_error(chan, false);
1324         nvkm_chan_put(&chan, flags);
1325         return 0;
1326 }
1327
1328 static int
1329 r535_gsp_msg_mmu_fault_queued(void *priv, u32 fn, void *repv, u32 repc)
1330 {
1331         struct nvkm_gsp *gsp = priv;
1332         struct nvkm_subdev *subdev = &gsp->subdev;
1333
1334         WARN_ON(repc != 0);
1335
1336         nvkm_error(subdev, "mmu fault queued\n");
1337         return 0;
1338 }
1339
1340 static int
1341 r535_gsp_msg_post_event(void *priv, u32 fn, void *repv, u32 repc)
1342 {
1343         struct nvkm_gsp *gsp = priv;
1344         struct nvkm_gsp_client *client;
1345         struct nvkm_subdev *subdev = &gsp->subdev;
1346         rpc_post_event_v17_00 *msg = repv;
1347
1348         if (WARN_ON(repc < sizeof(*msg)))
1349                 return -EINVAL;
1350         if (WARN_ON(repc != sizeof(*msg) + msg->eventDataSize))
1351                 return -EINVAL;
1352
1353         nvkm_debug(subdev, "event: %08x %08x %d %08x %08x %d %d\n",
1354                    msg->hClient, msg->hEvent, msg->notifyIndex, msg->data,
1355                    msg->status, msg->eventDataSize, msg->bNotifyList);
1356
1357         mutex_lock(&gsp->client_id.mutex);
1358         client = idr_find(&gsp->client_id.idr, msg->hClient & 0xffff);
1359         if (client) {
1360                 struct nvkm_gsp_event *event;
1361                 bool handled = false;
1362
1363                 list_for_each_entry(event, &client->events, head) {
1364                         if (event->object.handle == msg->hEvent) {
1365                                 event->func(event, msg->eventData, msg->eventDataSize);
1366                                 handled = true;
1367                         }
1368                 }
1369
1370                 if (!handled) {
1371                         nvkm_error(subdev, "event: cid 0x%08x event 0x%08x not found!\n",
1372                                    msg->hClient, msg->hEvent);
1373                 }
1374         } else {
1375                 nvkm_error(subdev, "event: cid 0x%08x not found!\n", msg->hClient);
1376         }
1377         mutex_unlock(&gsp->client_id.mutex);
1378         return 0;
1379 }
1380
1381 /**
1382  * r535_gsp_msg_run_cpu_sequencer() -- process I/O commands from the GSP
1383  *
1384  * The GSP sequencer is a list of I/O commands that the GSP can send to
1385  * the driver to perform for various purposes.  The most common usage is to
1386  * perform a special mid-initialization reset.
1387  */
1388 static int
1389 r535_gsp_msg_run_cpu_sequencer(void *priv, u32 fn, void *repv, u32 repc)
1390 {
1391         struct nvkm_gsp *gsp = priv;
1392         struct nvkm_subdev *subdev = &gsp->subdev;
1393         struct nvkm_device *device = subdev->device;
1394         rpc_run_cpu_sequencer_v17_00 *seq = repv;
1395         int ptr = 0, ret;
1396
1397         nvkm_debug(subdev, "seq: %08x %08x\n", seq->bufferSizeDWord, seq->cmdIndex);
1398
1399         while (ptr < seq->cmdIndex) {
1400                 GSP_SEQUENCER_BUFFER_CMD *cmd = (void *)&seq->commandBuffer[ptr];
1401
1402                 ptr += 1;
1403                 ptr += GSP_SEQUENCER_PAYLOAD_SIZE_DWORDS(cmd->opCode);
1404
1405                 switch (cmd->opCode) {
1406                 case GSP_SEQ_BUF_OPCODE_REG_WRITE: {
1407                         u32 addr = cmd->payload.regWrite.addr;
1408                         u32 data = cmd->payload.regWrite.val;
1409
1410                         nvkm_trace(subdev, "seq wr32 %06x %08x\n", addr, data);
1411                         nvkm_wr32(device, addr, data);
1412                 }
1413                         break;
1414                 case GSP_SEQ_BUF_OPCODE_REG_MODIFY: {
1415                         u32 addr = cmd->payload.regModify.addr;
1416                         u32 mask = cmd->payload.regModify.mask;
1417                         u32 data = cmd->payload.regModify.val;
1418
1419                         nvkm_trace(subdev, "seq mask %06x %08x %08x\n", addr, mask, data);
1420                         nvkm_mask(device, addr, mask, data);
1421                 }
1422                         break;
1423                 case GSP_SEQ_BUF_OPCODE_REG_POLL: {
1424                         u32 addr = cmd->payload.regPoll.addr;
1425                         u32 mask = cmd->payload.regPoll.mask;
1426                         u32 data = cmd->payload.regPoll.val;
1427                         u32 usec = cmd->payload.regPoll.timeout ?: 4000000;
1428                         //u32 error = cmd->payload.regPoll.error;
1429
1430                         nvkm_trace(subdev, "seq poll %06x %08x %08x %d\n", addr, mask, data, usec);
1431                         nvkm_rd32(device, addr);
1432                         nvkm_usec(device, usec,
1433                                 if ((nvkm_rd32(device, addr) & mask) == data)
1434                                         break;
1435                         );
1436                 }
1437                         break;
1438                 case GSP_SEQ_BUF_OPCODE_DELAY_US: {
1439                         u32 usec = cmd->payload.delayUs.val;
1440
1441                         nvkm_trace(subdev, "seq usec %d\n", usec);
1442                         udelay(usec);
1443                 }
1444                         break;
1445                 case GSP_SEQ_BUF_OPCODE_REG_STORE: {
1446                         u32 addr = cmd->payload.regStore.addr;
1447                         u32 slot = cmd->payload.regStore.index;
1448
1449                         seq->regSaveArea[slot] = nvkm_rd32(device, addr);
1450                         nvkm_trace(subdev, "seq save %08x -> %d: %08x\n", addr, slot,
1451                                    seq->regSaveArea[slot]);
1452                 }
1453                         break;
1454                 case GSP_SEQ_BUF_OPCODE_CORE_RESET:
1455                         nvkm_trace(subdev, "seq core reset\n");
1456                         nvkm_falcon_reset(&gsp->falcon);
1457                         nvkm_falcon_mask(&gsp->falcon, 0x624, 0x00000080, 0x00000080);
1458                         nvkm_falcon_wr32(&gsp->falcon, 0x10c, 0x00000000);
1459                         break;
1460                 case GSP_SEQ_BUF_OPCODE_CORE_START:
1461                         nvkm_trace(subdev, "seq core start\n");
1462                         if (nvkm_falcon_rd32(&gsp->falcon, 0x100) & 0x00000040)
1463                                 nvkm_falcon_wr32(&gsp->falcon, 0x130, 0x00000002);
1464                         else
1465                                 nvkm_falcon_wr32(&gsp->falcon, 0x100, 0x00000002);
1466                         break;
1467                 case GSP_SEQ_BUF_OPCODE_CORE_WAIT_FOR_HALT:
1468                         nvkm_trace(subdev, "seq core wait halt\n");
1469                         nvkm_msec(device, 2000,
1470                                 if (nvkm_falcon_rd32(&gsp->falcon, 0x100) & 0x00000010)
1471                                         break;
1472                         );
1473                         break;
1474                 case GSP_SEQ_BUF_OPCODE_CORE_RESUME: {
1475                         struct nvkm_sec2 *sec2 = device->sec2;
1476                         u32 mbox0;
1477
1478                         nvkm_trace(subdev, "seq core resume\n");
1479
1480                         ret = gsp->func->reset(gsp);
1481                         if (WARN_ON(ret))
1482                                 return ret;
1483
1484                         nvkm_falcon_wr32(&gsp->falcon, 0x040, lower_32_bits(gsp->libos.addr));
1485                         nvkm_falcon_wr32(&gsp->falcon, 0x044, upper_32_bits(gsp->libos.addr));
1486
1487                         nvkm_falcon_start(&sec2->falcon);
1488
1489                         if (nvkm_msec(device, 2000,
1490                                 if (nvkm_rd32(device, 0x1180f8) & 0x04000000)
1491                                         break;
1492                         ) < 0)
1493                                 return -ETIMEDOUT;
1494
1495                         mbox0 = nvkm_falcon_rd32(&sec2->falcon, 0x040);
1496                         if (WARN_ON(mbox0)) {
1497                                 nvkm_error(&gsp->subdev, "seq core resume sec2: 0x%x\n", mbox0);
1498                                 return -EIO;
1499                         }
1500
1501                         nvkm_falcon_wr32(&gsp->falcon, 0x080, gsp->boot.app_version);
1502
1503                         if (WARN_ON(!nvkm_falcon_riscv_active(&gsp->falcon)))
1504                                 return -EIO;
1505                 }
1506                         break;
1507                 default:
1508                         nvkm_error(subdev, "unknown sequencer opcode %08x\n", cmd->opCode);
1509                         return -EINVAL;
1510                 }
1511         }
1512
1513         return 0;
1514 }
1515
1516 static void
1517 nvkm_gsp_mem_dtor(struct nvkm_gsp *gsp, struct nvkm_gsp_mem *mem)
1518 {
1519         if (mem->data) {
1520                 dma_free_coherent(gsp->subdev.device->dev, mem->size, mem->data, mem->addr);
1521                 mem->data = NULL;
1522         }
1523 }
1524
1525 static int
1526 nvkm_gsp_mem_ctor(struct nvkm_gsp *gsp, u32 size, struct nvkm_gsp_mem *mem)
1527 {
1528         mem->size = size;
1529         mem->data = dma_alloc_coherent(gsp->subdev.device->dev, size, &mem->addr, GFP_KERNEL);
1530         if (WARN_ON(!mem->data))
1531                 return -ENOMEM;
1532
1533         return 0;
1534 }
1535
1536
1537 static int
1538 r535_gsp_booter_unload(struct nvkm_gsp *gsp, u32 mbox0, u32 mbox1)
1539 {
1540         struct nvkm_subdev *subdev = &gsp->subdev;
1541         struct nvkm_device *device = subdev->device;
1542         u32 wpr2_hi;
1543         int ret;
1544
1545         wpr2_hi = nvkm_rd32(device, 0x1fa828);
1546         if (!wpr2_hi) {
1547                 nvkm_debug(subdev, "WPR2 not set - skipping booter unload\n");
1548                 return 0;
1549         }
1550
1551         ret = nvkm_falcon_fw_boot(&gsp->booter.unload, &gsp->subdev, true, &mbox0, &mbox1, 0, 0);
1552         if (WARN_ON(ret))
1553                 return ret;
1554
1555         wpr2_hi = nvkm_rd32(device, 0x1fa828);
1556         if (WARN_ON(wpr2_hi))
1557                 return -EIO;
1558
1559         return 0;
1560 }
1561
1562 static int
1563 r535_gsp_booter_load(struct nvkm_gsp *gsp, u32 mbox0, u32 mbox1)
1564 {
1565         int ret;
1566
1567         ret = nvkm_falcon_fw_boot(&gsp->booter.load, &gsp->subdev, true, &mbox0, &mbox1, 0, 0);
1568         if (ret)
1569                 return ret;
1570
1571         nvkm_falcon_wr32(&gsp->falcon, 0x080, gsp->boot.app_version);
1572
1573         if (WARN_ON(!nvkm_falcon_riscv_active(&gsp->falcon)))
1574                 return -EIO;
1575
1576         return 0;
1577 }
1578
1579 static int
1580 r535_gsp_wpr_meta_init(struct nvkm_gsp *gsp)
1581 {
1582         GspFwWprMeta *meta;
1583         int ret;
1584
1585         ret = nvkm_gsp_mem_ctor(gsp, 0x1000, &gsp->wpr_meta);
1586         if (ret)
1587                 return ret;
1588
1589         meta = gsp->wpr_meta.data;
1590
1591         meta->magic = GSP_FW_WPR_META_MAGIC;
1592         meta->revision = GSP_FW_WPR_META_REVISION;
1593
1594         meta->sysmemAddrOfRadix3Elf = gsp->radix3.mem[0].addr;
1595         meta->sizeOfRadix3Elf = gsp->fb.wpr2.elf.size;
1596
1597         meta->sysmemAddrOfBootloader = gsp->boot.fw.addr;
1598         meta->sizeOfBootloader = gsp->boot.fw.size;
1599         meta->bootloaderCodeOffset = gsp->boot.code_offset;
1600         meta->bootloaderDataOffset = gsp->boot.data_offset;
1601         meta->bootloaderManifestOffset = gsp->boot.manifest_offset;
1602
1603         meta->sysmemAddrOfSignature = gsp->sig.addr;
1604         meta->sizeOfSignature = gsp->sig.size;
1605
1606         meta->gspFwRsvdStart = gsp->fb.heap.addr;
1607         meta->nonWprHeapOffset = gsp->fb.heap.addr;
1608         meta->nonWprHeapSize = gsp->fb.heap.size;
1609         meta->gspFwWprStart = gsp->fb.wpr2.addr;
1610         meta->gspFwHeapOffset = gsp->fb.wpr2.heap.addr;
1611         meta->gspFwHeapSize = gsp->fb.wpr2.heap.size;
1612         meta->gspFwOffset = gsp->fb.wpr2.elf.addr;
1613         meta->bootBinOffset = gsp->fb.wpr2.boot.addr;
1614         meta->frtsOffset = gsp->fb.wpr2.frts.addr;
1615         meta->frtsSize = gsp->fb.wpr2.frts.size;
1616         meta->gspFwWprEnd = ALIGN_DOWN(gsp->fb.bios.vga_workspace.addr, 0x20000);
1617         meta->fbSize = gsp->fb.size;
1618         meta->vgaWorkspaceOffset = gsp->fb.bios.vga_workspace.addr;
1619         meta->vgaWorkspaceSize = gsp->fb.bios.vga_workspace.size;
1620         meta->bootCount = 0;
1621         meta->partitionRpcAddr = 0;
1622         meta->partitionRpcRequestOffset = 0;
1623         meta->partitionRpcReplyOffset = 0;
1624         meta->verified = 0;
1625         return 0;
1626 }
1627
1628 static int
1629 r535_gsp_shared_init(struct nvkm_gsp *gsp)
1630 {
1631         struct {
1632                 msgqTxHeader tx;
1633                 msgqRxHeader rx;
1634         } *cmdq, *msgq;
1635         int ret, i;
1636
1637         gsp->shm.cmdq.size = 0x40000;
1638         gsp->shm.msgq.size = 0x40000;
1639
1640         gsp->shm.ptes.nr  = (gsp->shm.cmdq.size + gsp->shm.msgq.size) >> GSP_PAGE_SHIFT;
1641         gsp->shm.ptes.nr += DIV_ROUND_UP(gsp->shm.ptes.nr * sizeof(u64), GSP_PAGE_SIZE);
1642         gsp->shm.ptes.size = ALIGN(gsp->shm.ptes.nr * sizeof(u64), GSP_PAGE_SIZE);
1643
1644         ret = nvkm_gsp_mem_ctor(gsp, gsp->shm.ptes.size +
1645                                      gsp->shm.cmdq.size +
1646                                      gsp->shm.msgq.size,
1647                                 &gsp->shm.mem);
1648         if (ret)
1649                 return ret;
1650
1651         gsp->shm.ptes.ptr = gsp->shm.mem.data;
1652         gsp->shm.cmdq.ptr = (u8 *)gsp->shm.ptes.ptr + gsp->shm.ptes.size;
1653         gsp->shm.msgq.ptr = (u8 *)gsp->shm.cmdq.ptr + gsp->shm.cmdq.size;
1654
1655         for (i = 0; i < gsp->shm.ptes.nr; i++)
1656                 gsp->shm.ptes.ptr[i] = gsp->shm.mem.addr + (i << GSP_PAGE_SHIFT);
1657
1658         cmdq = gsp->shm.cmdq.ptr;
1659         cmdq->tx.version = 0;
1660         cmdq->tx.size = gsp->shm.cmdq.size;
1661         cmdq->tx.entryOff = GSP_PAGE_SIZE;
1662         cmdq->tx.msgSize = GSP_PAGE_SIZE;
1663         cmdq->tx.msgCount = (cmdq->tx.size - cmdq->tx.entryOff) / cmdq->tx.msgSize;
1664         cmdq->tx.writePtr = 0;
1665         cmdq->tx.flags = 1;
1666         cmdq->tx.rxHdrOff = offsetof(typeof(*cmdq), rx.readPtr);
1667
1668         msgq = gsp->shm.msgq.ptr;
1669
1670         gsp->cmdq.cnt = cmdq->tx.msgCount;
1671         gsp->cmdq.wptr = &cmdq->tx.writePtr;
1672         gsp->cmdq.rptr = &msgq->rx.readPtr;
1673         gsp->msgq.cnt = cmdq->tx.msgCount;
1674         gsp->msgq.wptr = &msgq->tx.writePtr;
1675         gsp->msgq.rptr = &cmdq->rx.readPtr;
1676         return 0;
1677 }
1678
1679 static int
1680 r535_gsp_rmargs_init(struct nvkm_gsp *gsp, bool resume)
1681 {
1682         GSP_ARGUMENTS_CACHED *args;
1683         int ret;
1684
1685         if (!resume) {
1686                 ret = r535_gsp_shared_init(gsp);
1687                 if (ret)
1688                         return ret;
1689
1690                 ret = nvkm_gsp_mem_ctor(gsp, 0x1000, &gsp->rmargs);
1691                 if (ret)
1692                         return ret;
1693         }
1694
1695         args = gsp->rmargs.data;
1696         args->messageQueueInitArguments.sharedMemPhysAddr = gsp->shm.mem.addr;
1697         args->messageQueueInitArguments.pageTableEntryCount = gsp->shm.ptes.nr;
1698         args->messageQueueInitArguments.cmdQueueOffset =
1699                 (u8 *)gsp->shm.cmdq.ptr - (u8 *)gsp->shm.mem.data;
1700         args->messageQueueInitArguments.statQueueOffset =
1701                 (u8 *)gsp->shm.msgq.ptr - (u8 *)gsp->shm.mem.data;
1702
1703         if (!resume) {
1704                 args->srInitArguments.oldLevel = 0;
1705                 args->srInitArguments.flags = 0;
1706                 args->srInitArguments.bInPMTransition = 0;
1707         } else {
1708                 args->srInitArguments.oldLevel = NV2080_CTRL_GPU_SET_POWER_STATE_GPU_LEVEL_3;
1709                 args->srInitArguments.flags = 0;
1710                 args->srInitArguments.bInPMTransition = 1;
1711         }
1712
1713         return 0;
1714 }
1715
1716 static inline u64
1717 r535_gsp_libos_id8(const char *name)
1718 {
1719         u64 id = 0;
1720
1721         for (int i = 0; i < sizeof(id) && *name; i++, name++)
1722                 id = (id << 8) | *name;
1723
1724         return id;
1725 }
1726
1727 /**
1728  * create_pte_array() - creates a PTE array of a physically contiguous buffer
1729  * @ptes: pointer to the array
1730  * @addr: base address of physically contiguous buffer (GSP_PAGE_SIZE aligned)
1731  * @size: size of the buffer
1732  *
1733  * GSP-RM sometimes expects physically-contiguous buffers to have an array of
1734  * "PTEs" for each page in that buffer.  Although in theory that allows for
1735  * the buffer to be physically discontiguous, GSP-RM does not currently
1736  * support that.
1737  *
1738  * In this case, the PTEs are DMA addresses of each page of the buffer.  Since
1739  * the buffer is physically contiguous, calculating all the PTEs is simple
1740  * math.
1741  *
1742  * See memdescGetPhysAddrsForGpu()
1743  */
1744 static void create_pte_array(u64 *ptes, dma_addr_t addr, size_t size)
1745 {
1746         unsigned int num_pages = DIV_ROUND_UP_ULL(size, GSP_PAGE_SIZE);
1747         unsigned int i;
1748
1749         for (i = 0; i < num_pages; i++)
1750                 ptes[i] = (u64)addr + (i << GSP_PAGE_SHIFT);
1751 }
1752
1753 /**
1754  * r535_gsp_libos_init() -- create the libos arguments structure
1755  *
1756  * The logging buffers are byte queues that contain encoded printf-like
1757  * messages from GSP-RM.  They need to be decoded by a special application
1758  * that can parse the buffers.
1759  *
1760  * The 'loginit' buffer contains logs from early GSP-RM init and
1761  * exception dumps.  The 'logrm' buffer contains the subsequent logs. Both are
1762  * written to directly by GSP-RM and can be any multiple of GSP_PAGE_SIZE.
1763  *
1764  * The physical address map for the log buffer is stored in the buffer
1765  * itself, starting with offset 1. Offset 0 contains the "put" pointer.
1766  *
1767  * The GSP only understands 4K pages (GSP_PAGE_SIZE), so even if the kernel is
1768  * configured for a larger page size (e.g. 64K pages), we need to give
1769  * the GSP an array of 4K pages. Fortunately, since the buffer is
1770  * physically contiguous, it's simple math to calculate the addresses.
1771  *
1772  * The buffers must be a multiple of GSP_PAGE_SIZE.  GSP-RM also currently
1773  * ignores the @kind field for LOGINIT, LOGINTR, and LOGRM, but expects the
1774  * buffers to be physically contiguous anyway.
1775  *
1776  * The memory allocated for the arguments must remain until the GSP sends the
1777  * init_done RPC.
1778  *
1779  * See _kgspInitLibosLoggingStructures (allocates memory for buffers)
1780  * See kgspSetupLibosInitArgs_IMPL (creates pLibosInitArgs[] array)
1781  */
1782 static int
1783 r535_gsp_libos_init(struct nvkm_gsp *gsp)
1784 {
1785         LibosMemoryRegionInitArgument *args;
1786         int ret;
1787
1788         ret = nvkm_gsp_mem_ctor(gsp, 0x1000, &gsp->libos);
1789         if (ret)
1790                 return ret;
1791
1792         args = gsp->libos.data;
1793
1794         ret = nvkm_gsp_mem_ctor(gsp, 0x10000, &gsp->loginit);
1795         if (ret)
1796                 return ret;
1797
1798         args[0].id8  = r535_gsp_libos_id8("LOGINIT");
1799         args[0].pa   = gsp->loginit.addr;
1800         args[0].size = gsp->loginit.size;
1801         args[0].kind = LIBOS_MEMORY_REGION_CONTIGUOUS;
1802         args[0].loc  = LIBOS_MEMORY_REGION_LOC_SYSMEM;
1803         create_pte_array(gsp->loginit.data + sizeof(u64), gsp->loginit.addr, gsp->loginit.size);
1804
1805         ret = nvkm_gsp_mem_ctor(gsp, 0x10000, &gsp->logintr);
1806         if (ret)
1807                 return ret;
1808
1809         args[1].id8  = r535_gsp_libos_id8("LOGINTR");
1810         args[1].pa   = gsp->logintr.addr;
1811         args[1].size = gsp->logintr.size;
1812         args[1].kind = LIBOS_MEMORY_REGION_CONTIGUOUS;
1813         args[1].loc  = LIBOS_MEMORY_REGION_LOC_SYSMEM;
1814         create_pte_array(gsp->logintr.data + sizeof(u64), gsp->logintr.addr, gsp->logintr.size);
1815
1816         ret = nvkm_gsp_mem_ctor(gsp, 0x10000, &gsp->logrm);
1817         if (ret)
1818                 return ret;
1819
1820         args[2].id8  = r535_gsp_libos_id8("LOGRM");
1821         args[2].pa   = gsp->logrm.addr;
1822         args[2].size = gsp->logrm.size;
1823         args[2].kind = LIBOS_MEMORY_REGION_CONTIGUOUS;
1824         args[2].loc  = LIBOS_MEMORY_REGION_LOC_SYSMEM;
1825         create_pte_array(gsp->logrm.data + sizeof(u64), gsp->logrm.addr, gsp->logrm.size);
1826
1827         ret = r535_gsp_rmargs_init(gsp, false);
1828         if (ret)
1829                 return ret;
1830
1831         args[3].id8  = r535_gsp_libos_id8("RMARGS");
1832         args[3].pa   = gsp->rmargs.addr;
1833         args[3].size = gsp->rmargs.size;
1834         args[3].kind = LIBOS_MEMORY_REGION_CONTIGUOUS;
1835         args[3].loc  = LIBOS_MEMORY_REGION_LOC_SYSMEM;
1836         return 0;
1837 }
1838
1839 void
1840 nvkm_gsp_sg_free(struct nvkm_device *device, struct sg_table *sgt)
1841 {
1842         struct scatterlist *sgl;
1843         int i;
1844
1845         dma_unmap_sgtable(device->dev, sgt, DMA_BIDIRECTIONAL, 0);
1846
1847         for_each_sgtable_sg(sgt, sgl, i) {
1848                 struct page *page = sg_page(sgl);
1849
1850                 __free_page(page);
1851         }
1852
1853         sg_free_table(sgt);
1854 }
1855
1856 int
1857 nvkm_gsp_sg(struct nvkm_device *device, u64 size, struct sg_table *sgt)
1858 {
1859         const u64 pages = DIV_ROUND_UP(size, PAGE_SIZE);
1860         struct scatterlist *sgl;
1861         int ret, i;
1862
1863         ret = sg_alloc_table(sgt, pages, GFP_KERNEL);
1864         if (ret)
1865                 return ret;
1866
1867         for_each_sgtable_sg(sgt, sgl, i) {
1868                 struct page *page = alloc_page(GFP_KERNEL);
1869
1870                 if (!page) {
1871                         nvkm_gsp_sg_free(device, sgt);
1872                         return -ENOMEM;
1873                 }
1874
1875                 sg_set_page(sgl, page, PAGE_SIZE, 0);
1876         }
1877
1878         ret = dma_map_sgtable(device->dev, sgt, DMA_BIDIRECTIONAL, 0);
1879         if (ret)
1880                 nvkm_gsp_sg_free(device, sgt);
1881
1882         return ret;
1883 }
1884
1885 static void
1886 nvkm_gsp_radix3_dtor(struct nvkm_gsp *gsp, struct nvkm_gsp_radix3 *rx3)
1887 {
1888         for (int i = ARRAY_SIZE(rx3->mem) - 1; i >= 0; i--)
1889                 nvkm_gsp_mem_dtor(gsp, &rx3->mem[i]);
1890 }
1891
1892 /**
1893  * nvkm_gsp_radix3_sg - build a radix3 table from a S/G list
1894  *
1895  * The GSP uses a three-level page table, called radix3, to map the firmware.
1896  * Each 64-bit "pointer" in the table is either the bus address of an entry in
1897  * the next table (for levels 0 and 1) or the bus address of the next page in
1898  * the GSP firmware image itself.
1899  *
1900  * Level 0 contains a single entry in one page that points to the first page
1901  * of level 1.
1902  *
1903  * Level 1, since it's also only one page in size, contains up to 512 entries,
1904  * one for each page in Level 2.
1905  *
1906  * Level 2 can be up to 512 pages in size, and each of those entries points to
1907  * the next page of the firmware image.  Since there can be up to 512*512
1908  * pages, that limits the size of the firmware to 512*512*GSP_PAGE_SIZE = 1GB.
1909  *
1910  * Internally, the GSP has its window into system memory, but the base
1911  * physical address of the aperture is not 0.  In fact, it varies depending on
1912  * the GPU architecture.  Since the GPU is a PCI device, this window is
1913  * accessed via DMA and is therefore bound by IOMMU translation.  The end
1914  * result is that GSP-RM must translate the bus addresses in the table to GSP
1915  * physical addresses.  All this should happen transparently.
1916  *
1917  * Returns 0 on success, or negative error code
1918  *
1919  * See kgspCreateRadix3_IMPL
1920  */
1921 static int
1922 nvkm_gsp_radix3_sg(struct nvkm_device *device, struct sg_table *sgt, u64 size,
1923                    struct nvkm_gsp_radix3 *rx3)
1924 {
1925         u64 addr;
1926
1927         for (int i = ARRAY_SIZE(rx3->mem) - 1; i >= 0; i--) {
1928                 u64 *ptes;
1929                 int idx;
1930
1931                 rx3->mem[i].size = ALIGN((size / GSP_PAGE_SIZE) * sizeof(u64), GSP_PAGE_SIZE);
1932                 rx3->mem[i].data = dma_alloc_coherent(device->dev, rx3->mem[i].size,
1933                                                       &rx3->mem[i].addr, GFP_KERNEL);
1934                 if (WARN_ON(!rx3->mem[i].data))
1935                         return -ENOMEM;
1936
1937                 ptes = rx3->mem[i].data;
1938                 if (i == 2) {
1939                         struct scatterlist *sgl;
1940
1941                         for_each_sgtable_dma_sg(sgt, sgl, idx) {
1942                                 for (int j = 0; j < sg_dma_len(sgl) / GSP_PAGE_SIZE; j++)
1943                                         *ptes++ = sg_dma_address(sgl) + (GSP_PAGE_SIZE * j);
1944                         }
1945                 } else {
1946                         for (int j = 0; j < size / GSP_PAGE_SIZE; j++)
1947                                 *ptes++ = addr + GSP_PAGE_SIZE * j;
1948                 }
1949
1950                 size = rx3->mem[i].size;
1951                 addr = rx3->mem[i].addr;
1952         }
1953
1954         return 0;
1955 }
1956
1957 int
1958 r535_gsp_fini(struct nvkm_gsp *gsp, bool suspend)
1959 {
1960         u32 mbox0 = 0xff, mbox1 = 0xff;
1961         int ret;
1962
1963         if (!gsp->running)
1964                 return 0;
1965
1966         if (suspend) {
1967                 GspFwWprMeta *meta = gsp->wpr_meta.data;
1968                 u64 len = meta->gspFwWprEnd - meta->gspFwWprStart;
1969                 GspFwSRMeta *sr;
1970
1971                 ret = nvkm_gsp_sg(gsp->subdev.device, len, &gsp->sr.sgt);
1972                 if (ret)
1973                         return ret;
1974
1975                 ret = nvkm_gsp_radix3_sg(gsp->subdev.device, &gsp->sr.sgt, len, &gsp->sr.radix3);
1976                 if (ret)
1977                         return ret;
1978
1979                 ret = nvkm_gsp_mem_ctor(gsp, sizeof(*sr), &gsp->sr.meta);
1980                 if (ret)
1981                         return ret;
1982
1983                 sr = gsp->sr.meta.data;
1984                 sr->magic = GSP_FW_SR_META_MAGIC;
1985                 sr->revision = GSP_FW_SR_META_REVISION;
1986                 sr->sysmemAddrOfSuspendResumeData = gsp->sr.radix3.mem[0].addr;
1987                 sr->sizeOfSuspendResumeData = len;
1988
1989                 mbox0 = lower_32_bits(gsp->sr.meta.addr);
1990                 mbox1 = upper_32_bits(gsp->sr.meta.addr);
1991         }
1992
1993         ret = r535_gsp_rpc_unloading_guest_driver(gsp, suspend);
1994         if (WARN_ON(ret))
1995                 return ret;
1996
1997         nvkm_msec(gsp->subdev.device, 2000,
1998                 if (nvkm_falcon_rd32(&gsp->falcon, 0x040) & 0x80000000)
1999                         break;
2000         );
2001
2002         nvkm_falcon_reset(&gsp->falcon);
2003
2004         ret = nvkm_gsp_fwsec_sb(gsp);
2005         WARN_ON(ret);
2006
2007         ret = r535_gsp_booter_unload(gsp, mbox0, mbox1);
2008         WARN_ON(ret);
2009
2010         gsp->running = false;
2011         return 0;
2012 }
2013
2014 int
2015 r535_gsp_init(struct nvkm_gsp *gsp)
2016 {
2017         u32 mbox0, mbox1;
2018         int ret;
2019
2020         if (!gsp->sr.meta.data) {
2021                 mbox0 = lower_32_bits(gsp->wpr_meta.addr);
2022                 mbox1 = upper_32_bits(gsp->wpr_meta.addr);
2023         } else {
2024                 r535_gsp_rmargs_init(gsp, true);
2025
2026                 mbox0 = lower_32_bits(gsp->sr.meta.addr);
2027                 mbox1 = upper_32_bits(gsp->sr.meta.addr);
2028         }
2029
2030         /* Execute booter to handle (eventually...) booting GSP-RM. */
2031         ret = r535_gsp_booter_load(gsp, mbox0, mbox1);
2032         if (WARN_ON(ret))
2033                 goto done;
2034
2035         ret = r535_gsp_rpc_poll(gsp, NV_VGPU_MSG_EVENT_GSP_INIT_DONE);
2036         if (ret)
2037                 goto done;
2038
2039         gsp->running = true;
2040
2041 done:
2042         if (gsp->sr.meta.data) {
2043                 nvkm_gsp_mem_dtor(gsp, &gsp->sr.meta);
2044                 nvkm_gsp_radix3_dtor(gsp, &gsp->sr.radix3);
2045                 nvkm_gsp_sg_free(gsp->subdev.device, &gsp->sr.sgt);
2046                 return ret;
2047         }
2048
2049         if (ret == 0)
2050                 ret = r535_gsp_postinit(gsp);
2051
2052         return ret;
2053 }
2054
2055 static int
2056 r535_gsp_rm_boot_ctor(struct nvkm_gsp *gsp)
2057 {
2058         const struct firmware *fw = gsp->fws.bl;
2059         const struct nvfw_bin_hdr *hdr;
2060         RM_RISCV_UCODE_DESC *desc;
2061         int ret;
2062
2063         hdr = nvfw_bin_hdr(&gsp->subdev, fw->data);
2064         desc = (void *)fw->data + hdr->header_offset;
2065
2066         ret = nvkm_gsp_mem_ctor(gsp, hdr->data_size, &gsp->boot.fw);
2067         if (ret)
2068                 return ret;
2069
2070         memcpy(gsp->boot.fw.data, fw->data + hdr->data_offset, hdr->data_size);
2071
2072         gsp->boot.code_offset = desc->monitorCodeOffset;
2073         gsp->boot.data_offset = desc->monitorDataOffset;
2074         gsp->boot.manifest_offset = desc->manifestOffset;
2075         gsp->boot.app_version = desc->appVersion;
2076         return 0;
2077 }
2078
2079 static const struct nvkm_firmware_func
2080 r535_gsp_fw = {
2081         .type = NVKM_FIRMWARE_IMG_SGT,
2082 };
2083
2084 static int
2085 r535_gsp_elf_section(struct nvkm_gsp *gsp, const char *name, const u8 **pdata, u64 *psize)
2086 {
2087         const u8 *img = gsp->fws.rm->data;
2088         const struct elf64_hdr *ehdr = (const struct elf64_hdr *)img;
2089         const struct elf64_shdr *shdr = (const struct elf64_shdr *)&img[ehdr->e_shoff];
2090         const char *names = &img[shdr[ehdr->e_shstrndx].sh_offset];
2091
2092         for (int i = 0; i < ehdr->e_shnum; i++, shdr++) {
2093                 if (!strcmp(&names[shdr->sh_name], name)) {
2094                         *pdata = &img[shdr->sh_offset];
2095                         *psize = shdr->sh_size;
2096                         return 0;
2097                 }
2098         }
2099
2100         nvkm_error(&gsp->subdev, "section '%s' not found\n", name);
2101         return -ENOENT;
2102 }
2103
2104 static void
2105 r535_gsp_dtor_fws(struct nvkm_gsp *gsp)
2106 {
2107         nvkm_firmware_put(gsp->fws.bl);
2108         gsp->fws.bl = NULL;
2109         nvkm_firmware_put(gsp->fws.booter.unload);
2110         gsp->fws.booter.unload = NULL;
2111         nvkm_firmware_put(gsp->fws.booter.load);
2112         gsp->fws.booter.load = NULL;
2113         nvkm_firmware_put(gsp->fws.rm);
2114         gsp->fws.rm = NULL;
2115 }
2116
2117 void
2118 r535_gsp_dtor(struct nvkm_gsp *gsp)
2119 {
2120         idr_destroy(&gsp->client_id.idr);
2121         mutex_destroy(&gsp->client_id.mutex);
2122
2123         nvkm_gsp_radix3_dtor(gsp, &gsp->radix3);
2124         nvkm_gsp_mem_dtor(gsp, &gsp->sig);
2125         nvkm_firmware_dtor(&gsp->fw);
2126
2127         nvkm_falcon_fw_dtor(&gsp->booter.unload);
2128         nvkm_falcon_fw_dtor(&gsp->booter.load);
2129
2130         mutex_destroy(&gsp->msgq.mutex);
2131         mutex_destroy(&gsp->cmdq.mutex);
2132
2133         r535_gsp_dtor_fws(gsp);
2134 }
2135
2136 int
2137 r535_gsp_oneinit(struct nvkm_gsp *gsp)
2138 {
2139         struct nvkm_device *device = gsp->subdev.device;
2140         const u8 *data;
2141         u64 size;
2142         int ret;
2143
2144         mutex_init(&gsp->cmdq.mutex);
2145         mutex_init(&gsp->msgq.mutex);
2146
2147         ret = gsp->func->booter.ctor(gsp, "booter-load", gsp->fws.booter.load,
2148                                      &device->sec2->falcon, &gsp->booter.load);
2149         if (ret)
2150                 return ret;
2151
2152         ret = gsp->func->booter.ctor(gsp, "booter-unload", gsp->fws.booter.unload,
2153                                      &device->sec2->falcon, &gsp->booter.unload);
2154         if (ret)
2155                 return ret;
2156
2157         /* Load GSP firmware from ELF image into DMA-accessible memory. */
2158         ret = r535_gsp_elf_section(gsp, ".fwimage", &data, &size);
2159         if (ret)
2160                 return ret;
2161
2162         ret = nvkm_firmware_ctor(&r535_gsp_fw, "gsp-rm", device, data, size, &gsp->fw);
2163         if (ret)
2164                 return ret;
2165
2166         /* Load relevant signature from ELF image. */
2167         ret = r535_gsp_elf_section(gsp, gsp->func->sig_section, &data, &size);
2168         if (ret)
2169                 return ret;
2170
2171         ret = nvkm_gsp_mem_ctor(gsp, ALIGN(size, 256), &gsp->sig);
2172         if (ret)
2173                 return ret;
2174
2175         memcpy(gsp->sig.data, data, size);
2176
2177         /* Build radix3 page table for ELF image. */
2178         ret = nvkm_gsp_radix3_sg(device, &gsp->fw.mem.sgt, gsp->fw.len, &gsp->radix3);
2179         if (ret)
2180                 return ret;
2181
2182         r535_gsp_msg_ntfy_add(gsp, NV_VGPU_MSG_EVENT_GSP_RUN_CPU_SEQUENCER,
2183                               r535_gsp_msg_run_cpu_sequencer, gsp);
2184         r535_gsp_msg_ntfy_add(gsp, NV_VGPU_MSG_EVENT_POST_EVENT, r535_gsp_msg_post_event, gsp);
2185         r535_gsp_msg_ntfy_add(gsp, NV_VGPU_MSG_EVENT_RC_TRIGGERED,
2186                               r535_gsp_msg_rc_triggered, gsp);
2187         r535_gsp_msg_ntfy_add(gsp, NV_VGPU_MSG_EVENT_MMU_FAULT_QUEUED,
2188                               r535_gsp_msg_mmu_fault_queued, gsp);
2189         r535_gsp_msg_ntfy_add(gsp, NV_VGPU_MSG_EVENT_OS_ERROR_LOG, r535_gsp_msg_os_error_log, gsp);
2190         r535_gsp_msg_ntfy_add(gsp, NV_VGPU_MSG_EVENT_PERF_BRIDGELESS_INFO_UPDATE, NULL, NULL);
2191         r535_gsp_msg_ntfy_add(gsp, NV_VGPU_MSG_EVENT_UCODE_LIBOS_PRINT, NULL, NULL);
2192         r535_gsp_msg_ntfy_add(gsp, NV_VGPU_MSG_EVENT_GSP_SEND_USER_SHARED_DATA, NULL, NULL);
2193         ret = r535_gsp_rm_boot_ctor(gsp);
2194         if (ret)
2195                 return ret;
2196
2197         /* Release FW images - we've copied them to DMA buffers now. */
2198         r535_gsp_dtor_fws(gsp);
2199
2200         /* Calculate FB layout. */
2201         gsp->fb.wpr2.frts.size = 0x100000;
2202         gsp->fb.wpr2.frts.addr = ALIGN_DOWN(gsp->fb.bios.addr, 0x20000) - gsp->fb.wpr2.frts.size;
2203
2204         gsp->fb.wpr2.boot.size = gsp->boot.fw.size;
2205         gsp->fb.wpr2.boot.addr = ALIGN_DOWN(gsp->fb.wpr2.frts.addr - gsp->fb.wpr2.boot.size, 0x1000);
2206
2207         gsp->fb.wpr2.elf.size = gsp->fw.len;
2208         gsp->fb.wpr2.elf.addr = ALIGN_DOWN(gsp->fb.wpr2.boot.addr - gsp->fb.wpr2.elf.size, 0x10000);
2209
2210         {
2211                 u32 fb_size_gb = DIV_ROUND_UP_ULL(gsp->fb.size, 1 << 30);
2212
2213                 gsp->fb.wpr2.heap.size =
2214                         gsp->func->wpr_heap.os_carveout_size +
2215                         gsp->func->wpr_heap.base_size +
2216                         ALIGN(GSP_FW_HEAP_PARAM_SIZE_PER_GB_FB * fb_size_gb, 1 << 20) +
2217                         ALIGN(GSP_FW_HEAP_PARAM_CLIENT_ALLOC_SIZE, 1 << 20);
2218
2219                 gsp->fb.wpr2.heap.size = max(gsp->fb.wpr2.heap.size, gsp->func->wpr_heap.min_size);
2220         }
2221
2222         gsp->fb.wpr2.heap.addr = ALIGN_DOWN(gsp->fb.wpr2.elf.addr - gsp->fb.wpr2.heap.size, 0x100000);
2223         gsp->fb.wpr2.heap.size = ALIGN_DOWN(gsp->fb.wpr2.elf.addr - gsp->fb.wpr2.heap.addr, 0x100000);
2224
2225         gsp->fb.wpr2.addr = ALIGN_DOWN(gsp->fb.wpr2.heap.addr - sizeof(GspFwWprMeta), 0x100000);
2226         gsp->fb.wpr2.size = gsp->fb.wpr2.frts.addr + gsp->fb.wpr2.frts.size - gsp->fb.wpr2.addr;
2227
2228         gsp->fb.heap.size = 0x100000;
2229         gsp->fb.heap.addr = gsp->fb.wpr2.addr - gsp->fb.heap.size;
2230
2231         ret = nvkm_gsp_fwsec_frts(gsp);
2232         if (WARN_ON(ret))
2233                 return ret;
2234
2235         ret = r535_gsp_libos_init(gsp);
2236         if (WARN_ON(ret))
2237                 return ret;
2238
2239         ret = r535_gsp_wpr_meta_init(gsp);
2240         if (WARN_ON(ret))
2241                 return ret;
2242
2243         ret = r535_gsp_rpc_set_system_info(gsp);
2244         if (WARN_ON(ret))
2245                 return ret;
2246
2247         ret = r535_gsp_rpc_set_registry(gsp);
2248         if (WARN_ON(ret))
2249                 return ret;
2250
2251         /* Reset GSP into RISC-V mode. */
2252         ret = gsp->func->reset(gsp);
2253         if (WARN_ON(ret))
2254                 return ret;
2255
2256         nvkm_falcon_wr32(&gsp->falcon, 0x040, lower_32_bits(gsp->libos.addr));
2257         nvkm_falcon_wr32(&gsp->falcon, 0x044, upper_32_bits(gsp->libos.addr));
2258
2259         mutex_init(&gsp->client_id.mutex);
2260         idr_init(&gsp->client_id.idr);
2261         return 0;
2262 }
2263
2264 static int
2265 r535_gsp_load_fw(struct nvkm_gsp *gsp, const char *name, const char *ver,
2266                  const struct firmware **pfw)
2267 {
2268         char fwname[64];
2269
2270         snprintf(fwname, sizeof(fwname), "gsp/%s-%s", name, ver);
2271         return nvkm_firmware_get(&gsp->subdev, fwname, 0, pfw);
2272 }
2273
2274 int
2275 r535_gsp_load(struct nvkm_gsp *gsp, int ver, const struct nvkm_gsp_fwif *fwif)
2276 {
2277         struct nvkm_subdev *subdev = &gsp->subdev;
2278         int ret;
2279
2280         if (!nvkm_boolopt(subdev->device->cfgopt, "NvGspRm", fwif->enable))
2281                 return -EINVAL;
2282
2283         if ((ret = r535_gsp_load_fw(gsp, "gsp", fwif->ver, &gsp->fws.rm)) ||
2284             (ret = r535_gsp_load_fw(gsp, "booter_load", fwif->ver, &gsp->fws.booter.load)) ||
2285             (ret = r535_gsp_load_fw(gsp, "booter_unload", fwif->ver, &gsp->fws.booter.unload)) ||
2286             (ret = r535_gsp_load_fw(gsp, "bootloader", fwif->ver, &gsp->fws.bl))) {
2287                 r535_gsp_dtor_fws(gsp);
2288                 return ret;
2289         }
2290
2291         return 0;
2292 }
2293
2294 #define NVKM_GSP_FIRMWARE(chip)                                  \
2295 MODULE_FIRMWARE("nvidia/"#chip"/gsp/booter_load-535.113.01.bin");   \
2296 MODULE_FIRMWARE("nvidia/"#chip"/gsp/booter_unload-535.113.01.bin"); \
2297 MODULE_FIRMWARE("nvidia/"#chip"/gsp/bootloader-535.113.01.bin");    \
2298 MODULE_FIRMWARE("nvidia/"#chip"/gsp/gsp-535.113.01.bin")
2299
2300 NVKM_GSP_FIRMWARE(tu102);
2301 NVKM_GSP_FIRMWARE(tu104);
2302 NVKM_GSP_FIRMWARE(tu106);
2303
2304 NVKM_GSP_FIRMWARE(tu116);
2305 NVKM_GSP_FIRMWARE(tu117);
2306
2307 NVKM_GSP_FIRMWARE(ga100);
2308
2309 NVKM_GSP_FIRMWARE(ga102);
2310 NVKM_GSP_FIRMWARE(ga103);
2311 NVKM_GSP_FIRMWARE(ga104);
2312 NVKM_GSP_FIRMWARE(ga106);
2313 NVKM_GSP_FIRMWARE(ga107);
2314
2315 NVKM_GSP_FIRMWARE(ad102);
2316 NVKM_GSP_FIRMWARE(ad103);
2317 NVKM_GSP_FIRMWARE(ad104);
2318 NVKM_GSP_FIRMWARE(ad106);
2319 NVKM_GSP_FIRMWARE(ad107);