drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c

   1 /*
   2  * Copyright 2016 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Author: Huang Rui
  23  *
  24  */
  25
  26 #include <linux/firmware.h>
  27 #include <drm/drmP.h>
  28 #include "amdgpu.h"
  29 #include "amdgpu_psp.h"
  30 #include "amdgpu_ucode.h"
  31 #include "soc15_common.h"
  32 #include "psp_v3_1.h"
  33 #include "psp_v10_0.h"
  34 #include "psp_v11_0.h"
  35
  36 static void psp_set_funcs(struct amdgpu_device *adev);
  37
  38 static int psp_early_init(void *handle)
  39 {
  40         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  41
  42         psp_set_funcs(adev);
  43
  44         return 0;
  45 }
  46
  47 static int psp_sw_init(void *handle)
  48 {
  49         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  50         struct psp_context *psp = &adev->psp;
  51         int ret;
  52
  53         switch (adev->asic_type) {
  54         case CHIP_VEGA10:
  55         case CHIP_VEGA12:
  56                 psp_v3_1_set_psp_funcs(psp);
  57                 break;
  58         case CHIP_RAVEN:
  59                 psp_v10_0_set_psp_funcs(psp);
  60                 break;
  61         case CHIP_VEGA20:
  62                 psp_v11_0_set_psp_funcs(psp);
  63                 break;
  64         default:
  65                 return -EINVAL;
  66         }
  67
  68         psp->adev = adev;
  69
  70         ret = psp_init_microcode(psp);
  71         if (ret) {
  72                 DRM_ERROR("Failed to load psp firmware!\n");
  73                 return ret;
  74         }
  75
  76         return 0;
  77 }
  78
  79 static int psp_sw_fini(void *handle)
  80 {
  81         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
  82
  83         release_firmware(adev->psp.sos_fw);
  84         adev->psp.sos_fw = NULL;
  85         release_firmware(adev->psp.asd_fw);
  86         adev->psp.asd_fw = NULL;
  87         if (adev->psp.ta_fw) {
  88                 release_firmware(adev->psp.ta_fw);
  89                 adev->psp.ta_fw = NULL;
  90         }
  91         return 0;
  92 }
  93
  94 int psp_wait_for(struct psp_context *psp, uint32_t reg_index,
  95                  uint32_t reg_val, uint32_t mask, bool check_changed)
  96 {
  97         uint32_t val;
  98         int i;
  99         struct amdgpu_device *adev = psp->adev;
 100
 101         for (i = 0; i < adev->usec_timeout; i++) {
 102                 val = RREG32(reg_index);
 103                 if (check_changed) {
 104                         if (val != reg_val)
 105                                 return 0;
 106                 } else {
 107                         if ((val & mask) == reg_val)
 108                                 return 0;
 109                 }
 110                 udelay(1);
 111         }
 112
 113         return -ETIME;
 114 }
 115
 116 static int
 117 psp_cmd_submit_buf(struct psp_context *psp,
 118                    struct amdgpu_firmware_info *ucode,
 119                    struct psp_gfx_cmd_resp *cmd, uint64_t fence_mc_addr)
 120 {
 121         int ret;
 122         int index;
 123         int timeout = 2000;
 124
 125         memset(psp->cmd_buf_mem, 0, PSP_CMD_BUFFER_SIZE);
 126
 127         memcpy(psp->cmd_buf_mem, cmd, sizeof(struct psp_gfx_cmd_resp));
 128
 129         index = atomic_inc_return(&psp->fence_value);
 130         ret = psp_cmd_submit(psp, ucode, psp->cmd_buf_mc_addr,
 131                              fence_mc_addr, index);
 132         if (ret) {
 133                 atomic_dec(&psp->fence_value);
 134                 return ret;
 135         }
 136
 137         while (*((unsigned int *)psp->fence_buf) != index) {
 138                 if (--timeout == 0)
 139                         break;
 140                 msleep(1);
 141         }
 142
 143         /* In some cases, psp response status is not 0 even there is no
 144          * problem while the command is submitted. Some version of PSP FW
 145          * doesn't write 0 to that field.
 146          * So here we would like to only print a warning instead of an error
 147          * during psp initialization to avoid breaking hw_init and it doesn't
 148          * return -EINVAL.
 149          */
 150         if (psp->cmd_buf_mem->resp.status || !timeout) {
 151                 if (ucode)
 152                         DRM_WARN("failed to load ucode id (%d) ",
 153                                   ucode->ucode_id);
 154                 DRM_WARN("psp command failed and response status is (%d)\n",
 155                           psp->cmd_buf_mem->resp.status);
 156                 if (!timeout)
 157                         return -EINVAL;
 158         }
 159
 160         /* get xGMI session id from response buffer */
 161         cmd->resp.session_id = psp->cmd_buf_mem->resp.session_id;
 162
 163         if (ucode) {
 164                 ucode->tmr_mc_addr_lo = psp->cmd_buf_mem->resp.fw_addr_lo;
 165                 ucode->tmr_mc_addr_hi = psp->cmd_buf_mem->resp.fw_addr_hi;
 166         }
 167
 168         return ret;
 169 }
 170
 171 static void psp_prep_tmr_cmd_buf(struct psp_context *psp,
 172                                  struct psp_gfx_cmd_resp *cmd,
 173                                  uint64_t tmr_mc, uint32_t size)
 174 {
 175         if (psp_support_vmr_ring(psp))
 176                 cmd->cmd_id = GFX_CMD_ID_SETUP_VMR;
 177         else
 178                 cmd->cmd_id = GFX_CMD_ID_SETUP_TMR;
 179         cmd->cmd.cmd_setup_tmr.buf_phy_addr_lo = lower_32_bits(tmr_mc);
 180         cmd->cmd.cmd_setup_tmr.buf_phy_addr_hi = upper_32_bits(tmr_mc);
 181         cmd->cmd.cmd_setup_tmr.buf_size = size;
 182 }
 183
 184 /* Set up Trusted Memory Region */
 185 static int psp_tmr_init(struct psp_context *psp)
 186 {
 187         int ret;
 188
 189         /*
 190          * According to HW engineer, they prefer the TMR address be "naturally
 191          * aligned" , e.g. the start address be an integer divide of TMR size.
 192          *
 193          * Note: this memory need be reserved till the driver
 194          * uninitializes.
 195          */
 196         ret = amdgpu_bo_create_kernel(psp->adev, PSP_TMR_SIZE, PSP_TMR_SIZE,
 197                                       AMDGPU_GEM_DOMAIN_VRAM,
 198                                       &psp->tmr_bo, &psp->tmr_mc_addr, &psp->tmr_buf);
 199
 200         return ret;
 201 }
 202
 203 static int psp_tmr_load(struct psp_context *psp)
 204 {
 205         int ret;
 206         struct psp_gfx_cmd_resp *cmd;
 207
 208         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 209         if (!cmd)
 210                 return -ENOMEM;
 211
 212         psp_prep_tmr_cmd_buf(psp, cmd, psp->tmr_mc_addr, PSP_TMR_SIZE);
 213         DRM_INFO("reserve 0x%x from 0x%llx for PSP TMR SIZE\n",
 214                         PSP_TMR_SIZE, psp->tmr_mc_addr);
 215
 216         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 217                                  psp->fence_buf_mc_addr);
 218         if (ret)
 219                 goto failed;
 220
 221         kfree(cmd);
 222
 223         return 0;
 224
 225 failed:
 226         kfree(cmd);
 227         return ret;
 228 }
 229
 230 static void psp_prep_asd_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 231                                  uint64_t asd_mc, uint64_t asd_mc_shared,
 232                                  uint32_t size, uint32_t shared_size)
 233 {
 234         cmd->cmd_id = GFX_CMD_ID_LOAD_ASD;
 235         cmd->cmd.cmd_load_ta.app_phy_addr_lo = lower_32_bits(asd_mc);
 236         cmd->cmd.cmd_load_ta.app_phy_addr_hi = upper_32_bits(asd_mc);
 237         cmd->cmd.cmd_load_ta.app_len = size;
 238
 239         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_lo = lower_32_bits(asd_mc_shared);
 240         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_hi = upper_32_bits(asd_mc_shared);
 241         cmd->cmd.cmd_load_ta.cmd_buf_len = shared_size;
 242 }
 243
 244 static int psp_asd_init(struct psp_context *psp)
 245 {
 246         int ret;
 247
 248         /*
 249          * Allocate 16k memory aligned to 4k from Frame Buffer (local
 250          * physical) for shared ASD <-> Driver
 251          */
 252         ret = amdgpu_bo_create_kernel(psp->adev, PSP_ASD_SHARED_MEM_SIZE,
 253                                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
 254                                       &psp->asd_shared_bo,
 255                                       &psp->asd_shared_mc_addr,
 256                                       &psp->asd_shared_buf);
 257
 258         return ret;
 259 }
 260
 261 static int psp_asd_load(struct psp_context *psp)
 262 {
 263         int ret;
 264         struct psp_gfx_cmd_resp *cmd;
 265
 266         /* If PSP version doesn't match ASD version, asd loading will be failed.
 267          * add workaround to bypass it for sriov now.
 268          * TODO: add version check to make it common
 269          */
 270         if (amdgpu_sriov_vf(psp->adev))
 271                 return 0;
 272
 273         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 274         if (!cmd)
 275                 return -ENOMEM;
 276
 277         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
 278         memcpy(psp->fw_pri_buf, psp->asd_start_addr, psp->asd_ucode_size);
 279
 280         psp_prep_asd_cmd_buf(cmd, psp->fw_pri_mc_addr, psp->asd_shared_mc_addr,
 281                              psp->asd_ucode_size, PSP_ASD_SHARED_MEM_SIZE);
 282
 283         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 284                                  psp->fence_buf_mc_addr);
 285
 286         kfree(cmd);
 287
 288         return ret;
 289 }
 290
 291 static void psp_prep_xgmi_ta_load_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 292                                           uint64_t xgmi_ta_mc, uint64_t xgmi_mc_shared,
 293                                           uint32_t xgmi_ta_size, uint32_t shared_size)
 294 {
 295         cmd->cmd_id = GFX_CMD_ID_LOAD_TA;
 296         cmd->cmd.cmd_load_ta.app_phy_addr_lo = lower_32_bits(xgmi_ta_mc);
 297         cmd->cmd.cmd_load_ta.app_phy_addr_hi = upper_32_bits(xgmi_ta_mc);
 298         cmd->cmd.cmd_load_ta.app_len = xgmi_ta_size;
 299
 300         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_lo = lower_32_bits(xgmi_mc_shared);
 301         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_hi = upper_32_bits(xgmi_mc_shared);
 302         cmd->cmd.cmd_load_ta.cmd_buf_len = shared_size;
 303 }
 304
 305 static int psp_xgmi_init_shared_buf(struct psp_context *psp)
 306 {
 307         int ret;
 308
 309         /*
 310          * Allocate 16k memory aligned to 4k from Frame Buffer (local
 311          * physical) for xgmi ta <-> Driver
 312          */
 313         ret = amdgpu_bo_create_kernel(psp->adev, PSP_XGMI_SHARED_MEM_SIZE,
 314                                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
 315                                       &psp->xgmi_context.xgmi_shared_bo,
 316                                       &psp->xgmi_context.xgmi_shared_mc_addr,
 317                                       &psp->xgmi_context.xgmi_shared_buf);
 318
 319         return ret;
 320 }
 321
 322 static int psp_xgmi_load(struct psp_context *psp)
 323 {
 324         int ret;
 325         struct psp_gfx_cmd_resp *cmd;
 326
 327         /*
 328          * TODO: bypass the loading in sriov for now
 329          */
 330         if (amdgpu_sriov_vf(psp->adev))
 331                 return 0;
 332
 333         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 334         if (!cmd)
 335                 return -ENOMEM;
 336
 337         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
 338         memcpy(psp->fw_pri_buf, psp->ta_xgmi_start_addr, psp->ta_xgmi_ucode_size);
 339
 340         psp_prep_xgmi_ta_load_cmd_buf(cmd, psp->fw_pri_mc_addr,
 341                                       psp->xgmi_context.xgmi_shared_mc_addr,
 342                                       psp->ta_xgmi_ucode_size, PSP_XGMI_SHARED_MEM_SIZE);
 343
 344         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 345                                  psp->fence_buf_mc_addr);
 346
 347         if (!ret) {
 348                 psp->xgmi_context.initialized = 1;
 349                 psp->xgmi_context.session_id = cmd->resp.session_id;
 350         }
 351
 352         kfree(cmd);
 353
 354         return ret;
 355 }
 356
 357 static void psp_prep_xgmi_ta_unload_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 358                                             uint32_t xgmi_session_id)
 359 {
 360         cmd->cmd_id = GFX_CMD_ID_UNLOAD_TA;
 361         cmd->cmd.cmd_unload_ta.session_id = xgmi_session_id;
 362 }
 363
 364 static int psp_xgmi_unload(struct psp_context *psp)
 365 {
 366         int ret;
 367         struct psp_gfx_cmd_resp *cmd;
 368
 369         /*
 370          * TODO: bypass the unloading in sriov for now
 371          */
 372         if (amdgpu_sriov_vf(psp->adev))
 373                 return 0;
 374
 375         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 376         if (!cmd)
 377                 return -ENOMEM;
 378
 379         psp_prep_xgmi_ta_unload_cmd_buf(cmd, psp->xgmi_context.session_id);
 380
 381         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 382                                  psp->fence_buf_mc_addr);
 383
 384         kfree(cmd);
 385
 386         return ret;
 387 }
 388
 389 static void psp_prep_xgmi_ta_invoke_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 390                                             uint32_t ta_cmd_id,
 391                                             uint32_t xgmi_session_id)
 392 {
 393         cmd->cmd_id = GFX_CMD_ID_INVOKE_CMD;
 394         cmd->cmd.cmd_invoke_cmd.session_id = xgmi_session_id;
 395         cmd->cmd.cmd_invoke_cmd.ta_cmd_id = ta_cmd_id;
 396         /* Note: cmd_invoke_cmd.buf is not used for now */
 397 }
 398
 399 int psp_xgmi_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
 400 {
 401         int ret;
 402         struct psp_gfx_cmd_resp *cmd;
 403
 404         /*
 405          * TODO: bypass the loading in sriov for now
 406         */
 407         if (amdgpu_sriov_vf(psp->adev))
 408                 return 0;
 409
 410         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 411         if (!cmd)
 412                 return -ENOMEM;
 413
 414         psp_prep_xgmi_ta_invoke_cmd_buf(cmd, ta_cmd_id,
 415                                         psp->xgmi_context.session_id);
 416
 417         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 418                                  psp->fence_buf_mc_addr);
 419
 420         kfree(cmd);
 421
 422         return ret;
 423 }
 424
 425 static int psp_xgmi_terminate(struct psp_context *psp)
 426 {
 427         int ret;
 428
 429         if (!psp->xgmi_context.initialized)
 430                 return 0;
 431
 432         ret = psp_xgmi_unload(psp);
 433         if (ret)
 434                 return ret;
 435
 436         psp->xgmi_context.initialized = 0;
 437
 438         /* free xgmi shared memory */
 439         amdgpu_bo_free_kernel(&psp->xgmi_context.xgmi_shared_bo,
 440                         &psp->xgmi_context.xgmi_shared_mc_addr,
 441                         &psp->xgmi_context.xgmi_shared_buf);
 442
 443         return 0;
 444 }
 445
 446 static int psp_xgmi_initialize(struct psp_context *psp)
 447 {
 448         struct ta_xgmi_shared_memory *xgmi_cmd;
 449         int ret;
 450
 451         if (!psp->adev->psp.ta_fw)
 452                 return -ENOENT;
 453
 454         if (!psp->xgmi_context.initialized) {
 455                 ret = psp_xgmi_init_shared_buf(psp);
 456                 if (ret)
 457                         return ret;
 458         }
 459
 460         /* Load XGMI TA */
 461         ret = psp_xgmi_load(psp);
 462         if (ret)
 463                 return ret;
 464
 465         /* Initialize XGMI session */
 466         xgmi_cmd = (struct ta_xgmi_shared_memory *)(psp->xgmi_context.xgmi_shared_buf);
 467         memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory));
 468         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__INITIALIZE;
 469
 470         ret = psp_xgmi_invoke(psp, xgmi_cmd->cmd_id);
 471
 472         return ret;
 473 }
 474
 475 // ras begin
 476 static void psp_prep_ras_ta_load_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 477                 uint64_t ras_ta_mc, uint64_t ras_mc_shared,
 478                 uint32_t ras_ta_size, uint32_t shared_size)
 479 {
 480         cmd->cmd_id = GFX_CMD_ID_LOAD_TA;
 481         cmd->cmd.cmd_load_ta.app_phy_addr_lo = lower_32_bits(ras_ta_mc);
 482         cmd->cmd.cmd_load_ta.app_phy_addr_hi = upper_32_bits(ras_ta_mc);
 483         cmd->cmd.cmd_load_ta.app_len = ras_ta_size;
 484
 485         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_lo = lower_32_bits(ras_mc_shared);
 486         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_hi = upper_32_bits(ras_mc_shared);
 487         cmd->cmd.cmd_load_ta.cmd_buf_len = shared_size;
 488 }
 489
 490 static int psp_ras_init_shared_buf(struct psp_context *psp)
 491 {
 492         int ret;
 493
 494         /*
 495          * Allocate 16k memory aligned to 4k from Frame Buffer (local
 496          * physical) for ras ta <-> Driver
 497          */
 498         ret = amdgpu_bo_create_kernel(psp->adev, PSP_RAS_SHARED_MEM_SIZE,
 499                         PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
 500                         &psp->ras.ras_shared_bo,
 501                         &psp->ras.ras_shared_mc_addr,
 502                         &psp->ras.ras_shared_buf);
 503
 504         return ret;
 505 }
 506
 507 static int psp_ras_load(struct psp_context *psp)
 508 {
 509         int ret;
 510         struct psp_gfx_cmd_resp *cmd;
 511
 512         /*
 513          * TODO: bypass the loading in sriov for now
 514          */
 515         if (amdgpu_sriov_vf(psp->adev))
 516                 return 0;
 517
 518         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 519         if (!cmd)
 520                 return -ENOMEM;
 521
 522         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
 523         memcpy(psp->fw_pri_buf, psp->ta_ras_start_addr, psp->ta_ras_ucode_size);
 524
 525         psp_prep_ras_ta_load_cmd_buf(cmd, psp->fw_pri_mc_addr,
 526                         psp->ras.ras_shared_mc_addr,
 527                         psp->ta_ras_ucode_size, PSP_RAS_SHARED_MEM_SIZE);
 528
 529         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 530                         psp->fence_buf_mc_addr);
 531
 532         if (!ret) {
 533                 psp->ras.ras_initialized = 1;
 534                 psp->ras.session_id = cmd->resp.session_id;
 535         }
 536
 537         kfree(cmd);
 538
 539         return ret;
 540 }
 541
 542 static void psp_prep_ras_ta_unload_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 543                                                 uint32_t ras_session_id)
 544 {
 545         cmd->cmd_id = GFX_CMD_ID_UNLOAD_TA;
 546         cmd->cmd.cmd_unload_ta.session_id = ras_session_id;
 547 }
 548
 549 static int psp_ras_unload(struct psp_context *psp)
 550 {
 551         int ret;
 552         struct psp_gfx_cmd_resp *cmd;
 553
 554         /*
 555          * TODO: bypass the unloading in sriov for now
 556          */
 557         if (amdgpu_sriov_vf(psp->adev))
 558                 return 0;
 559
 560         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 561         if (!cmd)
 562                 return -ENOMEM;
 563
 564         psp_prep_ras_ta_unload_cmd_buf(cmd, psp->ras.session_id);
 565
 566         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 567                         psp->fence_buf_mc_addr);
 568
 569         kfree(cmd);
 570
 571         return ret;
 572 }
 573
 574 static void psp_prep_ras_ta_invoke_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 575                 uint32_t ta_cmd_id,
 576                 uint32_t ras_session_id)
 577 {
 578         cmd->cmd_id = GFX_CMD_ID_INVOKE_CMD;
 579         cmd->cmd.cmd_invoke_cmd.session_id = ras_session_id;
 580         cmd->cmd.cmd_invoke_cmd.ta_cmd_id = ta_cmd_id;
 581         /* Note: cmd_invoke_cmd.buf is not used for now */
 582 }
 583
 584 int psp_ras_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
 585 {
 586         int ret;
 587         struct psp_gfx_cmd_resp *cmd;
 588
 589         /*
 590          * TODO: bypass the loading in sriov for now
 591          */
 592         if (amdgpu_sriov_vf(psp->adev))
 593                 return 0;
 594
 595         cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 596         if (!cmd)
 597                 return -ENOMEM;
 598
 599         psp_prep_ras_ta_invoke_cmd_buf(cmd, ta_cmd_id,
 600                         psp->ras.session_id);
 601
 602         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 603                         psp->fence_buf_mc_addr);
 604
 605         kfree(cmd);
 606
 607         return ret;
 608 }
 609
 610 int psp_ras_enable_features(struct psp_context *psp,
 611                 union ta_ras_cmd_input *info, bool enable)
 612 {
 613         struct ta_ras_shared_memory *ras_cmd;
 614         int ret;
 615
 616         if (!psp->ras.ras_initialized)
 617                 return -EINVAL;
 618
 619         ras_cmd = (struct ta_ras_shared_memory *)psp->ras.ras_shared_buf;
 620         memset(ras_cmd, 0, sizeof(struct ta_ras_shared_memory));
 621
 622         if (enable)
 623                 ras_cmd->cmd_id = TA_RAS_COMMAND__ENABLE_FEATURES;
 624         else
 625                 ras_cmd->cmd_id = TA_RAS_COMMAND__DISABLE_FEATURES;
 626
 627         ras_cmd->ras_in_message = *info;
 628
 629         ret = psp_ras_invoke(psp, ras_cmd->cmd_id);
 630         if (ret)
 631                 return -EINVAL;
 632
 633         return ras_cmd->ras_status;
 634 }
 635
 636 static int psp_ras_terminate(struct psp_context *psp)
 637 {
 638         int ret;
 639
 640         if (!psp->ras.ras_initialized)
 641                 return 0;
 642
 643         ret = psp_ras_unload(psp);
 644         if (ret)
 645                 return ret;
 646
 647         psp->ras.ras_initialized = 0;
 648
 649         /* free ras shared memory */
 650         amdgpu_bo_free_kernel(&psp->ras.ras_shared_bo,
 651                         &psp->ras.ras_shared_mc_addr,
 652                         &psp->ras.ras_shared_buf);
 653
 654         return 0;
 655 }
 656
 657 static int psp_ras_initialize(struct psp_context *psp)
 658 {
 659         int ret;
 660
 661         if (!psp->ras.ras_initialized) {
 662                 ret = psp_ras_init_shared_buf(psp);
 663                 if (ret)
 664                         return ret;
 665         }
 666
 667         ret = psp_ras_load(psp);
 668         if (ret)
 669                 return ret;
 670
 671         return 0;
 672 }
 673 // ras end
 674
 675 static int psp_hw_start(struct psp_context *psp)
 676 {
 677         struct amdgpu_device *adev = psp->adev;
 678         int ret;
 679
 680         if (!amdgpu_sriov_vf(adev) || !adev->in_gpu_reset) {
 681                 ret = psp_bootloader_load_sysdrv(psp);
 682                 if (ret) {
 683                         DRM_ERROR("PSP load sysdrv failed!\n");
 684                         return ret;
 685                 }
 686
 687                 ret = psp_bootloader_load_sos(psp);
 688                 if (ret) {
 689                         DRM_ERROR("PSP load sos failed!\n");
 690                         return ret;
 691                 }
 692         }
 693
 694         ret = psp_ring_create(psp, PSP_RING_TYPE__KM);
 695         if (ret) {
 696                 DRM_ERROR("PSP create ring failed!\n");
 697                 return ret;
 698         }
 699
 700         ret = psp_tmr_load(psp);
 701         if (ret) {
 702                 DRM_ERROR("PSP load tmr failed!\n");
 703                 return ret;
 704         }
 705
 706         ret = psp_asd_load(psp);
 707         if (ret) {
 708                 DRM_ERROR("PSP load asd failed!\n");
 709                 return ret;
 710         }
 711
 712         if (adev->gmc.xgmi.num_physical_nodes > 1) {
 713                 ret = psp_xgmi_initialize(psp);
 714                 /* Warning the XGMI seesion initialize failure
 715                  * Instead of stop driver initialization
 716                  */
 717                 if (ret)
 718                         dev_err(psp->adev->dev,
 719                                 "XGMI: Failed to initialize XGMI session\n");
 720         }
 721
 722
 723         if (psp->adev->psp.ta_fw) {
 724                 ret = psp_ras_initialize(psp);
 725                 if (ret)
 726                         dev_err(psp->adev->dev,
 727                                         "RAS: Failed to initialize RAS\n");
 728         }
 729
 730         return 0;
 731 }
 732
 733 static int psp_get_fw_type(struct amdgpu_firmware_info *ucode,
 734                            enum psp_gfx_fw_type *type)
 735 {
 736         switch (ucode->ucode_id) {
 737         case AMDGPU_UCODE_ID_SDMA0:
 738                 *type = GFX_FW_TYPE_SDMA0;
 739                 break;
 740         case AMDGPU_UCODE_ID_SDMA1:
 741                 *type = GFX_FW_TYPE_SDMA1;
 742                 break;
 743         case AMDGPU_UCODE_ID_CP_CE:
 744                 *type = GFX_FW_TYPE_CP_CE;
 745                 break;
 746         case AMDGPU_UCODE_ID_CP_PFP:
 747                 *type = GFX_FW_TYPE_CP_PFP;
 748                 break;
 749         case AMDGPU_UCODE_ID_CP_ME:
 750                 *type = GFX_FW_TYPE_CP_ME;
 751                 break;
 752         case AMDGPU_UCODE_ID_CP_MEC1:
 753                 *type = GFX_FW_TYPE_CP_MEC;
 754                 break;
 755         case AMDGPU_UCODE_ID_CP_MEC1_JT:
 756                 *type = GFX_FW_TYPE_CP_MEC_ME1;
 757                 break;
 758         case AMDGPU_UCODE_ID_CP_MEC2:
 759                 *type = GFX_FW_TYPE_CP_MEC;
 760                 break;
 761         case AMDGPU_UCODE_ID_CP_MEC2_JT:
 762                 *type = GFX_FW_TYPE_CP_MEC_ME2;
 763                 break;
 764         case AMDGPU_UCODE_ID_RLC_G:
 765                 *type = GFX_FW_TYPE_RLC_G;
 766                 break;
 767         case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL:
 768                 *type = GFX_FW_TYPE_RLC_RESTORE_LIST_SRM_CNTL;
 769                 break;
 770         case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM:
 771                 *type = GFX_FW_TYPE_RLC_RESTORE_LIST_GPM_MEM;
 772                 break;
 773         case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM:
 774                 *type = GFX_FW_TYPE_RLC_RESTORE_LIST_SRM_MEM;
 775                 break;
 776         case AMDGPU_UCODE_ID_SMC:
 777                 *type = GFX_FW_TYPE_SMU;
 778                 break;
 779         case AMDGPU_UCODE_ID_UVD:
 780                 *type = GFX_FW_TYPE_UVD;
 781                 break;
 782         case AMDGPU_UCODE_ID_UVD1:
 783                 *type = GFX_FW_TYPE_UVD1;
 784                 break;
 785         case AMDGPU_UCODE_ID_VCE:
 786                 *type = GFX_FW_TYPE_VCE;
 787                 break;
 788         case AMDGPU_UCODE_ID_VCN:
 789                 *type = GFX_FW_TYPE_VCN;
 790                 break;
 791         case AMDGPU_UCODE_ID_DMCU_ERAM:
 792                 *type = GFX_FW_TYPE_DMCU_ERAM;
 793                 break;
 794         case AMDGPU_UCODE_ID_DMCU_INTV:
 795                 *type = GFX_FW_TYPE_DMCU_ISR;
 796                 break;
 797         case AMDGPU_UCODE_ID_MAXIMUM:
 798         default:
 799                 return -EINVAL;
 800         }
 801
 802         return 0;
 803 }
 804
 805 static int psp_prep_load_ip_fw_cmd_buf(struct amdgpu_firmware_info *ucode,
 806                                        struct psp_gfx_cmd_resp *cmd)
 807 {
 808         int ret;
 809         uint64_t fw_mem_mc_addr = ucode->mc_addr;
 810
 811         memset(cmd, 0, sizeof(struct psp_gfx_cmd_resp));
 812
 813         cmd->cmd_id = GFX_CMD_ID_LOAD_IP_FW;
 814         cmd->cmd.cmd_load_ip_fw.fw_phy_addr_lo = lower_32_bits(fw_mem_mc_addr);
 815         cmd->cmd.cmd_load_ip_fw.fw_phy_addr_hi = upper_32_bits(fw_mem_mc_addr);
 816         cmd->cmd.cmd_load_ip_fw.fw_size = ucode->ucode_size;
 817
 818         ret = psp_get_fw_type(ucode, &cmd->cmd.cmd_load_ip_fw.fw_type);
 819         if (ret)
 820                 DRM_ERROR("Unknown firmware type\n");
 821
 822         return ret;
 823 }
 824
 825 static int psp_np_fw_load(struct psp_context *psp)
 826 {
 827         int i, ret;
 828         struct amdgpu_firmware_info *ucode;
 829         struct amdgpu_device* adev = psp->adev;
 830
 831         for (i = 0; i < adev->firmware.max_ucodes; i++) {
 832                 ucode = &adev->firmware.ucode[i];
 833                 if (!ucode->fw)
 834                         continue;
 835
 836                 if (ucode->ucode_id == AMDGPU_UCODE_ID_SMC &&
 837                     psp_smu_reload_quirk(psp))
 838                         continue;
 839                 if (amdgpu_sriov_vf(adev) &&
 840                    (ucode->ucode_id == AMDGPU_UCODE_ID_SDMA0
 841                     || ucode->ucode_id == AMDGPU_UCODE_ID_SDMA1
 842                     || ucode->ucode_id == AMDGPU_UCODE_ID_RLC_G))
 843                         /*skip ucode loading in SRIOV VF */
 844                         continue;
 845
 846                 ret = psp_prep_load_ip_fw_cmd_buf(ucode, psp->cmd);
 847                 if (ret)
 848                         return ret;
 849
 850                 ret = psp_cmd_submit_buf(psp, ucode, psp->cmd,
 851                                          psp->fence_buf_mc_addr);
 852                 if (ret)
 853                         return ret;
 854
 855 #if 0
 856                 /* check if firmware loaded sucessfully */
 857                 if (!amdgpu_psp_check_fw_loading_status(adev, i))
 858                         return -EINVAL;
 859 #endif
 860         }
 861
 862         return 0;
 863 }
 864
 865 static int psp_load_fw(struct amdgpu_device *adev)
 866 {
 867         int ret;
 868         struct psp_context *psp = &adev->psp;
 869
 870         if (amdgpu_sriov_vf(adev) && adev->in_gpu_reset) {
 871                 psp_ring_stop(psp, PSP_RING_TYPE__KM); /* should not destroy ring, only stop */
 872                 goto skip_memalloc;
 873         }
 874
 875         psp->cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 876         if (!psp->cmd)
 877                 return -ENOMEM;
 878
 879         ret = amdgpu_bo_create_kernel(adev, PSP_1_MEG, PSP_1_MEG,
 880                                         AMDGPU_GEM_DOMAIN_GTT,
 881                                         &psp->fw_pri_bo,
 882                                         &psp->fw_pri_mc_addr,
 883                                         &psp->fw_pri_buf);
 884         if (ret)
 885                 goto failed;
 886
 887         ret = amdgpu_bo_create_kernel(adev, PSP_FENCE_BUFFER_SIZE, PAGE_SIZE,
 888                                         AMDGPU_GEM_DOMAIN_VRAM,
 889                                         &psp->fence_buf_bo,
 890                                         &psp->fence_buf_mc_addr,
 891                                         &psp->fence_buf);
 892         if (ret)
 893                 goto failed;
 894
 895         ret = amdgpu_bo_create_kernel(adev, PSP_CMD_BUFFER_SIZE, PAGE_SIZE,
 896                                       AMDGPU_GEM_DOMAIN_VRAM,
 897                                       &psp->cmd_buf_bo, &psp->cmd_buf_mc_addr,
 898                                       (void **)&psp->cmd_buf_mem);
 899         if (ret)
 900                 goto failed;
 901
 902         memset(psp->fence_buf, 0, PSP_FENCE_BUFFER_SIZE);
 903
 904         ret = psp_ring_init(psp, PSP_RING_TYPE__KM);
 905         if (ret) {
 906                 DRM_ERROR("PSP ring init failed!\n");
 907                 goto failed;
 908         }
 909
 910         ret = psp_tmr_init(psp);
 911         if (ret) {
 912                 DRM_ERROR("PSP tmr init failed!\n");
 913                 goto failed;
 914         }
 915
 916         ret = psp_asd_init(psp);
 917         if (ret) {
 918                 DRM_ERROR("PSP asd init failed!\n");
 919                 goto failed;
 920         }
 921
 922 skip_memalloc:
 923         ret = psp_hw_start(psp);
 924         if (ret)
 925                 goto failed;
 926
 927         ret = psp_np_fw_load(psp);
 928         if (ret)
 929                 goto failed;
 930
 931         return 0;
 932
 933 failed:
 934         /*
 935          * all cleanup jobs (xgmi terminate, ras terminate,
 936          * ring destroy, cmd/fence/fw buffers destory,
 937          * psp->cmd destory) are delayed to psp_hw_fini
 938          */
 939         return ret;
 940 }
 941
 942 static int psp_hw_init(void *handle)
 943 {
 944         int ret;
 945         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 946
 947         mutex_lock(&adev->firmware.mutex);
 948         /*
 949          * This sequence is just used on hw_init only once, no need on
 950          * resume.
 951          */
 952         ret = amdgpu_ucode_init_bo(adev);
 953         if (ret)
 954                 goto failed;
 955
 956         ret = psp_load_fw(adev);
 957         if (ret) {
 958                 DRM_ERROR("PSP firmware loading failed\n");
 959                 goto failed;
 960         }
 961
 962         mutex_unlock(&adev->firmware.mutex);
 963         return 0;
 964
 965 failed:
 966         adev->firmware.load_type = AMDGPU_FW_LOAD_DIRECT;
 967         mutex_unlock(&adev->firmware.mutex);
 968         return -EINVAL;
 969 }
 970
 971 static int psp_hw_fini(void *handle)
 972 {
 973         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 974         struct psp_context *psp = &adev->psp;
 975
 976         if (adev->gmc.xgmi.num_physical_nodes > 1 &&
 977             psp->xgmi_context.initialized == 1)
 978                 psp_xgmi_terminate(psp);
 979
 980         if (psp->adev->psp.ta_fw)
 981                 psp_ras_terminate(psp);
 982
 983         psp_ring_destroy(psp, PSP_RING_TYPE__KM);
 984
 985         amdgpu_bo_free_kernel(&psp->tmr_bo, &psp->tmr_mc_addr, &psp->tmr_buf);
 986         amdgpu_bo_free_kernel(&psp->fw_pri_bo,
 987                               &psp->fw_pri_mc_addr, &psp->fw_pri_buf);
 988         amdgpu_bo_free_kernel(&psp->fence_buf_bo,
 989                               &psp->fence_buf_mc_addr, &psp->fence_buf);
 990         amdgpu_bo_free_kernel(&psp->asd_shared_bo, &psp->asd_shared_mc_addr,
 991                               &psp->asd_shared_buf);
 992         amdgpu_bo_free_kernel(&psp->cmd_buf_bo, &psp->cmd_buf_mc_addr,
 993                               (void **)&psp->cmd_buf_mem);
 994
 995         kfree(psp->cmd);
 996         psp->cmd = NULL;
 997
 998         return 0;
 999 }
1000
1001 static int psp_suspend(void *handle)
1002 {
1003         int ret;
1004         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1005         struct psp_context *psp = &adev->psp;
1006
1007         if (adev->gmc.xgmi.num_physical_nodes > 1 &&
1008             psp->xgmi_context.initialized == 1) {
1009                 ret = psp_xgmi_terminate(psp);
1010                 if (ret) {
1011                         DRM_ERROR("Failed to terminate xgmi ta\n");
1012                         return ret;
1013                 }
1014         }
1015
1016         if (psp->adev->psp.ta_fw) {
1017                 ret = psp_ras_terminate(psp);
1018                 if (ret) {
1019                         DRM_ERROR("Failed to terminate ras ta\n");
1020                         return ret;
1021                 }
1022         }
1023
1024         ret = psp_ring_stop(psp, PSP_RING_TYPE__KM);
1025         if (ret) {
1026                 DRM_ERROR("PSP ring stop failed\n");
1027                 return ret;
1028         }
1029
1030         return 0;
1031 }
1032
1033 static int psp_resume(void *handle)
1034 {
1035         int ret;
1036         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
1037         struct psp_context *psp = &adev->psp;
1038
1039         DRM_INFO("PSP is resuming...\n");
1040
1041         mutex_lock(&adev->firmware.mutex);
1042
1043         ret = psp_hw_start(psp);
1044         if (ret)
1045                 goto failed;
1046
1047         ret = psp_np_fw_load(psp);
1048         if (ret)
1049                 goto failed;
1050
1051         mutex_unlock(&adev->firmware.mutex);
1052
1053         return 0;
1054
1055 failed:
1056         DRM_ERROR("PSP resume failed\n");
1057         mutex_unlock(&adev->firmware.mutex);
1058         return ret;
1059 }
1060
1061 int psp_gpu_reset(struct amdgpu_device *adev)
1062 {
1063         if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP)
1064                 return 0;
1065
1066         return psp_mode1_reset(&adev->psp);
1067 }
1068
1069 static bool psp_check_fw_loading_status(struct amdgpu_device *adev,
1070                                         enum AMDGPU_UCODE_ID ucode_type)
1071 {
1072         struct amdgpu_firmware_info *ucode = NULL;
1073
1074         if (!adev->firmware.fw_size)
1075                 return false;
1076
1077         ucode = &adev->firmware.ucode[ucode_type];
1078         if (!ucode->fw || !ucode->ucode_size)
1079                 return false;
1080
1081         return psp_compare_sram_data(&adev->psp, ucode, ucode_type);
1082 }
1083
1084 static int psp_set_clockgating_state(void *handle,
1085                                      enum amd_clockgating_state state)
1086 {
1087         return 0;
1088 }
1089
1090 static int psp_set_powergating_state(void *handle,
1091                                      enum amd_powergating_state state)
1092 {
1093         return 0;
1094 }
1095
1096 const struct amd_ip_funcs psp_ip_funcs = {
1097         .name = "psp",
1098         .early_init = psp_early_init,
1099         .late_init = NULL,
1100         .sw_init = psp_sw_init,
1101         .sw_fini = psp_sw_fini,
1102         .hw_init = psp_hw_init,
1103         .hw_fini = psp_hw_fini,
1104         .suspend = psp_suspend,
1105         .resume = psp_resume,
1106         .is_idle = NULL,
1107         .check_soft_reset = NULL,
1108         .wait_for_idle = NULL,
1109         .soft_reset = NULL,
1110         .set_clockgating_state = psp_set_clockgating_state,
1111         .set_powergating_state = psp_set_powergating_state,
1112 };
1113
1114 static const struct amdgpu_psp_funcs psp_funcs = {
1115         .check_fw_loading_status = psp_check_fw_loading_status,
1116 };
1117
1118 static void psp_set_funcs(struct amdgpu_device *adev)
1119 {
1120         if (NULL == adev->firmware.funcs)
1121                 adev->firmware.funcs = &psp_funcs;
1122 }
1123
1124 const struct amdgpu_ip_block_version psp_v3_1_ip_block =
1125 {
1126         .type = AMD_IP_BLOCK_TYPE_PSP,
1127         .major = 3,
1128         .minor = 1,
1129         .rev = 0,
1130         .funcs = &psp_ip_funcs,
1131 };
1132
1133 const struct amdgpu_ip_block_version psp_v10_0_ip_block =
1134 {
1135         .type = AMD_IP_BLOCK_TYPE_PSP,
1136         .major = 10,
1137         .minor = 0,
1138         .rev = 0,
1139         .funcs = &psp_ip_funcs,
1140 };
1141
1142 const struct amdgpu_ip_block_version psp_v11_0_ip_block =
1143 {
1144         .type = AMD_IP_BLOCK_TYPE_PSP,
1145         .major = 11,
1146         .minor = 0,
1147         .rev = 0,
1148         .funcs = &psp_ip_funcs,
1149 };