drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c

   1 /*
   2  * Copyright 2016 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Author: Huang Rui
  23  *
  24  */
  25
  26 #include <linux/firmware.h>
  27 #include <drm/drm_drv.h>
  28
  29 #include "amdgpu.h"
  30 #include "amdgpu_psp.h"
  31 #include "amdgpu_ucode.h"
  32 #include "amdgpu_xgmi.h"
  33 #include "soc15_common.h"
  34 #include "psp_v3_1.h"
  35 #include "psp_v10_0.h"
  36 #include "psp_v11_0.h"
  37 #include "psp_v11_0_8.h"
  38 #include "psp_v12_0.h"
  39 #include "psp_v13_0.h"
  40 #include "psp_v13_0_4.h"
  41
  42 #include "amdgpu_ras.h"
  43 #include "amdgpu_securedisplay.h"
  44 #include "amdgpu_atomfirmware.h"
  45
  46 #define AMD_VBIOS_FILE_MAX_SIZE_B      (1024*1024*3)
  47
  48 static int psp_sysfs_init(struct amdgpu_device *adev);
  49 static void psp_sysfs_fini(struct amdgpu_device *adev);
  50
  51 static int psp_load_smu_fw(struct psp_context *psp);
  52 static int psp_rap_terminate(struct psp_context *psp);
  53 static int psp_securedisplay_terminate(struct psp_context *psp);
  54
  55 static int psp_ring_init(struct psp_context *psp,
  56                          enum psp_ring_type ring_type)
  57 {
  58         int ret = 0;
  59         struct psp_ring *ring;
  60         struct amdgpu_device *adev = psp->adev;
  61
  62         ring = &psp->km_ring;
  63
  64         ring->ring_type = ring_type;
  65
  66         /* allocate 4k Page of Local Frame Buffer memory for ring */
  67         ring->ring_size = 0x1000;
  68         ret = amdgpu_bo_create_kernel(adev, ring->ring_size, PAGE_SIZE,
  69                                       AMDGPU_GEM_DOMAIN_VRAM |
  70                                       AMDGPU_GEM_DOMAIN_GTT,
  71                                       &adev->firmware.rbuf,
  72                                       &ring->ring_mem_mc_addr,
  73                                       (void **)&ring->ring_mem);
  74         if (ret) {
  75                 ring->ring_size = 0;
  76                 return ret;
  77         }
  78
  79         return 0;
  80 }
  81
  82 /*
  83  * Due to DF Cstate management centralized to PMFW, the firmware
  84  * loading sequence will be updated as below:
  85  *   - Load KDB
  86  *   - Load SYS_DRV
  87  *   - Load tOS
  88  *   - Load PMFW
  89  *   - Setup TMR
  90  *   - Load other non-psp fw
  91  *   - Load ASD
  92  *   - Load XGMI/RAS/HDCP/DTM TA if any
  93  *
  94  * This new sequence is required for
  95  *   - Arcturus and onwards
  96  */
  97 static void psp_check_pmfw_centralized_cstate_management(struct psp_context *psp)
  98 {
  99         struct amdgpu_device *adev = psp->adev;
 100
 101         if (amdgpu_sriov_vf(adev)) {
 102                 psp->pmfw_centralized_cstate_management = false;
 103                 return;
 104         }
 105
 106         switch (adev->ip_versions[MP0_HWIP][0]) {
 107         case IP_VERSION(11, 0, 0):
 108         case IP_VERSION(11, 0, 4):
 109         case IP_VERSION(11, 0, 5):
 110         case IP_VERSION(11, 0, 7):
 111         case IP_VERSION(11, 0, 9):
 112         case IP_VERSION(11, 0, 11):
 113         case IP_VERSION(11, 0, 12):
 114         case IP_VERSION(11, 0, 13):
 115         case IP_VERSION(13, 0, 0):
 116         case IP_VERSION(13, 0, 2):
 117         case IP_VERSION(13, 0, 7):
 118                 psp->pmfw_centralized_cstate_management = true;
 119                 break;
 120         default:
 121                 psp->pmfw_centralized_cstate_management = false;
 122                 break;
 123         }
 124 }
 125
 126 static int psp_init_sriov_microcode(struct psp_context *psp)
 127 {
 128         struct amdgpu_device *adev = psp->adev;
 129         char ucode_prefix[30];
 130         int ret = 0;
 131
 132         amdgpu_ucode_ip_version_decode(adev, MP0_HWIP, ucode_prefix, sizeof(ucode_prefix));
 133
 134         switch (adev->ip_versions[MP0_HWIP][0]) {
 135         case IP_VERSION(9, 0, 0):
 136         case IP_VERSION(11, 0, 7):
 137         case IP_VERSION(11, 0, 9):
 138                 adev->virt.autoload_ucode_id = AMDGPU_UCODE_ID_CP_MEC2;
 139                 ret = psp_init_cap_microcode(psp, ucode_prefix);
 140                 break;
 141         case IP_VERSION(13, 0, 2):
 142                 adev->virt.autoload_ucode_id = AMDGPU_UCODE_ID_CP_MEC2;
 143                 ret = psp_init_cap_microcode(psp, ucode_prefix);
 144                 ret &= psp_init_ta_microcode(psp, ucode_prefix);
 145                 break;
 146         case IP_VERSION(13, 0, 0):
 147                 adev->virt.autoload_ucode_id = 0;
 148                 break;
 149         case IP_VERSION(13, 0, 6):
 150                 ret = psp_init_cap_microcode(psp, ucode_prefix);
 151                 break;
 152         case IP_VERSION(13, 0, 10):
 153                 adev->virt.autoload_ucode_id = AMDGPU_UCODE_ID_CP_MES1_DATA;
 154                 ret = psp_init_cap_microcode(psp, ucode_prefix);
 155                 break;
 156         default:
 157                 return -EINVAL;
 158         }
 159         return ret;
 160 }
 161
 162 static int psp_early_init(void *handle)
 163 {
 164         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 165         struct psp_context *psp = &adev->psp;
 166
 167         switch (adev->ip_versions[MP0_HWIP][0]) {
 168         case IP_VERSION(9, 0, 0):
 169                 psp_v3_1_set_psp_funcs(psp);
 170                 psp->autoload_supported = false;
 171                 break;
 172         case IP_VERSION(10, 0, 0):
 173         case IP_VERSION(10, 0, 1):
 174                 psp_v10_0_set_psp_funcs(psp);
 175                 psp->autoload_supported = false;
 176                 break;
 177         case IP_VERSION(11, 0, 2):
 178         case IP_VERSION(11, 0, 4):
 179                 psp_v11_0_set_psp_funcs(psp);
 180                 psp->autoload_supported = false;
 181                 break;
 182         case IP_VERSION(11, 0, 0):
 183         case IP_VERSION(11, 0, 5):
 184         case IP_VERSION(11, 0, 9):
 185         case IP_VERSION(11, 0, 7):
 186         case IP_VERSION(11, 0, 11):
 187         case IP_VERSION(11, 5, 0):
 188         case IP_VERSION(11, 0, 12):
 189         case IP_VERSION(11, 0, 13):
 190                 psp_v11_0_set_psp_funcs(psp);
 191                 psp->autoload_supported = true;
 192                 break;
 193         case IP_VERSION(11, 0, 3):
 194         case IP_VERSION(12, 0, 1):
 195                 psp_v12_0_set_psp_funcs(psp);
 196                 break;
 197         case IP_VERSION(13, 0, 2):
 198         case IP_VERSION(13, 0, 6):
 199                 psp_v13_0_set_psp_funcs(psp);
 200                 break;
 201         case IP_VERSION(13, 0, 1):
 202         case IP_VERSION(13, 0, 3):
 203         case IP_VERSION(13, 0, 5):
 204         case IP_VERSION(13, 0, 8):
 205         case IP_VERSION(13, 0, 10):
 206         case IP_VERSION(13, 0, 11):
 207                 psp_v13_0_set_psp_funcs(psp);
 208                 psp->autoload_supported = true;
 209                 break;
 210         case IP_VERSION(11, 0, 8):
 211                 if (adev->apu_flags & AMD_APU_IS_CYAN_SKILLFISH2) {
 212                         psp_v11_0_8_set_psp_funcs(psp);
 213                         psp->autoload_supported = false;
 214                 }
 215                 break;
 216         case IP_VERSION(13, 0, 0):
 217         case IP_VERSION(13, 0, 7):
 218                 psp_v13_0_set_psp_funcs(psp);
 219                 psp->autoload_supported = true;
 220                 break;
 221         case IP_VERSION(13, 0, 4):
 222                 psp_v13_0_4_set_psp_funcs(psp);
 223                 psp->autoload_supported = true;
 224                 break;
 225         default:
 226                 return -EINVAL;
 227         }
 228
 229         psp->adev = adev;
 230
 231         psp_check_pmfw_centralized_cstate_management(psp);
 232
 233         if (amdgpu_sriov_vf(adev))
 234                 return psp_init_sriov_microcode(psp);
 235         else
 236                 return psp_init_microcode(psp);
 237 }
 238
 239 void psp_ta_free_shared_buf(struct ta_mem_context *mem_ctx)
 240 {
 241         amdgpu_bo_free_kernel(&mem_ctx->shared_bo, &mem_ctx->shared_mc_addr,
 242                               &mem_ctx->shared_buf);
 243         mem_ctx->shared_bo = NULL;
 244 }
 245
 246 static void psp_free_shared_bufs(struct psp_context *psp)
 247 {
 248         void *tmr_buf;
 249         void **pptr;
 250
 251         /* free TMR memory buffer */
 252         pptr = amdgpu_sriov_vf(psp->adev) ? &tmr_buf : NULL;
 253         amdgpu_bo_free_kernel(&psp->tmr_bo, &psp->tmr_mc_addr, pptr);
 254         psp->tmr_bo = NULL;
 255
 256         /* free xgmi shared memory */
 257         psp_ta_free_shared_buf(&psp->xgmi_context.context.mem_context);
 258
 259         /* free ras shared memory */
 260         psp_ta_free_shared_buf(&psp->ras_context.context.mem_context);
 261
 262         /* free hdcp shared memory */
 263         psp_ta_free_shared_buf(&psp->hdcp_context.context.mem_context);
 264
 265         /* free dtm shared memory */
 266         psp_ta_free_shared_buf(&psp->dtm_context.context.mem_context);
 267
 268         /* free rap shared memory */
 269         psp_ta_free_shared_buf(&psp->rap_context.context.mem_context);
 270
 271         /* free securedisplay shared memory */
 272         psp_ta_free_shared_buf(&psp->securedisplay_context.context.mem_context);
 273
 274
 275 }
 276
 277 static void psp_memory_training_fini(struct psp_context *psp)
 278 {
 279         struct psp_memory_training_context *ctx = &psp->mem_train_ctx;
 280
 281         ctx->init = PSP_MEM_TRAIN_NOT_SUPPORT;
 282         kfree(ctx->sys_cache);
 283         ctx->sys_cache = NULL;
 284 }
 285
 286 static int psp_memory_training_init(struct psp_context *psp)
 287 {
 288         int ret;
 289         struct psp_memory_training_context *ctx = &psp->mem_train_ctx;
 290
 291         if (ctx->init != PSP_MEM_TRAIN_RESERVE_SUCCESS) {
 292                 DRM_DEBUG("memory training is not supported!\n");
 293                 return 0;
 294         }
 295
 296         ctx->sys_cache = kzalloc(ctx->train_data_size, GFP_KERNEL);
 297         if (ctx->sys_cache == NULL) {
 298                 DRM_ERROR("alloc mem_train_ctx.sys_cache failed!\n");
 299                 ret = -ENOMEM;
 300                 goto Err_out;
 301         }
 302
 303         DRM_DEBUG("train_data_size:%llx,p2c_train_data_offset:%llx,c2p_train_data_offset:%llx.\n",
 304                   ctx->train_data_size,
 305                   ctx->p2c_train_data_offset,
 306                   ctx->c2p_train_data_offset);
 307         ctx->init = PSP_MEM_TRAIN_INIT_SUCCESS;
 308         return 0;
 309
 310 Err_out:
 311         psp_memory_training_fini(psp);
 312         return ret;
 313 }
 314
 315 /*
 316  * Helper funciton to query psp runtime database entry
 317  *
 318  * @adev: amdgpu_device pointer
 319  * @entry_type: the type of psp runtime database entry
 320  * @db_entry: runtime database entry pointer
 321  *
 322  * Return false if runtime database doesn't exit or entry is invalid
 323  * or true if the specific database entry is found, and copy to @db_entry
 324  */
 325 static bool psp_get_runtime_db_entry(struct amdgpu_device *adev,
 326                                      enum psp_runtime_entry_type entry_type,
 327                                      void *db_entry)
 328 {
 329         uint64_t db_header_pos, db_dir_pos;
 330         struct psp_runtime_data_header db_header = {0};
 331         struct psp_runtime_data_directory db_dir = {0};
 332         bool ret = false;
 333         int i;
 334
 335         if (adev->ip_versions[MP0_HWIP][0] == IP_VERSION(13, 0, 6))
 336                 return false;
 337
 338         db_header_pos = adev->gmc.mc_vram_size - PSP_RUNTIME_DB_OFFSET;
 339         db_dir_pos = db_header_pos + sizeof(struct psp_runtime_data_header);
 340
 341         /* read runtime db header from vram */
 342         amdgpu_device_vram_access(adev, db_header_pos, (uint32_t *)&db_header,
 343                         sizeof(struct psp_runtime_data_header), false);
 344
 345         if (db_header.cookie != PSP_RUNTIME_DB_COOKIE_ID) {
 346                 /* runtime db doesn't exist, exit */
 347                 dev_dbg(adev->dev, "PSP runtime database doesn't exist\n");
 348                 return false;
 349         }
 350
 351         /* read runtime database entry from vram */
 352         amdgpu_device_vram_access(adev, db_dir_pos, (uint32_t *)&db_dir,
 353                         sizeof(struct psp_runtime_data_directory), false);
 354
 355         if (db_dir.entry_count >= PSP_RUNTIME_DB_DIAG_ENTRY_MAX_COUNT) {
 356                 /* invalid db entry count, exit */
 357                 dev_warn(adev->dev, "Invalid PSP runtime database entry count\n");
 358                 return false;
 359         }
 360
 361         /* look up for requested entry type */
 362         for (i = 0; i < db_dir.entry_count && !ret; i++) {
 363                 if (db_dir.entry_list[i].entry_type == entry_type) {
 364                         switch (entry_type) {
 365                         case PSP_RUNTIME_ENTRY_TYPE_BOOT_CONFIG:
 366                                 if (db_dir.entry_list[i].size < sizeof(struct psp_runtime_boot_cfg_entry)) {
 367                                         /* invalid db entry size */
 368                                         dev_warn(adev->dev, "Invalid PSP runtime database boot cfg entry size\n");
 369                                         return false;
 370                                 }
 371                                 /* read runtime database entry */
 372                                 amdgpu_device_vram_access(adev, db_header_pos + db_dir.entry_list[i].offset,
 373                                                           (uint32_t *)db_entry, sizeof(struct psp_runtime_boot_cfg_entry), false);
 374                                 ret = true;
 375                                 break;
 376                         case PSP_RUNTIME_ENTRY_TYPE_PPTABLE_ERR_STATUS:
 377                                 if (db_dir.entry_list[i].size < sizeof(struct psp_runtime_scpm_entry)) {
 378                                         /* invalid db entry size */
 379                                         dev_warn(adev->dev, "Invalid PSP runtime database scpm entry size\n");
 380                                         return false;
 381                                 }
 382                                 /* read runtime database entry */
 383                                 amdgpu_device_vram_access(adev, db_header_pos + db_dir.entry_list[i].offset,
 384                                                           (uint32_t *)db_entry, sizeof(struct psp_runtime_scpm_entry), false);
 385                                 ret = true;
 386                                 break;
 387                         default:
 388                                 ret = false;
 389                                 break;
 390                         }
 391                 }
 392         }
 393
 394         return ret;
 395 }
 396
 397 static int psp_sw_init(void *handle)
 398 {
 399         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 400         struct psp_context *psp = &adev->psp;
 401         int ret;
 402         struct psp_runtime_boot_cfg_entry boot_cfg_entry;
 403         struct psp_memory_training_context *mem_training_ctx = &psp->mem_train_ctx;
 404         struct psp_runtime_scpm_entry scpm_entry;
 405
 406         psp->cmd = kzalloc(sizeof(struct psp_gfx_cmd_resp), GFP_KERNEL);
 407         if (!psp->cmd) {
 408                 DRM_ERROR("Failed to allocate memory to command buffer!\n");
 409                 ret = -ENOMEM;
 410         }
 411
 412         adev->psp.xgmi_context.supports_extended_data =
 413                 !adev->gmc.xgmi.connected_to_cpu &&
 414                         adev->ip_versions[MP0_HWIP][0] == IP_VERSION(13, 0, 2);
 415
 416         memset(&scpm_entry, 0, sizeof(scpm_entry));
 417         if ((psp_get_runtime_db_entry(adev,
 418                                 PSP_RUNTIME_ENTRY_TYPE_PPTABLE_ERR_STATUS,
 419                                 &scpm_entry)) &&
 420             (scpm_entry.scpm_status != SCPM_DISABLE)) {
 421                 adev->scpm_enabled = true;
 422                 adev->scpm_status = scpm_entry.scpm_status;
 423         } else {
 424                 adev->scpm_enabled = false;
 425                 adev->scpm_status = SCPM_DISABLE;
 426         }
 427
 428         /* TODO: stop gpu driver services and print alarm if scpm is enabled with error status */
 429
 430         memset(&boot_cfg_entry, 0, sizeof(boot_cfg_entry));
 431         if (psp_get_runtime_db_entry(adev,
 432                                 PSP_RUNTIME_ENTRY_TYPE_BOOT_CONFIG,
 433                                 &boot_cfg_entry)) {
 434                 psp->boot_cfg_bitmask = boot_cfg_entry.boot_cfg_bitmask;
 435                 if ((psp->boot_cfg_bitmask) &
 436                     BOOT_CFG_FEATURE_TWO_STAGE_DRAM_TRAINING) {
 437                         /* If psp runtime database exists, then
 438                          * only enable two stage memory training
 439                          * when TWO_STAGE_DRAM_TRAINING bit is set
 440                          * in runtime database */
 441                         mem_training_ctx->enable_mem_training = true;
 442                 }
 443
 444         } else {
 445                 /* If psp runtime database doesn't exist or
 446                  * is invalid, force enable two stage memory
 447                  * training */
 448                 mem_training_ctx->enable_mem_training = true;
 449         }
 450
 451         if (mem_training_ctx->enable_mem_training) {
 452                 ret = psp_memory_training_init(psp);
 453                 if (ret) {
 454                         DRM_ERROR("Failed to initialize memory training!\n");
 455                         return ret;
 456                 }
 457
 458                 ret = psp_mem_training(psp, PSP_MEM_TRAIN_COLD_BOOT);
 459                 if (ret) {
 460                         DRM_ERROR("Failed to process memory training!\n");
 461                         return ret;
 462                 }
 463         }
 464
 465         if (adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 0) ||
 466             adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 7)) {
 467                 ret = psp_sysfs_init(adev);
 468                 if (ret)
 469                         return ret;
 470         }
 471
 472         ret = amdgpu_bo_create_kernel(adev, PSP_1_MEG, PSP_1_MEG,
 473                                       amdgpu_sriov_vf(adev) ?
 474                                       AMDGPU_GEM_DOMAIN_VRAM : AMDGPU_GEM_DOMAIN_GTT,
 475                                       &psp->fw_pri_bo,
 476                                       &psp->fw_pri_mc_addr,
 477                                       &psp->fw_pri_buf);
 478         if (ret)
 479                 return ret;
 480
 481         ret = amdgpu_bo_create_kernel(adev, PSP_FENCE_BUFFER_SIZE, PAGE_SIZE,
 482                                       AMDGPU_GEM_DOMAIN_VRAM |
 483                                       AMDGPU_GEM_DOMAIN_GTT,
 484                                       &psp->fence_buf_bo,
 485                                       &psp->fence_buf_mc_addr,
 486                                       &psp->fence_buf);
 487         if (ret)
 488                 goto failed1;
 489
 490         ret = amdgpu_bo_create_kernel(adev, PSP_CMD_BUFFER_SIZE, PAGE_SIZE,
 491                                       AMDGPU_GEM_DOMAIN_VRAM |
 492                                       AMDGPU_GEM_DOMAIN_GTT,
 493                                       &psp->cmd_buf_bo, &psp->cmd_buf_mc_addr,
 494                                       (void **)&psp->cmd_buf_mem);
 495         if (ret)
 496                 goto failed2;
 497
 498         return 0;
 499
 500 failed2:
 501         amdgpu_bo_free_kernel(&psp->fence_buf_bo,
 502                               &psp->fence_buf_mc_addr, &psp->fence_buf);
 503 failed1:
 504         amdgpu_bo_free_kernel(&psp->fw_pri_bo,
 505                               &psp->fw_pri_mc_addr, &psp->fw_pri_buf);
 506         return ret;
 507 }
 508
 509 static int psp_sw_fini(void *handle)
 510 {
 511         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 512         struct psp_context *psp = &adev->psp;
 513         struct psp_gfx_cmd_resp *cmd = psp->cmd;
 514
 515         psp_memory_training_fini(psp);
 516
 517         amdgpu_ucode_release(&psp->sos_fw);
 518         amdgpu_ucode_release(&psp->asd_fw);
 519         amdgpu_ucode_release(&psp->ta_fw);
 520         amdgpu_ucode_release(&psp->cap_fw);
 521         amdgpu_ucode_release(&psp->toc_fw);
 522
 523         if (adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 0) ||
 524             adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 7))
 525                 psp_sysfs_fini(adev);
 526
 527         kfree(cmd);
 528         cmd = NULL;
 529
 530         psp_free_shared_bufs(psp);
 531
 532         if (psp->km_ring.ring_mem)
 533                 amdgpu_bo_free_kernel(&adev->firmware.rbuf,
 534                                       &psp->km_ring.ring_mem_mc_addr,
 535                                       (void **)&psp->km_ring.ring_mem);
 536
 537         amdgpu_bo_free_kernel(&psp->fw_pri_bo,
 538                               &psp->fw_pri_mc_addr, &psp->fw_pri_buf);
 539         amdgpu_bo_free_kernel(&psp->fence_buf_bo,
 540                               &psp->fence_buf_mc_addr, &psp->fence_buf);
 541         amdgpu_bo_free_kernel(&psp->cmd_buf_bo, &psp->cmd_buf_mc_addr,
 542                               (void **)&psp->cmd_buf_mem);
 543
 544         return 0;
 545 }
 546
 547 int psp_wait_for(struct psp_context *psp, uint32_t reg_index,
 548                  uint32_t reg_val, uint32_t mask, bool check_changed)
 549 {
 550         uint32_t val;
 551         int i;
 552         struct amdgpu_device *adev = psp->adev;
 553
 554         if (psp->adev->no_hw_access)
 555                 return 0;
 556
 557         for (i = 0; i < adev->usec_timeout; i++) {
 558                 val = RREG32(reg_index);
 559                 if (check_changed) {
 560                         if (val != reg_val)
 561                                 return 0;
 562                 } else {
 563                         if ((val & mask) == reg_val)
 564                                 return 0;
 565                 }
 566                 udelay(1);
 567         }
 568
 569         return -ETIME;
 570 }
 571
 572 int psp_wait_for_spirom_update(struct psp_context *psp, uint32_t reg_index,
 573                                uint32_t reg_val, uint32_t mask, uint32_t msec_timeout)
 574 {
 575         uint32_t val;
 576         int i;
 577         struct amdgpu_device *adev = psp->adev;
 578
 579         if (psp->adev->no_hw_access)
 580                 return 0;
 581
 582         for (i = 0; i < msec_timeout; i++) {
 583                 val = RREG32(reg_index);
 584                 if ((val & mask) == reg_val)
 585                         return 0;
 586                 msleep(1);
 587         }
 588
 589         return -ETIME;
 590 }
 591
 592 static const char *psp_gfx_cmd_name(enum psp_gfx_cmd_id cmd_id)
 593 {
 594         switch (cmd_id) {
 595         case GFX_CMD_ID_LOAD_TA:
 596                 return "LOAD_TA";
 597         case GFX_CMD_ID_UNLOAD_TA:
 598                 return "UNLOAD_TA";
 599         case GFX_CMD_ID_INVOKE_CMD:
 600                 return "INVOKE_CMD";
 601         case GFX_CMD_ID_LOAD_ASD:
 602                 return "LOAD_ASD";
 603         case GFX_CMD_ID_SETUP_TMR:
 604                 return "SETUP_TMR";
 605         case GFX_CMD_ID_LOAD_IP_FW:
 606                 return "LOAD_IP_FW";
 607         case GFX_CMD_ID_DESTROY_TMR:
 608                 return "DESTROY_TMR";
 609         case GFX_CMD_ID_SAVE_RESTORE:
 610                 return "SAVE_RESTORE_IP_FW";
 611         case GFX_CMD_ID_SETUP_VMR:
 612                 return "SETUP_VMR";
 613         case GFX_CMD_ID_DESTROY_VMR:
 614                 return "DESTROY_VMR";
 615         case GFX_CMD_ID_PROG_REG:
 616                 return "PROG_REG";
 617         case GFX_CMD_ID_GET_FW_ATTESTATION:
 618                 return "GET_FW_ATTESTATION";
 619         case GFX_CMD_ID_LOAD_TOC:
 620                 return "ID_LOAD_TOC";
 621         case GFX_CMD_ID_AUTOLOAD_RLC:
 622                 return "AUTOLOAD_RLC";
 623         case GFX_CMD_ID_BOOT_CFG:
 624                 return "BOOT_CFG";
 625         default:
 626                 return "UNKNOWN CMD";
 627         }
 628 }
 629
 630 static int
 631 psp_cmd_submit_buf(struct psp_context *psp,
 632                    struct amdgpu_firmware_info *ucode,
 633                    struct psp_gfx_cmd_resp *cmd, uint64_t fence_mc_addr)
 634 {
 635         int ret;
 636         int index;
 637         int timeout = 20000;
 638         bool ras_intr = false;
 639         bool skip_unsupport = false;
 640
 641         if (psp->adev->no_hw_access)
 642                 return 0;
 643
 644         memset(psp->cmd_buf_mem, 0, PSP_CMD_BUFFER_SIZE);
 645
 646         memcpy(psp->cmd_buf_mem, cmd, sizeof(struct psp_gfx_cmd_resp));
 647
 648         index = atomic_inc_return(&psp->fence_value);
 649         ret = psp_ring_cmd_submit(psp, psp->cmd_buf_mc_addr, fence_mc_addr, index);
 650         if (ret) {
 651                 atomic_dec(&psp->fence_value);
 652                 goto exit;
 653         }
 654
 655         amdgpu_device_invalidate_hdp(psp->adev, NULL);
 656         while (*((unsigned int *)psp->fence_buf) != index) {
 657                 if (--timeout == 0)
 658                         break;
 659                 /*
 660                  * Shouldn't wait for timeout when err_event_athub occurs,
 661                  * because gpu reset thread triggered and lock resource should
 662                  * be released for psp resume sequence.
 663                  */
 664                 ras_intr = amdgpu_ras_intr_triggered();
 665                 if (ras_intr)
 666                         break;
 667                 usleep_range(10, 100);
 668                 amdgpu_device_invalidate_hdp(psp->adev, NULL);
 669         }
 670
 671         /* We allow TEE_ERROR_NOT_SUPPORTED for VMR command and PSP_ERR_UNKNOWN_COMMAND in SRIOV */
 672         skip_unsupport = (psp->cmd_buf_mem->resp.status == TEE_ERROR_NOT_SUPPORTED ||
 673                 psp->cmd_buf_mem->resp.status == PSP_ERR_UNKNOWN_COMMAND) && amdgpu_sriov_vf(psp->adev);
 674
 675         memcpy(&cmd->resp, &psp->cmd_buf_mem->resp, sizeof(struct psp_gfx_resp));
 676
 677         /* In some cases, psp response status is not 0 even there is no
 678          * problem while the command is submitted. Some version of PSP FW
 679          * doesn't write 0 to that field.
 680          * So here we would like to only print a warning instead of an error
 681          * during psp initialization to avoid breaking hw_init and it doesn't
 682          * return -EINVAL.
 683          */
 684         if (!skip_unsupport && (psp->cmd_buf_mem->resp.status || !timeout) && !ras_intr) {
 685                 if (ucode)
 686                         DRM_WARN("failed to load ucode %s(0x%X) ",
 687                                   amdgpu_ucode_name(ucode->ucode_id), ucode->ucode_id);
 688                 DRM_WARN("psp gfx command %s(0x%X) failed and response status is (0x%X)\n",
 689                          psp_gfx_cmd_name(psp->cmd_buf_mem->cmd_id), psp->cmd_buf_mem->cmd_id,
 690                          psp->cmd_buf_mem->resp.status);
 691                 /* If any firmware (including CAP) load fails under SRIOV, it should
 692                  * return failure to stop the VF from initializing.
 693                  * Also return failure in case of timeout
 694                  */
 695                 if ((ucode && amdgpu_sriov_vf(psp->adev)) || !timeout) {
 696                         ret = -EINVAL;
 697                         goto exit;
 698                 }
 699         }
 700
 701         if (ucode) {
 702                 ucode->tmr_mc_addr_lo = psp->cmd_buf_mem->resp.fw_addr_lo;
 703                 ucode->tmr_mc_addr_hi = psp->cmd_buf_mem->resp.fw_addr_hi;
 704         }
 705
 706 exit:
 707         return ret;
 708 }
 709
 710 static struct psp_gfx_cmd_resp *acquire_psp_cmd_buf(struct psp_context *psp)
 711 {
 712         struct psp_gfx_cmd_resp *cmd = psp->cmd;
 713
 714         mutex_lock(&psp->mutex);
 715
 716         memset(cmd, 0, sizeof(struct psp_gfx_cmd_resp));
 717
 718         return cmd;
 719 }
 720
 721 static void release_psp_cmd_buf(struct psp_context *psp)
 722 {
 723         mutex_unlock(&psp->mutex);
 724 }
 725
 726 static void psp_prep_tmr_cmd_buf(struct psp_context *psp,
 727                                  struct psp_gfx_cmd_resp *cmd,
 728                                  uint64_t tmr_mc, struct amdgpu_bo *tmr_bo)
 729 {
 730         struct amdgpu_device *adev = psp->adev;
 731         uint32_t size = 0;
 732         uint64_t tmr_pa = 0;
 733
 734         if (tmr_bo) {
 735                 size = amdgpu_bo_size(tmr_bo);
 736                 tmr_pa = amdgpu_gmc_vram_pa(adev, tmr_bo);
 737         }
 738
 739         if (amdgpu_sriov_vf(psp->adev))
 740                 cmd->cmd_id = GFX_CMD_ID_SETUP_VMR;
 741         else
 742                 cmd->cmd_id = GFX_CMD_ID_SETUP_TMR;
 743         cmd->cmd.cmd_setup_tmr.buf_phy_addr_lo = lower_32_bits(tmr_mc);
 744         cmd->cmd.cmd_setup_tmr.buf_phy_addr_hi = upper_32_bits(tmr_mc);
 745         cmd->cmd.cmd_setup_tmr.buf_size = size;
 746         cmd->cmd.cmd_setup_tmr.bitfield.virt_phy_addr = 1;
 747         cmd->cmd.cmd_setup_tmr.system_phy_addr_lo = lower_32_bits(tmr_pa);
 748         cmd->cmd.cmd_setup_tmr.system_phy_addr_hi = upper_32_bits(tmr_pa);
 749 }
 750
 751 static void psp_prep_load_toc_cmd_buf(struct psp_gfx_cmd_resp *cmd,
 752                                       uint64_t pri_buf_mc, uint32_t size)
 753 {
 754         cmd->cmd_id = GFX_CMD_ID_LOAD_TOC;
 755         cmd->cmd.cmd_load_toc.toc_phy_addr_lo = lower_32_bits(pri_buf_mc);
 756         cmd->cmd.cmd_load_toc.toc_phy_addr_hi = upper_32_bits(pri_buf_mc);
 757         cmd->cmd.cmd_load_toc.toc_size = size;
 758 }
 759
 760 /* Issue LOAD TOC cmd to PSP to part toc and calculate tmr size needed */
 761 static int psp_load_toc(struct psp_context *psp,
 762                         uint32_t *tmr_size)
 763 {
 764         int ret;
 765         struct psp_gfx_cmd_resp *cmd = acquire_psp_cmd_buf(psp);
 766
 767         /* Copy toc to psp firmware private buffer */
 768         psp_copy_fw(psp, psp->toc.start_addr, psp->toc.size_bytes);
 769
 770         psp_prep_load_toc_cmd_buf(cmd, psp->fw_pri_mc_addr, psp->toc.size_bytes);
 771
 772         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 773                                  psp->fence_buf_mc_addr);
 774         if (!ret)
 775                 *tmr_size = psp->cmd_buf_mem->resp.tmr_size;
 776
 777         release_psp_cmd_buf(psp);
 778
 779         return ret;
 780 }
 781
 782 static bool psp_boottime_tmr(struct psp_context *psp)
 783 {
 784         switch (psp->adev->ip_versions[MP0_HWIP][0]) {
 785         case IP_VERSION(13, 0, 6):
 786                 return true;
 787         default:
 788                 return false;
 789         }
 790 }
 791
 792 /* Set up Trusted Memory Region */
 793 static int psp_tmr_init(struct psp_context *psp)
 794 {
 795         int ret = 0;
 796         int tmr_size;
 797         void *tmr_buf;
 798         void **pptr;
 799
 800         /*
 801          * According to HW engineer, they prefer the TMR address be "naturally
 802          * aligned" , e.g. the start address be an integer divide of TMR size.
 803          *
 804          * Note: this memory need be reserved till the driver
 805          * uninitializes.
 806          */
 807         tmr_size = PSP_TMR_SIZE(psp->adev);
 808
 809         /* For ASICs support RLC autoload, psp will parse the toc
 810          * and calculate the total size of TMR needed */
 811         if (!amdgpu_sriov_vf(psp->adev) &&
 812             psp->toc.start_addr &&
 813             psp->toc.size_bytes &&
 814             psp->fw_pri_buf) {
 815                 ret = psp_load_toc(psp, &tmr_size);
 816                 if (ret) {
 817                         DRM_ERROR("Failed to load toc\n");
 818                         return ret;
 819                 }
 820         }
 821
 822         if (!psp->tmr_bo) {
 823                 pptr = amdgpu_sriov_vf(psp->adev) ? &tmr_buf : NULL;
 824                 ret = amdgpu_bo_create_kernel(psp->adev, tmr_size,
 825                                               PSP_TMR_ALIGNMENT,
 826                                               AMDGPU_HAS_VRAM(psp->adev) ?
 827                                               AMDGPU_GEM_DOMAIN_VRAM :
 828                                               AMDGPU_GEM_DOMAIN_GTT,
 829                                               &psp->tmr_bo, &psp->tmr_mc_addr,
 830                                               pptr);
 831         }
 832
 833         return ret;
 834 }
 835
 836 static bool psp_skip_tmr(struct psp_context *psp)
 837 {
 838         switch (psp->adev->ip_versions[MP0_HWIP][0]) {
 839         case IP_VERSION(11, 0, 9):
 840         case IP_VERSION(11, 0, 7):
 841         case IP_VERSION(13, 0, 2):
 842         case IP_VERSION(13, 0, 6):
 843         case IP_VERSION(13, 0, 10):
 844                 return true;
 845         default:
 846                 return false;
 847         }
 848 }
 849
 850 static int psp_tmr_load(struct psp_context *psp)
 851 {
 852         int ret;
 853         struct psp_gfx_cmd_resp *cmd;
 854
 855         /* For Navi12 and CHIP_SIENNA_CICHLID SRIOV, do not set up TMR.
 856          * Already set up by host driver.
 857          */
 858         if (amdgpu_sriov_vf(psp->adev) && psp_skip_tmr(psp))
 859                 return 0;
 860
 861         cmd = acquire_psp_cmd_buf(psp);
 862
 863         psp_prep_tmr_cmd_buf(psp, cmd, psp->tmr_mc_addr, psp->tmr_bo);
 864         if (psp->tmr_bo)
 865                 DRM_INFO("reserve 0x%lx from 0x%llx for PSP TMR\n",
 866                          amdgpu_bo_size(psp->tmr_bo), psp->tmr_mc_addr);
 867
 868         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 869                                  psp->fence_buf_mc_addr);
 870
 871         release_psp_cmd_buf(psp);
 872
 873         return ret;
 874 }
 875
 876 static void psp_prep_tmr_unload_cmd_buf(struct psp_context *psp,
 877                                         struct psp_gfx_cmd_resp *cmd)
 878 {
 879         if (amdgpu_sriov_vf(psp->adev))
 880                 cmd->cmd_id = GFX_CMD_ID_DESTROY_VMR;
 881         else
 882                 cmd->cmd_id = GFX_CMD_ID_DESTROY_TMR;
 883 }
 884
 885 static int psp_tmr_unload(struct psp_context *psp)
 886 {
 887         int ret;
 888         struct psp_gfx_cmd_resp *cmd;
 889
 890         /* skip TMR unload for Navi12 and CHIP_SIENNA_CICHLID SRIOV,
 891          * as TMR is not loaded at all
 892          */
 893         if (amdgpu_sriov_vf(psp->adev) && psp_skip_tmr(psp))
 894                 return 0;
 895
 896         cmd = acquire_psp_cmd_buf(psp);
 897
 898         psp_prep_tmr_unload_cmd_buf(psp, cmd);
 899         dev_dbg(psp->adev->dev, "free PSP TMR buffer\n");
 900
 901         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 902                                  psp->fence_buf_mc_addr);
 903
 904         release_psp_cmd_buf(psp);
 905
 906         return ret;
 907 }
 908
 909 static int psp_tmr_terminate(struct psp_context *psp)
 910 {
 911         return psp_tmr_unload(psp);
 912 }
 913
 914 int psp_get_fw_attestation_records_addr(struct psp_context *psp,
 915                                         uint64_t *output_ptr)
 916 {
 917         int ret;
 918         struct psp_gfx_cmd_resp *cmd;
 919
 920         if (!output_ptr)
 921                 return -EINVAL;
 922
 923         if (amdgpu_sriov_vf(psp->adev))
 924                 return 0;
 925
 926         cmd = acquire_psp_cmd_buf(psp);
 927
 928         cmd->cmd_id = GFX_CMD_ID_GET_FW_ATTESTATION;
 929
 930         ret = psp_cmd_submit_buf(psp, NULL, cmd,
 931                                  psp->fence_buf_mc_addr);
 932
 933         if (!ret) {
 934                 *output_ptr = ((uint64_t)cmd->resp.uresp.fwar_db_info.fwar_db_addr_lo) +
 935                               ((uint64_t)cmd->resp.uresp.fwar_db_info.fwar_db_addr_hi << 32);
 936         }
 937
 938         release_psp_cmd_buf(psp);
 939
 940         return ret;
 941 }
 942
 943 static int psp_boot_config_get(struct amdgpu_device *adev, uint32_t *boot_cfg)
 944 {
 945         struct psp_context *psp = &adev->psp;
 946         struct psp_gfx_cmd_resp *cmd;
 947         int ret;
 948
 949         if (amdgpu_sriov_vf(adev))
 950                 return 0;
 951
 952         cmd = acquire_psp_cmd_buf(psp);
 953
 954         cmd->cmd_id = GFX_CMD_ID_BOOT_CFG;
 955         cmd->cmd.boot_cfg.sub_cmd = BOOTCFG_CMD_GET;
 956
 957         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
 958         if (!ret) {
 959                 *boot_cfg =
 960                         (cmd->resp.uresp.boot_cfg.boot_cfg & BOOT_CONFIG_GECC) ? 1 : 0;
 961         }
 962
 963         release_psp_cmd_buf(psp);
 964
 965         return ret;
 966 }
 967
 968 static int psp_boot_config_set(struct amdgpu_device *adev, uint32_t boot_cfg)
 969 {
 970         int ret;
 971         struct psp_context *psp = &adev->psp;
 972         struct psp_gfx_cmd_resp *cmd;
 973
 974         if (amdgpu_sriov_vf(adev))
 975                 return 0;
 976
 977         cmd = acquire_psp_cmd_buf(psp);
 978
 979         cmd->cmd_id = GFX_CMD_ID_BOOT_CFG;
 980         cmd->cmd.boot_cfg.sub_cmd = BOOTCFG_CMD_SET;
 981         cmd->cmd.boot_cfg.boot_config = boot_cfg;
 982         cmd->cmd.boot_cfg.boot_config_valid = boot_cfg;
 983
 984         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
 985
 986         release_psp_cmd_buf(psp);
 987
 988         return ret;
 989 }
 990
 991 static int psp_rl_load(struct amdgpu_device *adev)
 992 {
 993         int ret;
 994         struct psp_context *psp = &adev->psp;
 995         struct psp_gfx_cmd_resp *cmd;
 996
 997         if (!is_psp_fw_valid(psp->rl))
 998                 return 0;
 999
1000         cmd = acquire_psp_cmd_buf(psp);
1001
1002         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
1003         memcpy(psp->fw_pri_buf, psp->rl.start_addr, psp->rl.size_bytes);
1004
1005         cmd->cmd_id = GFX_CMD_ID_LOAD_IP_FW;
1006         cmd->cmd.cmd_load_ip_fw.fw_phy_addr_lo = lower_32_bits(psp->fw_pri_mc_addr);
1007         cmd->cmd.cmd_load_ip_fw.fw_phy_addr_hi = upper_32_bits(psp->fw_pri_mc_addr);
1008         cmd->cmd.cmd_load_ip_fw.fw_size = psp->rl.size_bytes;
1009         cmd->cmd.cmd_load_ip_fw.fw_type = GFX_FW_TYPE_REG_LIST;
1010
1011         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
1012
1013         release_psp_cmd_buf(psp);
1014
1015         return ret;
1016 }
1017
1018 int psp_spatial_partition(struct psp_context *psp, int mode)
1019 {
1020         struct psp_gfx_cmd_resp *cmd;
1021         int ret;
1022
1023         if (amdgpu_sriov_vf(psp->adev))
1024                 return 0;
1025
1026         cmd = acquire_psp_cmd_buf(psp);
1027
1028         cmd->cmd_id = GFX_CMD_ID_SRIOV_SPATIAL_PART;
1029         cmd->cmd.cmd_spatial_part.mode = mode;
1030
1031         dev_info(psp->adev->dev, "Requesting %d partitions through PSP", mode);
1032         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
1033
1034         release_psp_cmd_buf(psp);
1035
1036         return ret;
1037 }
1038
1039 static int psp_asd_initialize(struct psp_context *psp)
1040 {
1041         int ret;
1042
1043         /* If PSP version doesn't match ASD version, asd loading will be failed.
1044          * add workaround to bypass it for sriov now.
1045          * TODO: add version check to make it common
1046          */
1047         if (amdgpu_sriov_vf(psp->adev) || !psp->asd_context.bin_desc.size_bytes)
1048                 return 0;
1049
1050         psp->asd_context.mem_context.shared_mc_addr  = 0;
1051         psp->asd_context.mem_context.shared_mem_size = PSP_ASD_SHARED_MEM_SIZE;
1052         psp->asd_context.ta_load_type                = GFX_CMD_ID_LOAD_ASD;
1053
1054         ret = psp_ta_load(psp, &psp->asd_context);
1055         if (!ret)
1056                 psp->asd_context.initialized = true;
1057
1058         return ret;
1059 }
1060
1061 static void psp_prep_ta_unload_cmd_buf(struct psp_gfx_cmd_resp *cmd,
1062                                        uint32_t session_id)
1063 {
1064         cmd->cmd_id = GFX_CMD_ID_UNLOAD_TA;
1065         cmd->cmd.cmd_unload_ta.session_id = session_id;
1066 }
1067
1068 int psp_ta_unload(struct psp_context *psp, struct ta_context *context)
1069 {
1070         int ret;
1071         struct psp_gfx_cmd_resp *cmd = acquire_psp_cmd_buf(psp);
1072
1073         psp_prep_ta_unload_cmd_buf(cmd, context->session_id);
1074
1075         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
1076
1077         context->resp_status = cmd->resp.status;
1078
1079         release_psp_cmd_buf(psp);
1080
1081         return ret;
1082 }
1083
1084 static int psp_asd_terminate(struct psp_context *psp)
1085 {
1086         int ret;
1087
1088         if (amdgpu_sriov_vf(psp->adev))
1089                 return 0;
1090
1091         if (!psp->asd_context.initialized)
1092                 return 0;
1093
1094         ret = psp_ta_unload(psp, &psp->asd_context);
1095         if (!ret)
1096                 psp->asd_context.initialized = false;
1097
1098         return ret;
1099 }
1100
1101 static void psp_prep_reg_prog_cmd_buf(struct psp_gfx_cmd_resp *cmd,
1102                 uint32_t id, uint32_t value)
1103 {
1104         cmd->cmd_id = GFX_CMD_ID_PROG_REG;
1105         cmd->cmd.cmd_setup_reg_prog.reg_value = value;
1106         cmd->cmd.cmd_setup_reg_prog.reg_id = id;
1107 }
1108
1109 int psp_reg_program(struct psp_context *psp, enum psp_reg_prog_id reg,
1110                 uint32_t value)
1111 {
1112         struct psp_gfx_cmd_resp *cmd;
1113         int ret = 0;
1114
1115         if (reg >= PSP_REG_LAST)
1116                 return -EINVAL;
1117
1118         cmd = acquire_psp_cmd_buf(psp);
1119
1120         psp_prep_reg_prog_cmd_buf(cmd, reg, value);
1121         ret = psp_cmd_submit_buf(psp, NULL, cmd, psp->fence_buf_mc_addr);
1122         if (ret)
1123                 DRM_ERROR("PSP failed to program reg id %d", reg);
1124
1125         release_psp_cmd_buf(psp);
1126
1127         return ret;
1128 }
1129
1130 static void psp_prep_ta_load_cmd_buf(struct psp_gfx_cmd_resp *cmd,
1131                                      uint64_t ta_bin_mc,
1132                                      struct ta_context *context)
1133 {
1134         cmd->cmd_id                             = context->ta_load_type;
1135         cmd->cmd.cmd_load_ta.app_phy_addr_lo    = lower_32_bits(ta_bin_mc);
1136         cmd->cmd.cmd_load_ta.app_phy_addr_hi    = upper_32_bits(ta_bin_mc);
1137         cmd->cmd.cmd_load_ta.app_len            = context->bin_desc.size_bytes;
1138
1139         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_lo =
1140                 lower_32_bits(context->mem_context.shared_mc_addr);
1141         cmd->cmd.cmd_load_ta.cmd_buf_phy_addr_hi =
1142                 upper_32_bits(context->mem_context.shared_mc_addr);
1143         cmd->cmd.cmd_load_ta.cmd_buf_len = context->mem_context.shared_mem_size;
1144 }
1145
1146 int psp_ta_init_shared_buf(struct psp_context *psp,
1147                                   struct ta_mem_context *mem_ctx)
1148 {
1149         /*
1150         * Allocate 16k memory aligned to 4k from Frame Buffer (local
1151         * physical) for ta to host memory
1152         */
1153         return amdgpu_bo_create_kernel(psp->adev, mem_ctx->shared_mem_size,
1154                                       PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM |
1155                                       AMDGPU_GEM_DOMAIN_GTT,
1156                                       &mem_ctx->shared_bo,
1157                                       &mem_ctx->shared_mc_addr,
1158                                       &mem_ctx->shared_buf);
1159 }
1160
1161 static void psp_prep_ta_invoke_cmd_buf(struct psp_gfx_cmd_resp *cmd,
1162                                        uint32_t ta_cmd_id,
1163                                        uint32_t session_id)
1164 {
1165         cmd->cmd_id                             = GFX_CMD_ID_INVOKE_CMD;
1166         cmd->cmd.cmd_invoke_cmd.session_id      = session_id;
1167         cmd->cmd.cmd_invoke_cmd.ta_cmd_id       = ta_cmd_id;
1168 }
1169
1170 int psp_ta_invoke(struct psp_context *psp,
1171                   uint32_t ta_cmd_id,
1172                   struct ta_context *context)
1173 {
1174         int ret;
1175         struct psp_gfx_cmd_resp *cmd = acquire_psp_cmd_buf(psp);
1176
1177         psp_prep_ta_invoke_cmd_buf(cmd, ta_cmd_id, context->session_id);
1178
1179         ret = psp_cmd_submit_buf(psp, NULL, cmd,
1180                                  psp->fence_buf_mc_addr);
1181
1182         context->resp_status = cmd->resp.status;
1183
1184         release_psp_cmd_buf(psp);
1185
1186         return ret;
1187 }
1188
1189 int psp_ta_load(struct psp_context *psp, struct ta_context *context)
1190 {
1191         int ret;
1192         struct psp_gfx_cmd_resp *cmd;
1193
1194         cmd = acquire_psp_cmd_buf(psp);
1195
1196         psp_copy_fw(psp, context->bin_desc.start_addr,
1197                     context->bin_desc.size_bytes);
1198
1199         psp_prep_ta_load_cmd_buf(cmd, psp->fw_pri_mc_addr, context);
1200
1201         ret = psp_cmd_submit_buf(psp, NULL, cmd,
1202                                  psp->fence_buf_mc_addr);
1203
1204         context->resp_status = cmd->resp.status;
1205
1206         if (!ret)
1207                 context->session_id = cmd->resp.session_id;
1208
1209         release_psp_cmd_buf(psp);
1210
1211         return ret;
1212 }
1213
1214 int psp_xgmi_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
1215 {
1216         return psp_ta_invoke(psp, ta_cmd_id, &psp->xgmi_context.context);
1217 }
1218
1219 int psp_xgmi_terminate(struct psp_context *psp)
1220 {
1221         int ret;
1222         struct amdgpu_device *adev = psp->adev;
1223
1224         /* XGMI TA unload currently is not supported on Arcturus/Aldebaran A+A */
1225         if (adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 4) ||
1226             (adev->ip_versions[MP0_HWIP][0] == IP_VERSION(13, 0, 2) &&
1227              adev->gmc.xgmi.connected_to_cpu))
1228                 return 0;
1229
1230         if (!psp->xgmi_context.context.initialized)
1231                 return 0;
1232
1233         ret = psp_ta_unload(psp, &psp->xgmi_context.context);
1234
1235         psp->xgmi_context.context.initialized = false;
1236
1237         return ret;
1238 }
1239
1240 int psp_xgmi_initialize(struct psp_context *psp, bool set_extended_data, bool load_ta)
1241 {
1242         struct ta_xgmi_shared_memory *xgmi_cmd;
1243         int ret;
1244
1245         if (!psp->ta_fw ||
1246             !psp->xgmi_context.context.bin_desc.size_bytes ||
1247             !psp->xgmi_context.context.bin_desc.start_addr)
1248                 return -ENOENT;
1249
1250         if (!load_ta)
1251                 goto invoke;
1252
1253         psp->xgmi_context.context.mem_context.shared_mem_size = PSP_XGMI_SHARED_MEM_SIZE;
1254         psp->xgmi_context.context.ta_load_type = GFX_CMD_ID_LOAD_TA;
1255
1256         if (!psp->xgmi_context.context.mem_context.shared_buf) {
1257                 ret = psp_ta_init_shared_buf(psp, &psp->xgmi_context.context.mem_context);
1258                 if (ret)
1259                         return ret;
1260         }
1261
1262         /* Load XGMI TA */
1263         ret = psp_ta_load(psp, &psp->xgmi_context.context);
1264         if (!ret)
1265                 psp->xgmi_context.context.initialized = true;
1266         else
1267                 return ret;
1268
1269 invoke:
1270         /* Initialize XGMI session */
1271         xgmi_cmd = (struct ta_xgmi_shared_memory *)(psp->xgmi_context.context.mem_context.shared_buf);
1272         memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory));
1273         xgmi_cmd->flag_extend_link_record = set_extended_data;
1274         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__INITIALIZE;
1275
1276         ret = psp_xgmi_invoke(psp, xgmi_cmd->cmd_id);
1277
1278         return ret;
1279 }
1280
1281 int psp_xgmi_get_hive_id(struct psp_context *psp, uint64_t *hive_id)
1282 {
1283         struct ta_xgmi_shared_memory *xgmi_cmd;
1284         int ret;
1285
1286         xgmi_cmd = (struct ta_xgmi_shared_memory *)psp->xgmi_context.context.mem_context.shared_buf;
1287         memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory));
1288
1289         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__GET_HIVE_ID;
1290
1291         /* Invoke xgmi ta to get hive id */
1292         ret = psp_xgmi_invoke(psp, xgmi_cmd->cmd_id);
1293         if (ret)
1294                 return ret;
1295
1296         *hive_id = xgmi_cmd->xgmi_out_message.get_hive_id.hive_id;
1297
1298         return 0;
1299 }
1300
1301 int psp_xgmi_get_node_id(struct psp_context *psp, uint64_t *node_id)
1302 {
1303         struct ta_xgmi_shared_memory *xgmi_cmd;
1304         int ret;
1305
1306         xgmi_cmd = (struct ta_xgmi_shared_memory *)psp->xgmi_context.context.mem_context.shared_buf;
1307         memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory));
1308
1309         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__GET_NODE_ID;
1310
1311         /* Invoke xgmi ta to get the node id */
1312         ret = psp_xgmi_invoke(psp, xgmi_cmd->cmd_id);
1313         if (ret)
1314                 return ret;
1315
1316         *node_id = xgmi_cmd->xgmi_out_message.get_node_id.node_id;
1317
1318         return 0;
1319 }
1320
1321 static bool psp_xgmi_peer_link_info_supported(struct psp_context *psp)
1322 {
1323         return (psp->adev->ip_versions[MP0_HWIP][0] == IP_VERSION(13, 0, 2) &&
1324                 psp->xgmi_context.context.bin_desc.fw_version >= 0x2000000b) ||
1325                 psp->adev->ip_versions[MP0_HWIP][0] >= IP_VERSION(13, 0, 6);
1326 }
1327
1328 /*
1329  * Chips that support extended topology information require the driver to
1330  * reflect topology information in the opposite direction.  This is
1331  * because the TA has already exceeded its link record limit and if the
1332  * TA holds bi-directional information, the driver would have to do
1333  * multiple fetches instead of just two.
1334  */
1335 static void psp_xgmi_reflect_topology_info(struct psp_context *psp,
1336                                         struct psp_xgmi_node_info node_info)
1337 {
1338         struct amdgpu_device *mirror_adev;
1339         struct amdgpu_hive_info *hive;
1340         uint64_t src_node_id = psp->adev->gmc.xgmi.node_id;
1341         uint64_t dst_node_id = node_info.node_id;
1342         uint8_t dst_num_hops = node_info.num_hops;
1343         uint8_t dst_num_links = node_info.num_links;
1344
1345         hive = amdgpu_get_xgmi_hive(psp->adev);
1346         list_for_each_entry(mirror_adev, &hive->device_list, gmc.xgmi.head) {
1347                 struct psp_xgmi_topology_info *mirror_top_info;
1348                 int j;
1349
1350                 if (mirror_adev->gmc.xgmi.node_id != dst_node_id)
1351                         continue;
1352
1353                 mirror_top_info = &mirror_adev->psp.xgmi_context.top_info;
1354                 for (j = 0; j < mirror_top_info->num_nodes; j++) {
1355                         if (mirror_top_info->nodes[j].node_id != src_node_id)
1356                                 continue;
1357
1358                         mirror_top_info->nodes[j].num_hops = dst_num_hops;
1359                         /*
1360                          * prevent 0 num_links value re-reflection since reflection
1361                          * criteria is based on num_hops (direct or indirect).
1362                          *
1363                          */
1364                         if (dst_num_links)
1365                                 mirror_top_info->nodes[j].num_links = dst_num_links;
1366
1367                         break;
1368                 }
1369
1370                 break;
1371         }
1372
1373         amdgpu_put_xgmi_hive(hive);
1374 }
1375
1376 int psp_xgmi_get_topology_info(struct psp_context *psp,
1377                                int number_devices,
1378                                struct psp_xgmi_topology_info *topology,
1379                                bool get_extended_data)
1380 {
1381         struct ta_xgmi_shared_memory *xgmi_cmd;
1382         struct ta_xgmi_cmd_get_topology_info_input *topology_info_input;
1383         struct ta_xgmi_cmd_get_topology_info_output *topology_info_output;
1384         int i;
1385         int ret;
1386
1387         if (!topology || topology->num_nodes > TA_XGMI__MAX_CONNECTED_NODES)
1388                 return -EINVAL;
1389
1390         xgmi_cmd = (struct ta_xgmi_shared_memory *)psp->xgmi_context.context.mem_context.shared_buf;
1391         memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory));
1392         xgmi_cmd->flag_extend_link_record = get_extended_data;
1393
1394         /* Fill in the shared memory with topology information as input */
1395         topology_info_input = &xgmi_cmd->xgmi_in_message.get_topology_info;
1396         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__GET_GET_TOPOLOGY_INFO;
1397         topology_info_input->num_nodes = number_devices;
1398
1399         for (i = 0; i < topology_info_input->num_nodes; i++) {
1400                 topology_info_input->nodes[i].node_id = topology->nodes[i].node_id;
1401                 topology_info_input->nodes[i].num_hops = topology->nodes[i].num_hops;
1402                 topology_info_input->nodes[i].is_sharing_enabled = topology->nodes[i].is_sharing_enabled;
1403                 topology_info_input->nodes[i].sdma_engine = topology->nodes[i].sdma_engine;
1404         }
1405
1406         /* Invoke xgmi ta to get the topology information */
1407         ret = psp_xgmi_invoke(psp, TA_COMMAND_XGMI__GET_GET_TOPOLOGY_INFO);
1408         if (ret)
1409                 return ret;
1410
1411         /* Read the output topology information from the shared memory */
1412         topology_info_output = &xgmi_cmd->xgmi_out_message.get_topology_info;
1413         topology->num_nodes = xgmi_cmd->xgmi_out_message.get_topology_info.num_nodes;
1414         for (i = 0; i < topology->num_nodes; i++) {
1415                 /* extended data will either be 0 or equal to non-extended data */
1416                 if (topology_info_output->nodes[i].num_hops)
1417                         topology->nodes[i].num_hops = topology_info_output->nodes[i].num_hops;
1418
1419                 /* non-extended data gets everything here so no need to update */
1420                 if (!get_extended_data) {
1421                         topology->nodes[i].node_id = topology_info_output->nodes[i].node_id;
1422                         topology->nodes[i].is_sharing_enabled =
1423                                         topology_info_output->nodes[i].is_sharing_enabled;
1424                         topology->nodes[i].sdma_engine =
1425                                         topology_info_output->nodes[i].sdma_engine;
1426                 }
1427
1428         }
1429
1430         /* Invoke xgmi ta again to get the link information */
1431         if (psp_xgmi_peer_link_info_supported(psp)) {
1432                 struct ta_xgmi_cmd_get_peer_link_info_output *link_info_output;
1433                 bool requires_reflection =
1434                         (psp->xgmi_context.supports_extended_data && get_extended_data) ||
1435                                 psp->adev->ip_versions[MP0_HWIP][0] == IP_VERSION(13, 0, 6);
1436
1437                 xgmi_cmd->cmd_id = TA_COMMAND_XGMI__GET_PEER_LINKS;
1438
1439                 ret = psp_xgmi_invoke(psp, TA_COMMAND_XGMI__GET_PEER_LINKS);
1440
1441                 if (ret)
1442                         return ret;
1443
1444                 link_info_output = &xgmi_cmd->xgmi_out_message.get_link_info;
1445                 for (i = 0; i < topology->num_nodes; i++) {
1446                         /* accumulate num_links on extended data */
1447                         topology->nodes[i].num_links = get_extended_data ?
1448                                         topology->nodes[i].num_links +
1449                                                         link_info_output->nodes[i].num_links :
1450                                         ((requires_reflection && topology->nodes[i].num_links) ? topology->nodes[i].num_links :
1451                                          link_info_output->nodes[i].num_links);
1452
1453                         /* reflect the topology information for bi-directionality */
1454                         if (requires_reflection && topology->nodes[i].num_hops)
1455                                 psp_xgmi_reflect_topology_info(psp, topology->nodes[i]);
1456                 }
1457         }
1458
1459         return 0;
1460 }
1461
1462 int psp_xgmi_set_topology_info(struct psp_context *psp,
1463                                int number_devices,
1464                                struct psp_xgmi_topology_info *topology)
1465 {
1466         struct ta_xgmi_shared_memory *xgmi_cmd;
1467         struct ta_xgmi_cmd_get_topology_info_input *topology_info_input;
1468         int i;
1469
1470         if (!topology || topology->num_nodes > TA_XGMI__MAX_CONNECTED_NODES)
1471                 return -EINVAL;
1472
1473         xgmi_cmd = (struct ta_xgmi_shared_memory *)psp->xgmi_context.context.mem_context.shared_buf;
1474         memset(xgmi_cmd, 0, sizeof(struct ta_xgmi_shared_memory));
1475
1476         topology_info_input = &xgmi_cmd->xgmi_in_message.get_topology_info;
1477         xgmi_cmd->cmd_id = TA_COMMAND_XGMI__SET_TOPOLOGY_INFO;
1478         topology_info_input->num_nodes = number_devices;
1479
1480         for (i = 0; i < topology_info_input->num_nodes; i++) {
1481                 topology_info_input->nodes[i].node_id = topology->nodes[i].node_id;
1482                 topology_info_input->nodes[i].num_hops = topology->nodes[i].num_hops;
1483                 topology_info_input->nodes[i].is_sharing_enabled = 1;
1484                 topology_info_input->nodes[i].sdma_engine = topology->nodes[i].sdma_engine;
1485         }
1486
1487         /* Invoke xgmi ta to set topology information */
1488         return psp_xgmi_invoke(psp, TA_COMMAND_XGMI__SET_TOPOLOGY_INFO);
1489 }
1490
1491 // ras begin
1492 static void psp_ras_ta_check_status(struct psp_context *psp)
1493 {
1494         struct ta_ras_shared_memory *ras_cmd =
1495                 (struct ta_ras_shared_memory *)psp->ras_context.context.mem_context.shared_buf;
1496
1497         switch (ras_cmd->ras_status) {
1498         case TA_RAS_STATUS__ERROR_UNSUPPORTED_IP:
1499                 dev_warn(psp->adev->dev,
1500                                 "RAS WARNING: cmd failed due to unsupported ip\n");
1501                 break;
1502         case TA_RAS_STATUS__ERROR_UNSUPPORTED_ERROR_INJ:
1503                 dev_warn(psp->adev->dev,
1504                                 "RAS WARNING: cmd failed due to unsupported error injection\n");
1505                 break;
1506         case TA_RAS_STATUS__SUCCESS:
1507                 break;
1508         case TA_RAS_STATUS__TEE_ERROR_ACCESS_DENIED:
1509                 if (ras_cmd->cmd_id == TA_RAS_COMMAND__TRIGGER_ERROR)
1510                         dev_warn(psp->adev->dev,
1511                                         "RAS WARNING: Inject error to critical region is not allowed\n");
1512                 break;
1513         default:
1514                 dev_warn(psp->adev->dev,
1515                                 "RAS WARNING: ras status = 0x%X\n", ras_cmd->ras_status);
1516                 break;
1517         }
1518 }
1519
1520 int psp_ras_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
1521 {
1522         struct ta_ras_shared_memory *ras_cmd;
1523         int ret;
1524
1525         ras_cmd = (struct ta_ras_shared_memory *)psp->ras_context.context.mem_context.shared_buf;
1526
1527         /*
1528          * TODO: bypass the loading in sriov for now
1529          */
1530         if (amdgpu_sriov_vf(psp->adev))
1531                 return 0;
1532
1533         ret = psp_ta_invoke(psp, ta_cmd_id, &psp->ras_context.context);
1534
1535         if (amdgpu_ras_intr_triggered())
1536                 return ret;
1537
1538         if (ras_cmd->if_version > RAS_TA_HOST_IF_VER) {
1539                 DRM_WARN("RAS: Unsupported Interface");
1540                 return -EINVAL;
1541         }
1542
1543         if (!ret) {
1544                 if (ras_cmd->ras_out_message.flags.err_inject_switch_disable_flag) {
1545                         dev_warn(psp->adev->dev, "ECC switch disabled\n");
1546
1547                         ras_cmd->ras_status = TA_RAS_STATUS__ERROR_RAS_NOT_AVAILABLE;
1548                 } else if (ras_cmd->ras_out_message.flags.reg_access_failure_flag)
1549                         dev_warn(psp->adev->dev,
1550                                  "RAS internal register access blocked\n");
1551
1552                 psp_ras_ta_check_status(psp);
1553         }
1554
1555         return ret;
1556 }
1557
1558 int psp_ras_enable_features(struct psp_context *psp,
1559                 union ta_ras_cmd_input *info, bool enable)
1560 {
1561         struct ta_ras_shared_memory *ras_cmd;
1562         int ret;
1563
1564         if (!psp->ras_context.context.initialized)
1565                 return -EINVAL;
1566
1567         ras_cmd = (struct ta_ras_shared_memory *)psp->ras_context.context.mem_context.shared_buf;
1568         memset(ras_cmd, 0, sizeof(struct ta_ras_shared_memory));
1569
1570         if (enable)
1571                 ras_cmd->cmd_id = TA_RAS_COMMAND__ENABLE_FEATURES;
1572         else
1573                 ras_cmd->cmd_id = TA_RAS_COMMAND__DISABLE_FEATURES;
1574
1575         ras_cmd->ras_in_message = *info;
1576
1577         ret = psp_ras_invoke(psp, ras_cmd->cmd_id);
1578         if (ret)
1579                 return -EINVAL;
1580
1581         return 0;
1582 }
1583
1584 int psp_ras_terminate(struct psp_context *psp)
1585 {
1586         int ret;
1587
1588         /*
1589          * TODO: bypass the terminate in sriov for now
1590          */
1591         if (amdgpu_sriov_vf(psp->adev))
1592                 return 0;
1593
1594         if (!psp->ras_context.context.initialized)
1595                 return 0;
1596
1597         ret = psp_ta_unload(psp, &psp->ras_context.context);
1598
1599         psp->ras_context.context.initialized = false;
1600
1601         return ret;
1602 }
1603
1604 int psp_ras_initialize(struct psp_context *psp)
1605 {
1606         int ret;
1607         uint32_t boot_cfg = 0xFF;
1608         struct amdgpu_device *adev = psp->adev;
1609         struct ta_ras_shared_memory *ras_cmd;
1610
1611         /*
1612          * TODO: bypass the initialize in sriov for now
1613          */
1614         if (amdgpu_sriov_vf(adev))
1615                 return 0;
1616
1617         if (!adev->psp.ras_context.context.bin_desc.size_bytes ||
1618             !adev->psp.ras_context.context.bin_desc.start_addr) {
1619                 dev_info(adev->dev, "RAS: optional ras ta ucode is not available\n");
1620                 return 0;
1621         }
1622
1623         if (amdgpu_atomfirmware_dynamic_boot_config_supported(adev)) {
1624                 /* query GECC enablement status from boot config
1625                  * boot_cfg: 1: GECC is enabled or 0: GECC is disabled
1626                  */
1627                 ret = psp_boot_config_get(adev, &boot_cfg);
1628                 if (ret)
1629                         dev_warn(adev->dev, "PSP get boot config failed\n");
1630
1631                 if (!amdgpu_ras_is_supported(psp->adev, AMDGPU_RAS_BLOCK__UMC)) {
1632                         if (!boot_cfg) {
1633                                 dev_info(adev->dev, "GECC is disabled\n");
1634                         } else {
1635                                 /* disable GECC in next boot cycle if ras is
1636                                  * disabled by module parameter amdgpu_ras_enable
1637                                  * and/or amdgpu_ras_mask, or boot_config_get call
1638                                  * is failed
1639                                  */
1640                                 ret = psp_boot_config_set(adev, 0);
1641                                 if (ret)
1642                                         dev_warn(adev->dev, "PSP set boot config failed\n");
1643                                 else
1644                                         dev_warn(adev->dev, "GECC will be disabled in next boot cycle if set amdgpu_ras_enable and/or amdgpu_ras_mask to 0x0\n");
1645                         }
1646                 } else {
1647                         if (boot_cfg == 1) {
1648                                 dev_info(adev->dev, "GECC is enabled\n");
1649                         } else {
1650                                 /* enable GECC in next boot cycle if it is disabled
1651                                  * in boot config, or force enable GECC if failed to
1652                                  * get boot configuration
1653                                  */
1654                                 ret = psp_boot_config_set(adev, BOOT_CONFIG_GECC);
1655                                 if (ret)
1656                                         dev_warn(adev->dev, "PSP set boot config failed\n");
1657                                 else
1658                                         dev_warn(adev->dev, "GECC will be enabled in next boot cycle\n");
1659                         }
1660                 }
1661         }
1662
1663         psp->ras_context.context.mem_context.shared_mem_size = PSP_RAS_SHARED_MEM_SIZE;
1664         psp->ras_context.context.ta_load_type = GFX_CMD_ID_LOAD_TA;
1665
1666         if (!psp->ras_context.context.mem_context.shared_buf) {
1667                 ret = psp_ta_init_shared_buf(psp, &psp->ras_context.context.mem_context);
1668                 if (ret)
1669                         return ret;
1670         }
1671
1672         ras_cmd = (struct ta_ras_shared_memory *)psp->ras_context.context.mem_context.shared_buf;
1673         memset(ras_cmd, 0, sizeof(struct ta_ras_shared_memory));
1674
1675         if (amdgpu_ras_is_poison_mode_supported(adev))
1676                 ras_cmd->ras_in_message.init_flags.poison_mode_en = 1;
1677         if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu)
1678                 ras_cmd->ras_in_message.init_flags.dgpu_mode = 1;
1679         ras_cmd->ras_in_message.init_flags.xcc_mask =
1680                 adev->gfx.xcc_mask;
1681         ras_cmd->ras_in_message.init_flags.channel_dis_num = hweight32(adev->gmc.m_half_use) * 2;
1682
1683         ret = psp_ta_load(psp, &psp->ras_context.context);
1684
1685         if (!ret && !ras_cmd->ras_status)
1686                 psp->ras_context.context.initialized = true;
1687         else {
1688                 if (ras_cmd->ras_status)
1689                         dev_warn(psp->adev->dev, "RAS Init Status: 0x%X\n", ras_cmd->ras_status);
1690
1691                 /* fail to load RAS TA */
1692                 psp->ras_context.context.initialized = false;
1693         }
1694
1695         return ret;
1696 }
1697
1698 int psp_ras_trigger_error(struct psp_context *psp,
1699                           struct ta_ras_trigger_error_input *info, uint32_t instance_mask)
1700 {
1701         struct ta_ras_shared_memory *ras_cmd;
1702         struct amdgpu_device *adev = psp->adev;
1703         int ret;
1704         uint32_t dev_mask;
1705
1706         if (!psp->ras_context.context.initialized)
1707                 return -EINVAL;
1708
1709         switch (info->block_id) {
1710         case TA_RAS_BLOCK__GFX:
1711                 dev_mask = GET_MASK(GC, instance_mask);
1712                 break;
1713         case TA_RAS_BLOCK__SDMA:
1714                 dev_mask = GET_MASK(SDMA0, instance_mask);
1715                 break;
1716         case TA_RAS_BLOCK__VCN:
1717         case TA_RAS_BLOCK__JPEG:
1718                 dev_mask = GET_MASK(VCN, instance_mask);
1719                 break;
1720         default:
1721                 dev_mask = instance_mask;
1722                 break;
1723         }
1724
1725         /* reuse sub_block_index for backward compatibility */
1726         dev_mask <<= AMDGPU_RAS_INST_SHIFT;
1727         dev_mask &= AMDGPU_RAS_INST_MASK;
1728         info->sub_block_index |= dev_mask;
1729
1730         ras_cmd = (struct ta_ras_shared_memory *)psp->ras_context.context.mem_context.shared_buf;
1731         memset(ras_cmd, 0, sizeof(struct ta_ras_shared_memory));
1732
1733         ras_cmd->cmd_id = TA_RAS_COMMAND__TRIGGER_ERROR;
1734         ras_cmd->ras_in_message.trigger_error = *info;
1735
1736         ret = psp_ras_invoke(psp, ras_cmd->cmd_id);
1737         if (ret)
1738                 return -EINVAL;
1739
1740         /* If err_event_athub occurs error inject was successful, however
1741            return status from TA is no long reliable */
1742         if (amdgpu_ras_intr_triggered())
1743                 return 0;
1744
1745         if (ras_cmd->ras_status == TA_RAS_STATUS__TEE_ERROR_ACCESS_DENIED)
1746                 return -EACCES;
1747         else if (ras_cmd->ras_status)
1748                 return -EINVAL;
1749
1750         return 0;
1751 }
1752 // ras end
1753
1754 // HDCP start
1755 static int psp_hdcp_initialize(struct psp_context *psp)
1756 {
1757         int ret;
1758
1759         /*
1760          * TODO: bypass the initialize in sriov for now
1761          */
1762         if (amdgpu_sriov_vf(psp->adev))
1763                 return 0;
1764
1765         if (!psp->hdcp_context.context.bin_desc.size_bytes ||
1766             !psp->hdcp_context.context.bin_desc.start_addr) {
1767                 dev_info(psp->adev->dev, "HDCP: optional hdcp ta ucode is not available\n");
1768                 return 0;
1769         }
1770
1771         psp->hdcp_context.context.mem_context.shared_mem_size = PSP_HDCP_SHARED_MEM_SIZE;
1772         psp->hdcp_context.context.ta_load_type = GFX_CMD_ID_LOAD_TA;
1773
1774         if (!psp->hdcp_context.context.mem_context.shared_buf) {
1775                 ret = psp_ta_init_shared_buf(psp, &psp->hdcp_context.context.mem_context);
1776                 if (ret)
1777                         return ret;
1778         }
1779
1780         ret = psp_ta_load(psp, &psp->hdcp_context.context);
1781         if (!ret) {
1782                 psp->hdcp_context.context.initialized = true;
1783                 mutex_init(&psp->hdcp_context.mutex);
1784         }
1785
1786         return ret;
1787 }
1788
1789 int psp_hdcp_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
1790 {
1791         /*
1792          * TODO: bypass the loading in sriov for now
1793          */
1794         if (amdgpu_sriov_vf(psp->adev))
1795                 return 0;
1796
1797         return psp_ta_invoke(psp, ta_cmd_id, &psp->hdcp_context.context);
1798 }
1799
1800 static int psp_hdcp_terminate(struct psp_context *psp)
1801 {
1802         int ret;
1803
1804         /*
1805          * TODO: bypass the terminate in sriov for now
1806          */
1807         if (amdgpu_sriov_vf(psp->adev))
1808                 return 0;
1809
1810         if (!psp->hdcp_context.context.initialized)
1811                 return 0;
1812
1813         ret = psp_ta_unload(psp, &psp->hdcp_context.context);
1814
1815         psp->hdcp_context.context.initialized = false;
1816
1817         return ret;
1818 }
1819 // HDCP end
1820
1821 // DTM start
1822 static int psp_dtm_initialize(struct psp_context *psp)
1823 {
1824         int ret;
1825
1826         /*
1827          * TODO: bypass the initialize in sriov for now
1828          */
1829         if (amdgpu_sriov_vf(psp->adev))
1830                 return 0;
1831
1832         if (!psp->dtm_context.context.bin_desc.size_bytes ||
1833             !psp->dtm_context.context.bin_desc.start_addr) {
1834                 dev_info(psp->adev->dev, "DTM: optional dtm ta ucode is not available\n");
1835                 return 0;
1836         }
1837
1838         psp->dtm_context.context.mem_context.shared_mem_size = PSP_DTM_SHARED_MEM_SIZE;
1839         psp->dtm_context.context.ta_load_type = GFX_CMD_ID_LOAD_TA;
1840
1841         if (!psp->dtm_context.context.mem_context.shared_buf) {
1842                 ret = psp_ta_init_shared_buf(psp, &psp->dtm_context.context.mem_context);
1843                 if (ret)
1844                         return ret;
1845         }
1846
1847         ret = psp_ta_load(psp, &psp->dtm_context.context);
1848         if (!ret) {
1849                 psp->dtm_context.context.initialized = true;
1850                 mutex_init(&psp->dtm_context.mutex);
1851         }
1852
1853         return ret;
1854 }
1855
1856 int psp_dtm_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
1857 {
1858         /*
1859          * TODO: bypass the loading in sriov for now
1860          */
1861         if (amdgpu_sriov_vf(psp->adev))
1862                 return 0;
1863
1864         return psp_ta_invoke(psp, ta_cmd_id, &psp->dtm_context.context);
1865 }
1866
1867 static int psp_dtm_terminate(struct psp_context *psp)
1868 {
1869         int ret;
1870
1871         /*
1872          * TODO: bypass the terminate in sriov for now
1873          */
1874         if (amdgpu_sriov_vf(psp->adev))
1875                 return 0;
1876
1877         if (!psp->dtm_context.context.initialized)
1878                 return 0;
1879
1880         ret = psp_ta_unload(psp, &psp->dtm_context.context);
1881
1882         psp->dtm_context.context.initialized = false;
1883
1884         return ret;
1885 }
1886 // DTM end
1887
1888 // RAP start
1889 static int psp_rap_initialize(struct psp_context *psp)
1890 {
1891         int ret;
1892         enum ta_rap_status status = TA_RAP_STATUS__SUCCESS;
1893
1894         /*
1895          * TODO: bypass the initialize in sriov for now
1896          */
1897         if (amdgpu_sriov_vf(psp->adev))
1898                 return 0;
1899
1900         if (!psp->rap_context.context.bin_desc.size_bytes ||
1901             !psp->rap_context.context.bin_desc.start_addr) {
1902                 dev_info(psp->adev->dev, "RAP: optional rap ta ucode is not available\n");
1903                 return 0;
1904         }
1905
1906         psp->rap_context.context.mem_context.shared_mem_size = PSP_RAP_SHARED_MEM_SIZE;
1907         psp->rap_context.context.ta_load_type = GFX_CMD_ID_LOAD_TA;
1908
1909         if (!psp->rap_context.context.mem_context.shared_buf) {
1910                 ret = psp_ta_init_shared_buf(psp, &psp->rap_context.context.mem_context);
1911                 if (ret)
1912                         return ret;
1913         }
1914
1915         ret = psp_ta_load(psp, &psp->rap_context.context);
1916         if (!ret) {
1917                 psp->rap_context.context.initialized = true;
1918                 mutex_init(&psp->rap_context.mutex);
1919         } else
1920                 return ret;
1921
1922         ret = psp_rap_invoke(psp, TA_CMD_RAP__INITIALIZE, &status);
1923         if (ret || status != TA_RAP_STATUS__SUCCESS) {
1924                 psp_rap_terminate(psp);
1925                 /* free rap shared memory */
1926                 psp_ta_free_shared_buf(&psp->rap_context.context.mem_context);
1927
1928                 dev_warn(psp->adev->dev, "RAP TA initialize fail (%d) status %d.\n",
1929                          ret, status);
1930
1931                 return ret;
1932         }
1933
1934         return 0;
1935 }
1936
1937 static int psp_rap_terminate(struct psp_context *psp)
1938 {
1939         int ret;
1940
1941         if (!psp->rap_context.context.initialized)
1942                 return 0;
1943
1944         ret = psp_ta_unload(psp, &psp->rap_context.context);
1945
1946         psp->rap_context.context.initialized = false;
1947
1948         return ret;
1949 }
1950
1951 int psp_rap_invoke(struct psp_context *psp, uint32_t ta_cmd_id, enum ta_rap_status *status)
1952 {
1953         struct ta_rap_shared_memory *rap_cmd;
1954         int ret = 0;
1955
1956         if (!psp->rap_context.context.initialized)
1957                 return 0;
1958
1959         if (ta_cmd_id != TA_CMD_RAP__INITIALIZE &&
1960             ta_cmd_id != TA_CMD_RAP__VALIDATE_L0)
1961                 return -EINVAL;
1962
1963         mutex_lock(&psp->rap_context.mutex);
1964
1965         rap_cmd = (struct ta_rap_shared_memory *)
1966                   psp->rap_context.context.mem_context.shared_buf;
1967         memset(rap_cmd, 0, sizeof(struct ta_rap_shared_memory));
1968
1969         rap_cmd->cmd_id = ta_cmd_id;
1970         rap_cmd->validation_method_id = METHOD_A;
1971
1972         ret = psp_ta_invoke(psp, rap_cmd->cmd_id, &psp->rap_context.context);
1973         if (ret)
1974                 goto out_unlock;
1975
1976         if (status)
1977                 *status = rap_cmd->rap_status;
1978
1979 out_unlock:
1980         mutex_unlock(&psp->rap_context.mutex);
1981
1982         return ret;
1983 }
1984 // RAP end
1985
1986 /* securedisplay start */
1987 static int psp_securedisplay_initialize(struct psp_context *psp)
1988 {
1989         int ret;
1990         struct ta_securedisplay_cmd *securedisplay_cmd;
1991
1992         /*
1993          * TODO: bypass the initialize in sriov for now
1994          */
1995         if (amdgpu_sriov_vf(psp->adev))
1996                 return 0;
1997
1998         if (!psp->securedisplay_context.context.bin_desc.size_bytes ||
1999             !psp->securedisplay_context.context.bin_desc.start_addr) {
2000                 dev_info(psp->adev->dev, "SECUREDISPLAY: securedisplay ta ucode is not available\n");
2001                 return 0;
2002         }
2003
2004         psp->securedisplay_context.context.mem_context.shared_mem_size =
2005                 PSP_SECUREDISPLAY_SHARED_MEM_SIZE;
2006         psp->securedisplay_context.context.ta_load_type = GFX_CMD_ID_LOAD_TA;
2007
2008         if (!psp->securedisplay_context.context.initialized) {
2009                 ret = psp_ta_init_shared_buf(psp,
2010                                              &psp->securedisplay_context.context.mem_context);
2011                 if (ret)
2012                         return ret;
2013         }
2014
2015         ret = psp_ta_load(psp, &psp->securedisplay_context.context);
2016         if (!ret) {
2017                 psp->securedisplay_context.context.initialized = true;
2018                 mutex_init(&psp->securedisplay_context.mutex);
2019         } else
2020                 return ret;
2021
2022         mutex_lock(&psp->securedisplay_context.mutex);
2023
2024         psp_prep_securedisplay_cmd_buf(psp, &securedisplay_cmd,
2025                         TA_SECUREDISPLAY_COMMAND__QUERY_TA);
2026
2027         ret = psp_securedisplay_invoke(psp, TA_SECUREDISPLAY_COMMAND__QUERY_TA);
2028
2029         mutex_unlock(&psp->securedisplay_context.mutex);
2030
2031         if (ret) {
2032                 psp_securedisplay_terminate(psp);
2033                 /* free securedisplay shared memory */
2034                 psp_ta_free_shared_buf(&psp->securedisplay_context.context.mem_context);
2035                 dev_err(psp->adev->dev, "SECUREDISPLAY TA initialize fail.\n");
2036                 return -EINVAL;
2037         }
2038
2039         if (securedisplay_cmd->status != TA_SECUREDISPLAY_STATUS__SUCCESS) {
2040                 psp_securedisplay_parse_resp_status(psp, securedisplay_cmd->status);
2041                 dev_err(psp->adev->dev, "SECUREDISPLAY: query securedisplay TA failed. ret 0x%x\n",
2042                         securedisplay_cmd->securedisplay_out_message.query_ta.query_cmd_ret);
2043                 /* don't try again */
2044                 psp->securedisplay_context.context.bin_desc.size_bytes = 0;
2045         }
2046
2047         return 0;
2048 }
2049
2050 static int psp_securedisplay_terminate(struct psp_context *psp)
2051 {
2052         int ret;
2053
2054         /*
2055          * TODO:bypass the terminate in sriov for now
2056          */
2057         if (amdgpu_sriov_vf(psp->adev))
2058                 return 0;
2059
2060         if (!psp->securedisplay_context.context.initialized)
2061                 return 0;
2062
2063         ret = psp_ta_unload(psp, &psp->securedisplay_context.context);
2064
2065         psp->securedisplay_context.context.initialized = false;
2066
2067         return ret;
2068 }
2069
2070 int psp_securedisplay_invoke(struct psp_context *psp, uint32_t ta_cmd_id)
2071 {
2072         int ret;
2073
2074         if (!psp->securedisplay_context.context.initialized)
2075                 return -EINVAL;
2076
2077         if (ta_cmd_id != TA_SECUREDISPLAY_COMMAND__QUERY_TA &&
2078             ta_cmd_id != TA_SECUREDISPLAY_COMMAND__SEND_ROI_CRC)
2079                 return -EINVAL;
2080
2081         ret = psp_ta_invoke(psp, ta_cmd_id, &psp->securedisplay_context.context);
2082
2083         return ret;
2084 }
2085 /* SECUREDISPLAY end */
2086
2087 static int psp_hw_start(struct psp_context *psp)
2088 {
2089         struct amdgpu_device *adev = psp->adev;
2090         int ret;
2091
2092         if (!amdgpu_sriov_vf(adev)) {
2093                 if ((is_psp_fw_valid(psp->kdb)) &&
2094                     (psp->funcs->bootloader_load_kdb != NULL)) {
2095                         ret = psp_bootloader_load_kdb(psp);
2096                         if (ret) {
2097                                 DRM_ERROR("PSP load kdb failed!\n");
2098                                 return ret;
2099                         }
2100                 }
2101
2102                 if ((is_psp_fw_valid(psp->spl)) &&
2103                     (psp->funcs->bootloader_load_spl != NULL)) {
2104                         ret = psp_bootloader_load_spl(psp);
2105                         if (ret) {
2106                                 DRM_ERROR("PSP load spl failed!\n");
2107                                 return ret;
2108                         }
2109                 }
2110
2111                 if ((is_psp_fw_valid(psp->sys)) &&
2112                     (psp->funcs->bootloader_load_sysdrv != NULL)) {
2113                         ret = psp_bootloader_load_sysdrv(psp);
2114                         if (ret) {
2115                                 DRM_ERROR("PSP load sys drv failed!\n");
2116                                 return ret;
2117                         }
2118                 }
2119
2120                 if ((is_psp_fw_valid(psp->soc_drv)) &&
2121                     (psp->funcs->bootloader_load_soc_drv != NULL)) {
2122                         ret = psp_bootloader_load_soc_drv(psp);
2123                         if (ret) {
2124                                 DRM_ERROR("PSP load soc drv failed!\n");
2125                                 return ret;
2126                         }
2127                 }
2128
2129                 if ((is_psp_fw_valid(psp->intf_drv)) &&
2130                     (psp->funcs->bootloader_load_intf_drv != NULL)) {
2131                         ret = psp_bootloader_load_intf_drv(psp);
2132                         if (ret) {
2133                                 DRM_ERROR("PSP load intf drv failed!\n");
2134                                 return ret;
2135                         }
2136                 }
2137
2138                 if ((is_psp_fw_valid(psp->dbg_drv)) &&
2139                     (psp->funcs->bootloader_load_dbg_drv != NULL)) {
2140                         ret = psp_bootloader_load_dbg_drv(psp);
2141                         if (ret) {
2142                                 DRM_ERROR("PSP load dbg drv failed!\n");
2143                                 return ret;
2144                         }
2145                 }
2146
2147                 if ((is_psp_fw_valid(psp->ras_drv)) &&
2148                     (psp->funcs->bootloader_load_ras_drv != NULL)) {
2149                         ret = psp_bootloader_load_ras_drv(psp);
2150                         if (ret) {
2151                                 DRM_ERROR("PSP load ras_drv failed!\n");
2152                                 return ret;
2153                         }
2154                 }
2155
2156                 if ((is_psp_fw_valid(psp->sos)) &&
2157                     (psp->funcs->bootloader_load_sos != NULL)) {
2158                         ret = psp_bootloader_load_sos(psp);
2159                         if (ret) {
2160                                 DRM_ERROR("PSP load sos failed!\n");
2161                                 return ret;
2162                         }
2163                 }
2164         }
2165
2166         ret = psp_ring_create(psp, PSP_RING_TYPE__KM);
2167         if (ret) {
2168                 DRM_ERROR("PSP create ring failed!\n");
2169                 return ret;
2170         }
2171
2172         if (amdgpu_sriov_vf(adev) && amdgpu_in_reset(adev))
2173                 goto skip_pin_bo;
2174
2175         if (!psp_boottime_tmr(psp)) {
2176                 ret = psp_tmr_init(psp);
2177                 if (ret) {
2178                         DRM_ERROR("PSP tmr init failed!\n");
2179                         return ret;
2180                 }
2181         }
2182
2183 skip_pin_bo:
2184         /*
2185          * For ASICs with DF Cstate management centralized
2186          * to PMFW, TMR setup should be performed after PMFW
2187          * loaded and before other non-psp firmware loaded.
2188          */
2189         if (psp->pmfw_centralized_cstate_management) {
2190                 ret = psp_load_smu_fw(psp);
2191                 if (ret)
2192                         return ret;
2193         }
2194
2195         ret = psp_tmr_load(psp);
2196         if (ret) {
2197                 DRM_ERROR("PSP load tmr failed!\n");
2198                 return ret;
2199         }
2200
2201         return 0;
2202 }
2203
2204 static int psp_get_fw_type(struct amdgpu_firmware_info *ucode,
2205                            enum psp_gfx_fw_type *type)
2206 {
2207         switch (ucode->ucode_id) {
2208         case AMDGPU_UCODE_ID_CAP:
2209                 *type = GFX_FW_TYPE_CAP;
2210                 break;
2211         case AMDGPU_UCODE_ID_SDMA0:
2212                 *type = GFX_FW_TYPE_SDMA0;
2213                 break;
2214         case AMDGPU_UCODE_ID_SDMA1:
2215                 *type = GFX_FW_TYPE_SDMA1;
2216                 break;
2217         case AMDGPU_UCODE_ID_SDMA2:
2218                 *type = GFX_FW_TYPE_SDMA2;
2219                 break;
2220         case AMDGPU_UCODE_ID_SDMA3:
2221                 *type = GFX_FW_TYPE_SDMA3;
2222                 break;
2223         case AMDGPU_UCODE_ID_SDMA4:
2224                 *type = GFX_FW_TYPE_SDMA4;
2225                 break;
2226         case AMDGPU_UCODE_ID_SDMA5:
2227                 *type = GFX_FW_TYPE_SDMA5;
2228                 break;
2229         case AMDGPU_UCODE_ID_SDMA6:
2230                 *type = GFX_FW_TYPE_SDMA6;
2231                 break;
2232         case AMDGPU_UCODE_ID_SDMA7:
2233                 *type = GFX_FW_TYPE_SDMA7;
2234                 break;
2235         case AMDGPU_UCODE_ID_CP_MES:
2236                 *type = GFX_FW_TYPE_CP_MES;
2237                 break;
2238         case AMDGPU_UCODE_ID_CP_MES_DATA:
2239                 *type = GFX_FW_TYPE_MES_STACK;
2240                 break;
2241         case AMDGPU_UCODE_ID_CP_MES1:
2242                 *type = GFX_FW_TYPE_CP_MES_KIQ;
2243                 break;
2244         case AMDGPU_UCODE_ID_CP_MES1_DATA:
2245                 *type = GFX_FW_TYPE_MES_KIQ_STACK;
2246                 break;
2247         case AMDGPU_UCODE_ID_CP_CE:
2248                 *type = GFX_FW_TYPE_CP_CE;
2249                 break;
2250         case AMDGPU_UCODE_ID_CP_PFP:
2251                 *type = GFX_FW_TYPE_CP_PFP;
2252                 break;
2253         case AMDGPU_UCODE_ID_CP_ME:
2254                 *type = GFX_FW_TYPE_CP_ME;
2255                 break;
2256         case AMDGPU_UCODE_ID_CP_MEC1:
2257                 *type = GFX_FW_TYPE_CP_MEC;
2258                 break;
2259         case AMDGPU_UCODE_ID_CP_MEC1_JT:
2260                 *type = GFX_FW_TYPE_CP_MEC_ME1;
2261                 break;
2262         case AMDGPU_UCODE_ID_CP_MEC2:
2263                 *type = GFX_FW_TYPE_CP_MEC;
2264                 break;
2265         case AMDGPU_UCODE_ID_CP_MEC2_JT:
2266                 *type = GFX_FW_TYPE_CP_MEC_ME2;
2267                 break;
2268         case AMDGPU_UCODE_ID_RLC_P:
2269                 *type = GFX_FW_TYPE_RLC_P;
2270                 break;
2271         case AMDGPU_UCODE_ID_RLC_V:
2272                 *type = GFX_FW_TYPE_RLC_V;
2273                 break;
2274         case AMDGPU_UCODE_ID_RLC_G:
2275                 *type = GFX_FW_TYPE_RLC_G;
2276                 break;
2277         case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_CNTL:
2278                 *type = GFX_FW_TYPE_RLC_RESTORE_LIST_SRM_CNTL;
2279                 break;
2280         case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_GPM_MEM:
2281                 *type = GFX_FW_TYPE_RLC_RESTORE_LIST_GPM_MEM;
2282                 break;
2283         case AMDGPU_UCODE_ID_RLC_RESTORE_LIST_SRM_MEM:
2284                 *type = GFX_FW_TYPE_RLC_RESTORE_LIST_SRM_MEM;
2285                 break;
2286         case AMDGPU_UCODE_ID_RLC_IRAM:
2287                 *type = GFX_FW_TYPE_RLC_IRAM;
2288                 break;
2289         case AMDGPU_UCODE_ID_RLC_DRAM:
2290                 *type = GFX_FW_TYPE_RLC_DRAM_BOOT;
2291                 break;
2292         case AMDGPU_UCODE_ID_GLOBAL_TAP_DELAYS:
2293                 *type = GFX_FW_TYPE_GLOBAL_TAP_DELAYS;
2294                 break;
2295         case AMDGPU_UCODE_ID_SE0_TAP_DELAYS:
2296                 *type = GFX_FW_TYPE_SE0_TAP_DELAYS;
2297                 break;
2298         case AMDGPU_UCODE_ID_SE1_TAP_DELAYS:
2299                 *type = GFX_FW_TYPE_SE1_TAP_DELAYS;
2300                 break;
2301         case AMDGPU_UCODE_ID_SE2_TAP_DELAYS:
2302                 *type = GFX_FW_TYPE_SE2_TAP_DELAYS;
2303                 break;
2304         case AMDGPU_UCODE_ID_SE3_TAP_DELAYS:
2305                 *type = GFX_FW_TYPE_SE3_TAP_DELAYS;
2306                 break;
2307         case AMDGPU_UCODE_ID_SMC:
2308                 *type = GFX_FW_TYPE_SMU;
2309                 break;
2310         case AMDGPU_UCODE_ID_PPTABLE:
2311                 *type = GFX_FW_TYPE_PPTABLE;
2312                 break;
2313         case AMDGPU_UCODE_ID_UVD:
2314                 *type = GFX_FW_TYPE_UVD;
2315                 break;
2316         case AMDGPU_UCODE_ID_UVD1:
2317                 *type = GFX_FW_TYPE_UVD1;
2318                 break;
2319         case AMDGPU_UCODE_ID_VCE:
2320                 *type = GFX_FW_TYPE_VCE;
2321                 break;
2322         case AMDGPU_UCODE_ID_VCN:
2323                 *type = GFX_FW_TYPE_VCN;
2324                 break;
2325         case AMDGPU_UCODE_ID_VCN1:
2326                 *type = GFX_FW_TYPE_VCN1;
2327                 break;
2328         case AMDGPU_UCODE_ID_DMCU_ERAM:
2329                 *type = GFX_FW_TYPE_DMCU_ERAM;
2330                 break;
2331         case AMDGPU_UCODE_ID_DMCU_INTV:
2332                 *type = GFX_FW_TYPE_DMCU_ISR;
2333                 break;
2334         case AMDGPU_UCODE_ID_VCN0_RAM:
2335                 *type = GFX_FW_TYPE_VCN0_RAM;
2336                 break;
2337         case AMDGPU_UCODE_ID_VCN1_RAM:
2338                 *type = GFX_FW_TYPE_VCN1_RAM;
2339                 break;
2340         case AMDGPU_UCODE_ID_DMCUB:
2341                 *type = GFX_FW_TYPE_DMUB;
2342                 break;
2343         case AMDGPU_UCODE_ID_SDMA_UCODE_TH0:
2344                 *type = GFX_FW_TYPE_SDMA_UCODE_TH0;
2345                 break;
2346         case AMDGPU_UCODE_ID_SDMA_UCODE_TH1:
2347                 *type = GFX_FW_TYPE_SDMA_UCODE_TH1;
2348                 break;
2349         case AMDGPU_UCODE_ID_IMU_I:
2350                 *type = GFX_FW_TYPE_IMU_I;
2351                 break;
2352         case AMDGPU_UCODE_ID_IMU_D:
2353                 *type = GFX_FW_TYPE_IMU_D;
2354                 break;
2355         case AMDGPU_UCODE_ID_CP_RS64_PFP:
2356                 *type = GFX_FW_TYPE_RS64_PFP;
2357                 break;
2358         case AMDGPU_UCODE_ID_CP_RS64_ME:
2359                 *type = GFX_FW_TYPE_RS64_ME;
2360                 break;
2361         case AMDGPU_UCODE_ID_CP_RS64_MEC:
2362                 *type = GFX_FW_TYPE_RS64_MEC;
2363                 break;
2364         case AMDGPU_UCODE_ID_CP_RS64_PFP_P0_STACK:
2365                 *type = GFX_FW_TYPE_RS64_PFP_P0_STACK;
2366                 break;
2367         case AMDGPU_UCODE_ID_CP_RS64_PFP_P1_STACK:
2368                 *type = GFX_FW_TYPE_RS64_PFP_P1_STACK;
2369                 break;
2370         case AMDGPU_UCODE_ID_CP_RS64_ME_P0_STACK:
2371                 *type = GFX_FW_TYPE_RS64_ME_P0_STACK;
2372                 break;
2373         case AMDGPU_UCODE_ID_CP_RS64_ME_P1_STACK:
2374                 *type = GFX_FW_TYPE_RS64_ME_P1_STACK;
2375                 break;
2376         case AMDGPU_UCODE_ID_CP_RS64_MEC_P0_STACK:
2377                 *type = GFX_FW_TYPE_RS64_MEC_P0_STACK;
2378                 break;
2379         case AMDGPU_UCODE_ID_CP_RS64_MEC_P1_STACK:
2380                 *type = GFX_FW_TYPE_RS64_MEC_P1_STACK;
2381                 break;
2382         case AMDGPU_UCODE_ID_CP_RS64_MEC_P2_STACK:
2383                 *type = GFX_FW_TYPE_RS64_MEC_P2_STACK;
2384                 break;
2385         case AMDGPU_UCODE_ID_CP_RS64_MEC_P3_STACK:
2386                 *type = GFX_FW_TYPE_RS64_MEC_P3_STACK;
2387                 break;
2388         case AMDGPU_UCODE_ID_MAXIMUM:
2389         default:
2390                 return -EINVAL;
2391         }
2392
2393         return 0;
2394 }
2395
2396 static void psp_print_fw_hdr(struct psp_context *psp,
2397                              struct amdgpu_firmware_info *ucode)
2398 {
2399         struct amdgpu_device *adev = psp->adev;
2400         struct common_firmware_header *hdr;
2401
2402         switch (ucode->ucode_id) {
2403         case AMDGPU_UCODE_ID_SDMA0:
2404         case AMDGPU_UCODE_ID_SDMA1:
2405         case AMDGPU_UCODE_ID_SDMA2:
2406         case AMDGPU_UCODE_ID_SDMA3:
2407         case AMDGPU_UCODE_ID_SDMA4:
2408         case AMDGPU_UCODE_ID_SDMA5:
2409         case AMDGPU_UCODE_ID_SDMA6:
2410         case AMDGPU_UCODE_ID_SDMA7:
2411                 hdr = (struct common_firmware_header *)
2412                         adev->sdma.instance[ucode->ucode_id - AMDGPU_UCODE_ID_SDMA0].fw->data;
2413                 amdgpu_ucode_print_sdma_hdr(hdr);
2414                 break;
2415         case AMDGPU_UCODE_ID_CP_CE:
2416                 hdr = (struct common_firmware_header *)adev->gfx.ce_fw->data;
2417                 amdgpu_ucode_print_gfx_hdr(hdr);
2418                 break;
2419         case AMDGPU_UCODE_ID_CP_PFP:
2420                 hdr = (struct common_firmware_header *)adev->gfx.pfp_fw->data;
2421                 amdgpu_ucode_print_gfx_hdr(hdr);
2422                 break;
2423         case AMDGPU_UCODE_ID_CP_ME:
2424                 hdr = (struct common_firmware_header *)adev->gfx.me_fw->data;
2425                 amdgpu_ucode_print_gfx_hdr(hdr);
2426                 break;
2427         case AMDGPU_UCODE_ID_CP_MEC1:
2428                 hdr = (struct common_firmware_header *)adev->gfx.mec_fw->data;
2429                 amdgpu_ucode_print_gfx_hdr(hdr);
2430                 break;
2431         case AMDGPU_UCODE_ID_RLC_G:
2432                 hdr = (struct common_firmware_header *)adev->gfx.rlc_fw->data;
2433                 amdgpu_ucode_print_rlc_hdr(hdr);
2434                 break;
2435         case AMDGPU_UCODE_ID_SMC:
2436                 hdr = (struct common_firmware_header *)adev->pm.fw->data;
2437                 amdgpu_ucode_print_smc_hdr(hdr);
2438                 break;
2439         default:
2440                 break;
2441         }
2442 }
2443
2444 static int psp_prep_load_ip_fw_cmd_buf(struct amdgpu_firmware_info *ucode,
2445                                        struct psp_gfx_cmd_resp *cmd)
2446 {
2447         int ret;
2448         uint64_t fw_mem_mc_addr = ucode->mc_addr;
2449
2450         cmd->cmd_id = GFX_CMD_ID_LOAD_IP_FW;
2451         cmd->cmd.cmd_load_ip_fw.fw_phy_addr_lo = lower_32_bits(fw_mem_mc_addr);
2452         cmd->cmd.cmd_load_ip_fw.fw_phy_addr_hi = upper_32_bits(fw_mem_mc_addr);
2453         cmd->cmd.cmd_load_ip_fw.fw_size = ucode->ucode_size;
2454
2455         ret = psp_get_fw_type(ucode, &cmd->cmd.cmd_load_ip_fw.fw_type);
2456         if (ret)
2457                 DRM_ERROR("Unknown firmware type\n");
2458
2459         return ret;
2460 }
2461
2462 static int psp_execute_non_psp_fw_load(struct psp_context *psp,
2463                                   struct amdgpu_firmware_info *ucode)
2464 {
2465         int ret = 0;
2466         struct psp_gfx_cmd_resp *cmd = acquire_psp_cmd_buf(psp);
2467
2468         ret = psp_prep_load_ip_fw_cmd_buf(ucode, cmd);
2469         if (!ret) {
2470                 ret = psp_cmd_submit_buf(psp, ucode, cmd,
2471                                          psp->fence_buf_mc_addr);
2472         }
2473
2474         release_psp_cmd_buf(psp);
2475
2476         return ret;
2477 }
2478
2479 static int psp_load_smu_fw(struct psp_context *psp)
2480 {
2481         int ret;
2482         struct amdgpu_device *adev = psp->adev;
2483         struct amdgpu_firmware_info *ucode =
2484                         &adev->firmware.ucode[AMDGPU_UCODE_ID_SMC];
2485         struct amdgpu_ras *ras = psp->ras_context.ras;
2486
2487         /*
2488          * Skip SMU FW reloading in case of using BACO for runpm only,
2489          * as SMU is always alive.
2490          */
2491         if (adev->in_runpm && (adev->pm.rpm_mode == AMDGPU_RUNPM_BACO))
2492                 return 0;
2493
2494         if (!ucode->fw || amdgpu_sriov_vf(psp->adev))
2495                 return 0;
2496
2497         if ((amdgpu_in_reset(adev) &&
2498              ras && adev->ras_enabled &&
2499              (adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 4) ||
2500               adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 2)))) {
2501                 ret = amdgpu_dpm_set_mp1_state(adev, PP_MP1_STATE_UNLOAD);
2502                 if (ret)
2503                         DRM_WARN("Failed to set MP1 state prepare for reload\n");
2504         }
2505
2506         ret = psp_execute_non_psp_fw_load(psp, ucode);
2507
2508         if (ret)
2509                 DRM_ERROR("PSP load smu failed!\n");
2510
2511         return ret;
2512 }
2513
2514 static bool fw_load_skip_check(struct psp_context *psp,
2515                                struct amdgpu_firmware_info *ucode)
2516 {
2517         if (!ucode->fw || !ucode->ucode_size)
2518                 return true;
2519
2520         if (ucode->ucode_id == AMDGPU_UCODE_ID_SMC &&
2521             (psp_smu_reload_quirk(psp) ||
2522              psp->autoload_supported ||
2523              psp->pmfw_centralized_cstate_management))
2524                 return true;
2525
2526         if (amdgpu_sriov_vf(psp->adev) &&
2527             amdgpu_virt_fw_load_skip_check(psp->adev, ucode->ucode_id))
2528                 return true;
2529
2530         if (psp->autoload_supported &&
2531             (ucode->ucode_id == AMDGPU_UCODE_ID_CP_MEC1_JT ||
2532              ucode->ucode_id == AMDGPU_UCODE_ID_CP_MEC2_JT))
2533                 /* skip mec JT when autoload is enabled */
2534                 return true;
2535
2536         return false;
2537 }
2538
2539 int psp_load_fw_list(struct psp_context *psp,
2540                      struct amdgpu_firmware_info **ucode_list, int ucode_count)
2541 {
2542         int ret = 0, i;
2543         struct amdgpu_firmware_info *ucode;
2544
2545         for (i = 0; i < ucode_count; ++i) {
2546                 ucode = ucode_list[i];
2547                 psp_print_fw_hdr(psp, ucode);
2548                 ret = psp_execute_non_psp_fw_load(psp, ucode);
2549                 if (ret)
2550                         return ret;
2551         }
2552         return ret;
2553 }
2554
2555 static int psp_load_non_psp_fw(struct psp_context *psp)
2556 {
2557         int i, ret;
2558         struct amdgpu_firmware_info *ucode;
2559         struct amdgpu_device *adev = psp->adev;
2560
2561         if (psp->autoload_supported &&
2562             !psp->pmfw_centralized_cstate_management) {
2563                 ret = psp_load_smu_fw(psp);
2564                 if (ret)
2565                         return ret;
2566         }
2567
2568         for (i = 0; i < adev->firmware.max_ucodes; i++) {
2569                 ucode = &adev->firmware.ucode[i];
2570
2571                 if (ucode->ucode_id == AMDGPU_UCODE_ID_SMC &&
2572                     !fw_load_skip_check(psp, ucode)) {
2573                         ret = psp_load_smu_fw(psp);
2574                         if (ret)
2575                                 return ret;
2576                         continue;
2577                 }
2578
2579                 if (fw_load_skip_check(psp, ucode))
2580                         continue;
2581
2582                 if (psp->autoload_supported &&
2583                     (adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 7) ||
2584                      adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 11) ||
2585                      adev->ip_versions[MP0_HWIP][0] == IP_VERSION(11, 0, 12)) &&
2586                     (ucode->ucode_id == AMDGPU_UCODE_ID_SDMA1 ||
2587                      ucode->ucode_id == AMDGPU_UCODE_ID_SDMA2 ||
2588                      ucode->ucode_id == AMDGPU_UCODE_ID_SDMA3))
2589                         /* PSP only receive one SDMA fw for sienna_cichlid,
2590                          * as all four sdma fw are same */
2591                         continue;
2592
2593                 psp_print_fw_hdr(psp, ucode);
2594
2595                 ret = psp_execute_non_psp_fw_load(psp, ucode);
2596                 if (ret)
2597                         return ret;
2598
2599                 /* Start rlc autoload after psp recieved all the gfx firmware */
2600                 if (psp->autoload_supported && ucode->ucode_id == (amdgpu_sriov_vf(adev) ?
2601                     adev->virt.autoload_ucode_id : AMDGPU_UCODE_ID_RLC_G)) {
2602                         ret = psp_rlc_autoload_start(psp);
2603                         if (ret) {
2604                                 DRM_ERROR("Failed to start rlc autoload\n");
2605                                 return ret;
2606                         }
2607                 }
2608         }
2609
2610         return 0;
2611 }
2612
2613 static int psp_load_fw(struct amdgpu_device *adev)
2614 {
2615         int ret;
2616         struct psp_context *psp = &adev->psp;
2617
2618         if (amdgpu_sriov_vf(adev) && amdgpu_in_reset(adev)) {
2619                 /* should not destroy ring, only stop */
2620                 psp_ring_stop(psp, PSP_RING_TYPE__KM);
2621         } else {
2622                 memset(psp->fence_buf, 0, PSP_FENCE_BUFFER_SIZE);
2623
2624                 ret = psp_ring_init(psp, PSP_RING_TYPE__KM);
2625                 if (ret) {
2626                         DRM_ERROR("PSP ring init failed!\n");
2627                         goto failed;
2628                 }
2629         }
2630
2631         ret = psp_hw_start(psp);
2632         if (ret)
2633                 goto failed;
2634
2635         ret = psp_load_non_psp_fw(psp);
2636         if (ret)
2637                 goto failed1;
2638
2639         ret = psp_asd_initialize(psp);
2640         if (ret) {
2641                 DRM_ERROR("PSP load asd failed!\n");
2642                 goto failed1;
2643         }
2644
2645         ret = psp_rl_load(adev);
2646         if (ret) {
2647                 DRM_ERROR("PSP load RL failed!\n");
2648                 goto failed1;
2649         }
2650
2651         if (amdgpu_sriov_vf(adev) && amdgpu_in_reset(adev)) {
2652                 if (adev->gmc.xgmi.num_physical_nodes > 1) {
2653                         ret = psp_xgmi_initialize(psp, false, true);
2654                         /* Warning the XGMI seesion initialize failure
2655                         * Instead of stop driver initialization
2656                         */
2657                         if (ret)
2658                                 dev_err(psp->adev->dev,
2659                                         "XGMI: Failed to initialize XGMI session\n");
2660                 }
2661         }
2662
2663         if (psp->ta_fw) {
2664                 ret = psp_ras_initialize(psp);
2665                 if (ret)
2666                         dev_err(psp->adev->dev,
2667                                         "RAS: Failed to initialize RAS\n");
2668
2669                 ret = psp_hdcp_initialize(psp);
2670                 if (ret)
2671                         dev_err(psp->adev->dev,
2672                                 "HDCP: Failed to initialize HDCP\n");
2673
2674                 ret = psp_dtm_initialize(psp);
2675                 if (ret)
2676                         dev_err(psp->adev->dev,
2677                                 "DTM: Failed to initialize DTM\n");
2678
2679                 ret = psp_rap_initialize(psp);
2680                 if (ret)
2681                         dev_err(psp->adev->dev,
2682                                 "RAP: Failed to initialize RAP\n");
2683
2684                 ret = psp_securedisplay_initialize(psp);
2685                 if (ret)
2686                         dev_err(psp->adev->dev,
2687                                 "SECUREDISPLAY: Failed to initialize SECUREDISPLAY\n");
2688         }
2689
2690         return 0;
2691
2692 failed1:
2693         psp_free_shared_bufs(psp);
2694 failed:
2695         /*
2696          * all cleanup jobs (xgmi terminate, ras terminate,
2697          * ring destroy, cmd/fence/fw buffers destory,
2698          * psp->cmd destory) are delayed to psp_hw_fini
2699          */
2700         psp_ring_destroy(psp, PSP_RING_TYPE__KM);
2701         return ret;
2702 }
2703
2704 static int psp_hw_init(void *handle)
2705 {
2706         int ret;
2707         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2708
2709         mutex_lock(&adev->firmware.mutex);
2710         /*
2711          * This sequence is just used on hw_init only once, no need on
2712          * resume.
2713          */
2714         ret = amdgpu_ucode_init_bo(adev);
2715         if (ret)
2716                 goto failed;
2717
2718         ret = psp_load_fw(adev);
2719         if (ret) {
2720                 DRM_ERROR("PSP firmware loading failed\n");
2721                 goto failed;
2722         }
2723
2724         mutex_unlock(&adev->firmware.mutex);
2725         return 0;
2726
2727 failed:
2728         adev->firmware.load_type = AMDGPU_FW_LOAD_DIRECT;
2729         mutex_unlock(&adev->firmware.mutex);
2730         return -EINVAL;
2731 }
2732
2733 static int psp_hw_fini(void *handle)
2734 {
2735         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2736         struct psp_context *psp = &adev->psp;
2737
2738         if (psp->ta_fw) {
2739                 psp_ras_terminate(psp);
2740                 psp_securedisplay_terminate(psp);
2741                 psp_rap_terminate(psp);
2742                 psp_dtm_terminate(psp);
2743                 psp_hdcp_terminate(psp);
2744
2745                 if (adev->gmc.xgmi.num_physical_nodes > 1)
2746                         psp_xgmi_terminate(psp);
2747         }
2748
2749         psp_asd_terminate(psp);
2750         psp_tmr_terminate(psp);
2751
2752         psp_ring_destroy(psp, PSP_RING_TYPE__KM);
2753
2754         return 0;
2755 }
2756
2757 static int psp_suspend(void *handle)
2758 {
2759         int ret = 0;
2760         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2761         struct psp_context *psp = &adev->psp;
2762
2763         if (adev->gmc.xgmi.num_physical_nodes > 1 &&
2764             psp->xgmi_context.context.initialized) {
2765                 ret = psp_xgmi_terminate(psp);
2766                 if (ret) {
2767                         DRM_ERROR("Failed to terminate xgmi ta\n");
2768                         goto out;
2769                 }
2770         }
2771
2772         if (psp->ta_fw) {
2773                 ret = psp_ras_terminate(psp);
2774                 if (ret) {
2775                         DRM_ERROR("Failed to terminate ras ta\n");
2776                         goto out;
2777                 }
2778                 ret = psp_hdcp_terminate(psp);
2779                 if (ret) {
2780                         DRM_ERROR("Failed to terminate hdcp ta\n");
2781                         goto out;
2782                 }
2783                 ret = psp_dtm_terminate(psp);
2784                 if (ret) {
2785                         DRM_ERROR("Failed to terminate dtm ta\n");
2786                         goto out;
2787                 }
2788                 ret = psp_rap_terminate(psp);
2789                 if (ret) {
2790                         DRM_ERROR("Failed to terminate rap ta\n");
2791                         goto out;
2792                 }
2793                 ret = psp_securedisplay_terminate(psp);
2794                 if (ret) {
2795                         DRM_ERROR("Failed to terminate securedisplay ta\n");
2796                         goto out;
2797                 }
2798         }
2799
2800         ret = psp_asd_terminate(psp);
2801         if (ret) {
2802                 DRM_ERROR("Failed to terminate asd\n");
2803                 goto out;
2804         }
2805
2806         ret = psp_tmr_terminate(psp);
2807         if (ret) {
2808                 DRM_ERROR("Failed to terminate tmr\n");
2809                 goto out;
2810         }
2811
2812         ret = psp_ring_stop(psp, PSP_RING_TYPE__KM);
2813         if (ret)
2814                 DRM_ERROR("PSP ring stop failed\n");
2815
2816 out:
2817         return ret;
2818 }
2819
2820 static int psp_resume(void *handle)
2821 {
2822         int ret;
2823         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
2824         struct psp_context *psp = &adev->psp;
2825
2826         DRM_INFO("PSP is resuming...\n");
2827
2828         if (psp->mem_train_ctx.enable_mem_training) {
2829                 ret = psp_mem_training(psp, PSP_MEM_TRAIN_RESUME);
2830                 if (ret) {
2831                         DRM_ERROR("Failed to process memory training!\n");
2832                         return ret;
2833                 }
2834         }
2835
2836         mutex_lock(&adev->firmware.mutex);
2837
2838         ret = psp_hw_start(psp);
2839         if (ret)
2840                 goto failed;
2841
2842         ret = psp_load_non_psp_fw(psp);
2843         if (ret)
2844                 goto failed;
2845
2846         ret = psp_asd_initialize(psp);
2847         if (ret) {
2848                 DRM_ERROR("PSP load asd failed!\n");
2849                 goto failed;
2850         }
2851
2852         ret = psp_rl_load(adev);
2853         if (ret) {
2854                 dev_err(adev->dev, "PSP load RL failed!\n");
2855                 goto failed;
2856         }
2857
2858         if (adev->gmc.xgmi.num_physical_nodes > 1) {
2859                 ret = psp_xgmi_initialize(psp, false, true);
2860                 /* Warning the XGMI seesion initialize failure
2861                  * Instead of stop driver initialization
2862                  */
2863                 if (ret)
2864                         dev_err(psp->adev->dev,
2865                                 "XGMI: Failed to initialize XGMI session\n");
2866         }
2867
2868         if (psp->ta_fw) {
2869                 ret = psp_ras_initialize(psp);
2870                 if (ret)
2871                         dev_err(psp->adev->dev,
2872                                         "RAS: Failed to initialize RAS\n");
2873
2874                 ret = psp_hdcp_initialize(psp);
2875                 if (ret)
2876                         dev_err(psp->adev->dev,
2877                                 "HDCP: Failed to initialize HDCP\n");
2878
2879                 ret = psp_dtm_initialize(psp);
2880                 if (ret)
2881                         dev_err(psp->adev->dev,
2882                                 "DTM: Failed to initialize DTM\n");
2883
2884                 ret = psp_rap_initialize(psp);
2885                 if (ret)
2886                         dev_err(psp->adev->dev,
2887                                 "RAP: Failed to initialize RAP\n");
2888
2889                 ret = psp_securedisplay_initialize(psp);
2890                 if (ret)
2891                         dev_err(psp->adev->dev,
2892                                 "SECUREDISPLAY: Failed to initialize SECUREDISPLAY\n");
2893         }
2894
2895         mutex_unlock(&adev->firmware.mutex);
2896
2897         return 0;
2898
2899 failed:
2900         DRM_ERROR("PSP resume failed\n");
2901         mutex_unlock(&adev->firmware.mutex);
2902         return ret;
2903 }
2904
2905 int psp_gpu_reset(struct amdgpu_device *adev)
2906 {
2907         int ret;
2908
2909         if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP)
2910                 return 0;
2911
2912         mutex_lock(&adev->psp.mutex);
2913         ret = psp_mode1_reset(&adev->psp);
2914         mutex_unlock(&adev->psp.mutex);
2915
2916         return ret;
2917 }
2918
2919 int psp_rlc_autoload_start(struct psp_context *psp)
2920 {
2921         int ret;
2922         struct psp_gfx_cmd_resp *cmd = acquire_psp_cmd_buf(psp);
2923
2924         cmd->cmd_id = GFX_CMD_ID_AUTOLOAD_RLC;
2925
2926         ret = psp_cmd_submit_buf(psp, NULL, cmd,
2927                                  psp->fence_buf_mc_addr);
2928
2929         release_psp_cmd_buf(psp);
2930
2931         return ret;
2932 }
2933
2934 int psp_update_vcn_sram(struct amdgpu_device *adev, int inst_idx,
2935                         uint64_t cmd_gpu_addr, int cmd_size)
2936 {
2937         struct amdgpu_firmware_info ucode = {0};
2938
2939         ucode.ucode_id = inst_idx ? AMDGPU_UCODE_ID_VCN1_RAM :
2940                 AMDGPU_UCODE_ID_VCN0_RAM;
2941         ucode.mc_addr = cmd_gpu_addr;
2942         ucode.ucode_size = cmd_size;
2943
2944         return psp_execute_non_psp_fw_load(&adev->psp, &ucode);
2945 }
2946
2947 int psp_ring_cmd_submit(struct psp_context *psp,
2948                         uint64_t cmd_buf_mc_addr,
2949                         uint64_t fence_mc_addr,
2950                         int index)
2951 {
2952         unsigned int psp_write_ptr_reg = 0;
2953         struct psp_gfx_rb_frame *write_frame;
2954         struct psp_ring *ring = &psp->km_ring;
2955         struct psp_gfx_rb_frame *ring_buffer_start = ring->ring_mem;
2956         struct psp_gfx_rb_frame *ring_buffer_end = ring_buffer_start +
2957                 ring->ring_size / sizeof(struct psp_gfx_rb_frame) - 1;
2958         struct amdgpu_device *adev = psp->adev;
2959         uint32_t ring_size_dw = ring->ring_size / 4;
2960         uint32_t rb_frame_size_dw = sizeof(struct psp_gfx_rb_frame) / 4;
2961
2962         /* KM (GPCOM) prepare write pointer */
2963         psp_write_ptr_reg = psp_ring_get_wptr(psp);
2964
2965         /* Update KM RB frame pointer to new frame */
2966         /* write_frame ptr increments by size of rb_frame in bytes */
2967         /* psp_write_ptr_reg increments by size of rb_frame in DWORDs */
2968         if ((psp_write_ptr_reg % ring_size_dw) == 0)
2969                 write_frame = ring_buffer_start;
2970         else
2971                 write_frame = ring_buffer_start + (psp_write_ptr_reg / rb_frame_size_dw);
2972         /* Check invalid write_frame ptr address */
2973         if ((write_frame < ring_buffer_start) || (ring_buffer_end < write_frame)) {
2974                 DRM_ERROR("ring_buffer_start = %p; ring_buffer_end = %p; write_frame = %p\n",
2975                           ring_buffer_start, ring_buffer_end, write_frame);
2976                 DRM_ERROR("write_frame is pointing to address out of bounds\n");
2977                 return -EINVAL;
2978         }
2979
2980         /* Initialize KM RB frame */
2981         memset(write_frame, 0, sizeof(struct psp_gfx_rb_frame));
2982
2983         /* Update KM RB frame */
2984         write_frame->cmd_buf_addr_hi = upper_32_bits(cmd_buf_mc_addr);
2985         write_frame->cmd_buf_addr_lo = lower_32_bits(cmd_buf_mc_addr);
2986         write_frame->fence_addr_hi = upper_32_bits(fence_mc_addr);
2987         write_frame->fence_addr_lo = lower_32_bits(fence_mc_addr);
2988         write_frame->fence_value = index;
2989         amdgpu_device_flush_hdp(adev, NULL);
2990
2991         /* Update the write Pointer in DWORDs */
2992         psp_write_ptr_reg = (psp_write_ptr_reg + rb_frame_size_dw) % ring_size_dw;
2993         psp_ring_set_wptr(psp, psp_write_ptr_reg);
2994         return 0;
2995 }
2996
2997 int psp_init_asd_microcode(struct psp_context *psp, const char *chip_name)
2998 {
2999         struct amdgpu_device *adev = psp->adev;
3000         char fw_name[PSP_FW_NAME_LEN];
3001         const struct psp_firmware_header_v1_0 *asd_hdr;
3002         int err = 0;
3003
3004         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_asd.bin", chip_name);
3005         err = amdgpu_ucode_request(adev, &adev->psp.asd_fw, fw_name);
3006         if (err)
3007                 goto out;
3008
3009         asd_hdr = (const struct psp_firmware_header_v1_0 *)adev->psp.asd_fw->data;
3010         adev->psp.asd_context.bin_desc.fw_version = le32_to_cpu(asd_hdr->header.ucode_version);
3011         adev->psp.asd_context.bin_desc.feature_version = le32_to_cpu(asd_hdr->sos.fw_version);
3012         adev->psp.asd_context.bin_desc.size_bytes = le32_to_cpu(asd_hdr->header.ucode_size_bytes);
3013         adev->psp.asd_context.bin_desc.start_addr = (uint8_t *)asd_hdr +
3014                                 le32_to_cpu(asd_hdr->header.ucode_array_offset_bytes);
3015         return 0;
3016 out:
3017         amdgpu_ucode_release(&adev->psp.asd_fw);
3018         return err;
3019 }
3020
3021 int psp_init_toc_microcode(struct psp_context *psp, const char *chip_name)
3022 {
3023         struct amdgpu_device *adev = psp->adev;
3024         char fw_name[PSP_FW_NAME_LEN];
3025         const struct psp_firmware_header_v1_0 *toc_hdr;
3026         int err = 0;
3027
3028         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_toc.bin", chip_name);
3029         err = amdgpu_ucode_request(adev, &adev->psp.toc_fw, fw_name);
3030         if (err)
3031                 goto out;
3032
3033         toc_hdr = (const struct psp_firmware_header_v1_0 *)adev->psp.toc_fw->data;
3034         adev->psp.toc.fw_version = le32_to_cpu(toc_hdr->header.ucode_version);
3035         adev->psp.toc.feature_version = le32_to_cpu(toc_hdr->sos.fw_version);
3036         adev->psp.toc.size_bytes = le32_to_cpu(toc_hdr->header.ucode_size_bytes);
3037         adev->psp.toc.start_addr = (uint8_t *)toc_hdr +
3038                                 le32_to_cpu(toc_hdr->header.ucode_array_offset_bytes);
3039         return 0;
3040 out:
3041         amdgpu_ucode_release(&adev->psp.toc_fw);
3042         return err;
3043 }
3044
3045 static int parse_sos_bin_descriptor(struct psp_context *psp,
3046                                    const struct psp_fw_bin_desc *desc,
3047                                    const struct psp_firmware_header_v2_0 *sos_hdr)
3048 {
3049         uint8_t *ucode_start_addr  = NULL;
3050
3051         if (!psp || !desc || !sos_hdr)
3052                 return -EINVAL;
3053
3054         ucode_start_addr  = (uint8_t *)sos_hdr +
3055                             le32_to_cpu(desc->offset_bytes) +
3056                             le32_to_cpu(sos_hdr->header.ucode_array_offset_bytes);
3057
3058         switch (desc->fw_type) {
3059         case PSP_FW_TYPE_PSP_SOS:
3060                 psp->sos.fw_version        = le32_to_cpu(desc->fw_version);
3061                 psp->sos.feature_version   = le32_to_cpu(desc->fw_version);
3062                 psp->sos.size_bytes        = le32_to_cpu(desc->size_bytes);
3063                 psp->sos.start_addr        = ucode_start_addr;
3064                 break;
3065         case PSP_FW_TYPE_PSP_SYS_DRV:
3066                 psp->sys.fw_version        = le32_to_cpu(desc->fw_version);
3067                 psp->sys.feature_version   = le32_to_cpu(desc->fw_version);
3068                 psp->sys.size_bytes        = le32_to_cpu(desc->size_bytes);
3069                 psp->sys.start_addr        = ucode_start_addr;
3070                 break;
3071         case PSP_FW_TYPE_PSP_KDB:
3072                 psp->kdb.fw_version        = le32_to_cpu(desc->fw_version);
3073                 psp->kdb.feature_version   = le32_to_cpu(desc->fw_version);
3074                 psp->kdb.size_bytes        = le32_to_cpu(desc->size_bytes);
3075                 psp->kdb.start_addr        = ucode_start_addr;
3076                 break;
3077         case PSP_FW_TYPE_PSP_TOC:
3078                 psp->toc.fw_version        = le32_to_cpu(desc->fw_version);
3079                 psp->toc.feature_version   = le32_to_cpu(desc->fw_version);
3080                 psp->toc.size_bytes        = le32_to_cpu(desc->size_bytes);
3081                 psp->toc.start_addr        = ucode_start_addr;
3082                 break;
3083         case PSP_FW_TYPE_PSP_SPL:
3084                 psp->spl.fw_version        = le32_to_cpu(desc->fw_version);
3085                 psp->spl.feature_version   = le32_to_cpu(desc->fw_version);
3086                 psp->spl.size_bytes        = le32_to_cpu(desc->size_bytes);
3087                 psp->spl.start_addr        = ucode_start_addr;
3088                 break;
3089         case PSP_FW_TYPE_PSP_RL:
3090                 psp->rl.fw_version         = le32_to_cpu(desc->fw_version);
3091                 psp->rl.feature_version    = le32_to_cpu(desc->fw_version);
3092                 psp->rl.size_bytes         = le32_to_cpu(desc->size_bytes);
3093                 psp->rl.start_addr         = ucode_start_addr;
3094                 break;
3095         case PSP_FW_TYPE_PSP_SOC_DRV:
3096                 psp->soc_drv.fw_version         = le32_to_cpu(desc->fw_version);
3097                 psp->soc_drv.feature_version    = le32_to_cpu(desc->fw_version);
3098                 psp->soc_drv.size_bytes         = le32_to_cpu(desc->size_bytes);
3099                 psp->soc_drv.start_addr         = ucode_start_addr;
3100                 break;
3101         case PSP_FW_TYPE_PSP_INTF_DRV:
3102                 psp->intf_drv.fw_version        = le32_to_cpu(desc->fw_version);
3103                 psp->intf_drv.feature_version   = le32_to_cpu(desc->fw_version);
3104                 psp->intf_drv.size_bytes        = le32_to_cpu(desc->size_bytes);
3105                 psp->intf_drv.start_addr        = ucode_start_addr;
3106                 break;
3107         case PSP_FW_TYPE_PSP_DBG_DRV:
3108                 psp->dbg_drv.fw_version         = le32_to_cpu(desc->fw_version);
3109                 psp->dbg_drv.feature_version    = le32_to_cpu(desc->fw_version);
3110                 psp->dbg_drv.size_bytes         = le32_to_cpu(desc->size_bytes);
3111                 psp->dbg_drv.start_addr         = ucode_start_addr;
3112                 break;
3113         case PSP_FW_TYPE_PSP_RAS_DRV:
3114                 psp->ras_drv.fw_version         = le32_to_cpu(desc->fw_version);
3115                 psp->ras_drv.feature_version    = le32_to_cpu(desc->fw_version);
3116                 psp->ras_drv.size_bytes         = le32_to_cpu(desc->size_bytes);
3117                 psp->ras_drv.start_addr         = ucode_start_addr;
3118                 break;
3119         default:
3120                 dev_warn(psp->adev->dev, "Unsupported PSP FW type: %d\n", desc->fw_type);
3121                 break;
3122         }
3123
3124         return 0;
3125 }
3126
3127 static int psp_init_sos_base_fw(struct amdgpu_device *adev)
3128 {
3129         const struct psp_firmware_header_v1_0 *sos_hdr;
3130         const struct psp_firmware_header_v1_3 *sos_hdr_v1_3;
3131         uint8_t *ucode_array_start_addr;
3132
3133         sos_hdr = (const struct psp_firmware_header_v1_0 *)adev->psp.sos_fw->data;
3134         ucode_array_start_addr = (uint8_t *)sos_hdr +
3135                 le32_to_cpu(sos_hdr->header.ucode_array_offset_bytes);
3136
3137         if (adev->gmc.xgmi.connected_to_cpu ||
3138             (adev->ip_versions[MP0_HWIP][0] != IP_VERSION(13, 0, 2))) {
3139                 adev->psp.sos.fw_version = le32_to_cpu(sos_hdr->header.ucode_version);
3140                 adev->psp.sos.feature_version = le32_to_cpu(sos_hdr->sos.fw_version);
3141
3142                 adev->psp.sys.size_bytes = le32_to_cpu(sos_hdr->sos.offset_bytes);
3143                 adev->psp.sys.start_addr = ucode_array_start_addr;
3144
3145                 adev->psp.sos.size_bytes = le32_to_cpu(sos_hdr->sos.size_bytes);
3146                 adev->psp.sos.start_addr = ucode_array_start_addr +
3147                                 le32_to_cpu(sos_hdr->sos.offset_bytes);
3148         } else {
3149                 /* Load alternate PSP SOS FW */
3150                 sos_hdr_v1_3 = (const struct psp_firmware_header_v1_3 *)adev->psp.sos_fw->data;
3151
3152                 adev->psp.sos.fw_version = le32_to_cpu(sos_hdr_v1_3->sos_aux.fw_version);
3153                 adev->psp.sos.feature_version = le32_to_cpu(sos_hdr_v1_3->sos_aux.fw_version);
3154
3155                 adev->psp.sys.size_bytes = le32_to_cpu(sos_hdr_v1_3->sys_drv_aux.size_bytes);
3156                 adev->psp.sys.start_addr = ucode_array_start_addr +
3157                         le32_to_cpu(sos_hdr_v1_3->sys_drv_aux.offset_bytes);
3158
3159                 adev->psp.sos.size_bytes = le32_to_cpu(sos_hdr_v1_3->sos_aux.size_bytes);
3160                 adev->psp.sos.start_addr = ucode_array_start_addr +
3161                         le32_to_cpu(sos_hdr_v1_3->sos_aux.offset_bytes);
3162         }
3163
3164         if ((adev->psp.sys.size_bytes == 0) || (adev->psp.sos.size_bytes == 0)) {
3165                 dev_warn(adev->dev, "PSP SOS FW not available");
3166                 return -EINVAL;
3167         }
3168
3169         return 0;
3170 }
3171
3172 int psp_init_sos_microcode(struct psp_context *psp, const char *chip_name)
3173 {
3174         struct amdgpu_device *adev = psp->adev;
3175         char fw_name[PSP_FW_NAME_LEN];
3176         const struct psp_firmware_header_v1_0 *sos_hdr;
3177         const struct psp_firmware_header_v1_1 *sos_hdr_v1_1;
3178         const struct psp_firmware_header_v1_2 *sos_hdr_v1_2;
3179         const struct psp_firmware_header_v1_3 *sos_hdr_v1_3;
3180         const struct psp_firmware_header_v2_0 *sos_hdr_v2_0;
3181         int err = 0;
3182         uint8_t *ucode_array_start_addr;
3183         int fw_index = 0;
3184
3185         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_sos.bin", chip_name);
3186         err = amdgpu_ucode_request(adev, &adev->psp.sos_fw, fw_name);
3187         if (err)
3188                 goto out;
3189
3190         sos_hdr = (const struct psp_firmware_header_v1_0 *)adev->psp.sos_fw->data;
3191         ucode_array_start_addr = (uint8_t *)sos_hdr +
3192                 le32_to_cpu(sos_hdr->header.ucode_array_offset_bytes);
3193         amdgpu_ucode_print_psp_hdr(&sos_hdr->header);
3194
3195         switch (sos_hdr->header.header_version_major) {
3196         case 1:
3197                 err = psp_init_sos_base_fw(adev);
3198                 if (err)
3199                         goto out;
3200
3201                 if (sos_hdr->header.header_version_minor == 1) {
3202                         sos_hdr_v1_1 = (const struct psp_firmware_header_v1_1 *)adev->psp.sos_fw->data;
3203                         adev->psp.toc.size_bytes = le32_to_cpu(sos_hdr_v1_1->toc.size_bytes);
3204                         adev->psp.toc.start_addr = (uint8_t *)adev->psp.sys.start_addr +
3205                                         le32_to_cpu(sos_hdr_v1_1->toc.offset_bytes);
3206                         adev->psp.kdb.size_bytes = le32_to_cpu(sos_hdr_v1_1->kdb.size_bytes);
3207                         adev->psp.kdb.start_addr = (uint8_t *)adev->psp.sys.start_addr +
3208                                         le32_to_cpu(sos_hdr_v1_1->kdb.offset_bytes);
3209                 }
3210                 if (sos_hdr->header.header_version_minor == 2) {
3211                         sos_hdr_v1_2 = (const struct psp_firmware_header_v1_2 *)adev->psp.sos_fw->data;
3212                         adev->psp.kdb.size_bytes = le32_to_cpu(sos_hdr_v1_2->kdb.size_bytes);
3213                         adev->psp.kdb.start_addr = (uint8_t *)adev->psp.sys.start_addr +
3214                                                     le32_to_cpu(sos_hdr_v1_2->kdb.offset_bytes);
3215                 }
3216                 if (sos_hdr->header.header_version_minor == 3) {
3217                         sos_hdr_v1_3 = (const struct psp_firmware_header_v1_3 *)adev->psp.sos_fw->data;
3218                         adev->psp.toc.size_bytes = le32_to_cpu(sos_hdr_v1_3->v1_1.toc.size_bytes);
3219                         adev->psp.toc.start_addr = ucode_array_start_addr +
3220                                 le32_to_cpu(sos_hdr_v1_3->v1_1.toc.offset_bytes);
3221                         adev->psp.kdb.size_bytes = le32_to_cpu(sos_hdr_v1_3->v1_1.kdb.size_bytes);
3222                         adev->psp.kdb.start_addr = ucode_array_start_addr +
3223                                 le32_to_cpu(sos_hdr_v1_3->v1_1.kdb.offset_bytes);
3224                         adev->psp.spl.size_bytes = le32_to_cpu(sos_hdr_v1_3->spl.size_bytes);
3225                         adev->psp.spl.start_addr = ucode_array_start_addr +
3226                                 le32_to_cpu(sos_hdr_v1_3->spl.offset_bytes);
3227                         adev->psp.rl.size_bytes = le32_to_cpu(sos_hdr_v1_3->rl.size_bytes);
3228                         adev->psp.rl.start_addr = ucode_array_start_addr +
3229                                 le32_to_cpu(sos_hdr_v1_3->rl.offset_bytes);
3230                 }
3231                 break;
3232         case 2:
3233                 sos_hdr_v2_0 = (const struct psp_firmware_header_v2_0 *)adev->psp.sos_fw->data;
3234
3235                 if (le32_to_cpu(sos_hdr_v2_0->psp_fw_bin_count) >= UCODE_MAX_PSP_PACKAGING) {
3236                         dev_err(adev->dev, "packed SOS count exceeds maximum limit\n");
3237                         err = -EINVAL;
3238                         goto out;
3239                 }
3240
3241                 for (fw_index = 0; fw_index < le32_to_cpu(sos_hdr_v2_0->psp_fw_bin_count); fw_index++) {
3242                         err = parse_sos_bin_descriptor(psp,
3243                                                        &sos_hdr_v2_0->psp_fw_bin[fw_index],
3244                                                        sos_hdr_v2_0);
3245                         if (err)
3246                                 goto out;
3247                 }
3248                 break;
3249         default:
3250                 dev_err(adev->dev,
3251                         "unsupported psp sos firmware\n");
3252                 err = -EINVAL;
3253                 goto out;
3254         }
3255
3256         return 0;
3257 out:
3258         amdgpu_ucode_release(&adev->psp.sos_fw);
3259
3260         return err;
3261 }
3262
3263 static int parse_ta_bin_descriptor(struct psp_context *psp,
3264                                    const struct psp_fw_bin_desc *desc,
3265                                    const struct ta_firmware_header_v2_0 *ta_hdr)
3266 {
3267         uint8_t *ucode_start_addr  = NULL;
3268
3269         if (!psp || !desc || !ta_hdr)
3270                 return -EINVAL;
3271
3272         ucode_start_addr  = (uint8_t *)ta_hdr +
3273                             le32_to_cpu(desc->offset_bytes) +
3274                             le32_to_cpu(ta_hdr->header.ucode_array_offset_bytes);
3275
3276         switch (desc->fw_type) {
3277         case TA_FW_TYPE_PSP_ASD:
3278                 psp->asd_context.bin_desc.fw_version        = le32_to_cpu(desc->fw_version);
3279                 psp->asd_context.bin_desc.feature_version   = le32_to_cpu(desc->fw_version);
3280                 psp->asd_context.bin_desc.size_bytes        = le32_to_cpu(desc->size_bytes);
3281                 psp->asd_context.bin_desc.start_addr        = ucode_start_addr;
3282                 break;
3283         case TA_FW_TYPE_PSP_XGMI:
3284                 psp->xgmi_context.context.bin_desc.fw_version       = le32_to_cpu(desc->fw_version);
3285                 psp->xgmi_context.context.bin_desc.size_bytes       = le32_to_cpu(desc->size_bytes);
3286                 psp->xgmi_context.context.bin_desc.start_addr       = ucode_start_addr;
3287                 break;
3288         case TA_FW_TYPE_PSP_RAS:
3289                 psp->ras_context.context.bin_desc.fw_version        = le32_to_cpu(desc->fw_version);
3290                 psp->ras_context.context.bin_desc.size_bytes        = le32_to_cpu(desc->size_bytes);
3291                 psp->ras_context.context.bin_desc.start_addr        = ucode_start_addr;
3292                 break;
3293         case TA_FW_TYPE_PSP_HDCP:
3294                 psp->hdcp_context.context.bin_desc.fw_version       = le32_to_cpu(desc->fw_version);
3295                 psp->hdcp_context.context.bin_desc.size_bytes       = le32_to_cpu(desc->size_bytes);
3296                 psp->hdcp_context.context.bin_desc.start_addr       = ucode_start_addr;
3297                 break;
3298         case TA_FW_TYPE_PSP_DTM:
3299                 psp->dtm_context.context.bin_desc.fw_version       = le32_to_cpu(desc->fw_version);
3300                 psp->dtm_context.context.bin_desc.size_bytes       = le32_to_cpu(desc->size_bytes);
3301                 psp->dtm_context.context.bin_desc.start_addr       = ucode_start_addr;
3302                 break;
3303         case TA_FW_TYPE_PSP_RAP:
3304                 psp->rap_context.context.bin_desc.fw_version       = le32_to_cpu(desc->fw_version);
3305                 psp->rap_context.context.bin_desc.size_bytes       = le32_to_cpu(desc->size_bytes);
3306                 psp->rap_context.context.bin_desc.start_addr       = ucode_start_addr;
3307                 break;
3308         case TA_FW_TYPE_PSP_SECUREDISPLAY:
3309                 psp->securedisplay_context.context.bin_desc.fw_version =
3310                         le32_to_cpu(desc->fw_version);
3311                 psp->securedisplay_context.context.bin_desc.size_bytes =
3312                         le32_to_cpu(desc->size_bytes);
3313                 psp->securedisplay_context.context.bin_desc.start_addr =
3314                         ucode_start_addr;
3315                 break;
3316         default:
3317                 dev_warn(psp->adev->dev, "Unsupported TA type: %d\n", desc->fw_type);
3318                 break;
3319         }
3320
3321         return 0;
3322 }
3323
3324 static int parse_ta_v1_microcode(struct psp_context *psp)
3325 {
3326         const struct ta_firmware_header_v1_0 *ta_hdr;
3327         struct amdgpu_device *adev = psp->adev;
3328
3329         ta_hdr = (const struct ta_firmware_header_v1_0 *) adev->psp.ta_fw->data;
3330
3331         if (le16_to_cpu(ta_hdr->header.header_version_major) != 1)
3332                 return -EINVAL;
3333
3334         adev->psp.xgmi_context.context.bin_desc.fw_version =
3335                 le32_to_cpu(ta_hdr->xgmi.fw_version);
3336         adev->psp.xgmi_context.context.bin_desc.size_bytes =
3337                 le32_to_cpu(ta_hdr->xgmi.size_bytes);
3338         adev->psp.xgmi_context.context.bin_desc.start_addr =
3339                 (uint8_t *)ta_hdr +
3340                 le32_to_cpu(ta_hdr->header.ucode_array_offset_bytes);
3341
3342         adev->psp.ras_context.context.bin_desc.fw_version =
3343                 le32_to_cpu(ta_hdr->ras.fw_version);
3344         adev->psp.ras_context.context.bin_desc.size_bytes =
3345                 le32_to_cpu(ta_hdr->ras.size_bytes);
3346         adev->psp.ras_context.context.bin_desc.start_addr =
3347                 (uint8_t *)adev->psp.xgmi_context.context.bin_desc.start_addr +
3348                 le32_to_cpu(ta_hdr->ras.offset_bytes);
3349
3350         adev->psp.hdcp_context.context.bin_desc.fw_version =
3351                 le32_to_cpu(ta_hdr->hdcp.fw_version);
3352         adev->psp.hdcp_context.context.bin_desc.size_bytes =
3353                 le32_to_cpu(ta_hdr->hdcp.size_bytes);
3354         adev->psp.hdcp_context.context.bin_desc.start_addr =
3355                 (uint8_t *)ta_hdr +
3356                 le32_to_cpu(ta_hdr->header.ucode_array_offset_bytes);
3357
3358         adev->psp.dtm_context.context.bin_desc.fw_version =
3359                 le32_to_cpu(ta_hdr->dtm.fw_version);
3360         adev->psp.dtm_context.context.bin_desc.size_bytes =
3361                 le32_to_cpu(ta_hdr->dtm.size_bytes);
3362         adev->psp.dtm_context.context.bin_desc.start_addr =
3363                 (uint8_t *)adev->psp.hdcp_context.context.bin_desc.start_addr +
3364                 le32_to_cpu(ta_hdr->dtm.offset_bytes);
3365
3366         adev->psp.securedisplay_context.context.bin_desc.fw_version =
3367                 le32_to_cpu(ta_hdr->securedisplay.fw_version);
3368         adev->psp.securedisplay_context.context.bin_desc.size_bytes =
3369                 le32_to_cpu(ta_hdr->securedisplay.size_bytes);
3370         adev->psp.securedisplay_context.context.bin_desc.start_addr =
3371                 (uint8_t *)adev->psp.hdcp_context.context.bin_desc.start_addr +
3372                 le32_to_cpu(ta_hdr->securedisplay.offset_bytes);
3373
3374         adev->psp.ta_fw_version = le32_to_cpu(ta_hdr->header.ucode_version);
3375
3376         return 0;
3377 }
3378
3379 static int parse_ta_v2_microcode(struct psp_context *psp)
3380 {
3381         const struct ta_firmware_header_v2_0 *ta_hdr;
3382         struct amdgpu_device *adev = psp->adev;
3383         int err = 0;
3384         int ta_index = 0;
3385
3386         ta_hdr = (const struct ta_firmware_header_v2_0 *)adev->psp.ta_fw->data;
3387
3388         if (le16_to_cpu(ta_hdr->header.header_version_major) != 2)
3389                 return -EINVAL;
3390
3391         if (le32_to_cpu(ta_hdr->ta_fw_bin_count) >= UCODE_MAX_PSP_PACKAGING) {
3392                 dev_err(adev->dev, "packed TA count exceeds maximum limit\n");
3393                 return -EINVAL;
3394         }
3395
3396         for (ta_index = 0; ta_index < le32_to_cpu(ta_hdr->ta_fw_bin_count); ta_index++) {
3397                 err = parse_ta_bin_descriptor(psp,
3398                                               &ta_hdr->ta_fw_bin[ta_index],
3399                                               ta_hdr);
3400                 if (err)
3401                         return err;
3402         }
3403
3404         return 0;
3405 }
3406
3407 int psp_init_ta_microcode(struct psp_context *psp, const char *chip_name)
3408 {
3409         const struct common_firmware_header *hdr;
3410         struct amdgpu_device *adev = psp->adev;
3411         char fw_name[PSP_FW_NAME_LEN];
3412         int err;
3413
3414         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_ta.bin", chip_name);
3415         err = amdgpu_ucode_request(adev, &adev->psp.ta_fw, fw_name);
3416         if (err)
3417                 return err;
3418
3419         hdr = (const struct common_firmware_header *)adev->psp.ta_fw->data;
3420         switch (le16_to_cpu(hdr->header_version_major)) {
3421         case 1:
3422                 err = parse_ta_v1_microcode(psp);
3423                 break;
3424         case 2:
3425                 err = parse_ta_v2_microcode(psp);
3426                 break;
3427         default:
3428                 dev_err(adev->dev, "unsupported TA header version\n");
3429                 err = -EINVAL;
3430         }
3431
3432         if (err)
3433                 amdgpu_ucode_release(&adev->psp.ta_fw);
3434
3435         return err;
3436 }
3437
3438 int psp_init_cap_microcode(struct psp_context *psp, const char *chip_name)
3439 {
3440         struct amdgpu_device *adev = psp->adev;
3441         char fw_name[PSP_FW_NAME_LEN];
3442         const struct psp_firmware_header_v1_0 *cap_hdr_v1_0;
3443         struct amdgpu_firmware_info *info = NULL;
3444         int err = 0;
3445
3446         if (!amdgpu_sriov_vf(adev)) {
3447                 dev_err(adev->dev, "cap microcode should only be loaded under SRIOV\n");
3448                 return -EINVAL;
3449         }
3450
3451         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_cap.bin", chip_name);
3452         err = amdgpu_ucode_request(adev, &adev->psp.cap_fw, fw_name);
3453         if (err) {
3454                 if (err == -ENODEV) {
3455                         dev_warn(adev->dev, "cap microcode does not exist, skip\n");
3456                         err = 0;
3457                         goto out;
3458                 }
3459                 dev_err(adev->dev, "fail to initialize cap microcode\n");
3460         }
3461
3462         info = &adev->firmware.ucode[AMDGPU_UCODE_ID_CAP];
3463         info->ucode_id = AMDGPU_UCODE_ID_CAP;
3464         info->fw = adev->psp.cap_fw;
3465         cap_hdr_v1_0 = (const struct psp_firmware_header_v1_0 *)
3466                 adev->psp.cap_fw->data;
3467         adev->firmware.fw_size += ALIGN(
3468                         le32_to_cpu(cap_hdr_v1_0->header.ucode_size_bytes), PAGE_SIZE);
3469         adev->psp.cap_fw_version = le32_to_cpu(cap_hdr_v1_0->header.ucode_version);
3470         adev->psp.cap_feature_version = le32_to_cpu(cap_hdr_v1_0->sos.fw_version);
3471         adev->psp.cap_ucode_size = le32_to_cpu(cap_hdr_v1_0->header.ucode_size_bytes);
3472
3473         return 0;
3474
3475 out:
3476         amdgpu_ucode_release(&adev->psp.cap_fw);
3477         return err;
3478 }
3479
3480 static int psp_set_clockgating_state(void *handle,
3481                                      enum amd_clockgating_state state)
3482 {
3483         return 0;
3484 }
3485
3486 static int psp_set_powergating_state(void *handle,
3487                                      enum amd_powergating_state state)
3488 {
3489         return 0;
3490 }
3491
3492 static ssize_t psp_usbc_pd_fw_sysfs_read(struct device *dev,
3493                                          struct device_attribute *attr,
3494                                          char *buf)
3495 {
3496         struct drm_device *ddev = dev_get_drvdata(dev);
3497         struct amdgpu_device *adev = drm_to_adev(ddev);
3498         uint32_t fw_ver;
3499         int ret;
3500
3501         if (!adev->ip_blocks[AMD_IP_BLOCK_TYPE_PSP].status.late_initialized) {
3502                 DRM_INFO("PSP block is not ready yet.");
3503                 return -EBUSY;
3504         }
3505
3506         mutex_lock(&adev->psp.mutex);
3507         ret = psp_read_usbc_pd_fw(&adev->psp, &fw_ver);
3508         mutex_unlock(&adev->psp.mutex);
3509
3510         if (ret) {
3511                 DRM_ERROR("Failed to read USBC PD FW, err = %d", ret);
3512                 return ret;
3513         }
3514
3515         return sysfs_emit(buf, "%x\n", fw_ver);
3516 }
3517
3518 static ssize_t psp_usbc_pd_fw_sysfs_write(struct device *dev,
3519                                                        struct device_attribute *attr,
3520                                                        const char *buf,
3521                                                        size_t count)
3522 {
3523         struct drm_device *ddev = dev_get_drvdata(dev);
3524         struct amdgpu_device *adev = drm_to_adev(ddev);
3525         int ret, idx;
3526         char fw_name[100];
3527         const struct firmware *usbc_pd_fw;
3528         struct amdgpu_bo *fw_buf_bo = NULL;
3529         uint64_t fw_pri_mc_addr;
3530         void *fw_pri_cpu_addr;
3531
3532         if (!adev->ip_blocks[AMD_IP_BLOCK_TYPE_PSP].status.late_initialized) {
3533                 DRM_INFO("PSP block is not ready yet.");
3534                 return -EBUSY;
3535         }
3536
3537         if (!drm_dev_enter(ddev, &idx))
3538                 return -ENODEV;
3539
3540         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s", buf);
3541         ret = request_firmware(&usbc_pd_fw, fw_name, adev->dev);
3542         if (ret)
3543                 goto fail;
3544
3545         /* LFB address which is aligned to 1MB boundary per PSP request */
3546         ret = amdgpu_bo_create_kernel(adev, usbc_pd_fw->size, 0x100000,
3547                                       AMDGPU_GEM_DOMAIN_VRAM |
3548                                       AMDGPU_GEM_DOMAIN_GTT,
3549                                       &fw_buf_bo, &fw_pri_mc_addr,
3550                                       &fw_pri_cpu_addr);
3551         if (ret)
3552                 goto rel_buf;
3553
3554         memcpy_toio(fw_pri_cpu_addr, usbc_pd_fw->data, usbc_pd_fw->size);
3555
3556         mutex_lock(&adev->psp.mutex);
3557         ret = psp_load_usbc_pd_fw(&adev->psp, fw_pri_mc_addr);
3558         mutex_unlock(&adev->psp.mutex);
3559
3560         amdgpu_bo_free_kernel(&fw_buf_bo, &fw_pri_mc_addr, &fw_pri_cpu_addr);
3561
3562 rel_buf:
3563         release_firmware(usbc_pd_fw);
3564 fail:
3565         if (ret) {
3566                 DRM_ERROR("Failed to load USBC PD FW, err = %d", ret);
3567                 count = ret;
3568         }
3569
3570         drm_dev_exit(idx);
3571         return count;
3572 }
3573
3574 void psp_copy_fw(struct psp_context *psp, uint8_t *start_addr, uint32_t bin_size)
3575 {
3576         int idx;
3577
3578         if (!drm_dev_enter(adev_to_drm(psp->adev), &idx))
3579                 return;
3580
3581         memset(psp->fw_pri_buf, 0, PSP_1_MEG);
3582         memcpy(psp->fw_pri_buf, start_addr, bin_size);
3583
3584         drm_dev_exit(idx);
3585 }
3586
3587 static DEVICE_ATTR(usbc_pd_fw, 0644,
3588                    psp_usbc_pd_fw_sysfs_read,
3589                    psp_usbc_pd_fw_sysfs_write);
3590
3591 int is_psp_fw_valid(struct psp_bin_desc bin)
3592 {
3593         return bin.size_bytes;
3594 }
3595
3596 static ssize_t amdgpu_psp_vbflash_write(struct file *filp, struct kobject *kobj,
3597                                         struct bin_attribute *bin_attr,
3598                                         char *buffer, loff_t pos, size_t count)
3599 {
3600         struct device *dev = kobj_to_dev(kobj);
3601         struct drm_device *ddev = dev_get_drvdata(dev);
3602         struct amdgpu_device *adev = drm_to_adev(ddev);
3603
3604         adev->psp.vbflash_done = false;
3605
3606         /* Safeguard against memory drain */
3607         if (adev->psp.vbflash_image_size > AMD_VBIOS_FILE_MAX_SIZE_B) {
3608                 dev_err(adev->dev, "File size cannot exceed %u", AMD_VBIOS_FILE_MAX_SIZE_B);
3609                 kvfree(adev->psp.vbflash_tmp_buf);
3610                 adev->psp.vbflash_tmp_buf = NULL;
3611                 adev->psp.vbflash_image_size = 0;
3612                 return -ENOMEM;
3613         }
3614
3615         /* TODO Just allocate max for now and optimize to realloc later if needed */
3616         if (!adev->psp.vbflash_tmp_buf) {
3617                 adev->psp.vbflash_tmp_buf = kvmalloc(AMD_VBIOS_FILE_MAX_SIZE_B, GFP_KERNEL);
3618                 if (!adev->psp.vbflash_tmp_buf)
3619                         return -ENOMEM;
3620         }
3621
3622         mutex_lock(&adev->psp.mutex);
3623         memcpy(adev->psp.vbflash_tmp_buf + pos, buffer, count);
3624         adev->psp.vbflash_image_size += count;
3625         mutex_unlock(&adev->psp.mutex);
3626
3627         dev_info(adev->dev, "VBIOS flash write PSP done");
3628
3629         return count;
3630 }
3631
3632 static ssize_t amdgpu_psp_vbflash_read(struct file *filp, struct kobject *kobj,
3633                                        struct bin_attribute *bin_attr, char *buffer,
3634                                        loff_t pos, size_t count)
3635 {
3636         struct device *dev = kobj_to_dev(kobj);
3637         struct drm_device *ddev = dev_get_drvdata(dev);
3638         struct amdgpu_device *adev = drm_to_adev(ddev);
3639         struct amdgpu_bo *fw_buf_bo = NULL;
3640         uint64_t fw_pri_mc_addr;
3641         void *fw_pri_cpu_addr;
3642         int ret;
3643
3644         if (adev->psp.vbflash_image_size == 0)
3645                 return -EINVAL;
3646
3647         dev_info(adev->dev, "VBIOS flash to PSP started");
3648
3649         ret = amdgpu_bo_create_kernel(adev, adev->psp.vbflash_image_size,
3650                                         AMDGPU_GPU_PAGE_SIZE,
3651                                         AMDGPU_GEM_DOMAIN_VRAM,
3652                                         &fw_buf_bo,
3653                                         &fw_pri_mc_addr,
3654                                         &fw_pri_cpu_addr);
3655         if (ret)
3656                 goto rel_buf;
3657
3658         memcpy_toio(fw_pri_cpu_addr, adev->psp.vbflash_tmp_buf, adev->psp.vbflash_image_size);
3659
3660         mutex_lock(&adev->psp.mutex);
3661         ret = psp_update_spirom(&adev->psp, fw_pri_mc_addr);
3662         mutex_unlock(&adev->psp.mutex);
3663
3664         amdgpu_bo_free_kernel(&fw_buf_bo, &fw_pri_mc_addr, &fw_pri_cpu_addr);
3665
3666 rel_buf:
3667         kvfree(adev->psp.vbflash_tmp_buf);
3668         adev->psp.vbflash_tmp_buf = NULL;
3669         adev->psp.vbflash_image_size = 0;
3670
3671         if (ret) {
3672                 dev_err(adev->dev, "Failed to load VBIOS FW, err = %d", ret);
3673                 return ret;
3674         }
3675
3676         dev_info(adev->dev, "VBIOS flash to PSP done");
3677         return 0;
3678 }
3679
3680 static ssize_t amdgpu_psp_vbflash_status(struct device *dev,
3681                                          struct device_attribute *attr,
3682                                          char *buf)
3683 {
3684         struct drm_device *ddev = dev_get_drvdata(dev);
3685         struct amdgpu_device *adev = drm_to_adev(ddev);
3686         uint32_t vbflash_status;
3687
3688         vbflash_status = psp_vbflash_status(&adev->psp);
3689         if (!adev->psp.vbflash_done)
3690                 vbflash_status = 0;
3691         else if (adev->psp.vbflash_done && !(vbflash_status & 0x80000000))
3692                 vbflash_status = 1;
3693
3694         return sysfs_emit(buf, "0x%x\n", vbflash_status);
3695 }
3696
3697 static const struct bin_attribute psp_vbflash_bin_attr = {
3698         .attr = {.name = "psp_vbflash", .mode = 0660},
3699         .size = 0,
3700         .write = amdgpu_psp_vbflash_write,
3701         .read = amdgpu_psp_vbflash_read,
3702 };
3703
3704 static DEVICE_ATTR(psp_vbflash_status, 0440, amdgpu_psp_vbflash_status, NULL);
3705
3706 int amdgpu_psp_sysfs_init(struct amdgpu_device *adev)
3707 {
3708         int ret = 0;
3709
3710         if (amdgpu_sriov_vf(adev))
3711                 return -EINVAL;
3712
3713         switch (adev->ip_versions[MP0_HWIP][0]) {
3714         case IP_VERSION(13, 0, 0):
3715         case IP_VERSION(13, 0, 7):
3716         case IP_VERSION(13, 0, 10):
3717                 ret = sysfs_create_bin_file(&adev->dev->kobj, &psp_vbflash_bin_attr);
3718                 if (ret)
3719                         dev_err(adev->dev, "Failed to create device file psp_vbflash");
3720                 ret = device_create_file(adev->dev, &dev_attr_psp_vbflash_status);
3721                 if (ret)
3722                         dev_err(adev->dev, "Failed to create device file psp_vbflash_status");
3723                 return ret;
3724         default:
3725                 return 0;
3726         }
3727 }
3728
3729 const struct amd_ip_funcs psp_ip_funcs = {
3730         .name = "psp",
3731         .early_init = psp_early_init,
3732         .late_init = NULL,
3733         .sw_init = psp_sw_init,
3734         .sw_fini = psp_sw_fini,
3735         .hw_init = psp_hw_init,
3736         .hw_fini = psp_hw_fini,
3737         .suspend = psp_suspend,
3738         .resume = psp_resume,
3739         .is_idle = NULL,
3740         .check_soft_reset = NULL,
3741         .wait_for_idle = NULL,
3742         .soft_reset = NULL,
3743         .set_clockgating_state = psp_set_clockgating_state,
3744         .set_powergating_state = psp_set_powergating_state,
3745 };
3746
3747 static int psp_sysfs_init(struct amdgpu_device *adev)
3748 {
3749         int ret = device_create_file(adev->dev, &dev_attr_usbc_pd_fw);
3750
3751         if (ret)
3752                 DRM_ERROR("Failed to create USBC PD FW control file!");
3753
3754         return ret;
3755 }
3756
3757 void amdgpu_psp_sysfs_fini(struct amdgpu_device *adev)
3758 {
3759         sysfs_remove_bin_file(&adev->dev->kobj, &psp_vbflash_bin_attr);
3760         device_remove_file(adev->dev, &dev_attr_psp_vbflash_status);
3761 }
3762
3763 static void psp_sysfs_fini(struct amdgpu_device *adev)
3764 {
3765         device_remove_file(adev->dev, &dev_attr_usbc_pd_fw);
3766 }
3767
3768 const struct amdgpu_ip_block_version psp_v3_1_ip_block = {
3769         .type = AMD_IP_BLOCK_TYPE_PSP,
3770         .major = 3,
3771         .minor = 1,
3772         .rev = 0,
3773         .funcs = &psp_ip_funcs,
3774 };
3775
3776 const struct amdgpu_ip_block_version psp_v10_0_ip_block = {
3777         .type = AMD_IP_BLOCK_TYPE_PSP,
3778         .major = 10,
3779         .minor = 0,
3780         .rev = 0,
3781         .funcs = &psp_ip_funcs,
3782 };
3783
3784 const struct amdgpu_ip_block_version psp_v11_0_ip_block = {
3785         .type = AMD_IP_BLOCK_TYPE_PSP,
3786         .major = 11,
3787         .minor = 0,
3788         .rev = 0,
3789         .funcs = &psp_ip_funcs,
3790 };
3791
3792 const struct amdgpu_ip_block_version psp_v11_0_8_ip_block = {
3793         .type = AMD_IP_BLOCK_TYPE_PSP,
3794         .major = 11,
3795         .minor = 0,
3796         .rev = 8,
3797         .funcs = &psp_ip_funcs,
3798 };
3799
3800 const struct amdgpu_ip_block_version psp_v12_0_ip_block = {
3801         .type = AMD_IP_BLOCK_TYPE_PSP,
3802         .major = 12,
3803         .minor = 0,
3804         .rev = 0,
3805         .funcs = &psp_ip_funcs,
3806 };
3807
3808 const struct amdgpu_ip_block_version psp_v13_0_ip_block = {
3809         .type = AMD_IP_BLOCK_TYPE_PSP,
3810         .major = 13,
3811         .minor = 0,
3812         .rev = 0,
3813         .funcs = &psp_ip_funcs,
3814 };
3815
3816 const struct amdgpu_ip_block_version psp_v13_0_4_ip_block = {
3817         .type = AMD_IP_BLOCK_TYPE_PSP,
3818         .major = 13,
3819         .minor = 0,
3820         .rev = 4,
3821         .funcs = &psp_ip_funcs,
3822 };