drivers/gpu/drm/imagination/pvr_fw.c

   1 // SPDX-License-Identifier: GPL-2.0-only OR MIT
   2 /* Copyright (c) 2023 Imagination Technologies Ltd. */
   3
   4 #include "pvr_ccb.h"
   5 #include "pvr_device.h"
   6 #include "pvr_device_info.h"
   7 #include "pvr_fw.h"
   8 #include "pvr_fw_info.h"
   9 #include "pvr_fw_startstop.h"
  10 #include "pvr_fw_trace.h"
  11 #include "pvr_gem.h"
  12 #include "pvr_power.h"
  13 #include "pvr_rogue_fwif_dev_info.h"
  14 #include "pvr_rogue_heap_config.h"
  15 #include "pvr_vm.h"
  16
  17 #include <drm/drm_drv.h>
  18 #include <drm/drm_managed.h>
  19 #include <drm/drm_mm.h>
  20 #include <linux/clk.h>
  21 #include <linux/firmware.h>
  22 #include <linux/math.h>
  23 #include <linux/minmax.h>
  24 #include <linux/sizes.h>
  25
  26 #define FW_MAX_SUPPORTED_MAJOR_VERSION 1
  27
  28 #define FW_BOOT_TIMEOUT_USEC 5000000
  29
  30 /* Config heap occupies top 192k of the firmware heap. */
  31 #define PVR_ROGUE_FW_CONFIG_HEAP_GRANULARITY SZ_64K
  32 #define PVR_ROGUE_FW_CONFIG_HEAP_SIZE (3 * PVR_ROGUE_FW_CONFIG_HEAP_GRANULARITY)
  33
  34 /* Main firmware allocations should come from the remainder of the heap. */
  35 #define PVR_ROGUE_FW_MAIN_HEAP_BASE ROGUE_FW_HEAP_BASE
  36
  37 /* Offsets from start of configuration area of FW heap. */
  38 #define PVR_ROGUE_FWIF_CONNECTION_CTL_OFFSET 0
  39 #define PVR_ROGUE_FWIF_OSINIT_OFFSET \
  40         (PVR_ROGUE_FWIF_CONNECTION_CTL_OFFSET + PVR_ROGUE_FW_CONFIG_HEAP_GRANULARITY)
  41 #define PVR_ROGUE_FWIF_SYSINIT_OFFSET \
  42         (PVR_ROGUE_FWIF_OSINIT_OFFSET + PVR_ROGUE_FW_CONFIG_HEAP_GRANULARITY)
  43
  44 #define PVR_ROGUE_FAULT_PAGE_SIZE SZ_4K
  45
  46 #define PVR_SYNC_OBJ_SIZE sizeof(u32)
  47
  48 const struct pvr_fw_layout_entry *
  49 pvr_fw_find_layout_entry(struct pvr_device *pvr_dev, enum pvr_fw_section_id id)
  50 {
  51         const struct pvr_fw_layout_entry *layout_entries = pvr_dev->fw_dev.layout_entries;
  52         u32 num_layout_entries = pvr_dev->fw_dev.header->layout_entry_num;
  53         u32 entry;
  54
  55         for (entry = 0; entry < num_layout_entries; entry++) {
  56                 if (layout_entries[entry].id == id)
  57                         return &layout_entries[entry];
  58         }
  59
  60         return NULL;
  61 }
  62
  63 static const struct pvr_fw_layout_entry *
  64 pvr_fw_find_private_data(struct pvr_device *pvr_dev)
  65 {
  66         const struct pvr_fw_layout_entry *layout_entries = pvr_dev->fw_dev.layout_entries;
  67         u32 num_layout_entries = pvr_dev->fw_dev.header->layout_entry_num;
  68         u32 entry;
  69
  70         for (entry = 0; entry < num_layout_entries; entry++) {
  71                 if (layout_entries[entry].id == META_PRIVATE_DATA ||
  72                     layout_entries[entry].id == MIPS_PRIVATE_DATA ||
  73                     layout_entries[entry].id == RISCV_PRIVATE_DATA)
  74                         return &layout_entries[entry];
  75         }
  76
  77         return NULL;
  78 }
  79
  80 #define DEV_INFO_MASK_SIZE(x) DIV_ROUND_UP(x, 64)
  81
  82 /**
  83  * pvr_fw_validate() - Parse firmware header and check compatibility
  84  * @pvr_dev: Device pointer.
  85  *
  86  * Returns:
  87  *  * 0 on success, or
  88  *  * -EINVAL if firmware is incompatible.
  89  */
  90 static int
  91 pvr_fw_validate(struct pvr_device *pvr_dev)
  92 {
  93         struct drm_device *drm_dev = from_pvr_device(pvr_dev);
  94         const struct firmware *firmware = pvr_dev->fw_dev.firmware;
  95         const struct pvr_fw_layout_entry *layout_entries;
  96         const struct pvr_fw_info_header *header;
  97         const u8 *fw = firmware->data;
  98         u32 fw_offset = firmware->size - SZ_4K;
  99         u32 layout_table_size;
 100         u32 entry;
 101
 102         if (firmware->size < SZ_4K || (firmware->size % FW_BLOCK_SIZE))
 103                 return -EINVAL;
 104
 105         header = (const struct pvr_fw_info_header *)&fw[fw_offset];
 106
 107         if (header->info_version != PVR_FW_INFO_VERSION) {
 108                 drm_err(drm_dev, "Unsupported fw info version %u\n",
 109                         header->info_version);
 110                 return -EINVAL;
 111         }
 112
 113         if (header->header_len != sizeof(struct pvr_fw_info_header) ||
 114             header->layout_entry_size != sizeof(struct pvr_fw_layout_entry) ||
 115             header->layout_entry_num > PVR_FW_INFO_MAX_NUM_ENTRIES) {
 116                 drm_err(drm_dev, "FW info format mismatch\n");
 117                 return -EINVAL;
 118         }
 119
 120         if (!(header->flags & PVR_FW_FLAGS_OPEN_SOURCE) ||
 121             header->fw_version_major > FW_MAX_SUPPORTED_MAJOR_VERSION ||
 122             header->fw_version_major == 0) {
 123                 drm_err(drm_dev, "Unsupported FW version %u.%u (build: %u%s)\n",
 124                         header->fw_version_major, header->fw_version_minor,
 125                         header->fw_version_build,
 126                         (header->flags & PVR_FW_FLAGS_OPEN_SOURCE) ? " OS" : "");
 127                 return -EINVAL;
 128         }
 129
 130         if (pvr_gpu_id_to_packed_bvnc(&pvr_dev->gpu_id) != header->bvnc) {
 131                 struct pvr_gpu_id fw_gpu_id;
 132
 133                 packed_bvnc_to_pvr_gpu_id(header->bvnc, &fw_gpu_id);
 134                 drm_err(drm_dev, "FW built for incorrect GPU ID %i.%i.%i.%i (expected %i.%i.%i.%i)\n",
 135                         fw_gpu_id.b, fw_gpu_id.v, fw_gpu_id.n, fw_gpu_id.c,
 136                         pvr_dev->gpu_id.b, pvr_dev->gpu_id.v, pvr_dev->gpu_id.n, pvr_dev->gpu_id.c);
 137                 return -EINVAL;
 138         }
 139
 140         fw_offset += header->header_len;
 141         layout_table_size =
 142                 header->layout_entry_size * header->layout_entry_num;
 143         if ((fw_offset + layout_table_size) > firmware->size)
 144                 return -EINVAL;
 145
 146         layout_entries = (const struct pvr_fw_layout_entry *)&fw[fw_offset];
 147         for (entry = 0; entry < header->layout_entry_num; entry++) {
 148                 u32 start_addr = layout_entries[entry].base_addr;
 149                 u32 end_addr = start_addr + layout_entries[entry].alloc_size;
 150
 151                 if (start_addr >= end_addr)
 152                         return -EINVAL;
 153         }
 154
 155         fw_offset = (firmware->size - SZ_4K) - header->device_info_size;
 156
 157         drm_info(drm_dev, "FW version v%u.%u (build %u OS)\n", header->fw_version_major,
 158                  header->fw_version_minor, header->fw_version_build);
 159
 160         pvr_dev->fw_version.major = header->fw_version_major;
 161         pvr_dev->fw_version.minor = header->fw_version_minor;
 162
 163         pvr_dev->fw_dev.header = header;
 164         pvr_dev->fw_dev.layout_entries = layout_entries;
 165
 166         return 0;
 167 }
 168
 169 static int
 170 pvr_fw_get_device_info(struct pvr_device *pvr_dev)
 171 {
 172         const struct firmware *firmware = pvr_dev->fw_dev.firmware;
 173         struct pvr_fw_device_info_header *header;
 174         const u8 *fw = firmware->data;
 175         const u64 *dev_info;
 176         u32 fw_offset;
 177
 178         fw_offset = (firmware->size - SZ_4K) - pvr_dev->fw_dev.header->device_info_size;
 179
 180         header = (struct pvr_fw_device_info_header *)&fw[fw_offset];
 181         dev_info = (u64 *)(header + 1);
 182
 183         pvr_device_info_set_quirks(pvr_dev, dev_info, header->brn_mask_size);
 184         dev_info += header->brn_mask_size;
 185
 186         pvr_device_info_set_enhancements(pvr_dev, dev_info, header->ern_mask_size);
 187         dev_info += header->ern_mask_size;
 188
 189         return pvr_device_info_set_features(pvr_dev, dev_info, header->feature_mask_size,
 190                                             header->feature_param_size);
 191 }
 192
 193 static void
 194 layout_get_sizes(struct pvr_device *pvr_dev)
 195 {
 196         const struct pvr_fw_layout_entry *layout_entries = pvr_dev->fw_dev.layout_entries;
 197         u32 num_layout_entries = pvr_dev->fw_dev.header->layout_entry_num;
 198         struct pvr_fw_mem *fw_mem = &pvr_dev->fw_dev.mem;
 199
 200         fw_mem->code_alloc_size = 0;
 201         fw_mem->data_alloc_size = 0;
 202         fw_mem->core_code_alloc_size = 0;
 203         fw_mem->core_data_alloc_size = 0;
 204
 205         /* Extract section sizes from FW layout table. */
 206         for (u32 entry = 0; entry < num_layout_entries; entry++) {
 207                 switch (layout_entries[entry].type) {
 208                 case FW_CODE:
 209                         fw_mem->code_alloc_size += layout_entries[entry].alloc_size;
 210                         break;
 211                 case FW_DATA:
 212                         fw_mem->data_alloc_size += layout_entries[entry].alloc_size;
 213                         break;
 214                 case FW_COREMEM_CODE:
 215                         fw_mem->core_code_alloc_size +=
 216                                 layout_entries[entry].alloc_size;
 217                         break;
 218                 case FW_COREMEM_DATA:
 219                         fw_mem->core_data_alloc_size +=
 220                                 layout_entries[entry].alloc_size;
 221                         break;
 222                 case NONE:
 223                         break;
 224                 }
 225         }
 226 }
 227
 228 int
 229 pvr_fw_find_mmu_segment(struct pvr_device *pvr_dev, u32 addr, u32 size, void *fw_code_ptr,
 230                         void *fw_data_ptr, void *fw_core_code_ptr, void *fw_core_data_ptr,
 231                         void **host_addr_out)
 232 {
 233         const struct pvr_fw_layout_entry *layout_entries = pvr_dev->fw_dev.layout_entries;
 234         u32 num_layout_entries = pvr_dev->fw_dev.header->layout_entry_num;
 235         u32 end_addr = addr + size;
 236         int entry = 0;
 237
 238         /* Ensure requested range is not zero, and size is not causing addr to overflow. */
 239         if (end_addr <= addr)
 240                 return -EINVAL;
 241
 242         for (entry = 0; entry < num_layout_entries; entry++) {
 243                 u32 entry_start_addr = layout_entries[entry].base_addr;
 244                 u32 entry_end_addr = entry_start_addr + layout_entries[entry].alloc_size;
 245
 246                 if (addr >= entry_start_addr && addr < entry_end_addr &&
 247                     end_addr > entry_start_addr && end_addr <= entry_end_addr) {
 248                         switch (layout_entries[entry].type) {
 249                         case FW_CODE:
 250                                 *host_addr_out = fw_code_ptr;
 251                                 break;
 252
 253                         case FW_DATA:
 254                                 *host_addr_out = fw_data_ptr;
 255                                 break;
 256
 257                         case FW_COREMEM_CODE:
 258                                 *host_addr_out = fw_core_code_ptr;
 259                                 break;
 260
 261                         case FW_COREMEM_DATA:
 262                                 *host_addr_out = fw_core_data_ptr;
 263                                 break;
 264
 265                         default:
 266                                 return -EINVAL;
 267                         }
 268                         /* Direct Mem write to mapped memory */
 269                         addr -= layout_entries[entry].base_addr;
 270                         addr += layout_entries[entry].alloc_offset;
 271
 272                         /*
 273                          * Add offset to pointer to FW allocation only if that
 274                          * allocation is available
 275                          */
 276                         *(u8 **)host_addr_out += addr;
 277                         return 0;
 278                 }
 279         }
 280
 281         return -EINVAL;
 282 }
 283
 284 static int
 285 pvr_fw_create_fwif_connection_ctl(struct pvr_device *pvr_dev)
 286 {
 287         struct drm_device *drm_dev = from_pvr_device(pvr_dev);
 288         struct pvr_fw_device *fw_dev = &pvr_dev->fw_dev;
 289
 290         fw_dev->fwif_connection_ctl =
 291                 pvr_fw_object_create_and_map_offset(pvr_dev,
 292                                                     fw_dev->fw_heap_info.config_offset +
 293                                                     PVR_ROGUE_FWIF_CONNECTION_CTL_OFFSET,
 294                                                     sizeof(*fw_dev->fwif_connection_ctl),
 295                                                     PVR_BO_FW_FLAGS_DEVICE_UNCACHED,
 296                                                     NULL, NULL,
 297                                                     &fw_dev->mem.fwif_connection_ctl_obj);
 298         if (IS_ERR(fw_dev->fwif_connection_ctl)) {
 299                 drm_err(drm_dev,
 300                         "Unable to allocate FWIF connection control memory\n");
 301                 return PTR_ERR(fw_dev->fwif_connection_ctl);
 302         }
 303
 304         return 0;
 305 }
 306
 307 static void
 308 pvr_fw_fini_fwif_connection_ctl(struct pvr_device *pvr_dev)
 309 {
 310         struct pvr_fw_device *fw_dev = &pvr_dev->fw_dev;
 311
 312         pvr_fw_object_unmap_and_destroy(fw_dev->mem.fwif_connection_ctl_obj);
 313 }
 314
 315 static void
 316 fw_osinit_init(void *cpu_ptr, void *priv)
 317 {
 318         struct rogue_fwif_osinit *fwif_osinit = cpu_ptr;
 319         struct pvr_device *pvr_dev = priv;
 320         struct pvr_fw_device *fw_dev = &pvr_dev->fw_dev;
 321         struct pvr_fw_mem *fw_mem = &fw_dev->mem;
 322
 323         fwif_osinit->kernel_ccbctl_fw_addr = pvr_dev->kccb.ccb.ctrl_fw_addr;
 324         fwif_osinit->kernel_ccb_fw_addr = pvr_dev->kccb.ccb.ccb_fw_addr;
 325         pvr_fw_object_get_fw_addr(pvr_dev->kccb.rtn_obj,
 326                                   &fwif_osinit->kernel_ccb_rtn_slots_fw_addr);
 327
 328         fwif_osinit->firmware_ccbctl_fw_addr = pvr_dev->fwccb.ctrl_fw_addr;
 329         fwif_osinit->firmware_ccb_fw_addr = pvr_dev->fwccb.ccb_fw_addr;
 330
 331         fwif_osinit->work_est_firmware_ccbctl_fw_addr = 0;
 332         fwif_osinit->work_est_firmware_ccb_fw_addr = 0;
 333
 334         pvr_fw_object_get_fw_addr(fw_mem->hwrinfobuf_obj,
 335                                   &fwif_osinit->rogue_fwif_hwr_info_buf_ctl_fw_addr);
 336         pvr_fw_object_get_fw_addr(fw_mem->osdata_obj, &fwif_osinit->fw_os_data_fw_addr);
 337
 338         fwif_osinit->hwr_debug_dump_limit = 0;
 339
 340         rogue_fwif_compchecks_bvnc_init(&fwif_osinit->rogue_comp_checks.hw_bvnc);
 341         rogue_fwif_compchecks_bvnc_init(&fwif_osinit->rogue_comp_checks.fw_bvnc);
 342 }
 343
 344 static void
 345 fw_osdata_init(void *cpu_ptr, void *priv)
 346 {
 347         struct rogue_fwif_osdata *fwif_osdata = cpu_ptr;
 348         struct pvr_device *pvr_dev = priv;
 349         struct pvr_fw_mem *fw_mem = &pvr_dev->fw_dev.mem;
 350
 351         pvr_fw_object_get_fw_addr(fw_mem->power_sync_obj, &fwif_osdata->power_sync_fw_addr);
 352 }
 353
 354 static void
 355 fw_fault_page_init(void *cpu_ptr, void *priv)
 356 {
 357         u32 *fault_page = cpu_ptr;
 358
 359         for (int i = 0; i < PVR_ROGUE_FAULT_PAGE_SIZE / sizeof(*fault_page); i++)
 360                 fault_page[i] = 0xdeadbee0;
 361 }
 362
 363 static void
 364 fw_sysinit_init(void *cpu_ptr, void *priv)
 365 {
 366         struct rogue_fwif_sysinit *fwif_sysinit = cpu_ptr;
 367         struct pvr_device *pvr_dev = priv;
 368         struct pvr_fw_device *fw_dev = &pvr_dev->fw_dev;
 369         struct pvr_fw_mem *fw_mem = &fw_dev->mem;
 370         dma_addr_t fault_dma_addr = 0;
 371         u32 clock_speed_hz = clk_get_rate(pvr_dev->core_clk);
 372
 373         WARN_ON(!clock_speed_hz);
 374
 375         WARN_ON(pvr_fw_object_get_dma_addr(fw_mem->fault_page_obj, 0, &fault_dma_addr));
 376         fwif_sysinit->fault_phys_addr = (u64)fault_dma_addr;
 377
 378         fwif_sysinit->pds_exec_base = ROGUE_PDSCODEDATA_HEAP_BASE;
 379         fwif_sysinit->usc_exec_base = ROGUE_USCCODE_HEAP_BASE;
 380
 381         pvr_fw_object_get_fw_addr(fw_mem->runtime_cfg_obj, &fwif_sysinit->runtime_cfg_fw_addr);
 382         pvr_fw_object_get_fw_addr(fw_dev->fw_trace.tracebuf_ctrl_obj,
 383                                   &fwif_sysinit->trace_buf_ctl_fw_addr);
 384         pvr_fw_object_get_fw_addr(fw_mem->sysdata_obj, &fwif_sysinit->fw_sys_data_fw_addr);
 385         pvr_fw_object_get_fw_addr(fw_mem->gpu_util_fwcb_obj,
 386                                   &fwif_sysinit->gpu_util_fw_cb_ctl_fw_addr);
 387         if (fw_mem->core_data_obj) {
 388                 pvr_fw_object_get_fw_addr(fw_mem->core_data_obj,
 389                                           &fwif_sysinit->coremem_data_store.fw_addr);
 390         }
 391
 392         /* Currently unsupported. */
 393         fwif_sysinit->counter_dump_ctl.buffer_fw_addr = 0;
 394         fwif_sysinit->counter_dump_ctl.size_in_dwords = 0;
 395
 396         /* Skip alignment checks. */
 397         fwif_sysinit->align_checks = 0;
 398
 399         fwif_sysinit->filter_flags = 0;
 400         fwif_sysinit->hw_perf_filter = 0;
 401         fwif_sysinit->firmware_perf = FW_PERF_CONF_NONE;
 402         fwif_sysinit->initial_core_clock_speed = clock_speed_hz;
 403         fwif_sysinit->active_pm_latency_ms = 0;
 404         fwif_sysinit->gpio_validation_mode = ROGUE_FWIF_GPIO_VAL_OFF;
 405         fwif_sysinit->firmware_started = false;
 406         fwif_sysinit->marker_val = 1;
 407
 408         memset(&fwif_sysinit->bvnc_km_feature_flags, 0,
 409                sizeof(fwif_sysinit->bvnc_km_feature_flags));
 410 }
 411
 412 #define ROGUE_FWIF_SLC_MIN_SIZE_FOR_DM_OVERLAP_KB 4
 413
 414 static void
 415 fw_sysdata_init(void *cpu_ptr, void *priv)
 416 {
 417         struct rogue_fwif_sysdata *fwif_sysdata = cpu_ptr;
 418         struct pvr_device *pvr_dev = priv;
 419         u32 slc_size_in_kilobytes = 0;
 420         u32 config_flags = 0;
 421
 422         WARN_ON(PVR_FEATURE_VALUE(pvr_dev, slc_size_in_kilobytes, &slc_size_in_kilobytes));
 423
 424         if (slc_size_in_kilobytes < ROGUE_FWIF_SLC_MIN_SIZE_FOR_DM_OVERLAP_KB)
 425                 config_flags |= ROGUE_FWIF_INICFG_DISABLE_DM_OVERLAP;
 426
 427         fwif_sysdata->config_flags = config_flags;
 428 }
 429
 430 static void
 431 fw_runtime_cfg_init(void *cpu_ptr, void *priv)
 432 {
 433         struct rogue_fwif_runtime_cfg *runtime_cfg = cpu_ptr;
 434         struct pvr_device *pvr_dev = priv;
 435         u32 clock_speed_hz = clk_get_rate(pvr_dev->core_clk);
 436
 437         WARN_ON(!clock_speed_hz);
 438
 439         runtime_cfg->core_clock_speed = clock_speed_hz;
 440         runtime_cfg->active_pm_latency_ms = 0;
 441         runtime_cfg->active_pm_latency_persistant = true;
 442         WARN_ON(PVR_FEATURE_VALUE(pvr_dev, num_clusters,
 443                                   &runtime_cfg->default_dusts_num_init) != 0);
 444 }
 445
 446 static void
 447 fw_gpu_util_fwcb_init(void *cpu_ptr, void *priv)
 448 {
 449         struct rogue_fwif_gpu_util_fwcb *gpu_util_fwcb = cpu_ptr;
 450
 451         gpu_util_fwcb->last_word = PVR_FWIF_GPU_UTIL_STATE_IDLE;
 452 }
 453
 454 static int
 455 pvr_fw_create_structures(struct pvr_device *pvr_dev)
 456 {
 457         struct drm_device *drm_dev = from_pvr_device(pvr_dev);
 458         struct pvr_fw_device *fw_dev = &pvr_dev->fw_dev;
 459         struct pvr_fw_mem *fw_mem = &fw_dev->mem;
 460         int err;
 461
 462         fw_dev->power_sync = pvr_fw_object_create_and_map(pvr_dev, sizeof(*fw_dev->power_sync),
 463                                                           PVR_BO_FW_FLAGS_DEVICE_UNCACHED,
 464                                                           NULL, NULL, &fw_mem->power_sync_obj);
 465         if (IS_ERR(fw_dev->power_sync)) {
 466                 drm_err(drm_dev, "Unable to allocate FW power_sync structure\n");
 467                 return PTR_ERR(fw_dev->power_sync);
 468         }
 469
 470         fw_dev->hwrinfobuf = pvr_fw_object_create_and_map(pvr_dev, sizeof(*fw_dev->hwrinfobuf),
 471                                                           PVR_BO_FW_FLAGS_DEVICE_UNCACHED,
 472                                                           NULL, NULL, &fw_mem->hwrinfobuf_obj);
 473         if (IS_ERR(fw_dev->hwrinfobuf)) {
 474                 drm_err(drm_dev,
 475                         "Unable to allocate FW hwrinfobuf structure\n");
 476                 err = PTR_ERR(fw_dev->hwrinfobuf);
 477                 goto err_release_power_sync;
 478         }
 479
 480         err = pvr_fw_object_create(pvr_dev, PVR_SYNC_OBJ_SIZE,
 481                                    PVR_BO_FW_FLAGS_DEVICE_UNCACHED,
 482                                    NULL, NULL, &fw_mem->mmucache_sync_obj);
 483         if (err) {
 484                 drm_err(drm_dev,
 485                         "Unable to allocate MMU cache sync object\n");
 486                 goto err_release_hwrinfobuf;
 487         }
 488
 489         fw_dev->fwif_sysdata = pvr_fw_object_create_and_map(pvr_dev,
 490                                                             sizeof(*fw_dev->fwif_sysdata),
 491                                                             PVR_BO_FW_FLAGS_DEVICE_UNCACHED,
 492                                                             fw_sysdata_init, pvr_dev,
 493                                                             &fw_mem->sysdata_obj);
 494         if (IS_ERR(fw_dev->fwif_sysdata)) {
 495                 drm_err(drm_dev, "Unable to allocate FW SYSDATA structure\n");
 496                 err = PTR_ERR(fw_dev->fwif_sysdata);
 497                 goto err_release_mmucache_sync_obj;
 498         }
 499
 500         err = pvr_fw_object_create(pvr_dev, PVR_ROGUE_FAULT_PAGE_SIZE,
 501                                    PVR_BO_FW_FLAGS_DEVICE_UNCACHED,
 502                                    fw_fault_page_init, NULL, &fw_mem->fault_page_obj);
 503         if (err) {
 504                 drm_err(drm_dev, "Unable to allocate FW fault page\n");
 505                 goto err_release_sysdata;
 506         }
 507
 508         err = pvr_fw_object_create(pvr_dev, sizeof(struct rogue_fwif_gpu_util_fwcb),
 509                                    PVR_BO_FW_FLAGS_DEVICE_UNCACHED,
 510                                    fw_gpu_util_fwcb_init, pvr_dev, &fw_mem->gpu_util_fwcb_obj);
 511         if (err) {
 512                 drm_err(drm_dev, "Unable to allocate GPU util FWCB\n");
 513                 goto err_release_fault_page;
 514         }
 515
 516         err = pvr_fw_object_create(pvr_dev, sizeof(struct rogue_fwif_runtime_cfg),
 517                                    PVR_BO_FW_FLAGS_DEVICE_UNCACHED,
 518                                    fw_runtime_cfg_init, pvr_dev, &fw_mem->runtime_cfg_obj);
 519         if (err) {
 520                 drm_err(drm_dev, "Unable to allocate FW runtime config\n");
 521                 goto err_release_gpu_util_fwcb;
 522         }
 523
 524         err = pvr_fw_trace_init(pvr_dev);
 525         if (err)
 526                 goto err_release_runtime_cfg;
 527
 528         fw_dev->fwif_osdata = pvr_fw_object_create_and_map(pvr_dev,
 529                                                            sizeof(*fw_dev->fwif_osdata),
 530                                                            PVR_BO_FW_FLAGS_DEVICE_UNCACHED,
 531                                                            fw_osdata_init, pvr_dev,
 532                                                            &fw_mem->osdata_obj);
 533         if (IS_ERR(fw_dev->fwif_osdata)) {
 534                 drm_err(drm_dev, "Unable to allocate FW OSDATA structure\n");
 535                 err = PTR_ERR(fw_dev->fwif_osdata);
 536                 goto err_fw_trace_fini;
 537         }
 538
 539         fw_dev->fwif_osinit =
 540                 pvr_fw_object_create_and_map_offset(pvr_dev,
 541                                                     fw_dev->fw_heap_info.config_offset +
 542                                                     PVR_ROGUE_FWIF_OSINIT_OFFSET,
 543                                                     sizeof(*fw_dev->fwif_osinit),
 544                                                     PVR_BO_FW_FLAGS_DEVICE_UNCACHED,
 545                                                     fw_osinit_init, pvr_dev, &fw_mem->osinit_obj);
 546         if (IS_ERR(fw_dev->fwif_osinit)) {
 547                 drm_err(drm_dev, "Unable to allocate FW OSINIT structure\n");
 548                 err = PTR_ERR(fw_dev->fwif_osinit);
 549                 goto err_release_osdata;
 550         }
 551
 552         fw_dev->fwif_sysinit =
 553                 pvr_fw_object_create_and_map_offset(pvr_dev,
 554                                                     fw_dev->fw_heap_info.config_offset +
 555                                                     PVR_ROGUE_FWIF_SYSINIT_OFFSET,
 556                                                     sizeof(*fw_dev->fwif_sysinit),
 557                                                     PVR_BO_FW_FLAGS_DEVICE_UNCACHED,
 558                                                     fw_sysinit_init, pvr_dev, &fw_mem->sysinit_obj);
 559         if (IS_ERR(fw_dev->fwif_sysinit)) {
 560                 drm_err(drm_dev, "Unable to allocate FW SYSINIT structure\n");
 561                 err = PTR_ERR(fw_dev->fwif_sysinit);
 562                 goto err_release_osinit;
 563         }
 564
 565         return 0;
 566
 567 err_release_osinit:
 568         pvr_fw_object_unmap_and_destroy(fw_mem->osinit_obj);
 569
 570 err_release_osdata:
 571         pvr_fw_object_unmap_and_destroy(fw_mem->osdata_obj);
 572
 573 err_fw_trace_fini:
 574         pvr_fw_trace_fini(pvr_dev);
 575
 576 err_release_runtime_cfg:
 577         pvr_fw_object_destroy(fw_mem->runtime_cfg_obj);
 578
 579 err_release_gpu_util_fwcb:
 580         pvr_fw_object_destroy(fw_mem->gpu_util_fwcb_obj);
 581
 582 err_release_fault_page:
 583         pvr_fw_object_destroy(fw_mem->fault_page_obj);
 584
 585 err_release_sysdata:
 586         pvr_fw_object_unmap_and_destroy(fw_mem->sysdata_obj);
 587
 588 err_release_mmucache_sync_obj:
 589         pvr_fw_object_destroy(fw_mem->mmucache_sync_obj);
 590
 591 err_release_hwrinfobuf:
 592         pvr_fw_object_unmap_and_destroy(fw_mem->hwrinfobuf_obj);
 593
 594 err_release_power_sync:
 595         pvr_fw_object_unmap_and_destroy(fw_mem->power_sync_obj);
 596
 597         return err;
 598 }
 599
 600 static void
 601 pvr_fw_destroy_structures(struct pvr_device *pvr_dev)
 602 {
 603         struct pvr_fw_device *fw_dev = &pvr_dev->fw_dev;
 604         struct pvr_fw_mem *fw_mem = &fw_dev->mem;
 605
 606         pvr_fw_trace_fini(pvr_dev);
 607         pvr_fw_object_destroy(fw_mem->runtime_cfg_obj);
 608         pvr_fw_object_destroy(fw_mem->gpu_util_fwcb_obj);
 609         pvr_fw_object_destroy(fw_mem->fault_page_obj);
 610         pvr_fw_object_unmap_and_destroy(fw_mem->sysdata_obj);
 611         pvr_fw_object_unmap_and_destroy(fw_mem->sysinit_obj);
 612
 613         pvr_fw_object_destroy(fw_mem->mmucache_sync_obj);
 614         pvr_fw_object_unmap_and_destroy(fw_mem->hwrinfobuf_obj);
 615         pvr_fw_object_unmap_and_destroy(fw_mem->power_sync_obj);
 616         pvr_fw_object_unmap_and_destroy(fw_mem->osdata_obj);
 617         pvr_fw_object_unmap_and_destroy(fw_mem->osinit_obj);
 618 }
 619
 620 /**
 621  * pvr_fw_process() - Process firmware image, allocate FW memory and create boot
 622  *                    arguments
 623  * @pvr_dev: Device pointer.
 624  *
 625  * Returns:
 626  *  * 0 on success, or
 627  *  * Any error returned by pvr_fw_object_create_and_map_offset(), or
 628  *  * Any error returned by pvr_fw_object_create_and_map().
 629  */
 630 static int
 631 pvr_fw_process(struct pvr_device *pvr_dev)
 632 {
 633         struct drm_device *drm_dev = from_pvr_device(pvr_dev);
 634         struct pvr_fw_mem *fw_mem = &pvr_dev->fw_dev.mem;
 635         const u8 *fw = pvr_dev->fw_dev.firmware->data;
 636         const struct pvr_fw_layout_entry *private_data;
 637         u8 *fw_code_ptr;
 638         u8 *fw_data_ptr;
 639         u8 *fw_core_code_ptr;
 640         u8 *fw_core_data_ptr;
 641         int err;
 642
 643         layout_get_sizes(pvr_dev);
 644
 645         private_data = pvr_fw_find_private_data(pvr_dev);
 646         if (!private_data)
 647                 return -EINVAL;
 648
 649         /* Allocate and map memory for firmware sections. */
 650
 651         /*
 652          * Code allocation must be at the start of the firmware heap, otherwise
 653          * firmware processor will be unable to boot.
 654          *
 655          * This has the useful side-effect that for every other object in the
 656          * driver, a firmware address of 0 is invalid.
 657          */
 658         fw_code_ptr = pvr_fw_object_create_and_map_offset(pvr_dev, 0, fw_mem->code_alloc_size,
 659                                                           PVR_BO_FW_FLAGS_DEVICE_UNCACHED,
 660                                                           NULL, NULL, &fw_mem->code_obj);
 661         if (IS_ERR(fw_code_ptr)) {
 662                 drm_err(drm_dev, "Unable to allocate FW code memory\n");
 663                 return PTR_ERR(fw_code_ptr);
 664         }
 665
 666         if (pvr_dev->fw_dev.defs->has_fixed_data_addr()) {
 667                 u32 base_addr = private_data->base_addr & pvr_dev->fw_dev.fw_heap_info.offset_mask;
 668
 669                 fw_data_ptr =
 670                         pvr_fw_object_create_and_map_offset(pvr_dev, base_addr,
 671                                                             fw_mem->data_alloc_size,
 672                                                             PVR_BO_FW_FLAGS_DEVICE_UNCACHED,
 673                                                             NULL, NULL, &fw_mem->data_obj);
 674         } else {
 675                 fw_data_ptr = pvr_fw_object_create_and_map(pvr_dev, fw_mem->data_alloc_size,
 676                                                            PVR_BO_FW_FLAGS_DEVICE_UNCACHED,
 677                                                            NULL, NULL, &fw_mem->data_obj);
 678         }
 679         if (IS_ERR(fw_data_ptr)) {
 680                 drm_err(drm_dev, "Unable to allocate FW data memory\n");
 681                 err = PTR_ERR(fw_data_ptr);
 682                 goto err_free_fw_code_obj;
 683         }
 684
 685         /* Core code and data sections are optional. */
 686         if (fw_mem->core_code_alloc_size) {
 687                 fw_core_code_ptr =
 688                         pvr_fw_object_create_and_map(pvr_dev, fw_mem->core_code_alloc_size,
 689                                                      PVR_BO_FW_FLAGS_DEVICE_UNCACHED,
 690                                                      NULL, NULL, &fw_mem->core_code_obj);
 691                 if (IS_ERR(fw_core_code_ptr)) {
 692                         drm_err(drm_dev,
 693                                 "Unable to allocate FW core code memory\n");
 694                         err = PTR_ERR(fw_core_code_ptr);
 695                         goto err_free_fw_data_obj;
 696                 }
 697         } else {
 698                 fw_core_code_ptr = NULL;
 699         }
 700
 701         if (fw_mem->core_data_alloc_size) {
 702                 fw_core_data_ptr =
 703                         pvr_fw_object_create_and_map(pvr_dev, fw_mem->core_data_alloc_size,
 704                                                      PVR_BO_FW_FLAGS_DEVICE_UNCACHED,
 705                                                      NULL, NULL, &fw_mem->core_data_obj);
 706                 if (IS_ERR(fw_core_data_ptr)) {
 707                         drm_err(drm_dev,
 708                                 "Unable to allocate FW core data memory\n");
 709                         err = PTR_ERR(fw_core_data_ptr);
 710                         goto err_free_fw_core_code_obj;
 711                 }
 712         } else {
 713                 fw_core_data_ptr = NULL;
 714         }
 715
 716         fw_mem->code = kzalloc(fw_mem->code_alloc_size, GFP_KERNEL);
 717         fw_mem->data = kzalloc(fw_mem->data_alloc_size, GFP_KERNEL);
 718         if (fw_mem->core_code_alloc_size)
 719                 fw_mem->core_code = kzalloc(fw_mem->core_code_alloc_size, GFP_KERNEL);
 720         if (fw_mem->core_data_alloc_size)
 721                 fw_mem->core_data = kzalloc(fw_mem->core_data_alloc_size, GFP_KERNEL);
 722
 723         if (!fw_mem->code || !fw_mem->data ||
 724             (!fw_mem->core_code && fw_mem->core_code_alloc_size) ||
 725             (!fw_mem->core_data && fw_mem->core_data_alloc_size)) {
 726                 err = -ENOMEM;
 727                 goto err_free_kdata;
 728         }
 729
 730         err = pvr_dev->fw_dev.defs->fw_process(pvr_dev, fw,
 731                                                fw_mem->code, fw_mem->data, fw_mem->core_code,
 732                                                fw_mem->core_data, fw_mem->core_code_alloc_size);
 733
 734         if (err)
 735                 goto err_free_fw_core_data_obj;
 736
 737         memcpy(fw_code_ptr, fw_mem->code, fw_mem->code_alloc_size);
 738         memcpy(fw_data_ptr, fw_mem->data, fw_mem->data_alloc_size);
 739         if (fw_mem->core_code)
 740                 memcpy(fw_core_code_ptr, fw_mem->core_code, fw_mem->core_code_alloc_size);
 741         if (fw_mem->core_data)
 742                 memcpy(fw_core_data_ptr, fw_mem->core_data, fw_mem->core_data_alloc_size);
 743
 744         /* We're finished with the firmware section memory on the CPU, unmap. */
 745         if (fw_core_data_ptr)
 746                 pvr_fw_object_vunmap(fw_mem->core_data_obj);
 747         if (fw_core_code_ptr)
 748                 pvr_fw_object_vunmap(fw_mem->core_code_obj);
 749         pvr_fw_object_vunmap(fw_mem->data_obj);
 750         fw_data_ptr = NULL;
 751         pvr_fw_object_vunmap(fw_mem->code_obj);
 752         fw_code_ptr = NULL;
 753
 754         err = pvr_fw_create_fwif_connection_ctl(pvr_dev);
 755         if (err)
 756                 goto err_free_fw_core_data_obj;
 757
 758         return 0;
 759
 760 err_free_kdata:
 761         kfree(fw_mem->core_data);
 762         kfree(fw_mem->core_code);
 763         kfree(fw_mem->data);
 764         kfree(fw_mem->code);
 765
 766 err_free_fw_core_data_obj:
 767         if (fw_core_data_ptr)
 768                 pvr_fw_object_unmap_and_destroy(fw_mem->core_data_obj);
 769
 770 err_free_fw_core_code_obj:
 771         if (fw_core_code_ptr)
 772                 pvr_fw_object_unmap_and_destroy(fw_mem->core_code_obj);
 773
 774 err_free_fw_data_obj:
 775         if (fw_data_ptr)
 776                 pvr_fw_object_vunmap(fw_mem->data_obj);
 777         pvr_fw_object_destroy(fw_mem->data_obj);
 778
 779 err_free_fw_code_obj:
 780         if (fw_code_ptr)
 781                 pvr_fw_object_vunmap(fw_mem->code_obj);
 782         pvr_fw_object_destroy(fw_mem->code_obj);
 783
 784         return err;
 785 }
 786
 787 static int
 788 pvr_copy_to_fw(struct pvr_fw_object *dest_obj, u8 *src_ptr, u32 size)
 789 {
 790         u8 *dest_ptr = pvr_fw_object_vmap(dest_obj);
 791
 792         if (IS_ERR(dest_ptr))
 793                 return PTR_ERR(dest_ptr);
 794
 795         memcpy(dest_ptr, src_ptr, size);
 796
 797         pvr_fw_object_vunmap(dest_obj);
 798
 799         return 0;
 800 }
 801
 802 static int
 803 pvr_fw_reinit_code_data(struct pvr_device *pvr_dev)
 804 {
 805         struct pvr_fw_mem *fw_mem = &pvr_dev->fw_dev.mem;
 806         int err;
 807
 808         err = pvr_copy_to_fw(fw_mem->code_obj, fw_mem->code, fw_mem->code_alloc_size);
 809         if (err)
 810                 return err;
 811
 812         err = pvr_copy_to_fw(fw_mem->data_obj, fw_mem->data, fw_mem->data_alloc_size);
 813         if (err)
 814                 return err;
 815
 816         if (fw_mem->core_code) {
 817                 err = pvr_copy_to_fw(fw_mem->core_code_obj, fw_mem->core_code,
 818                                      fw_mem->core_code_alloc_size);
 819                 if (err)
 820                         return err;
 821         }
 822
 823         if (fw_mem->core_data) {
 824                 err = pvr_copy_to_fw(fw_mem->core_data_obj, fw_mem->core_data,
 825                                      fw_mem->core_data_alloc_size);
 826                 if (err)
 827                         return err;
 828         }
 829
 830         return 0;
 831 }
 832
 833 static void
 834 pvr_fw_cleanup(struct pvr_device *pvr_dev)
 835 {
 836         struct pvr_fw_mem *fw_mem = &pvr_dev->fw_dev.mem;
 837
 838         pvr_fw_fini_fwif_connection_ctl(pvr_dev);
 839         if (fw_mem->core_code_obj)
 840                 pvr_fw_object_destroy(fw_mem->core_code_obj);
 841         if (fw_mem->core_data_obj)
 842                 pvr_fw_object_destroy(fw_mem->core_data_obj);
 843         pvr_fw_object_destroy(fw_mem->code_obj);
 844         pvr_fw_object_destroy(fw_mem->data_obj);
 845 }
 846
 847 /**
 848  * pvr_wait_for_fw_boot() - Wait for firmware to finish booting
 849  * @pvr_dev: Target PowerVR device.
 850  *
 851  * Returns:
 852  *  * 0 on success, or
 853  *  * -%ETIMEDOUT if firmware fails to boot within timeout.
 854  */
 855 int
 856 pvr_wait_for_fw_boot(struct pvr_device *pvr_dev)
 857 {
 858         ktime_t deadline = ktime_add_us(ktime_get(), FW_BOOT_TIMEOUT_USEC);
 859         struct pvr_fw_device *fw_dev = &pvr_dev->fw_dev;
 860
 861         while (ktime_to_ns(ktime_sub(deadline, ktime_get())) > 0) {
 862                 if (READ_ONCE(fw_dev->fwif_sysinit->firmware_started))
 863                         return 0;
 864         }
 865
 866         return -ETIMEDOUT;
 867 }
 868
 869 /*
 870  * pvr_fw_heap_info_init() - Calculate size and masks for FW heap
 871  * @pvr_dev: Target PowerVR device.
 872  * @log2_size: Log2 of raw heap size.
 873  * @reserved_size: Size of reserved area of heap, in bytes. May be zero.
 874  */
 875 void
 876 pvr_fw_heap_info_init(struct pvr_device *pvr_dev, u32 log2_size, u32 reserved_size)
 877 {
 878         struct pvr_fw_device *fw_dev = &pvr_dev->fw_dev;
 879
 880         fw_dev->fw_heap_info.gpu_addr = PVR_ROGUE_FW_MAIN_HEAP_BASE;
 881         fw_dev->fw_heap_info.log2_size = log2_size;
 882         fw_dev->fw_heap_info.reserved_size = reserved_size;
 883         fw_dev->fw_heap_info.raw_size = 1 << fw_dev->fw_heap_info.log2_size;
 884         fw_dev->fw_heap_info.offset_mask = fw_dev->fw_heap_info.raw_size - 1;
 885         fw_dev->fw_heap_info.config_offset = fw_dev->fw_heap_info.raw_size -
 886                                              PVR_ROGUE_FW_CONFIG_HEAP_SIZE;
 887         fw_dev->fw_heap_info.size = fw_dev->fw_heap_info.raw_size -
 888                                     (PVR_ROGUE_FW_CONFIG_HEAP_SIZE + reserved_size);
 889 }
 890
 891 /**
 892  * pvr_fw_validate_init_device_info() - Validate firmware and initialise device information
 893  * @pvr_dev: Target PowerVR device.
 894  *
 895  * This function must be called before querying device information.
 896  *
 897  * Returns:
 898  *  * 0 on success, or
 899  *  * -%EINVAL if firmware validation fails.
 900  */
 901 int
 902 pvr_fw_validate_init_device_info(struct pvr_device *pvr_dev)
 903 {
 904         int err;
 905
 906         err = pvr_fw_validate(pvr_dev);
 907         if (err)
 908                 return err;
 909
 910         return pvr_fw_get_device_info(pvr_dev);
 911 }
 912
 913 /**
 914  * pvr_fw_init() - Initialise and boot firmware
 915  * @pvr_dev: Target PowerVR device
 916  *
 917  * On successful completion of the function the PowerVR device will be
 918  * initialised and ready to use.
 919  *
 920  * Returns:
 921  *  * 0 on success,
 922  *  * -%EINVAL on invalid firmware image,
 923  *  * -%ENOMEM on out of memory, or
 924  *  * -%ETIMEDOUT if firmware processor fails to boot or on register poll timeout.
 925  */
 926 int
 927 pvr_fw_init(struct pvr_device *pvr_dev)
 928 {
 929         u32 kccb_size_log2 = ROGUE_FWIF_KCCB_NUMCMDS_LOG2_DEFAULT;
 930         u32 kccb_rtn_size = (1 << kccb_size_log2) * sizeof(*pvr_dev->kccb.rtn);
 931         struct pvr_fw_device *fw_dev = &pvr_dev->fw_dev;
 932         int err;
 933
 934         if (fw_dev->processor_type == PVR_FW_PROCESSOR_TYPE_META)
 935                 fw_dev->defs = &pvr_fw_defs_meta;
 936         else
 937                 return -EINVAL;
 938
 939         err = fw_dev->defs->init(pvr_dev);
 940         if (err)
 941                 return err;
 942
 943         drm_mm_init(&fw_dev->fw_mm, ROGUE_FW_HEAP_BASE, fw_dev->fw_heap_info.raw_size);
 944         fw_dev->fw_mm_base = ROGUE_FW_HEAP_BASE;
 945         spin_lock_init(&fw_dev->fw_mm_lock);
 946
 947         INIT_LIST_HEAD(&fw_dev->fw_objs.list);
 948         err = drmm_mutex_init(from_pvr_device(pvr_dev), &fw_dev->fw_objs.lock);
 949         if (err)
 950                 goto err_mm_takedown;
 951
 952         err = pvr_fw_process(pvr_dev);
 953         if (err)
 954                 goto err_mm_takedown;
 955
 956         /* Initialise KCCB and FWCCB. */
 957         err = pvr_kccb_init(pvr_dev);
 958         if (err)
 959                 goto err_fw_cleanup;
 960
 961         err = pvr_fwccb_init(pvr_dev);
 962         if (err)
 963                 goto err_kccb_fini;
 964
 965         /* Allocate memory for KCCB return slots. */
 966         pvr_dev->kccb.rtn = pvr_fw_object_create_and_map(pvr_dev, kccb_rtn_size,
 967                                                          PVR_BO_FW_FLAGS_DEVICE_UNCACHED,
 968                                                          NULL, NULL, &pvr_dev->kccb.rtn_obj);
 969         if (IS_ERR(pvr_dev->kccb.rtn)) {
 970                 err = PTR_ERR(pvr_dev->kccb.rtn);
 971                 goto err_fwccb_fini;
 972         }
 973
 974         err = pvr_fw_create_structures(pvr_dev);
 975         if (err)
 976                 goto err_kccb_rtn_release;
 977
 978         err = pvr_fw_start(pvr_dev);
 979         if (err)
 980                 goto err_destroy_structures;
 981
 982         err = pvr_wait_for_fw_boot(pvr_dev);
 983         if (err) {
 984                 drm_err(from_pvr_device(pvr_dev), "Firmware failed to boot\n");
 985                 goto err_fw_stop;
 986         }
 987
 988         fw_dev->booted = true;
 989
 990         return 0;
 991
 992 err_fw_stop:
 993         pvr_fw_stop(pvr_dev);
 994
 995 err_destroy_structures:
 996         pvr_fw_destroy_structures(pvr_dev);
 997
 998 err_kccb_rtn_release:
 999         pvr_fw_object_unmap_and_destroy(pvr_dev->kccb.rtn_obj);
1000
1001 err_fwccb_fini:
1002         pvr_ccb_fini(&pvr_dev->fwccb);
1003
1004 err_kccb_fini:
1005         pvr_kccb_fini(pvr_dev);
1006
1007 err_fw_cleanup:
1008         pvr_fw_cleanup(pvr_dev);
1009
1010 err_mm_takedown:
1011         drm_mm_takedown(&fw_dev->fw_mm);
1012
1013         if (fw_dev->defs->fini)
1014                 fw_dev->defs->fini(pvr_dev);
1015
1016         return err;
1017 }
1018
1019 /**
1020  * pvr_fw_fini() - Shutdown firmware processor and free associated memory
1021  * @pvr_dev: Target PowerVR device
1022  */
1023 void
1024 pvr_fw_fini(struct pvr_device *pvr_dev)
1025 {
1026         struct pvr_fw_device *fw_dev = &pvr_dev->fw_dev;
1027
1028         fw_dev->booted = false;
1029
1030         pvr_fw_destroy_structures(pvr_dev);
1031         pvr_fw_object_unmap_and_destroy(pvr_dev->kccb.rtn_obj);
1032
1033         /*
1034          * Ensure FWCCB worker has finished executing before destroying FWCCB. The IRQ handler has
1035          * been unregistered at this point so no new work should be being submitted.
1036          */
1037         pvr_ccb_fini(&pvr_dev->fwccb);
1038         pvr_kccb_fini(pvr_dev);
1039         pvr_fw_cleanup(pvr_dev);
1040
1041         mutex_lock(&pvr_dev->fw_dev.fw_objs.lock);
1042         WARN_ON(!list_empty(&pvr_dev->fw_dev.fw_objs.list));
1043         mutex_unlock(&pvr_dev->fw_dev.fw_objs.lock);
1044
1045         drm_mm_takedown(&fw_dev->fw_mm);
1046
1047         if (fw_dev->defs->fini)
1048                 fw_dev->defs->fini(pvr_dev);
1049 }
1050
1051 /**
1052  * pvr_fw_mts_schedule() - Schedule work via an MTS kick
1053  * @pvr_dev: Target PowerVR device
1054  * @val: Kick mask. Should be a combination of %ROGUE_CR_MTS_SCHEDULE_*
1055  */
1056 void
1057 pvr_fw_mts_schedule(struct pvr_device *pvr_dev, u32 val)
1058 {
1059         /* Ensure memory is flushed before kicking MTS. */
1060         wmb();
1061
1062         pvr_cr_write32(pvr_dev, ROGUE_CR_MTS_SCHEDULE, val);
1063
1064         /* Ensure the MTS kick goes through before continuing. */
1065         mb();
1066 }
1067
1068 /**
1069  * pvr_fw_structure_cleanup() - Send FW cleanup request for an object
1070  * @pvr_dev: Target PowerVR device.
1071  * @type: Type of object to cleanup. Must be one of &enum rogue_fwif_cleanup_type.
1072  * @fw_obj: Pointer to FW object containing object to cleanup.
1073  * @offset: Offset within FW object of object to cleanup.
1074  *
1075  * Returns:
1076  *  * 0 on success,
1077  *  * -EBUSY if object is busy,
1078  *  * -ETIMEDOUT on timeout, or
1079  *  * -EIO if device is lost.
1080  */
1081 int
1082 pvr_fw_structure_cleanup(struct pvr_device *pvr_dev, u32 type, struct pvr_fw_object *fw_obj,
1083                          u32 offset)
1084 {
1085         struct rogue_fwif_kccb_cmd cmd;
1086         int slot_nr;
1087         int idx;
1088         int err;
1089         u32 rtn;
1090
1091         struct rogue_fwif_cleanup_request *cleanup_req = &cmd.cmd_data.cleanup_data;
1092
1093         down_read(&pvr_dev->reset_sem);
1094
1095         if (!drm_dev_enter(from_pvr_device(pvr_dev), &idx)) {
1096                 err = -EIO;
1097                 goto err_up_read;
1098         }
1099
1100         cmd.cmd_type = ROGUE_FWIF_KCCB_CMD_CLEANUP;
1101         cmd.kccb_flags = 0;
1102         cleanup_req->cleanup_type = type;
1103
1104         switch (type) {
1105         case ROGUE_FWIF_CLEANUP_FWCOMMONCONTEXT:
1106                 pvr_fw_object_get_fw_addr_offset(fw_obj, offset,
1107                                                  &cleanup_req->cleanup_data.context_fw_addr);
1108                 break;
1109         case ROGUE_FWIF_CLEANUP_HWRTDATA:
1110                 pvr_fw_object_get_fw_addr_offset(fw_obj, offset,
1111                                                  &cleanup_req->cleanup_data.hwrt_data_fw_addr);
1112                 break;
1113         case ROGUE_FWIF_CLEANUP_FREELIST:
1114                 pvr_fw_object_get_fw_addr_offset(fw_obj, offset,
1115                                                  &cleanup_req->cleanup_data.freelist_fw_addr);
1116                 break;
1117         default:
1118                 err = -EINVAL;
1119                 goto err_drm_dev_exit;
1120         }
1121
1122         err = pvr_kccb_send_cmd(pvr_dev, &cmd, &slot_nr);
1123         if (err)
1124                 goto err_drm_dev_exit;
1125
1126         err = pvr_kccb_wait_for_completion(pvr_dev, slot_nr, HZ, &rtn);
1127         if (err)
1128                 goto err_drm_dev_exit;
1129
1130         if (rtn & ROGUE_FWIF_KCCB_RTN_SLOT_CLEANUP_BUSY)
1131                 err = -EBUSY;
1132
1133 err_drm_dev_exit:
1134         drm_dev_exit(idx);
1135
1136 err_up_read:
1137         up_read(&pvr_dev->reset_sem);
1138
1139         return err;
1140 }
1141
1142 /**
1143  * pvr_fw_object_fw_map() - Map a FW object in firmware address space
1144  * @pvr_dev: Device pointer.
1145  * @fw_obj: FW object to map.
1146  * @dev_addr: Desired address in device space, if a specific address is
1147  *            required. 0 otherwise.
1148  *
1149  * Returns:
1150  *  * 0 on success, or
1151  *  * -%EINVAL if @fw_obj is already mapped but has no references, or
1152  *  * Any error returned by DRM.
1153  */
1154 static int
1155 pvr_fw_object_fw_map(struct pvr_device *pvr_dev, struct pvr_fw_object *fw_obj, u64 dev_addr)
1156 {
1157         struct pvr_gem_object *pvr_obj = fw_obj->gem;
1158         struct drm_gem_object *gem_obj = gem_from_pvr_gem(pvr_obj);
1159         struct pvr_fw_device *fw_dev = &pvr_dev->fw_dev;
1160
1161         int err;
1162
1163         spin_lock(&fw_dev->fw_mm_lock);
1164
1165         if (drm_mm_node_allocated(&fw_obj->fw_mm_node)) {
1166                 err = -EINVAL;
1167                 goto err_unlock;
1168         }
1169
1170         if (!dev_addr) {
1171                 /*
1172                  * Allocate from the main heap only (firmware heap minus
1173                  * config space).
1174                  */
1175                 err = drm_mm_insert_node_in_range(&fw_dev->fw_mm, &fw_obj->fw_mm_node,
1176                                                   gem_obj->size, 0, 0,
1177                                                   fw_dev->fw_heap_info.gpu_addr,
1178                                                   fw_dev->fw_heap_info.gpu_addr +
1179                                                   fw_dev->fw_heap_info.size, 0);
1180                 if (err)
1181                         goto err_unlock;
1182         } else {
1183                 fw_obj->fw_mm_node.start = dev_addr;
1184                 fw_obj->fw_mm_node.size = gem_obj->size;
1185                 err = drm_mm_reserve_node(&fw_dev->fw_mm, &fw_obj->fw_mm_node);
1186                 if (err)
1187                         goto err_unlock;
1188         }
1189
1190         spin_unlock(&fw_dev->fw_mm_lock);
1191
1192         /* Map object on GPU. */
1193         err = fw_dev->defs->vm_map(pvr_dev, fw_obj);
1194         if (err)
1195                 goto err_remove_node;
1196
1197         fw_obj->fw_addr_offset = (u32)(fw_obj->fw_mm_node.start - fw_dev->fw_mm_base);
1198
1199         return 0;
1200
1201 err_remove_node:
1202         spin_lock(&fw_dev->fw_mm_lock);
1203         drm_mm_remove_node(&fw_obj->fw_mm_node);
1204
1205 err_unlock:
1206         spin_unlock(&fw_dev->fw_mm_lock);
1207
1208         return err;
1209 }
1210
1211 /**
1212  * pvr_fw_object_fw_unmap() - Unmap a previously mapped FW object
1213  * @fw_obj: FW object to unmap.
1214  *
1215  * Returns:
1216  *  * 0 on success, or
1217  *  * -%EINVAL if object is not currently mapped.
1218  */
1219 static int
1220 pvr_fw_object_fw_unmap(struct pvr_fw_object *fw_obj)
1221 {
1222         struct pvr_gem_object *pvr_obj = fw_obj->gem;
1223         struct drm_gem_object *gem_obj = gem_from_pvr_gem(pvr_obj);
1224         struct pvr_device *pvr_dev = to_pvr_device(gem_obj->dev);
1225         struct pvr_fw_device *fw_dev = &pvr_dev->fw_dev;
1226
1227         fw_dev->defs->vm_unmap(pvr_dev, fw_obj);
1228
1229         spin_lock(&fw_dev->fw_mm_lock);
1230
1231         if (!drm_mm_node_allocated(&fw_obj->fw_mm_node)) {
1232                 spin_unlock(&fw_dev->fw_mm_lock);
1233                 return -EINVAL;
1234         }
1235
1236         drm_mm_remove_node(&fw_obj->fw_mm_node);
1237
1238         spin_unlock(&fw_dev->fw_mm_lock);
1239
1240         return 0;
1241 }
1242
1243 static void *
1244 pvr_fw_object_create_and_map_common(struct pvr_device *pvr_dev, size_t size,
1245                                     u64 flags, u64 dev_addr,
1246                                     void (*init)(void *cpu_ptr, void *priv),
1247                                     void *init_priv, struct pvr_fw_object **fw_obj_out)
1248 {
1249         struct pvr_fw_object *fw_obj;
1250         void *cpu_ptr;
1251         int err;
1252
1253         /* %DRM_PVR_BO_PM_FW_PROTECT is implicit for FW objects. */
1254         flags |= DRM_PVR_BO_PM_FW_PROTECT;
1255
1256         fw_obj = kzalloc(sizeof(*fw_obj), GFP_KERNEL);
1257         if (!fw_obj)
1258                 return ERR_PTR(-ENOMEM);
1259
1260         INIT_LIST_HEAD(&fw_obj->node);
1261         fw_obj->init = init;
1262         fw_obj->init_priv = init_priv;
1263
1264         fw_obj->gem = pvr_gem_object_create(pvr_dev, size, flags);
1265         if (IS_ERR(fw_obj->gem)) {
1266                 err = PTR_ERR(fw_obj->gem);
1267                 fw_obj->gem = NULL;
1268                 goto err_put_object;
1269         }
1270
1271         err = pvr_fw_object_fw_map(pvr_dev, fw_obj, dev_addr);
1272         if (err)
1273                 goto err_put_object;
1274
1275         cpu_ptr = pvr_fw_object_vmap(fw_obj);
1276         if (IS_ERR(cpu_ptr)) {
1277                 err = PTR_ERR(cpu_ptr);
1278                 goto err_put_object;
1279         }
1280
1281         *fw_obj_out = fw_obj;
1282
1283         if (fw_obj->init)
1284                 fw_obj->init(cpu_ptr, fw_obj->init_priv);
1285
1286         mutex_lock(&pvr_dev->fw_dev.fw_objs.lock);
1287         list_add_tail(&fw_obj->node, &pvr_dev->fw_dev.fw_objs.list);
1288         mutex_unlock(&pvr_dev->fw_dev.fw_objs.lock);
1289
1290         return cpu_ptr;
1291
1292 err_put_object:
1293         pvr_fw_object_destroy(fw_obj);
1294
1295         return ERR_PTR(err);
1296 }
1297
1298 /**
1299  * pvr_fw_object_create() - Create a FW object and map to firmware
1300  * @pvr_dev: PowerVR device pointer.
1301  * @size: Size of object, in bytes.
1302  * @flags: Options which affect both this operation and future mapping
1303  * operations performed on the returned object. Must be a combination of
1304  * DRM_PVR_BO_* and/or PVR_BO_* flags.
1305  * @init: Initialisation callback.
1306  * @init_priv: Private pointer to pass to initialisation callback.
1307  * @fw_obj_out: Pointer to location to store created object pointer.
1308  *
1309  * %DRM_PVR_BO_DEVICE_PM_FW_PROTECT is implied for all FW objects. Consequently,
1310  * this function will fail if @flags has %DRM_PVR_BO_CPU_ALLOW_USERSPACE_ACCESS
1311  * set.
1312  *
1313  * Returns:
1314  *  * 0 on success, or
1315  *  * Any error returned by pvr_fw_object_create_common().
1316  */
1317 int
1318 pvr_fw_object_create(struct pvr_device *pvr_dev, size_t size, u64 flags,
1319                      void (*init)(void *cpu_ptr, void *priv), void *init_priv,
1320                      struct pvr_fw_object **fw_obj_out)
1321 {
1322         void *cpu_ptr;
1323
1324         cpu_ptr = pvr_fw_object_create_and_map_common(pvr_dev, size, flags, 0, init, init_priv,
1325                                                       fw_obj_out);
1326         if (IS_ERR(cpu_ptr))
1327                 return PTR_ERR(cpu_ptr);
1328
1329         pvr_fw_object_vunmap(*fw_obj_out);
1330
1331         return 0;
1332 }
1333
1334 /**
1335  * pvr_fw_object_create_and_map() - Create a FW object and map to firmware and CPU
1336  * @pvr_dev: PowerVR device pointer.
1337  * @size: Size of object, in bytes.
1338  * @flags: Options which affect both this operation and future mapping
1339  * operations performed on the returned object. Must be a combination of
1340  * DRM_PVR_BO_* and/or PVR_BO_* flags.
1341  * @init: Initialisation callback.
1342  * @init_priv: Private pointer to pass to initialisation callback.
1343  * @fw_obj_out: Pointer to location to store created object pointer.
1344  *
1345  * %DRM_PVR_BO_DEVICE_PM_FW_PROTECT is implied for all FW objects. Consequently,
1346  * this function will fail if @flags has %DRM_PVR_BO_CPU_ALLOW_USERSPACE_ACCESS
1347  * set.
1348  *
1349  * Caller is responsible for calling pvr_fw_object_vunmap() to release the CPU
1350  * mapping.
1351  *
1352  * Returns:
1353  *  * Pointer to CPU mapping of newly created object, or
1354  *  * Any error returned by pvr_fw_object_create(), or
1355  *  * Any error returned by pvr_fw_object_vmap().
1356  */
1357 void *
1358 pvr_fw_object_create_and_map(struct pvr_device *pvr_dev, size_t size, u64 flags,
1359                              void (*init)(void *cpu_ptr, void *priv),
1360                              void *init_priv, struct pvr_fw_object **fw_obj_out)
1361 {
1362         return pvr_fw_object_create_and_map_common(pvr_dev, size, flags, 0, init, init_priv,
1363                                                    fw_obj_out);
1364 }
1365
1366 /**
1367  * pvr_fw_object_create_and_map_offset() - Create a FW object and map to
1368  * firmware at the provided offset and to the CPU.
1369  * @pvr_dev: PowerVR device pointer.
1370  * @dev_offset: Base address of desired FW mapping, offset from start of FW heap.
1371  * @size: Size of object, in bytes.
1372  * @flags: Options which affect both this operation and future mapping
1373  * operations performed on the returned object. Must be a combination of
1374  * DRM_PVR_BO_* and/or PVR_BO_* flags.
1375  * @init: Initialisation callback.
1376  * @init_priv: Private pointer to pass to initialisation callback.
1377  * @fw_obj_out: Pointer to location to store created object pointer.
1378  *
1379  * %DRM_PVR_BO_DEVICE_PM_FW_PROTECT is implied for all FW objects. Consequently,
1380  * this function will fail if @flags has %DRM_PVR_BO_CPU_ALLOW_USERSPACE_ACCESS
1381  * set.
1382  *
1383  * Caller is responsible for calling pvr_fw_object_vunmap() to release the CPU
1384  * mapping.
1385  *
1386  * Returns:
1387  *  * Pointer to CPU mapping of newly created object, or
1388  *  * Any error returned by pvr_fw_object_create(), or
1389  *  * Any error returned by pvr_fw_object_vmap().
1390  */
1391 void *
1392 pvr_fw_object_create_and_map_offset(struct pvr_device *pvr_dev,
1393                                     u32 dev_offset, size_t size, u64 flags,
1394                                     void (*init)(void *cpu_ptr, void *priv),
1395                                     void *init_priv, struct pvr_fw_object **fw_obj_out)
1396 {
1397         u64 dev_addr = pvr_dev->fw_dev.fw_mm_base + dev_offset;
1398
1399         return pvr_fw_object_create_and_map_common(pvr_dev, size, flags, dev_addr, init, init_priv,
1400                                                    fw_obj_out);
1401 }
1402
1403 /**
1404  * pvr_fw_object_destroy() - Destroy a pvr_fw_object
1405  * @fw_obj: Pointer to object to destroy.
1406  */
1407 void pvr_fw_object_destroy(struct pvr_fw_object *fw_obj)
1408 {
1409         struct pvr_gem_object *pvr_obj = fw_obj->gem;
1410         struct drm_gem_object *gem_obj = gem_from_pvr_gem(pvr_obj);
1411         struct pvr_device *pvr_dev = to_pvr_device(gem_obj->dev);
1412
1413         mutex_lock(&pvr_dev->fw_dev.fw_objs.lock);
1414         list_del(&fw_obj->node);
1415         mutex_unlock(&pvr_dev->fw_dev.fw_objs.lock);
1416
1417         if (drm_mm_node_allocated(&fw_obj->fw_mm_node)) {
1418                 /* If we can't unmap, leak the memory. */
1419                 if (WARN_ON(pvr_fw_object_fw_unmap(fw_obj)))
1420                         return;
1421         }
1422
1423         if (fw_obj->gem)
1424                 pvr_gem_object_put(fw_obj->gem);
1425
1426         kfree(fw_obj);
1427 }
1428
1429 /**
1430  * pvr_fw_object_get_fw_addr_offset() - Return address of object in firmware address space, with
1431  * given offset.
1432  * @fw_obj: Pointer to object.
1433  * @offset: Desired offset from start of object.
1434  * @fw_addr_out: Location to store address to.
1435  */
1436 void pvr_fw_object_get_fw_addr_offset(struct pvr_fw_object *fw_obj, u32 offset, u32 *fw_addr_out)
1437 {
1438         struct pvr_gem_object *pvr_obj = fw_obj->gem;
1439         struct pvr_device *pvr_dev = to_pvr_device(gem_from_pvr_gem(pvr_obj)->dev);
1440
1441         *fw_addr_out = pvr_dev->fw_dev.defs->get_fw_addr_with_offset(fw_obj, offset);
1442 }
1443
1444 /*
1445  * pvr_fw_hard_reset() - Re-initialise the FW code and data segments, and reset all global FW
1446  *                       structures
1447  * @pvr_dev: Device pointer
1448  *
1449  * If this function returns an error then the caller must regard the device as lost.
1450  *
1451  * Returns:
1452  *  * 0 on success, or
1453  *  * Any error returned by pvr_fw_init_dev_structures() or pvr_fw_reset_all().
1454  */
1455 int
1456 pvr_fw_hard_reset(struct pvr_device *pvr_dev)
1457 {
1458         struct list_head *pos;
1459         int err;
1460
1461         /* Reset all FW objects */
1462         mutex_lock(&pvr_dev->fw_dev.fw_objs.lock);
1463
1464         list_for_each(pos, &pvr_dev->fw_dev.fw_objs.list) {
1465                 struct pvr_fw_object *fw_obj = container_of(pos, struct pvr_fw_object, node);
1466                 void *cpu_ptr = pvr_fw_object_vmap(fw_obj);
1467
1468                 WARN_ON(IS_ERR(cpu_ptr));
1469
1470                 if (!(fw_obj->gem->flags & PVR_BO_FW_NO_CLEAR_ON_RESET)) {
1471                         memset(cpu_ptr, 0, pvr_gem_object_size(fw_obj->gem));
1472
1473                         if (fw_obj->init)
1474                                 fw_obj->init(cpu_ptr, fw_obj->init_priv);
1475                 }
1476
1477                 pvr_fw_object_vunmap(fw_obj);
1478         }
1479
1480         mutex_unlock(&pvr_dev->fw_dev.fw_objs.lock);
1481
1482         err = pvr_fw_reinit_code_data(pvr_dev);
1483         if (err)
1484                 return err;
1485
1486         return 0;
1487 }