drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c

   1 // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
   2 /* Copyright(c) 2020 - 2021 Intel Corporation */
   3 #include <linux/iopoll.h>
   4 #include <adf_accel_devices.h>
   5 #include <adf_admin.h>
   6 #include <adf_cfg.h>
   7 #include <adf_cfg_services.h>
   8 #include <adf_clock.h>
   9 #include <adf_common_drv.h>
  10 #include <adf_fw_config.h>
  11 #include <adf_gen4_config.h>
  12 #include <adf_gen4_dc.h>
  13 #include <adf_gen4_hw_data.h>
  14 #include <adf_gen4_pfvf.h>
  15 #include <adf_gen4_pm.h>
  16 #include "adf_gen4_ras.h"
  17 #include <adf_gen4_timer.h>
  18 #include <adf_gen4_tl.h>
  19 #include "adf_4xxx_hw_data.h"
  20 #include "icp_qat_hw.h"
  21
  22 #define ADF_AE_GROUP_0          GENMASK(3, 0)
  23 #define ADF_AE_GROUP_1          GENMASK(7, 4)
  24 #define ADF_AE_GROUP_2          BIT(8)
  25
  26 #define ENA_THD_MASK_ASYM       GENMASK(1, 0)
  27 #define ENA_THD_MASK_ASYM_401XX GENMASK(5, 0)
  28 #define ENA_THD_MASK_SYM        GENMASK(6, 0)
  29 #define ENA_THD_MASK_DC         GENMASK(1, 0)
  30
  31 static const char * const adf_4xxx_fw_objs[] = {
  32         [ADF_FW_SYM_OBJ] =  ADF_4XXX_SYM_OBJ,
  33         [ADF_FW_ASYM_OBJ] =  ADF_4XXX_ASYM_OBJ,
  34         [ADF_FW_DC_OBJ] =  ADF_4XXX_DC_OBJ,
  35         [ADF_FW_ADMIN_OBJ] = ADF_4XXX_ADMIN_OBJ,
  36 };
  37
  38 static const char * const adf_402xx_fw_objs[] = {
  39         [ADF_FW_SYM_OBJ] =  ADF_402XX_SYM_OBJ,
  40         [ADF_FW_ASYM_OBJ] =  ADF_402XX_ASYM_OBJ,
  41         [ADF_FW_DC_OBJ] =  ADF_402XX_DC_OBJ,
  42         [ADF_FW_ADMIN_OBJ] = ADF_402XX_ADMIN_OBJ,
  43 };
  44
  45 static const struct adf_fw_config adf_fw_cy_config[] = {
  46         {ADF_AE_GROUP_1, ADF_FW_SYM_OBJ},
  47         {ADF_AE_GROUP_0, ADF_FW_ASYM_OBJ},
  48         {ADF_AE_GROUP_2, ADF_FW_ADMIN_OBJ},
  49 };
  50
  51 static const struct adf_fw_config adf_fw_dc_config[] = {
  52         {ADF_AE_GROUP_1, ADF_FW_DC_OBJ},
  53         {ADF_AE_GROUP_0, ADF_FW_DC_OBJ},
  54         {ADF_AE_GROUP_2, ADF_FW_ADMIN_OBJ},
  55 };
  56
  57 static const struct adf_fw_config adf_fw_sym_config[] = {
  58         {ADF_AE_GROUP_1, ADF_FW_SYM_OBJ},
  59         {ADF_AE_GROUP_0, ADF_FW_SYM_OBJ},
  60         {ADF_AE_GROUP_2, ADF_FW_ADMIN_OBJ},
  61 };
  62
  63 static const struct adf_fw_config adf_fw_asym_config[] = {
  64         {ADF_AE_GROUP_1, ADF_FW_ASYM_OBJ},
  65         {ADF_AE_GROUP_0, ADF_FW_ASYM_OBJ},
  66         {ADF_AE_GROUP_2, ADF_FW_ADMIN_OBJ},
  67 };
  68
  69 static const struct adf_fw_config adf_fw_asym_dc_config[] = {
  70         {ADF_AE_GROUP_1, ADF_FW_ASYM_OBJ},
  71         {ADF_AE_GROUP_0, ADF_FW_DC_OBJ},
  72         {ADF_AE_GROUP_2, ADF_FW_ADMIN_OBJ},
  73 };
  74
  75 static const struct adf_fw_config adf_fw_sym_dc_config[] = {
  76         {ADF_AE_GROUP_1, ADF_FW_SYM_OBJ},
  77         {ADF_AE_GROUP_0, ADF_FW_DC_OBJ},
  78         {ADF_AE_GROUP_2, ADF_FW_ADMIN_OBJ},
  79 };
  80
  81 static const struct adf_fw_config adf_fw_dcc_config[] = {
  82         {ADF_AE_GROUP_1, ADF_FW_DC_OBJ},
  83         {ADF_AE_GROUP_0, ADF_FW_SYM_OBJ},
  84         {ADF_AE_GROUP_2, ADF_FW_ADMIN_OBJ},
  85 };
  86
  87 static_assert(ARRAY_SIZE(adf_fw_cy_config) == ARRAY_SIZE(adf_fw_dc_config));
  88 static_assert(ARRAY_SIZE(adf_fw_cy_config) == ARRAY_SIZE(adf_fw_sym_config));
  89 static_assert(ARRAY_SIZE(adf_fw_cy_config) == ARRAY_SIZE(adf_fw_asym_config));
  90 static_assert(ARRAY_SIZE(adf_fw_cy_config) == ARRAY_SIZE(adf_fw_asym_dc_config));
  91 static_assert(ARRAY_SIZE(adf_fw_cy_config) == ARRAY_SIZE(adf_fw_sym_dc_config));
  92 static_assert(ARRAY_SIZE(adf_fw_cy_config) == ARRAY_SIZE(adf_fw_dcc_config));
  93
  94 static struct adf_hw_device_class adf_4xxx_class = {
  95         .name = ADF_4XXX_DEVICE_NAME,
  96         .type = DEV_4XXX,
  97         .instances = 0,
  98 };
  99
 100 static u32 get_ae_mask(struct adf_hw_device_data *self)
 101 {
 102         u32 me_disable = self->fuses;
 103
 104         return ~me_disable & ADF_4XXX_ACCELENGINES_MASK;
 105 }
 106
 107 static u32 get_accel_cap(struct adf_accel_dev *accel_dev)
 108 {
 109         struct pci_dev *pdev = accel_dev->accel_pci_dev.pci_dev;
 110         u32 capabilities_sym, capabilities_asym, capabilities_dc;
 111         u32 capabilities_dcc;
 112         u32 fusectl1;
 113
 114         /* Read accelerator capabilities mask */
 115         pci_read_config_dword(pdev, ADF_GEN4_FUSECTL1_OFFSET, &fusectl1);
 116
 117         capabilities_sym = ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC |
 118                           ICP_ACCEL_CAPABILITIES_CIPHER |
 119                           ICP_ACCEL_CAPABILITIES_AUTHENTICATION |
 120                           ICP_ACCEL_CAPABILITIES_SHA3 |
 121                           ICP_ACCEL_CAPABILITIES_SHA3_EXT |
 122                           ICP_ACCEL_CAPABILITIES_HKDF |
 123                           ICP_ACCEL_CAPABILITIES_CHACHA_POLY |
 124                           ICP_ACCEL_CAPABILITIES_AESGCM_SPC |
 125                           ICP_ACCEL_CAPABILITIES_SM3 |
 126                           ICP_ACCEL_CAPABILITIES_SM4 |
 127                           ICP_ACCEL_CAPABILITIES_AES_V2;
 128
 129         /* A set bit in fusectl1 means the feature is OFF in this SKU */
 130         if (fusectl1 & ICP_ACCEL_GEN4_MASK_CIPHER_SLICE) {
 131                 capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC;
 132                 capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_HKDF;
 133                 capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_CIPHER;
 134         }
 135
 136         if (fusectl1 & ICP_ACCEL_GEN4_MASK_UCS_SLICE) {
 137                 capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_CHACHA_POLY;
 138                 capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_AESGCM_SPC;
 139                 capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_AES_V2;
 140                 capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_CIPHER;
 141         }
 142
 143         if (fusectl1 & ICP_ACCEL_GEN4_MASK_AUTH_SLICE) {
 144                 capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_AUTHENTICATION;
 145                 capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_SHA3;
 146                 capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_SHA3_EXT;
 147                 capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_CIPHER;
 148         }
 149
 150         if (fusectl1 & ICP_ACCEL_GEN4_MASK_SMX_SLICE) {
 151                 capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_SM3;
 152                 capabilities_sym &= ~ICP_ACCEL_CAPABILITIES_SM4;
 153         }
 154
 155         capabilities_asym = ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC |
 156                           ICP_ACCEL_CAPABILITIES_CIPHER |
 157                           ICP_ACCEL_CAPABILITIES_SM2 |
 158                           ICP_ACCEL_CAPABILITIES_ECEDMONT;
 159
 160         if (fusectl1 & ICP_ACCEL_GEN4_MASK_PKE_SLICE) {
 161                 capabilities_asym &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_ASYMMETRIC;
 162                 capabilities_asym &= ~ICP_ACCEL_CAPABILITIES_SM2;
 163                 capabilities_asym &= ~ICP_ACCEL_CAPABILITIES_ECEDMONT;
 164         }
 165
 166         capabilities_dc = ICP_ACCEL_CAPABILITIES_COMPRESSION |
 167                           ICP_ACCEL_CAPABILITIES_LZ4_COMPRESSION |
 168                           ICP_ACCEL_CAPABILITIES_LZ4S_COMPRESSION |
 169                           ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY64;
 170
 171         if (fusectl1 & ICP_ACCEL_GEN4_MASK_COMPRESS_SLICE) {
 172                 capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_COMPRESSION;
 173                 capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_LZ4_COMPRESSION;
 174                 capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_LZ4S_COMPRESSION;
 175                 capabilities_dc &= ~ICP_ACCEL_CAPABILITIES_CNV_INTEGRITY64;
 176         }
 177
 178         switch (adf_get_service_enabled(accel_dev)) {
 179         case SVC_CY:
 180         case SVC_CY2:
 181                 return capabilities_sym | capabilities_asym;
 182         case SVC_DC:
 183                 return capabilities_dc;
 184         case SVC_DCC:
 185                 /*
 186                  * Sym capabilities are available for chaining operations,
 187                  * but sym crypto instances cannot be supported
 188                  */
 189                 capabilities_dcc = capabilities_dc | capabilities_sym;
 190                 capabilities_dcc &= ~ICP_ACCEL_CAPABILITIES_CRYPTO_SYMMETRIC;
 191                 return capabilities_dcc;
 192         case SVC_SYM:
 193                 return capabilities_sym;
 194         case SVC_ASYM:
 195                 return capabilities_asym;
 196         case SVC_ASYM_DC:
 197         case SVC_DC_ASYM:
 198                 return capabilities_asym | capabilities_dc;
 199         case SVC_SYM_DC:
 200         case SVC_DC_SYM:
 201                 return capabilities_sym | capabilities_dc;
 202         default:
 203                 return 0;
 204         }
 205 }
 206
 207 static const u32 *adf_get_arbiter_mapping(struct adf_accel_dev *accel_dev)
 208 {
 209         if (adf_gen4_init_thd2arb_map(accel_dev))
 210                 dev_warn(&GET_DEV(accel_dev),
 211                          "Generate of the thread to arbiter map failed");
 212
 213         return GET_HW_DATA(accel_dev)->thd_to_arb_map;
 214 }
 215
 216 static void adf_init_rl_data(struct adf_rl_hw_data *rl_data)
 217 {
 218         rl_data->pciout_tb_offset = ADF_GEN4_RL_TOKEN_PCIEOUT_BUCKET_OFFSET;
 219         rl_data->pciin_tb_offset = ADF_GEN4_RL_TOKEN_PCIEIN_BUCKET_OFFSET;
 220         rl_data->r2l_offset = ADF_GEN4_RL_R2L_OFFSET;
 221         rl_data->l2c_offset = ADF_GEN4_RL_L2C_OFFSET;
 222         rl_data->c2s_offset = ADF_GEN4_RL_C2S_OFFSET;
 223
 224         rl_data->pcie_scale_div = ADF_4XXX_RL_PCIE_SCALE_FACTOR_DIV;
 225         rl_data->pcie_scale_mul = ADF_4XXX_RL_PCIE_SCALE_FACTOR_MUL;
 226         rl_data->dcpr_correction = ADF_4XXX_RL_DCPR_CORRECTION;
 227         rl_data->max_tp[ADF_SVC_ASYM] = ADF_4XXX_RL_MAX_TP_ASYM;
 228         rl_data->max_tp[ADF_SVC_SYM] = ADF_4XXX_RL_MAX_TP_SYM;
 229         rl_data->max_tp[ADF_SVC_DC] = ADF_4XXX_RL_MAX_TP_DC;
 230         rl_data->scan_interval = ADF_4XXX_RL_SCANS_PER_SEC;
 231         rl_data->scale_ref = ADF_4XXX_RL_SLICE_REF;
 232 }
 233
 234 static u32 uof_get_num_objs(struct adf_accel_dev *accel_dev)
 235 {
 236         return ARRAY_SIZE(adf_fw_cy_config);
 237 }
 238
 239 static const struct adf_fw_config *get_fw_config(struct adf_accel_dev *accel_dev)
 240 {
 241         switch (adf_get_service_enabled(accel_dev)) {
 242         case SVC_CY:
 243         case SVC_CY2:
 244                 return adf_fw_cy_config;
 245         case SVC_DC:
 246                 return adf_fw_dc_config;
 247         case SVC_DCC:
 248                 return adf_fw_dcc_config;
 249         case SVC_SYM:
 250                 return adf_fw_sym_config;
 251         case SVC_ASYM:
 252                 return adf_fw_asym_config;
 253         case SVC_ASYM_DC:
 254         case SVC_DC_ASYM:
 255                 return adf_fw_asym_dc_config;
 256         case SVC_SYM_DC:
 257         case SVC_DC_SYM:
 258                 return adf_fw_sym_dc_config;
 259         default:
 260                 return NULL;
 261         }
 262 }
 263
 264 static int get_rp_group(struct adf_accel_dev *accel_dev, u32 ae_mask)
 265 {
 266         switch (ae_mask) {
 267         case ADF_AE_GROUP_0:
 268                 return RP_GROUP_0;
 269         case ADF_AE_GROUP_1:
 270                 return RP_GROUP_1;
 271         default:
 272                 dev_dbg(&GET_DEV(accel_dev), "ae_mask not recognized");
 273                 return -EINVAL;
 274         }
 275 }
 276
 277 static u32 get_ena_thd_mask(struct adf_accel_dev *accel_dev, u32 obj_num)
 278 {
 279         const struct adf_fw_config *fw_config;
 280
 281         if (obj_num >= uof_get_num_objs(accel_dev))
 282                 return ADF_GEN4_ENA_THD_MASK_ERROR;
 283
 284         fw_config = get_fw_config(accel_dev);
 285         if (!fw_config)
 286                 return ADF_GEN4_ENA_THD_MASK_ERROR;
 287
 288         switch (fw_config[obj_num].obj) {
 289         case ADF_FW_ASYM_OBJ:
 290                 return ENA_THD_MASK_ASYM;
 291         case ADF_FW_SYM_OBJ:
 292                 return ENA_THD_MASK_SYM;
 293         case ADF_FW_DC_OBJ:
 294                 return ENA_THD_MASK_DC;
 295         default:
 296                 return ADF_GEN4_ENA_THD_MASK_ERROR;
 297         }
 298 }
 299
 300 static u32 get_ena_thd_mask_401xx(struct adf_accel_dev *accel_dev, u32 obj_num)
 301 {
 302         const struct adf_fw_config *fw_config;
 303
 304         if (obj_num >= uof_get_num_objs(accel_dev))
 305                 return ADF_GEN4_ENA_THD_MASK_ERROR;
 306
 307         fw_config = get_fw_config(accel_dev);
 308         if (!fw_config)
 309                 return ADF_GEN4_ENA_THD_MASK_ERROR;
 310
 311         switch (fw_config[obj_num].obj) {
 312         case ADF_FW_ASYM_OBJ:
 313                 return ENA_THD_MASK_ASYM_401XX;
 314         case ADF_FW_SYM_OBJ:
 315                 return ENA_THD_MASK_SYM;
 316         case ADF_FW_DC_OBJ:
 317                 return ENA_THD_MASK_DC;
 318         default:
 319                 return ADF_GEN4_ENA_THD_MASK_ERROR;
 320         }
 321 }
 322
 323 static u16 get_ring_to_svc_map(struct adf_accel_dev *accel_dev)
 324 {
 325         enum adf_cfg_service_type rps[RP_GROUP_COUNT];
 326         const struct adf_fw_config *fw_config;
 327         u16 ring_to_svc_map;
 328         int i, j;
 329
 330         fw_config = get_fw_config(accel_dev);
 331         if (!fw_config)
 332                 return 0;
 333
 334         for (i = 0; i < RP_GROUP_COUNT; i++) {
 335                 switch (fw_config[i].ae_mask) {
 336                 case ADF_AE_GROUP_0:
 337                         j = RP_GROUP_0;
 338                         break;
 339                 case ADF_AE_GROUP_1:
 340                         j = RP_GROUP_1;
 341                         break;
 342                 default:
 343                         return 0;
 344                 }
 345
 346                 switch (fw_config[i].obj) {
 347                 case ADF_FW_SYM_OBJ:
 348                         rps[j] = SYM;
 349                         break;
 350                 case ADF_FW_ASYM_OBJ:
 351                         rps[j] = ASYM;
 352                         break;
 353                 case ADF_FW_DC_OBJ:
 354                         rps[j] = COMP;
 355                         break;
 356                 default:
 357                         rps[j] = 0;
 358                         break;
 359                 }
 360         }
 361
 362         ring_to_svc_map = rps[RP_GROUP_0] << ADF_CFG_SERV_RING_PAIR_0_SHIFT |
 363                           rps[RP_GROUP_1] << ADF_CFG_SERV_RING_PAIR_1_SHIFT |
 364                           rps[RP_GROUP_0] << ADF_CFG_SERV_RING_PAIR_2_SHIFT |
 365                           rps[RP_GROUP_1] << ADF_CFG_SERV_RING_PAIR_3_SHIFT;
 366
 367         return ring_to_svc_map;
 368 }
 369
 370 static const char *uof_get_name(struct adf_accel_dev *accel_dev, u32 obj_num,
 371                                 const char * const fw_objs[], int num_objs)
 372 {
 373         const struct adf_fw_config *fw_config;
 374         int id;
 375
 376         fw_config = get_fw_config(accel_dev);
 377         if (fw_config)
 378                 id = fw_config[obj_num].obj;
 379         else
 380                 id = -EINVAL;
 381
 382         if (id < 0 || id > num_objs)
 383                 return NULL;
 384
 385         return fw_objs[id];
 386 }
 387
 388 static const char *uof_get_name_4xxx(struct adf_accel_dev *accel_dev, u32 obj_num)
 389 {
 390         int num_fw_objs = ARRAY_SIZE(adf_4xxx_fw_objs);
 391
 392         return uof_get_name(accel_dev, obj_num, adf_4xxx_fw_objs, num_fw_objs);
 393 }
 394
 395 static const char *uof_get_name_402xx(struct adf_accel_dev *accel_dev, u32 obj_num)
 396 {
 397         int num_fw_objs = ARRAY_SIZE(adf_402xx_fw_objs);
 398
 399         return uof_get_name(accel_dev, obj_num, adf_402xx_fw_objs, num_fw_objs);
 400 }
 401
 402 static u32 uof_get_ae_mask(struct adf_accel_dev *accel_dev, u32 obj_num)
 403 {
 404         const struct adf_fw_config *fw_config;
 405
 406         fw_config = get_fw_config(accel_dev);
 407         if (!fw_config)
 408                 return 0;
 409
 410         return fw_config[obj_num].ae_mask;
 411 }
 412
 413 static void adf_gen4_set_err_mask(struct adf_dev_err_mask *dev_err_mask)
 414 {
 415         dev_err_mask->cppagentcmdpar_mask = ADF_4XXX_HICPPAGENTCMDPARERRLOG_MASK;
 416         dev_err_mask->parerr_ath_cph_mask = ADF_4XXX_PARITYERRORMASK_ATH_CPH_MASK;
 417         dev_err_mask->parerr_cpr_xlt_mask = ADF_4XXX_PARITYERRORMASK_CPR_XLT_MASK;
 418         dev_err_mask->parerr_dcpr_ucs_mask = ADF_4XXX_PARITYERRORMASK_DCPR_UCS_MASK;
 419         dev_err_mask->parerr_pke_mask = ADF_4XXX_PARITYERRORMASK_PKE_MASK;
 420         dev_err_mask->ssmfeatren_mask = ADF_4XXX_SSMFEATREN_MASK;
 421 }
 422
 423 void adf_init_hw_data_4xxx(struct adf_hw_device_data *hw_data, u32 dev_id)
 424 {
 425         hw_data->dev_class = &adf_4xxx_class;
 426         hw_data->instance_id = adf_4xxx_class.instances++;
 427         hw_data->num_banks = ADF_GEN4_ETR_MAX_BANKS;
 428         hw_data->num_banks_per_vf = ADF_GEN4_NUM_BANKS_PER_VF;
 429         hw_data->num_rings_per_bank = ADF_GEN4_NUM_RINGS_PER_BANK;
 430         hw_data->num_accel = ADF_GEN4_MAX_ACCELERATORS;
 431         hw_data->num_engines = ADF_4XXX_MAX_ACCELENGINES;
 432         hw_data->num_logical_accel = 1;
 433         hw_data->tx_rx_gap = ADF_GEN4_RX_RINGS_OFFSET;
 434         hw_data->tx_rings_mask = ADF_GEN4_TX_RINGS_MASK;
 435         hw_data->ring_to_svc_map = ADF_GEN4_DEFAULT_RING_TO_SRV_MAP;
 436         hw_data->alloc_irq = adf_isr_resource_alloc;
 437         hw_data->free_irq = adf_isr_resource_free;
 438         hw_data->enable_error_correction = adf_gen4_enable_error_correction;
 439         hw_data->get_accel_mask = adf_gen4_get_accel_mask;
 440         hw_data->get_ae_mask = get_ae_mask;
 441         hw_data->get_num_accels = adf_gen4_get_num_accels;
 442         hw_data->get_num_aes = adf_gen4_get_num_aes;
 443         hw_data->get_sram_bar_id = adf_gen4_get_sram_bar_id;
 444         hw_data->get_etr_bar_id = adf_gen4_get_etr_bar_id;
 445         hw_data->get_misc_bar_id = adf_gen4_get_misc_bar_id;
 446         hw_data->get_arb_info = adf_gen4_get_arb_info;
 447         hw_data->get_admin_info = adf_gen4_get_admin_info;
 448         hw_data->get_accel_cap = get_accel_cap;
 449         hw_data->get_sku = adf_gen4_get_sku;
 450         hw_data->init_admin_comms = adf_init_admin_comms;
 451         hw_data->exit_admin_comms = adf_exit_admin_comms;
 452         hw_data->send_admin_init = adf_send_admin_init;
 453         hw_data->init_arb = adf_init_arb;
 454         hw_data->exit_arb = adf_exit_arb;
 455         hw_data->get_arb_mapping = adf_get_arbiter_mapping;
 456         hw_data->enable_ints = adf_gen4_enable_ints;
 457         hw_data->init_device = adf_gen4_init_device;
 458         hw_data->reset_device = adf_reset_flr;
 459         hw_data->admin_ae_mask = ADF_4XXX_ADMIN_AE_MASK;
 460         hw_data->num_rps = ADF_GEN4_MAX_RPS;
 461         switch (dev_id) {
 462         case ADF_402XX_PCI_DEVICE_ID:
 463                 hw_data->fw_name = ADF_402XX_FW;
 464                 hw_data->fw_mmp_name = ADF_402XX_MMP;
 465                 hw_data->uof_get_name = uof_get_name_402xx;
 466                 hw_data->get_ena_thd_mask = get_ena_thd_mask;
 467                 break;
 468         case ADF_401XX_PCI_DEVICE_ID:
 469                 hw_data->fw_name = ADF_4XXX_FW;
 470                 hw_data->fw_mmp_name = ADF_4XXX_MMP;
 471                 hw_data->uof_get_name = uof_get_name_4xxx;
 472                 hw_data->get_ena_thd_mask = get_ena_thd_mask_401xx;
 473                 break;
 474         default:
 475                 hw_data->fw_name = ADF_4XXX_FW;
 476                 hw_data->fw_mmp_name = ADF_4XXX_MMP;
 477                 hw_data->uof_get_name = uof_get_name_4xxx;
 478                 hw_data->get_ena_thd_mask = get_ena_thd_mask;
 479                 break;
 480         }
 481         hw_data->uof_get_num_objs = uof_get_num_objs;
 482         hw_data->uof_get_ae_mask = uof_get_ae_mask;
 483         hw_data->get_rp_group = get_rp_group;
 484         hw_data->set_msix_rttable = adf_gen4_set_msix_default_rttable;
 485         hw_data->set_ssm_wdtimer = adf_gen4_set_ssm_wdtimer;
 486         hw_data->get_ring_to_svc_map = get_ring_to_svc_map;
 487         hw_data->disable_iov = adf_disable_sriov;
 488         hw_data->ring_pair_reset = adf_gen4_ring_pair_reset;
 489         hw_data->enable_pm = adf_gen4_enable_pm;
 490         hw_data->handle_pm_interrupt = adf_gen4_handle_pm_interrupt;
 491         hw_data->dev_config = adf_gen4_dev_config;
 492         hw_data->start_timer = adf_gen4_timer_start;
 493         hw_data->stop_timer = adf_gen4_timer_stop;
 494         hw_data->get_hb_clock = adf_gen4_get_heartbeat_clock;
 495         hw_data->num_hb_ctrs = ADF_NUM_HB_CNT_PER_AE;
 496         hw_data->clock_frequency = ADF_4XXX_AE_FREQ;
 497
 498         adf_gen4_set_err_mask(&hw_data->dev_err_mask);
 499         adf_gen4_init_hw_csr_ops(&hw_data->csr_ops);
 500         adf_gen4_init_pf_pfvf_ops(&hw_data->pfvf_ops);
 501         adf_gen4_init_dc_ops(&hw_data->dc_ops);
 502         adf_gen4_init_ras_ops(&hw_data->ras_ops);
 503         adf_gen4_init_tl_data(&hw_data->tl_data);
 504         adf_init_rl_data(&hw_data->rl_data);
 505 }
 506
 507 void adf_clean_hw_data_4xxx(struct adf_hw_device_data *hw_data)
 508 {
 509         hw_data->dev_class->instances--;
 510 }