drivers/net/wireless/intel/iwlwifi/pcie/trans.c

   1 /******************************************************************************
   2  *
   3  * This file is provided under a dual BSD/GPLv2 license.  When using or
   4  * redistributing this file, you may do so under either license.
   5  *
   6  * GPL LICENSE SUMMARY
   7  *
   8  * Copyright(c) 2007 - 2015 Intel Corporation. All rights reserved.
   9  * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
  10  * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
  11  * Copyright(c) 2018 Intel Corporation
  12  *
  13  * This program is free software; you can redistribute it and/or modify
  14  * it under the terms of version 2 of the GNU General Public License as
  15  * published by the Free Software Foundation.
  16  *
  17  * This program is distributed in the hope that it will be useful, but
  18  * WITHOUT ANY WARRANTY; without even the implied warranty of
  19  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  20  * General Public License for more details.
  21  *
  22  * The full GNU General Public License is included in this distribution
  23  * in the file called COPYING.
  24  *
  25  * Contact Information:
  26  *  Intel Linux Wireless <linuxwifi@intel.com>
  27  * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
  28  *
  29  * BSD LICENSE
  30  *
  31  * Copyright(c) 2005 - 2015 Intel Corporation. All rights reserved.
  32  * Copyright(c) 2013 - 2015 Intel Mobile Communications GmbH
  33  * Copyright(c) 2016 - 2017 Intel Deutschland GmbH
  34  * Copyright(c) 2018 Intel Corporation
  35  * All rights reserved.
  36  *
  37  * Redistribution and use in source and binary forms, with or without
  38  * modification, are permitted provided that the following conditions
  39  * are met:
  40  *
  41  *  * Redistributions of source code must retain the above copyright
  42  *    notice, this list of conditions and the following disclaimer.
  43  *  * Redistributions in binary form must reproduce the above copyright
  44  *    notice, this list of conditions and the following disclaimer in
  45  *    the documentation and/or other materials provided with the
  46  *    distribution.
  47  *  * Neither the name Intel Corporation nor the names of its
  48  *    contributors may be used to endorse or promote products derived
  49  *    from this software without specific prior written permission.
  50  *
  51  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  52  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  53  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  54  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  55  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  56  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  57  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  58  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  59  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  60  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  61  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  62  *
  63  *****************************************************************************/
  64 #include <linux/pci.h>
  65 #include <linux/pci-aspm.h>
  66 #include <linux/interrupt.h>
  67 #include <linux/debugfs.h>
  68 #include <linux/sched.h>
  69 #include <linux/bitops.h>
  70 #include <linux/gfp.h>
  71 #include <linux/vmalloc.h>
  72 #include <linux/pm_runtime.h>
  73 #include <linux/module.h>
  74
  75 #include "iwl-drv.h"
  76 #include "iwl-trans.h"
  77 #include "iwl-csr.h"
  78 #include "iwl-prph.h"
  79 #include "iwl-scd.h"
  80 #include "iwl-agn-hw.h"
  81 #include "fw/error-dump.h"
  82 #include "fw/dbg.h"
  83 #include "internal.h"
  84 #include "iwl-fh.h"
  85
  86 /* extended range in FW SRAM */
  87 #define IWL_FW_MEM_EXTENDED_START       0x40000
  88 #define IWL_FW_MEM_EXTENDED_END         0x57FFF
  89
  90 void iwl_trans_pcie_dump_regs(struct iwl_trans *trans)
  91 {
  92 #define PCI_DUMP_SIZE   64
  93 #define PREFIX_LEN      32
  94         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
  95         struct pci_dev *pdev = trans_pcie->pci_dev;
  96         u32 i, pos, alloc_size, *ptr, *buf;
  97         char *prefix;
  98
  99         if (trans_pcie->pcie_dbg_dumped_once)
 100                 return;
 101
 102         /* Should be a multiple of 4 */
 103         BUILD_BUG_ON(PCI_DUMP_SIZE > 4096 || PCI_DUMP_SIZE & 0x3);
 104         /* Alloc a max size buffer */
 105         if (PCI_ERR_ROOT_ERR_SRC +  4 > PCI_DUMP_SIZE)
 106                 alloc_size = PCI_ERR_ROOT_ERR_SRC +  4 + PREFIX_LEN;
 107         else
 108                 alloc_size = PCI_DUMP_SIZE + PREFIX_LEN;
 109         buf = kmalloc(alloc_size, GFP_ATOMIC);
 110         if (!buf)
 111                 return;
 112         prefix = (char *)buf + alloc_size - PREFIX_LEN;
 113
 114         IWL_ERR(trans, "iwlwifi transaction failed, dumping registers\n");
 115
 116         /* Print wifi device registers */
 117         sprintf(prefix, "iwlwifi %s: ", pci_name(pdev));
 118         IWL_ERR(trans, "iwlwifi device config registers:\n");
 119         for (i = 0, ptr = buf; i < PCI_DUMP_SIZE; i += 4, ptr++)
 120                 if (pci_read_config_dword(pdev, i, ptr))
 121                         goto err_read;
 122         print_hex_dump(KERN_ERR, prefix, DUMP_PREFIX_OFFSET, 32, 4, buf, i, 0);
 123
 124         IWL_ERR(trans, "iwlwifi device memory mapped registers:\n");
 125         for (i = 0, ptr = buf; i < PCI_DUMP_SIZE; i += 4, ptr++)
 126                 *ptr = iwl_read32(trans, i);
 127         print_hex_dump(KERN_ERR, prefix, DUMP_PREFIX_OFFSET, 32, 4, buf, i, 0);
 128
 129         pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ERR);
 130         if (pos) {
 131                 IWL_ERR(trans, "iwlwifi device AER capability structure:\n");
 132                 for (i = 0, ptr = buf; i < PCI_ERR_ROOT_COMMAND; i += 4, ptr++)
 133                         if (pci_read_config_dword(pdev, pos + i, ptr))
 134                                 goto err_read;
 135                 print_hex_dump(KERN_ERR, prefix, DUMP_PREFIX_OFFSET,
 136                                32, 4, buf, i, 0);
 137         }
 138
 139         /* Print parent device registers next */
 140         if (!pdev->bus->self)
 141                 goto out;
 142
 143         pdev = pdev->bus->self;
 144         sprintf(prefix, "iwlwifi %s: ", pci_name(pdev));
 145
 146         IWL_ERR(trans, "iwlwifi parent port (%s) config registers:\n",
 147                 pci_name(pdev));
 148         for (i = 0, ptr = buf; i < PCI_DUMP_SIZE; i += 4, ptr++)
 149                 if (pci_read_config_dword(pdev, i, ptr))
 150                         goto err_read;
 151         print_hex_dump(KERN_ERR, prefix, DUMP_PREFIX_OFFSET, 32, 4, buf, i, 0);
 152
 153         /* Print root port AER registers */
 154         pos = 0;
 155         pdev = pcie_find_root_port(pdev);
 156         if (pdev)
 157                 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_ERR);
 158         if (pos) {
 159                 IWL_ERR(trans, "iwlwifi root port (%s) AER cap structure:\n",
 160                         pci_name(pdev));
 161                 sprintf(prefix, "iwlwifi %s: ", pci_name(pdev));
 162                 for (i = 0, ptr = buf; i <= PCI_ERR_ROOT_ERR_SRC; i += 4, ptr++)
 163                         if (pci_read_config_dword(pdev, pos + i, ptr))
 164                                 goto err_read;
 165                 print_hex_dump(KERN_ERR, prefix, DUMP_PREFIX_OFFSET, 32,
 166                                4, buf, i, 0);
 167         }
 168         goto out;
 169
 170 err_read:
 171         print_hex_dump(KERN_ERR, prefix, DUMP_PREFIX_OFFSET, 32, 4, buf, i, 0);
 172         IWL_ERR(trans, "Read failed at 0x%X\n", i);
 173 out:
 174         trans_pcie->pcie_dbg_dumped_once = 1;
 175         kfree(buf);
 176 }
 177
 178 static void iwl_trans_pcie_sw_reset(struct iwl_trans *trans)
 179 {
 180         /* Reset entire device - do controller reset (results in SHRD_HW_RST) */
 181         iwl_set_bit(trans, trans->cfg->csr->addr_sw_reset,
 182                     BIT(trans->cfg->csr->flag_sw_reset));
 183         usleep_range(5000, 6000);
 184 }
 185
 186 static void iwl_pcie_free_fw_monitor(struct iwl_trans *trans)
 187 {
 188         int i;
 189
 190         for (i = 0; i < trans->num_blocks; i++) {
 191                 dma_free_coherent(trans->dev, trans->fw_mon[i].size,
 192                                   trans->fw_mon[i].block,
 193                                   trans->fw_mon[i].physical);
 194                 trans->fw_mon[i].block = NULL;
 195                 trans->fw_mon[i].physical = 0;
 196                 trans->fw_mon[i].size = 0;
 197                 trans->num_blocks--;
 198         }
 199 }
 200
 201 static void iwl_pcie_alloc_fw_monitor_block(struct iwl_trans *trans,
 202                                             u8 max_power, u8 min_power)
 203 {
 204         void *cpu_addr = NULL;
 205         dma_addr_t phys = 0;
 206         u32 size = 0;
 207         u8 power;
 208
 209         for (power = max_power; power >= min_power; power--) {
 210                 size = BIT(power);
 211                 cpu_addr = dma_alloc_coherent(trans->dev, size, &phys,
 212                                               GFP_KERNEL | __GFP_NOWARN |
 213                                               __GFP_ZERO | __GFP_COMP);
 214                 if (!cpu_addr)
 215                         continue;
 216
 217                 IWL_INFO(trans,
 218                          "Allocated 0x%08x bytes for firmware monitor.\n",
 219                          size);
 220                 break;
 221         }
 222
 223         if (WARN_ON_ONCE(!cpu_addr))
 224                 return;
 225
 226         if (power != max_power)
 227                 IWL_ERR(trans,
 228                         "Sorry - debug buffer is only %luK while you requested %luK\n",
 229                         (unsigned long)BIT(power - 10),
 230                         (unsigned long)BIT(max_power - 10));
 231
 232         trans->fw_mon[trans->num_blocks].block = cpu_addr;
 233         trans->fw_mon[trans->num_blocks].physical = phys;
 234         trans->fw_mon[trans->num_blocks].size = size;
 235         trans->num_blocks++;
 236 }
 237
 238 void iwl_pcie_alloc_fw_monitor(struct iwl_trans *trans, u8 max_power)
 239 {
 240         if (!max_power) {
 241                 /* default max_power is maximum */
 242                 max_power = 26;
 243         } else {
 244                 max_power += 11;
 245         }
 246
 247         if (WARN(max_power > 26,
 248                  "External buffer size for monitor is too big %d, check the FW TLV\n",
 249                  max_power))
 250                 return;
 251
 252         /*
 253          * This function allocats the default fw monitor.
 254          * The optional additional ones will be allocated in runtime
 255          */
 256         if (trans->num_blocks)
 257                 return;
 258
 259         iwl_pcie_alloc_fw_monitor_block(trans, max_power, 11);
 260 }
 261
 262 static u32 iwl_trans_pcie_read_shr(struct iwl_trans *trans, u32 reg)
 263 {
 264         iwl_write32(trans, HEEP_CTRL_WRD_PCIEX_CTRL_REG,
 265                     ((reg & 0x0000ffff) | (2 << 28)));
 266         return iwl_read32(trans, HEEP_CTRL_WRD_PCIEX_DATA_REG);
 267 }
 268
 269 static void iwl_trans_pcie_write_shr(struct iwl_trans *trans, u32 reg, u32 val)
 270 {
 271         iwl_write32(trans, HEEP_CTRL_WRD_PCIEX_DATA_REG, val);
 272         iwl_write32(trans, HEEP_CTRL_WRD_PCIEX_CTRL_REG,
 273                     ((reg & 0x0000ffff) | (3 << 28)));
 274 }
 275
 276 static void iwl_pcie_set_pwr(struct iwl_trans *trans, bool vaux)
 277 {
 278         if (trans->cfg->apmg_not_supported)
 279                 return;
 280
 281         if (vaux && pci_pme_capable(to_pci_dev(trans->dev), PCI_D3cold))
 282                 iwl_set_bits_mask_prph(trans, APMG_PS_CTRL_REG,
 283                                        APMG_PS_CTRL_VAL_PWR_SRC_VAUX,
 284                                        ~APMG_PS_CTRL_MSK_PWR_SRC);
 285         else
 286                 iwl_set_bits_mask_prph(trans, APMG_PS_CTRL_REG,
 287                                        APMG_PS_CTRL_VAL_PWR_SRC_VMAIN,
 288                                        ~APMG_PS_CTRL_MSK_PWR_SRC);
 289 }
 290
 291 /* PCI registers */
 292 #define PCI_CFG_RETRY_TIMEOUT   0x041
 293
 294 void iwl_pcie_apm_config(struct iwl_trans *trans)
 295 {
 296         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 297         u16 lctl;
 298         u16 cap;
 299
 300         /*
 301          * HW bug W/A for instability in PCIe bus L0S->L1 transition.
 302          * Check if BIOS (or OS) enabled L1-ASPM on this device.
 303          * If so (likely), disable L0S, so device moves directly L0->L1;
 304          *    costs negligible amount of power savings.
 305          * If not (unlikely), enable L0S, so there is at least some
 306          *    power savings, even without L1.
 307          */
 308         pcie_capability_read_word(trans_pcie->pci_dev, PCI_EXP_LNKCTL, &lctl);
 309         if (lctl & PCI_EXP_LNKCTL_ASPM_L1)
 310                 iwl_set_bit(trans, CSR_GIO_REG, CSR_GIO_REG_VAL_L0S_ENABLED);
 311         else
 312                 iwl_clear_bit(trans, CSR_GIO_REG, CSR_GIO_REG_VAL_L0S_ENABLED);
 313         trans->pm_support = !(lctl & PCI_EXP_LNKCTL_ASPM_L0S);
 314
 315         pcie_capability_read_word(trans_pcie->pci_dev, PCI_EXP_DEVCTL2, &cap);
 316         trans->ltr_enabled = cap & PCI_EXP_DEVCTL2_LTR_EN;
 317         IWL_DEBUG_POWER(trans, "L1 %sabled - LTR %sabled\n",
 318                         (lctl & PCI_EXP_LNKCTL_ASPM_L1) ? "En" : "Dis",
 319                         trans->ltr_enabled ? "En" : "Dis");
 320 }
 321
 322 /*
 323  * Start up NIC's basic functionality after it has been reset
 324  * (e.g. after platform boot, or shutdown via iwl_pcie_apm_stop())
 325  * NOTE:  This does not load uCode nor start the embedded processor
 326  */
 327 static int iwl_pcie_apm_init(struct iwl_trans *trans)
 328 {
 329         int ret;
 330
 331         IWL_DEBUG_INFO(trans, "Init card's basic functions\n");
 332
 333         /*
 334          * Use "set_bit" below rather than "write", to preserve any hardware
 335          * bits already set by default after reset.
 336          */
 337
 338         /* Disable L0S exit timer (platform NMI Work/Around) */
 339         if (trans->cfg->device_family < IWL_DEVICE_FAMILY_8000)
 340                 iwl_set_bit(trans, CSR_GIO_CHICKEN_BITS,
 341                             CSR_GIO_CHICKEN_BITS_REG_BIT_DIS_L0S_EXIT_TIMER);
 342
 343         /*
 344          * Disable L0s without affecting L1;
 345          *  don't wait for ICH L0s (ICH bug W/A)
 346          */
 347         iwl_set_bit(trans, CSR_GIO_CHICKEN_BITS,
 348                     CSR_GIO_CHICKEN_BITS_REG_BIT_L1A_NO_L0S_RX);
 349
 350         /* Set FH wait threshold to maximum (HW error during stress W/A) */
 351         iwl_set_bit(trans, CSR_DBG_HPET_MEM_REG, CSR_DBG_HPET_MEM_REG_VAL);
 352
 353         /*
 354          * Enable HAP INTA (interrupt from management bus) to
 355          * wake device's PCI Express link L1a -> L0s
 356          */
 357         iwl_set_bit(trans, CSR_HW_IF_CONFIG_REG,
 358                     CSR_HW_IF_CONFIG_REG_BIT_HAP_WAKE_L1A);
 359
 360         iwl_pcie_apm_config(trans);
 361
 362         /* Configure analog phase-lock-loop before activating to D0A */
 363         if (trans->cfg->base_params->pll_cfg)
 364                 iwl_set_bit(trans, CSR_ANA_PLL_CFG, CSR50_ANA_PLL_CFG_VAL);
 365
 366         /*
 367          * Set "initialization complete" bit to move adapter from
 368          * D0U* --> D0A* (powered-up active) state.
 369          */
 370         iwl_set_bit(trans, CSR_GP_CNTRL,
 371                     BIT(trans->cfg->csr->flag_init_done));
 372
 373         /*
 374          * Wait for clock stabilization; once stabilized, access to
 375          * device-internal resources is supported, e.g. iwl_write_prph()
 376          * and accesses to uCode SRAM.
 377          */
 378         ret = iwl_poll_bit(trans, CSR_GP_CNTRL,
 379                            BIT(trans->cfg->csr->flag_mac_clock_ready),
 380                            BIT(trans->cfg->csr->flag_mac_clock_ready),
 381                            25000);
 382         if (ret < 0) {
 383                 IWL_ERR(trans, "Failed to init the card\n");
 384                 return ret;
 385         }
 386
 387         if (trans->cfg->host_interrupt_operation_mode) {
 388                 /*
 389                  * This is a bit of an abuse - This is needed for 7260 / 3160
 390                  * only check host_interrupt_operation_mode even if this is
 391                  * not related to host_interrupt_operation_mode.
 392                  *
 393                  * Enable the oscillator to count wake up time for L1 exit. This
 394                  * consumes slightly more power (100uA) - but allows to be sure
 395                  * that we wake up from L1 on time.
 396                  *
 397                  * This looks weird: read twice the same register, discard the
 398                  * value, set a bit, and yet again, read that same register
 399                  * just to discard the value. But that's the way the hardware
 400                  * seems to like it.
 401                  */
 402                 iwl_read_prph(trans, OSC_CLK);
 403                 iwl_read_prph(trans, OSC_CLK);
 404                 iwl_set_bits_prph(trans, OSC_CLK, OSC_CLK_FORCE_CONTROL);
 405                 iwl_read_prph(trans, OSC_CLK);
 406                 iwl_read_prph(trans, OSC_CLK);
 407         }
 408
 409         /*
 410          * Enable DMA clock and wait for it to stabilize.
 411          *
 412          * Write to "CLK_EN_REG"; "1" bits enable clocks, while "0"
 413          * bits do not disable clocks.  This preserves any hardware
 414          * bits already set by default in "CLK_CTRL_REG" after reset.
 415          */
 416         if (!trans->cfg->apmg_not_supported) {
 417                 iwl_write_prph(trans, APMG_CLK_EN_REG,
 418                                APMG_CLK_VAL_DMA_CLK_RQT);
 419                 udelay(20);
 420
 421                 /* Disable L1-Active */
 422                 iwl_set_bits_prph(trans, APMG_PCIDEV_STT_REG,
 423                                   APMG_PCIDEV_STT_VAL_L1_ACT_DIS);
 424
 425                 /* Clear the interrupt in APMG if the NIC is in RFKILL */
 426                 iwl_write_prph(trans, APMG_RTC_INT_STT_REG,
 427                                APMG_RTC_INT_STT_RFKILL);
 428         }
 429
 430         set_bit(STATUS_DEVICE_ENABLED, &trans->status);
 431
 432         return 0;
 433 }
 434
 435 /*
 436  * Enable LP XTAL to avoid HW bug where device may consume much power if
 437  * FW is not loaded after device reset. LP XTAL is disabled by default
 438  * after device HW reset. Do it only if XTAL is fed by internal source.
 439  * Configure device's "persistence" mode to avoid resetting XTAL again when
 440  * SHRD_HW_RST occurs in S3.
 441  */
 442 static void iwl_pcie_apm_lp_xtal_enable(struct iwl_trans *trans)
 443 {
 444         int ret;
 445         u32 apmg_gp1_reg;
 446         u32 apmg_xtal_cfg_reg;
 447         u32 dl_cfg_reg;
 448
 449         /* Force XTAL ON */
 450         __iwl_trans_pcie_set_bit(trans, CSR_GP_CNTRL,
 451                                  CSR_GP_CNTRL_REG_FLAG_XTAL_ON);
 452
 453         iwl_trans_pcie_sw_reset(trans);
 454
 455         /*
 456          * Set "initialization complete" bit to move adapter from
 457          * D0U* --> D0A* (powered-up active) state.
 458          */
 459         iwl_set_bit(trans, CSR_GP_CNTRL,
 460                     BIT(trans->cfg->csr->flag_init_done));
 461
 462         /*
 463          * Wait for clock stabilization; once stabilized, access to
 464          * device-internal resources is possible.
 465          */
 466         ret = iwl_poll_bit(trans, CSR_GP_CNTRL,
 467                            BIT(trans->cfg->csr->flag_mac_clock_ready),
 468                            BIT(trans->cfg->csr->flag_mac_clock_ready),
 469                            25000);
 470         if (WARN_ON(ret < 0)) {
 471                 IWL_ERR(trans, "Access time out - failed to enable LP XTAL\n");
 472                 /* Release XTAL ON request */
 473                 __iwl_trans_pcie_clear_bit(trans, CSR_GP_CNTRL,
 474                                            CSR_GP_CNTRL_REG_FLAG_XTAL_ON);
 475                 return;
 476         }
 477
 478         /*
 479          * Clear "disable persistence" to avoid LP XTAL resetting when
 480          * SHRD_HW_RST is applied in S3.
 481          */
 482         iwl_clear_bits_prph(trans, APMG_PCIDEV_STT_REG,
 483                                     APMG_PCIDEV_STT_VAL_PERSIST_DIS);
 484
 485         /*
 486          * Force APMG XTAL to be active to prevent its disabling by HW
 487          * caused by APMG idle state.
 488          */
 489         apmg_xtal_cfg_reg = iwl_trans_pcie_read_shr(trans,
 490                                                     SHR_APMG_XTAL_CFG_REG);
 491         iwl_trans_pcie_write_shr(trans, SHR_APMG_XTAL_CFG_REG,
 492                                  apmg_xtal_cfg_reg |
 493                                  SHR_APMG_XTAL_CFG_XTAL_ON_REQ);
 494
 495         iwl_trans_pcie_sw_reset(trans);
 496
 497         /* Enable LP XTAL by indirect access through CSR */
 498         apmg_gp1_reg = iwl_trans_pcie_read_shr(trans, SHR_APMG_GP1_REG);
 499         iwl_trans_pcie_write_shr(trans, SHR_APMG_GP1_REG, apmg_gp1_reg |
 500                                  SHR_APMG_GP1_WF_XTAL_LP_EN |
 501                                  SHR_APMG_GP1_CHICKEN_BIT_SELECT);
 502
 503         /* Clear delay line clock power up */
 504         dl_cfg_reg = iwl_trans_pcie_read_shr(trans, SHR_APMG_DL_CFG_REG);
 505         iwl_trans_pcie_write_shr(trans, SHR_APMG_DL_CFG_REG, dl_cfg_reg &
 506                                  ~SHR_APMG_DL_CFG_DL_CLOCK_POWER_UP);
 507
 508         /*
 509          * Enable persistence mode to avoid LP XTAL resetting when
 510          * SHRD_HW_RST is applied in S3.
 511          */
 512         iwl_set_bit(trans, CSR_HW_IF_CONFIG_REG,
 513                     CSR_HW_IF_CONFIG_REG_PERSIST_MODE);
 514
 515         /*
 516          * Clear "initialization complete" bit to move adapter from
 517          * D0A* (powered-up Active) --> D0U* (Uninitialized) state.
 518          */
 519         iwl_clear_bit(trans, CSR_GP_CNTRL,
 520                       BIT(trans->cfg->csr->flag_init_done));
 521
 522         /* Activates XTAL resources monitor */
 523         __iwl_trans_pcie_set_bit(trans, CSR_MONITOR_CFG_REG,
 524                                  CSR_MONITOR_XTAL_RESOURCES);
 525
 526         /* Release XTAL ON request */
 527         __iwl_trans_pcie_clear_bit(trans, CSR_GP_CNTRL,
 528                                    CSR_GP_CNTRL_REG_FLAG_XTAL_ON);
 529         udelay(10);
 530
 531         /* Release APMG XTAL */
 532         iwl_trans_pcie_write_shr(trans, SHR_APMG_XTAL_CFG_REG,
 533                                  apmg_xtal_cfg_reg &
 534                                  ~SHR_APMG_XTAL_CFG_XTAL_ON_REQ);
 535 }
 536
 537 void iwl_pcie_apm_stop_master(struct iwl_trans *trans)
 538 {
 539         int ret;
 540
 541         /* stop device's busmaster DMA activity */
 542         iwl_set_bit(trans, trans->cfg->csr->addr_sw_reset,
 543                     BIT(trans->cfg->csr->flag_stop_master));
 544
 545         ret = iwl_poll_bit(trans, trans->cfg->csr->addr_sw_reset,
 546                            BIT(trans->cfg->csr->flag_master_dis),
 547                            BIT(trans->cfg->csr->flag_master_dis), 100);
 548         if (ret < 0)
 549                 IWL_WARN(trans, "Master Disable Timed Out, 100 usec\n");
 550
 551         IWL_DEBUG_INFO(trans, "stop master\n");
 552 }
 553
 554 static void iwl_pcie_apm_stop(struct iwl_trans *trans, bool op_mode_leave)
 555 {
 556         IWL_DEBUG_INFO(trans, "Stop card, put in low power state\n");
 557
 558         if (op_mode_leave) {
 559                 if (!test_bit(STATUS_DEVICE_ENABLED, &trans->status))
 560                         iwl_pcie_apm_init(trans);
 561
 562                 /* inform ME that we are leaving */
 563                 if (trans->cfg->device_family == IWL_DEVICE_FAMILY_7000)
 564                         iwl_set_bits_prph(trans, APMG_PCIDEV_STT_REG,
 565                                           APMG_PCIDEV_STT_VAL_WAKE_ME);
 566                 else if (trans->cfg->device_family >= IWL_DEVICE_FAMILY_8000) {
 567                         iwl_set_bit(trans, CSR_DBG_LINK_PWR_MGMT_REG,
 568                                     CSR_RESET_LINK_PWR_MGMT_DISABLED);
 569                         iwl_set_bit(trans, CSR_HW_IF_CONFIG_REG,
 570                                     CSR_HW_IF_CONFIG_REG_PREPARE |
 571                                     CSR_HW_IF_CONFIG_REG_ENABLE_PME);
 572                         mdelay(1);
 573                         iwl_clear_bit(trans, CSR_DBG_LINK_PWR_MGMT_REG,
 574                                       CSR_RESET_LINK_PWR_MGMT_DISABLED);
 575                 }
 576                 mdelay(5);
 577         }
 578
 579         clear_bit(STATUS_DEVICE_ENABLED, &trans->status);
 580
 581         /* Stop device's DMA activity */
 582         iwl_pcie_apm_stop_master(trans);
 583
 584         if (trans->cfg->lp_xtal_workaround) {
 585                 iwl_pcie_apm_lp_xtal_enable(trans);
 586                 return;
 587         }
 588
 589         iwl_trans_pcie_sw_reset(trans);
 590
 591         /*
 592          * Clear "initialization complete" bit to move adapter from
 593          * D0A* (powered-up Active) --> D0U* (Uninitialized) state.
 594          */
 595         iwl_clear_bit(trans, CSR_GP_CNTRL,
 596                       BIT(trans->cfg->csr->flag_init_done));
 597 }
 598
 599 static int iwl_pcie_nic_init(struct iwl_trans *trans)
 600 {
 601         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 602         int ret;
 603
 604         /* nic_init */
 605         spin_lock(&trans_pcie->irq_lock);
 606         ret = iwl_pcie_apm_init(trans);
 607         spin_unlock(&trans_pcie->irq_lock);
 608
 609         if (ret)
 610                 return ret;
 611
 612         iwl_pcie_set_pwr(trans, false);
 613
 614         iwl_op_mode_nic_config(trans->op_mode);
 615
 616         /* Allocate the RX queue, or reset if it is already allocated */
 617         iwl_pcie_rx_init(trans);
 618
 619         /* Allocate or reset and init all Tx and Command queues */
 620         if (iwl_pcie_tx_init(trans))
 621                 return -ENOMEM;
 622
 623         if (trans->cfg->base_params->shadow_reg_enable) {
 624                 /* enable shadow regs in HW */
 625                 iwl_set_bit(trans, CSR_MAC_SHADOW_REG_CTRL, 0x800FFFFF);
 626                 IWL_DEBUG_INFO(trans, "Enabling shadow registers in device\n");
 627         }
 628
 629         return 0;
 630 }
 631
 632 #define HW_READY_TIMEOUT (50)
 633
 634 /* Note: returns poll_bit return value, which is >= 0 if success */
 635 static int iwl_pcie_set_hw_ready(struct iwl_trans *trans)
 636 {
 637         int ret;
 638
 639         iwl_set_bit(trans, CSR_HW_IF_CONFIG_REG,
 640                     CSR_HW_IF_CONFIG_REG_BIT_NIC_READY);
 641
 642         /* See if we got it */
 643         ret = iwl_poll_bit(trans, CSR_HW_IF_CONFIG_REG,
 644                            CSR_HW_IF_CONFIG_REG_BIT_NIC_READY,
 645                            CSR_HW_IF_CONFIG_REG_BIT_NIC_READY,
 646                            HW_READY_TIMEOUT);
 647
 648         if (ret >= 0)
 649                 iwl_set_bit(trans, CSR_MBOX_SET_REG, CSR_MBOX_SET_REG_OS_ALIVE);
 650
 651         IWL_DEBUG_INFO(trans, "hardware%s ready\n", ret < 0 ? " not" : "");
 652         return ret;
 653 }
 654
 655 /* Note: returns standard 0/-ERROR code */
 656 int iwl_pcie_prepare_card_hw(struct iwl_trans *trans)
 657 {
 658         int ret;
 659         int t = 0;
 660         int iter;
 661
 662         IWL_DEBUG_INFO(trans, "iwl_trans_prepare_card_hw enter\n");
 663
 664         ret = iwl_pcie_set_hw_ready(trans);
 665         /* If the card is ready, exit 0 */
 666         if (ret >= 0)
 667                 return 0;
 668
 669         iwl_set_bit(trans, CSR_DBG_LINK_PWR_MGMT_REG,
 670                     CSR_RESET_LINK_PWR_MGMT_DISABLED);
 671         usleep_range(1000, 2000);
 672
 673         for (iter = 0; iter < 10; iter++) {
 674                 /* If HW is not ready, prepare the conditions to check again */
 675                 iwl_set_bit(trans, CSR_HW_IF_CONFIG_REG,
 676                             CSR_HW_IF_CONFIG_REG_PREPARE);
 677
 678                 do {
 679                         ret = iwl_pcie_set_hw_ready(trans);
 680                         if (ret >= 0)
 681                                 return 0;
 682
 683                         usleep_range(200, 1000);
 684                         t += 200;
 685                 } while (t < 150000);
 686                 msleep(25);
 687         }
 688
 689         IWL_ERR(trans, "Couldn't prepare the card\n");
 690
 691         return ret;
 692 }
 693
 694 /*
 695  * ucode
 696  */
 697 static void iwl_pcie_load_firmware_chunk_fh(struct iwl_trans *trans,
 698                                             u32 dst_addr, dma_addr_t phy_addr,
 699                                             u32 byte_cnt)
 700 {
 701         iwl_write32(trans, FH_TCSR_CHNL_TX_CONFIG_REG(FH_SRVC_CHNL),
 702                     FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_PAUSE);
 703
 704         iwl_write32(trans, FH_SRVC_CHNL_SRAM_ADDR_REG(FH_SRVC_CHNL),
 705                     dst_addr);
 706
 707         iwl_write32(trans, FH_TFDIB_CTRL0_REG(FH_SRVC_CHNL),
 708                     phy_addr & FH_MEM_TFDIB_DRAM_ADDR_LSB_MSK);
 709
 710         iwl_write32(trans, FH_TFDIB_CTRL1_REG(FH_SRVC_CHNL),
 711                     (iwl_get_dma_hi_addr(phy_addr)
 712                         << FH_MEM_TFDIB_REG1_ADDR_BITSHIFT) | byte_cnt);
 713
 714         iwl_write32(trans, FH_TCSR_CHNL_TX_BUF_STS_REG(FH_SRVC_CHNL),
 715                     BIT(FH_TCSR_CHNL_TX_BUF_STS_REG_POS_TB_NUM) |
 716                     BIT(FH_TCSR_CHNL_TX_BUF_STS_REG_POS_TB_IDX) |
 717                     FH_TCSR_CHNL_TX_BUF_STS_REG_VAL_TFDB_VALID);
 718
 719         iwl_write32(trans, FH_TCSR_CHNL_TX_CONFIG_REG(FH_SRVC_CHNL),
 720                     FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_ENABLE |
 721                     FH_TCSR_TX_CONFIG_REG_VAL_DMA_CREDIT_DISABLE |
 722                     FH_TCSR_TX_CONFIG_REG_VAL_CIRQ_HOST_ENDTFD);
 723 }
 724
 725 static int iwl_pcie_load_firmware_chunk(struct iwl_trans *trans,
 726                                         u32 dst_addr, dma_addr_t phy_addr,
 727                                         u32 byte_cnt)
 728 {
 729         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
 730         unsigned long flags;
 731         int ret;
 732
 733         trans_pcie->ucode_write_complete = false;
 734
 735         if (!iwl_trans_grab_nic_access(trans, &flags))
 736                 return -EIO;
 737
 738         iwl_pcie_load_firmware_chunk_fh(trans, dst_addr, phy_addr,
 739                                         byte_cnt);
 740         iwl_trans_release_nic_access(trans, &flags);
 741
 742         ret = wait_event_timeout(trans_pcie->ucode_write_waitq,
 743                                  trans_pcie->ucode_write_complete, 5 * HZ);
 744         if (!ret) {
 745                 IWL_ERR(trans, "Failed to load firmware chunk!\n");
 746                 iwl_trans_pcie_dump_regs(trans);
 747                 return -ETIMEDOUT;
 748         }
 749
 750         return 0;
 751 }
 752
 753 static int iwl_pcie_load_section(struct iwl_trans *trans, u8 section_num,
 754                             const struct fw_desc *section)
 755 {
 756         u8 *v_addr;
 757         dma_addr_t p_addr;
 758         u32 offset, chunk_sz = min_t(u32, FH_MEM_TB_MAX_LENGTH, section->len);
 759         int ret = 0;
 760
 761         IWL_DEBUG_FW(trans, "[%d] uCode section being loaded...\n",
 762                      section_num);
 763
 764         v_addr = dma_alloc_coherent(trans->dev, chunk_sz, &p_addr,
 765                                     GFP_KERNEL | __GFP_NOWARN);
 766         if (!v_addr) {
 767                 IWL_DEBUG_INFO(trans, "Falling back to small chunks of DMA\n");
 768                 chunk_sz = PAGE_SIZE;
 769                 v_addr = dma_alloc_coherent(trans->dev, chunk_sz,
 770                                             &p_addr, GFP_KERNEL);
 771                 if (!v_addr)
 772                         return -ENOMEM;
 773         }
 774
 775         for (offset = 0; offset < section->len; offset += chunk_sz) {
 776                 u32 copy_size, dst_addr;
 777                 bool extended_addr = false;
 778
 779                 copy_size = min_t(u32, chunk_sz, section->len - offset);
 780                 dst_addr = section->offset + offset;
 781
 782                 if (dst_addr >= IWL_FW_MEM_EXTENDED_START &&
 783                     dst_addr <= IWL_FW_MEM_EXTENDED_END)
 784                         extended_addr = true;
 785
 786                 if (extended_addr)
 787                         iwl_set_bits_prph(trans, LMPM_CHICK,
 788                                           LMPM_CHICK_EXTENDED_ADDR_SPACE);
 789
 790                 memcpy(v_addr, (u8 *)section->data + offset, copy_size);
 791                 ret = iwl_pcie_load_firmware_chunk(trans, dst_addr, p_addr,
 792                                                    copy_size);
 793
 794                 if (extended_addr)
 795                         iwl_clear_bits_prph(trans, LMPM_CHICK,
 796                                             LMPM_CHICK_EXTENDED_ADDR_SPACE);
 797
 798                 if (ret) {
 799                         IWL_ERR(trans,
 800                                 "Could not load the [%d] uCode section\n",
 801                                 section_num);
 802                         break;
 803                 }
 804         }
 805
 806         dma_free_coherent(trans->dev, chunk_sz, v_addr, p_addr);
 807         return ret;
 808 }
 809
 810 static int iwl_pcie_load_cpu_sections_8000(struct iwl_trans *trans,
 811                                            const struct fw_img *image,
 812                                            int cpu,
 813                                            int *first_ucode_section)
 814 {
 815         int shift_param;
 816         int i, ret = 0, sec_num = 0x1;
 817         u32 val, last_read_idx = 0;
 818
 819         if (cpu == 1) {
 820                 shift_param = 0;
 821                 *first_ucode_section = 0;
 822         } else {
 823                 shift_param = 16;
 824                 (*first_ucode_section)++;
 825         }
 826
 827         for (i = *first_ucode_section; i < image->num_sec; i++) {
 828                 last_read_idx = i;
 829
 830                 /*
 831                  * CPU1_CPU2_SEPARATOR_SECTION delimiter - separate between
 832                  * CPU1 to CPU2.
 833                  * PAGING_SEPARATOR_SECTION delimiter - separate between
 834                  * CPU2 non paged to CPU2 paging sec.
 835                  */
 836                 if (!image->sec[i].data ||
 837                     image->sec[i].offset == CPU1_CPU2_SEPARATOR_SECTION ||
 838                     image->sec[i].offset == PAGING_SEPARATOR_SECTION) {
 839                         IWL_DEBUG_FW(trans,
 840                                      "Break since Data not valid or Empty section, sec = %d\n",
 841                                      i);
 842                         break;
 843                 }
 844
 845                 ret = iwl_pcie_load_section(trans, i, &image->sec[i]);
 846                 if (ret)
 847                         return ret;
 848
 849                 /* Notify ucode of loaded section number and status */
 850                 val = iwl_read_direct32(trans, FH_UCODE_LOAD_STATUS);
 851                 val = val | (sec_num << shift_param);
 852                 iwl_write_direct32(trans, FH_UCODE_LOAD_STATUS, val);
 853
 854                 sec_num = (sec_num << 1) | 0x1;
 855         }
 856
 857         *first_ucode_section = last_read_idx;
 858
 859         iwl_enable_interrupts(trans);
 860
 861         if (trans->cfg->use_tfh) {
 862                 if (cpu == 1)
 863                         iwl_write_prph(trans, UREG_UCODE_LOAD_STATUS,
 864                                        0xFFFF);
 865                 else
 866                         iwl_write_prph(trans, UREG_UCODE_LOAD_STATUS,
 867                                        0xFFFFFFFF);
 868         } else {
 869                 if (cpu == 1)
 870                         iwl_write_direct32(trans, FH_UCODE_LOAD_STATUS,
 871                                            0xFFFF);
 872                 else
 873                         iwl_write_direct32(trans, FH_UCODE_LOAD_STATUS,
 874                                            0xFFFFFFFF);
 875         }
 876
 877         return 0;
 878 }
 879
 880 static int iwl_pcie_load_cpu_sections(struct iwl_trans *trans,
 881                                       const struct fw_img *image,
 882                                       int cpu,
 883                                       int *first_ucode_section)
 884 {
 885         int i, ret = 0;
 886         u32 last_read_idx = 0;
 887
 888         if (cpu == 1)
 889                 *first_ucode_section = 0;
 890         else
 891                 (*first_ucode_section)++;
 892
 893         for (i = *first_ucode_section; i < image->num_sec; i++) {
 894                 last_read_idx = i;
 895
 896                 /*
 897                  * CPU1_CPU2_SEPARATOR_SECTION delimiter - separate between
 898                  * CPU1 to CPU2.
 899                  * PAGING_SEPARATOR_SECTION delimiter - separate between
 900                  * CPU2 non paged to CPU2 paging sec.
 901                  */
 902                 if (!image->sec[i].data ||
 903                     image->sec[i].offset == CPU1_CPU2_SEPARATOR_SECTION ||
 904                     image->sec[i].offset == PAGING_SEPARATOR_SECTION) {
 905                         IWL_DEBUG_FW(trans,
 906                                      "Break since Data not valid or Empty section, sec = %d\n",
 907                                      i);
 908                         break;
 909                 }
 910
 911                 ret = iwl_pcie_load_section(trans, i, &image->sec[i]);
 912                 if (ret)
 913                         return ret;
 914         }
 915
 916         *first_ucode_section = last_read_idx;
 917
 918         return 0;
 919 }
 920
 921 void iwl_pcie_apply_destination(struct iwl_trans *trans)
 922 {
 923         const struct iwl_fw_dbg_dest_tlv_v1 *dest = trans->dbg_dest_tlv;
 924         int i;
 925
 926         IWL_INFO(trans, "Applying debug destination %s\n",
 927                  get_fw_dbg_mode_string(dest->monitor_mode));
 928
 929         if (dest->monitor_mode == EXTERNAL_MODE)
 930                 iwl_pcie_alloc_fw_monitor(trans, dest->size_power);
 931         else
 932                 IWL_WARN(trans, "PCI should have external buffer debug\n");
 933
 934         for (i = 0; i < trans->dbg_dest_reg_num; i++) {
 935                 u32 addr = le32_to_cpu(dest->reg_ops[i].addr);
 936                 u32 val = le32_to_cpu(dest->reg_ops[i].val);
 937
 938                 switch (dest->reg_ops[i].op) {
 939                 case CSR_ASSIGN:
 940                         iwl_write32(trans, addr, val);
 941                         break;
 942                 case CSR_SETBIT:
 943                         iwl_set_bit(trans, addr, BIT(val));
 944                         break;
 945                 case CSR_CLEARBIT:
 946                         iwl_clear_bit(trans, addr, BIT(val));
 947                         break;
 948                 case PRPH_ASSIGN:
 949                         iwl_write_prph(trans, addr, val);
 950                         break;
 951                 case PRPH_SETBIT:
 952                         iwl_set_bits_prph(trans, addr, BIT(val));
 953                         break;
 954                 case PRPH_CLEARBIT:
 955                         iwl_clear_bits_prph(trans, addr, BIT(val));
 956                         break;
 957                 case PRPH_BLOCKBIT:
 958                         if (iwl_read_prph(trans, addr) & BIT(val)) {
 959                                 IWL_ERR(trans,
 960                                         "BIT(%u) in address 0x%x is 1, stopping FW configuration\n",
 961                                         val, addr);
 962                                 goto monitor;
 963                         }
 964                         break;
 965                 default:
 966                         IWL_ERR(trans, "FW debug - unknown OP %d\n",
 967                                 dest->reg_ops[i].op);
 968                         break;
 969                 }
 970         }
 971
 972 monitor:
 973         if (dest->monitor_mode == EXTERNAL_MODE && trans->fw_mon[0].size) {
 974                 iwl_write_prph(trans, le32_to_cpu(dest->base_reg),
 975                                trans->fw_mon[0].physical >> dest->base_shift);
 976                 if (trans->cfg->device_family >= IWL_DEVICE_FAMILY_8000)
 977                         iwl_write_prph(trans, le32_to_cpu(dest->end_reg),
 978                                        (trans->fw_mon[0].physical +
 979                                         trans->fw_mon[0].size - 256) >>
 980                                                 dest->end_shift);
 981                 else
 982                         iwl_write_prph(trans, le32_to_cpu(dest->end_reg),
 983                                        (trans->fw_mon[0].physical +
 984                                         trans->fw_mon[0].size) >>
 985                                                 dest->end_shift);
 986         }
 987 }
 988
 989 static int iwl_pcie_load_given_ucode(struct iwl_trans *trans,
 990                                 const struct fw_img *image)
 991 {
 992         int ret = 0;
 993         int first_ucode_section;
 994
 995         IWL_DEBUG_FW(trans, "working with %s CPU\n",
 996                      image->is_dual_cpus ? "Dual" : "Single");
 997
 998         /* load to FW the binary non secured sections of CPU1 */
 999         ret = iwl_pcie_load_cpu_sections(trans, image, 1, &first_ucode_section);
1000         if (ret)
1001                 return ret;
1002
1003         if (image->is_dual_cpus) {
1004                 /* set CPU2 header address */
1005                 iwl_write_prph(trans,
1006                                LMPM_SECURE_UCODE_LOAD_CPU2_HDR_ADDR,
1007                                LMPM_SECURE_CPU2_HDR_MEM_SPACE);
1008
1009                 /* load to FW the binary sections of CPU2 */
1010                 ret = iwl_pcie_load_cpu_sections(trans, image, 2,
1011                                                  &first_ucode_section);
1012                 if (ret)
1013                         return ret;
1014         }
1015
1016         /* supported for 7000 only for the moment */
1017         if (iwlwifi_mod_params.fw_monitor &&
1018             trans->cfg->device_family == IWL_DEVICE_FAMILY_7000) {
1019                 iwl_pcie_alloc_fw_monitor(trans, 0);
1020
1021                 if (trans->fw_mon[0].size) {
1022                         iwl_write_prph(trans, MON_BUFF_BASE_ADDR,
1023                                        trans->fw_mon[0].physical >> 4);
1024                         iwl_write_prph(trans, MON_BUFF_END_ADDR,
1025                                        (trans->fw_mon[0].physical +
1026                                         trans->fw_mon[0].size) >> 4);
1027                 }
1028         } else if (trans->dbg_dest_tlv) {
1029                 iwl_pcie_apply_destination(trans);
1030         }
1031
1032         iwl_enable_interrupts(trans);
1033
1034         /* release CPU reset */
1035         iwl_write32(trans, CSR_RESET, 0);
1036
1037         return 0;
1038 }
1039
1040 static int iwl_pcie_load_given_ucode_8000(struct iwl_trans *trans,
1041                                           const struct fw_img *image)
1042 {
1043         int ret = 0;
1044         int first_ucode_section;
1045
1046         IWL_DEBUG_FW(trans, "working with %s CPU\n",
1047                      image->is_dual_cpus ? "Dual" : "Single");
1048
1049         if (trans->dbg_dest_tlv)
1050                 iwl_pcie_apply_destination(trans);
1051
1052         IWL_DEBUG_POWER(trans, "Original WFPM value = 0x%08X\n",
1053                         iwl_read_prph(trans, WFPM_GP2));
1054
1055         /*
1056          * Set default value. On resume reading the values that were
1057          * zeored can provide debug data on the resume flow.
1058          * This is for debugging only and has no functional impact.
1059          */
1060         iwl_write_prph(trans, WFPM_GP2, 0x01010101);
1061
1062         /* configure the ucode to be ready to get the secured image */
1063         /* release CPU reset */
1064         iwl_write_prph(trans, RELEASE_CPU_RESET, RELEASE_CPU_RESET_BIT);
1065
1066         /* load to FW the binary Secured sections of CPU1 */
1067         ret = iwl_pcie_load_cpu_sections_8000(trans, image, 1,
1068                                               &first_ucode_section);
1069         if (ret)
1070                 return ret;
1071
1072         /* load to FW the binary sections of CPU2 */
1073         return iwl_pcie_load_cpu_sections_8000(trans, image, 2,
1074                                                &first_ucode_section);
1075 }
1076
1077 bool iwl_pcie_check_hw_rf_kill(struct iwl_trans *trans)
1078 {
1079         struct iwl_trans_pcie *trans_pcie =  IWL_TRANS_GET_PCIE_TRANS(trans);
1080         bool hw_rfkill = iwl_is_rfkill_set(trans);
1081         bool prev = test_bit(STATUS_RFKILL_OPMODE, &trans->status);
1082         bool report;
1083
1084         if (hw_rfkill) {
1085                 set_bit(STATUS_RFKILL_HW, &trans->status);
1086                 set_bit(STATUS_RFKILL_OPMODE, &trans->status);
1087         } else {
1088                 clear_bit(STATUS_RFKILL_HW, &trans->status);
1089                 if (trans_pcie->opmode_down)
1090                         clear_bit(STATUS_RFKILL_OPMODE, &trans->status);
1091         }
1092
1093         report = test_bit(STATUS_RFKILL_OPMODE, &trans->status);
1094
1095         if (prev != report)
1096                 iwl_trans_pcie_rf_kill(trans, report);
1097
1098         return hw_rfkill;
1099 }
1100
1101 struct iwl_causes_list {
1102         u32 cause_num;
1103         u32 mask_reg;
1104         u8 addr;
1105 };
1106
1107 static struct iwl_causes_list causes_list[] = {
1108         {MSIX_FH_INT_CAUSES_D2S_CH0_NUM,        CSR_MSIX_FH_INT_MASK_AD, 0},
1109         {MSIX_FH_INT_CAUSES_D2S_CH1_NUM,        CSR_MSIX_FH_INT_MASK_AD, 0x1},
1110         {MSIX_FH_INT_CAUSES_S2D,                CSR_MSIX_FH_INT_MASK_AD, 0x3},
1111         {MSIX_FH_INT_CAUSES_FH_ERR,             CSR_MSIX_FH_INT_MASK_AD, 0x5},
1112         {MSIX_HW_INT_CAUSES_REG_ALIVE,          CSR_MSIX_HW_INT_MASK_AD, 0x10},
1113         {MSIX_HW_INT_CAUSES_REG_WAKEUP,         CSR_MSIX_HW_INT_MASK_AD, 0x11},
1114         {MSIX_HW_INT_CAUSES_REG_CT_KILL,        CSR_MSIX_HW_INT_MASK_AD, 0x16},
1115         {MSIX_HW_INT_CAUSES_REG_RF_KILL,        CSR_MSIX_HW_INT_MASK_AD, 0x17},
1116         {MSIX_HW_INT_CAUSES_REG_PERIODIC,       CSR_MSIX_HW_INT_MASK_AD, 0x18},
1117         {MSIX_HW_INT_CAUSES_REG_SW_ERR,         CSR_MSIX_HW_INT_MASK_AD, 0x29},
1118         {MSIX_HW_INT_CAUSES_REG_SCD,            CSR_MSIX_HW_INT_MASK_AD, 0x2A},
1119         {MSIX_HW_INT_CAUSES_REG_FH_TX,          CSR_MSIX_HW_INT_MASK_AD, 0x2B},
1120         {MSIX_HW_INT_CAUSES_REG_HW_ERR,         CSR_MSIX_HW_INT_MASK_AD, 0x2D},
1121         {MSIX_HW_INT_CAUSES_REG_HAP,            CSR_MSIX_HW_INT_MASK_AD, 0x2E},
1122 };
1123
1124 static struct iwl_causes_list causes_list_v2[] = {
1125         {MSIX_FH_INT_CAUSES_D2S_CH0_NUM,        CSR_MSIX_FH_INT_MASK_AD, 0},
1126         {MSIX_FH_INT_CAUSES_D2S_CH1_NUM,        CSR_MSIX_FH_INT_MASK_AD, 0x1},
1127         {MSIX_FH_INT_CAUSES_S2D,                CSR_MSIX_FH_INT_MASK_AD, 0x3},
1128         {MSIX_FH_INT_CAUSES_FH_ERR,             CSR_MSIX_FH_INT_MASK_AD, 0x5},
1129         {MSIX_HW_INT_CAUSES_REG_ALIVE,          CSR_MSIX_HW_INT_MASK_AD, 0x10},
1130         {MSIX_HW_INT_CAUSES_REG_IPC,            CSR_MSIX_HW_INT_MASK_AD, 0x11},
1131         {MSIX_HW_INT_CAUSES_REG_SW_ERR_V2,      CSR_MSIX_HW_INT_MASK_AD, 0x15},
1132         {MSIX_HW_INT_CAUSES_REG_CT_KILL,        CSR_MSIX_HW_INT_MASK_AD, 0x16},
1133         {MSIX_HW_INT_CAUSES_REG_RF_KILL,        CSR_MSIX_HW_INT_MASK_AD, 0x17},
1134         {MSIX_HW_INT_CAUSES_REG_PERIODIC,       CSR_MSIX_HW_INT_MASK_AD, 0x18},
1135         {MSIX_HW_INT_CAUSES_REG_SCD,            CSR_MSIX_HW_INT_MASK_AD, 0x2A},
1136         {MSIX_HW_INT_CAUSES_REG_FH_TX,          CSR_MSIX_HW_INT_MASK_AD, 0x2B},
1137         {MSIX_HW_INT_CAUSES_REG_HW_ERR,         CSR_MSIX_HW_INT_MASK_AD, 0x2D},
1138         {MSIX_HW_INT_CAUSES_REG_HAP,            CSR_MSIX_HW_INT_MASK_AD, 0x2E},
1139 };
1140
1141 static void iwl_pcie_map_non_rx_causes(struct iwl_trans *trans)
1142 {
1143         struct iwl_trans_pcie *trans_pcie =  IWL_TRANS_GET_PCIE_TRANS(trans);
1144         int val = trans_pcie->def_irq | MSIX_NON_AUTO_CLEAR_CAUSE;
1145         int i, arr_size =
1146                 (trans->cfg->device_family < IWL_DEVICE_FAMILY_22560) ?
1147                 ARRAY_SIZE(causes_list) : ARRAY_SIZE(causes_list_v2);
1148
1149         /*
1150          * Access all non RX causes and map them to the default irq.
1151          * In case we are missing at least one interrupt vector,
1152          * the first interrupt vector will serve non-RX and FBQ causes.
1153          */
1154         for (i = 0; i < arr_size; i++) {
1155                 struct iwl_causes_list *causes =
1156                         (trans->cfg->device_family < IWL_DEVICE_FAMILY_22560) ?
1157                         causes_list : causes_list_v2;
1158
1159                 iwl_write8(trans, CSR_MSIX_IVAR(causes[i].addr), val);
1160                 iwl_clear_bit(trans, causes[i].mask_reg,
1161                               causes[i].cause_num);
1162         }
1163 }
1164
1165 static void iwl_pcie_map_rx_causes(struct iwl_trans *trans)
1166 {
1167         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1168         u32 offset =
1169                 trans_pcie->shared_vec_mask & IWL_SHARED_IRQ_FIRST_RSS ? 1 : 0;
1170         u32 val, idx;
1171
1172         /*
1173          * The first RX queue - fallback queue, which is designated for
1174          * management frame, command responses etc, is always mapped to the
1175          * first interrupt vector. The other RX queues are mapped to
1176          * the other (N - 2) interrupt vectors.
1177          */
1178         val = BIT(MSIX_FH_INT_CAUSES_Q(0));
1179         for (idx = 1; idx < trans->num_rx_queues; idx++) {
1180                 iwl_write8(trans, CSR_MSIX_RX_IVAR(idx),
1181                            MSIX_FH_INT_CAUSES_Q(idx - offset));
1182                 val |= BIT(MSIX_FH_INT_CAUSES_Q(idx));
1183         }
1184         iwl_write32(trans, CSR_MSIX_FH_INT_MASK_AD, ~val);
1185
1186         val = MSIX_FH_INT_CAUSES_Q(0);
1187         if (trans_pcie->shared_vec_mask & IWL_SHARED_IRQ_NON_RX)
1188                 val |= MSIX_NON_AUTO_CLEAR_CAUSE;
1189         iwl_write8(trans, CSR_MSIX_RX_IVAR(0), val);
1190
1191         if (trans_pcie->shared_vec_mask & IWL_SHARED_IRQ_FIRST_RSS)
1192                 iwl_write8(trans, CSR_MSIX_RX_IVAR(1), val);
1193 }
1194
1195 void iwl_pcie_conf_msix_hw(struct iwl_trans_pcie *trans_pcie)
1196 {
1197         struct iwl_trans *trans = trans_pcie->trans;
1198
1199         if (!trans_pcie->msix_enabled) {
1200                 if (trans->cfg->mq_rx_supported &&
1201                     test_bit(STATUS_DEVICE_ENABLED, &trans->status))
1202                         iwl_write_prph(trans, UREG_CHICK,
1203                                        UREG_CHICK_MSI_ENABLE);
1204                 return;
1205         }
1206         /*
1207          * The IVAR table needs to be configured again after reset,
1208          * but if the device is disabled, we can't write to
1209          * prph.
1210          */
1211         if (test_bit(STATUS_DEVICE_ENABLED, &trans->status))
1212                 iwl_write_prph(trans, UREG_CHICK, UREG_CHICK_MSIX_ENABLE);
1213
1214         /*
1215          * Each cause from the causes list above and the RX causes is
1216          * represented as a byte in the IVAR table. The first nibble
1217          * represents the bound interrupt vector of the cause, the second
1218          * represents no auto clear for this cause. This will be set if its
1219          * interrupt vector is bound to serve other causes.
1220          */
1221         iwl_pcie_map_rx_causes(trans);
1222
1223         iwl_pcie_map_non_rx_causes(trans);
1224 }
1225
1226 static void iwl_pcie_init_msix(struct iwl_trans_pcie *trans_pcie)
1227 {
1228         struct iwl_trans *trans = trans_pcie->trans;
1229
1230         iwl_pcie_conf_msix_hw(trans_pcie);
1231
1232         if (!trans_pcie->msix_enabled)
1233                 return;
1234
1235         trans_pcie->fh_init_mask = ~iwl_read32(trans, CSR_MSIX_FH_INT_MASK_AD);
1236         trans_pcie->fh_mask = trans_pcie->fh_init_mask;
1237         trans_pcie->hw_init_mask = ~iwl_read32(trans, CSR_MSIX_HW_INT_MASK_AD);
1238         trans_pcie->hw_mask = trans_pcie->hw_init_mask;
1239 }
1240
1241 static void _iwl_trans_pcie_stop_device(struct iwl_trans *trans, bool low_power)
1242 {
1243         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1244
1245         lockdep_assert_held(&trans_pcie->mutex);
1246
1247         if (trans_pcie->is_down)
1248                 return;
1249
1250         trans_pcie->is_down = true;
1251
1252         /* Stop dbgc before stopping device */
1253         _iwl_fw_dbg_stop_recording(trans, NULL);
1254
1255         /* tell the device to stop sending interrupts */
1256         iwl_disable_interrupts(trans);
1257
1258         /* device going down, Stop using ICT table */
1259         iwl_pcie_disable_ict(trans);
1260
1261         /*
1262          * If a HW restart happens during firmware loading,
1263          * then the firmware loading might call this function
1264          * and later it might be called again due to the
1265          * restart. So don't process again if the device is
1266          * already dead.
1267          */
1268         if (test_and_clear_bit(STATUS_DEVICE_ENABLED, &trans->status)) {
1269                 IWL_DEBUG_INFO(trans,
1270                                "DEVICE_ENABLED bit was set and is now cleared\n");
1271                 iwl_pcie_tx_stop(trans);
1272                 iwl_pcie_rx_stop(trans);
1273
1274                 /* Power-down device's busmaster DMA clocks */
1275                 if (!trans->cfg->apmg_not_supported) {
1276                         iwl_write_prph(trans, APMG_CLK_DIS_REG,
1277                                        APMG_CLK_VAL_DMA_CLK_RQT);
1278                         udelay(5);
1279                 }
1280         }
1281
1282         /* Make sure (redundant) we've released our request to stay awake */
1283         iwl_clear_bit(trans, CSR_GP_CNTRL,
1284                       BIT(trans->cfg->csr->flag_mac_access_req));
1285
1286         /* Stop the device, and put it in low power state */
1287         iwl_pcie_apm_stop(trans, false);
1288
1289         iwl_trans_pcie_sw_reset(trans);
1290
1291         /*
1292          * Upon stop, the IVAR table gets erased, so msi-x won't
1293          * work. This causes a bug in RF-KILL flows, since the interrupt
1294          * that enables radio won't fire on the correct irq, and the
1295          * driver won't be able to handle the interrupt.
1296          * Configure the IVAR table again after reset.
1297          */
1298         iwl_pcie_conf_msix_hw(trans_pcie);
1299
1300         /*
1301          * Upon stop, the APM issues an interrupt if HW RF kill is set.
1302          * This is a bug in certain verions of the hardware.
1303          * Certain devices also keep sending HW RF kill interrupt all
1304          * the time, unless the interrupt is ACKed even if the interrupt
1305          * should be masked. Re-ACK all the interrupts here.
1306          */
1307         iwl_disable_interrupts(trans);
1308
1309         /* clear all status bits */
1310         clear_bit(STATUS_SYNC_HCMD_ACTIVE, &trans->status);
1311         clear_bit(STATUS_INT_ENABLED, &trans->status);
1312         clear_bit(STATUS_TPOWER_PMI, &trans->status);
1313
1314         /*
1315          * Even if we stop the HW, we still want the RF kill
1316          * interrupt
1317          */
1318         iwl_enable_rfkill_int(trans);
1319
1320         /* re-take ownership to prevent other users from stealing the device */
1321         iwl_pcie_prepare_card_hw(trans);
1322 }
1323
1324 void iwl_pcie_synchronize_irqs(struct iwl_trans *trans)
1325 {
1326         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1327
1328         if (trans_pcie->msix_enabled) {
1329                 int i;
1330
1331                 for (i = 0; i < trans_pcie->alloc_vecs; i++)
1332                         synchronize_irq(trans_pcie->msix_entries[i].vector);
1333         } else {
1334                 synchronize_irq(trans_pcie->pci_dev->irq);
1335         }
1336 }
1337
1338 static int iwl_trans_pcie_start_fw(struct iwl_trans *trans,
1339                                    const struct fw_img *fw, bool run_in_rfkill)
1340 {
1341         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1342         bool hw_rfkill;
1343         int ret;
1344
1345         /* This may fail if AMT took ownership of the device */
1346         if (iwl_pcie_prepare_card_hw(trans)) {
1347                 IWL_WARN(trans, "Exit HW not ready\n");
1348                 ret = -EIO;
1349                 goto out;
1350         }
1351
1352         iwl_enable_rfkill_int(trans);
1353
1354         iwl_write32(trans, CSR_INT, 0xFFFFFFFF);
1355
1356         /*
1357          * We enabled the RF-Kill interrupt and the handler may very
1358          * well be running. Disable the interrupts to make sure no other
1359          * interrupt can be fired.
1360          */
1361         iwl_disable_interrupts(trans);
1362
1363         /* Make sure it finished running */
1364         iwl_pcie_synchronize_irqs(trans);
1365
1366         mutex_lock(&trans_pcie->mutex);
1367
1368         /* If platform's RF_KILL switch is NOT set to KILL */
1369         hw_rfkill = iwl_pcie_check_hw_rf_kill(trans);
1370         if (hw_rfkill && !run_in_rfkill) {
1371                 ret = -ERFKILL;
1372                 goto out;
1373         }
1374
1375         /* Someone called stop_device, don't try to start_fw */
1376         if (trans_pcie->is_down) {
1377                 IWL_WARN(trans,
1378                          "Can't start_fw since the HW hasn't been started\n");
1379                 ret = -EIO;
1380                 goto out;
1381         }
1382
1383         /* make sure rfkill handshake bits are cleared */
1384         iwl_write32(trans, CSR_UCODE_DRV_GP1_CLR, CSR_UCODE_SW_BIT_RFKILL);
1385         iwl_write32(trans, CSR_UCODE_DRV_GP1_CLR,
1386                     CSR_UCODE_DRV_GP1_BIT_CMD_BLOCKED);
1387
1388         /* clear (again), then enable host interrupts */
1389         iwl_write32(trans, CSR_INT, 0xFFFFFFFF);
1390
1391         ret = iwl_pcie_nic_init(trans);
1392         if (ret) {
1393                 IWL_ERR(trans, "Unable to init nic\n");
1394                 goto out;
1395         }
1396
1397         /*
1398          * Now, we load the firmware and don't want to be interrupted, even
1399          * by the RF-Kill interrupt (hence mask all the interrupt besides the
1400          * FH_TX interrupt which is needed to load the firmware). If the
1401          * RF-Kill switch is toggled, we will find out after having loaded
1402          * the firmware and return the proper value to the caller.
1403          */
1404         iwl_enable_fw_load_int(trans);
1405
1406         /* really make sure rfkill handshake bits are cleared */
1407         iwl_write32(trans, CSR_UCODE_DRV_GP1_CLR, CSR_UCODE_SW_BIT_RFKILL);
1408         iwl_write32(trans, CSR_UCODE_DRV_GP1_CLR, CSR_UCODE_SW_BIT_RFKILL);
1409
1410         /* Load the given image to the HW */
1411         if (trans->cfg->device_family >= IWL_DEVICE_FAMILY_8000)
1412                 ret = iwl_pcie_load_given_ucode_8000(trans, fw);
1413         else
1414                 ret = iwl_pcie_load_given_ucode(trans, fw);
1415
1416         /* re-check RF-Kill state since we may have missed the interrupt */
1417         hw_rfkill = iwl_pcie_check_hw_rf_kill(trans);
1418         if (hw_rfkill && !run_in_rfkill)
1419                 ret = -ERFKILL;
1420
1421 out:
1422         mutex_unlock(&trans_pcie->mutex);
1423         return ret;
1424 }
1425
1426 static void iwl_trans_pcie_fw_alive(struct iwl_trans *trans, u32 scd_addr)
1427 {
1428         iwl_pcie_reset_ict(trans);
1429         iwl_pcie_tx_start(trans, scd_addr);
1430 }
1431
1432 void iwl_trans_pcie_handle_stop_rfkill(struct iwl_trans *trans,
1433                                        bool was_in_rfkill)
1434 {
1435         bool hw_rfkill;
1436
1437         /*
1438          * Check again since the RF kill state may have changed while
1439          * all the interrupts were disabled, in this case we couldn't
1440          * receive the RF kill interrupt and update the state in the
1441          * op_mode.
1442          * Don't call the op_mode if the rkfill state hasn't changed.
1443          * This allows the op_mode to call stop_device from the rfkill
1444          * notification without endless recursion. Under very rare
1445          * circumstances, we might have a small recursion if the rfkill
1446          * state changed exactly now while we were called from stop_device.
1447          * This is very unlikely but can happen and is supported.
1448          */
1449         hw_rfkill = iwl_is_rfkill_set(trans);
1450         if (hw_rfkill) {
1451                 set_bit(STATUS_RFKILL_HW, &trans->status);
1452                 set_bit(STATUS_RFKILL_OPMODE, &trans->status);
1453         } else {
1454                 clear_bit(STATUS_RFKILL_HW, &trans->status);
1455                 clear_bit(STATUS_RFKILL_OPMODE, &trans->status);
1456         }
1457         if (hw_rfkill != was_in_rfkill)
1458                 iwl_trans_pcie_rf_kill(trans, hw_rfkill);
1459 }
1460
1461 static void iwl_trans_pcie_stop_device(struct iwl_trans *trans, bool low_power)
1462 {
1463         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1464         bool was_in_rfkill;
1465
1466         mutex_lock(&trans_pcie->mutex);
1467         trans_pcie->opmode_down = true;
1468         was_in_rfkill = test_bit(STATUS_RFKILL_OPMODE, &trans->status);
1469         _iwl_trans_pcie_stop_device(trans, low_power);
1470         iwl_trans_pcie_handle_stop_rfkill(trans, was_in_rfkill);
1471         mutex_unlock(&trans_pcie->mutex);
1472 }
1473
1474 void iwl_trans_pcie_rf_kill(struct iwl_trans *trans, bool state)
1475 {
1476         struct iwl_trans_pcie __maybe_unused *trans_pcie =
1477                 IWL_TRANS_GET_PCIE_TRANS(trans);
1478
1479         lockdep_assert_held(&trans_pcie->mutex);
1480
1481         IWL_WARN(trans, "reporting RF_KILL (radio %s)\n",
1482                  state ? "disabled" : "enabled");
1483         if (iwl_op_mode_hw_rf_kill(trans->op_mode, state)) {
1484                 if (trans->cfg->gen2)
1485                         _iwl_trans_pcie_gen2_stop_device(trans, true);
1486                 else
1487                         _iwl_trans_pcie_stop_device(trans, true);
1488         }
1489 }
1490
1491 static void iwl_trans_pcie_d3_suspend(struct iwl_trans *trans, bool test,
1492                                       bool reset)
1493 {
1494         if (!reset) {
1495                 /* Enable persistence mode to avoid reset */
1496                 iwl_set_bit(trans, CSR_HW_IF_CONFIG_REG,
1497                             CSR_HW_IF_CONFIG_REG_PERSIST_MODE);
1498         }
1499
1500         iwl_disable_interrupts(trans);
1501
1502         /*
1503          * in testing mode, the host stays awake and the
1504          * hardware won't be reset (not even partially)
1505          */
1506         if (test)
1507                 return;
1508
1509         iwl_pcie_disable_ict(trans);
1510
1511         iwl_pcie_synchronize_irqs(trans);
1512
1513         iwl_clear_bit(trans, CSR_GP_CNTRL,
1514                       BIT(trans->cfg->csr->flag_mac_access_req));
1515         iwl_clear_bit(trans, CSR_GP_CNTRL,
1516                       BIT(trans->cfg->csr->flag_init_done));
1517
1518         iwl_pcie_enable_rx_wake(trans, false);
1519
1520         if (reset) {
1521                 /*
1522                  * reset TX queues -- some of their registers reset during S3
1523                  * so if we don't reset everything here the D3 image would try
1524                  * to execute some invalid memory upon resume
1525                  */
1526                 iwl_trans_pcie_tx_reset(trans);
1527         }
1528
1529         iwl_pcie_set_pwr(trans, true);
1530 }
1531
1532 static int iwl_trans_pcie_d3_resume(struct iwl_trans *trans,
1533                                     enum iwl_d3_status *status,
1534                                     bool test,  bool reset)
1535 {
1536         struct iwl_trans_pcie *trans_pcie =  IWL_TRANS_GET_PCIE_TRANS(trans);
1537         u32 val;
1538         int ret;
1539
1540         if (test) {
1541                 iwl_enable_interrupts(trans);
1542                 *status = IWL_D3_STATUS_ALIVE;
1543                 return 0;
1544         }
1545
1546         iwl_pcie_enable_rx_wake(trans, true);
1547
1548         iwl_set_bit(trans, CSR_GP_CNTRL,
1549                     BIT(trans->cfg->csr->flag_mac_access_req));
1550         iwl_set_bit(trans, CSR_GP_CNTRL,
1551                     BIT(trans->cfg->csr->flag_init_done));
1552
1553         if (trans->cfg->device_family >= IWL_DEVICE_FAMILY_8000)
1554                 udelay(2);
1555
1556         ret = iwl_poll_bit(trans, CSR_GP_CNTRL,
1557                            BIT(trans->cfg->csr->flag_mac_clock_ready),
1558                            BIT(trans->cfg->csr->flag_mac_clock_ready),
1559                            25000);
1560         if (ret < 0) {
1561                 IWL_ERR(trans, "Failed to resume the device (mac ready)\n");
1562                 return ret;
1563         }
1564
1565         /*
1566          * Reconfigure IVAR table in case of MSIX or reset ict table in
1567          * MSI mode since HW reset erased it.
1568          * Also enables interrupts - none will happen as
1569          * the device doesn't know we're waking it up, only when
1570          * the opmode actually tells it after this call.
1571          */
1572         iwl_pcie_conf_msix_hw(trans_pcie);
1573         if (!trans_pcie->msix_enabled)
1574                 iwl_pcie_reset_ict(trans);
1575         iwl_enable_interrupts(trans);
1576
1577         iwl_pcie_set_pwr(trans, false);
1578
1579         if (!reset) {
1580                 iwl_clear_bit(trans, CSR_GP_CNTRL,
1581                               BIT(trans->cfg->csr->flag_mac_access_req));
1582         } else {
1583                 iwl_trans_pcie_tx_reset(trans);
1584
1585                 ret = iwl_pcie_rx_init(trans);
1586                 if (ret) {
1587                         IWL_ERR(trans,
1588                                 "Failed to resume the device (RX reset)\n");
1589                         return ret;
1590                 }
1591         }
1592
1593         IWL_DEBUG_POWER(trans, "WFPM value upon resume = 0x%08X\n",
1594                         iwl_read_prph(trans, WFPM_GP2));
1595
1596         val = iwl_read32(trans, CSR_RESET);
1597         if (val & CSR_RESET_REG_FLAG_NEVO_RESET)
1598                 *status = IWL_D3_STATUS_RESET;
1599         else
1600                 *status = IWL_D3_STATUS_ALIVE;
1601
1602         return 0;
1603 }
1604
1605 static void iwl_pcie_set_interrupt_capa(struct pci_dev *pdev,
1606                                         struct iwl_trans *trans)
1607 {
1608         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1609         int max_irqs, num_irqs, i, ret;
1610         u16 pci_cmd;
1611
1612         if (!trans->cfg->mq_rx_supported)
1613                 goto enable_msi;
1614
1615         max_irqs = min_t(u32, num_online_cpus() + 2, IWL_MAX_RX_HW_QUEUES);
1616         for (i = 0; i < max_irqs; i++)
1617                 trans_pcie->msix_entries[i].entry = i;
1618
1619         num_irqs = pci_enable_msix_range(pdev, trans_pcie->msix_entries,
1620                                          MSIX_MIN_INTERRUPT_VECTORS,
1621                                          max_irqs);
1622         if (num_irqs < 0) {
1623                 IWL_DEBUG_INFO(trans,
1624                                "Failed to enable msi-x mode (ret %d). Moving to msi mode.\n",
1625                                num_irqs);
1626                 goto enable_msi;
1627         }
1628         trans_pcie->def_irq = (num_irqs == max_irqs) ? num_irqs - 1 : 0;
1629
1630         IWL_DEBUG_INFO(trans,
1631                        "MSI-X enabled. %d interrupt vectors were allocated\n",
1632                        num_irqs);
1633
1634         /*
1635          * In case the OS provides fewer interrupts than requested, different
1636          * causes will share the same interrupt vector as follows:
1637          * One interrupt less: non rx causes shared with FBQ.
1638          * Two interrupts less: non rx causes shared with FBQ and RSS.
1639          * More than two interrupts: we will use fewer RSS queues.
1640          */
1641         if (num_irqs <= max_irqs - 2) {
1642                 trans_pcie->trans->num_rx_queues = num_irqs + 1;
1643                 trans_pcie->shared_vec_mask = IWL_SHARED_IRQ_NON_RX |
1644                         IWL_SHARED_IRQ_FIRST_RSS;
1645         } else if (num_irqs == max_irqs - 1) {
1646                 trans_pcie->trans->num_rx_queues = num_irqs;
1647                 trans_pcie->shared_vec_mask = IWL_SHARED_IRQ_NON_RX;
1648         } else {
1649                 trans_pcie->trans->num_rx_queues = num_irqs - 1;
1650         }
1651         WARN_ON(trans_pcie->trans->num_rx_queues > IWL_MAX_RX_HW_QUEUES);
1652
1653         trans_pcie->alloc_vecs = num_irqs;
1654         trans_pcie->msix_enabled = true;
1655         return;
1656
1657 enable_msi:
1658         ret = pci_enable_msi(pdev);
1659         if (ret) {
1660                 dev_err(&pdev->dev, "pci_enable_msi failed - %d\n", ret);
1661                 /* enable rfkill interrupt: hw bug w/a */
1662                 pci_read_config_word(pdev, PCI_COMMAND, &pci_cmd);
1663                 if (pci_cmd & PCI_COMMAND_INTX_DISABLE) {
1664                         pci_cmd &= ~PCI_COMMAND_INTX_DISABLE;
1665                         pci_write_config_word(pdev, PCI_COMMAND, pci_cmd);
1666                 }
1667         }
1668 }
1669
1670 static void iwl_pcie_irq_set_affinity(struct iwl_trans *trans)
1671 {
1672         int iter_rx_q, i, ret, cpu, offset;
1673         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1674
1675         i = trans_pcie->shared_vec_mask & IWL_SHARED_IRQ_FIRST_RSS ? 0 : 1;
1676         iter_rx_q = trans_pcie->trans->num_rx_queues - 1 + i;
1677         offset = 1 + i;
1678         for (; i < iter_rx_q ; i++) {
1679                 /*
1680                  * Get the cpu prior to the place to search
1681                  * (i.e. return will be > i - 1).
1682                  */
1683                 cpu = cpumask_next(i - offset, cpu_online_mask);
1684                 cpumask_set_cpu(cpu, &trans_pcie->affinity_mask[i]);
1685                 ret = irq_set_affinity_hint(trans_pcie->msix_entries[i].vector,
1686                                             &trans_pcie->affinity_mask[i]);
1687                 if (ret)
1688                         IWL_ERR(trans_pcie->trans,
1689                                 "Failed to set affinity mask for IRQ %d\n",
1690                                 i);
1691         }
1692 }
1693
1694 static int iwl_pcie_init_msix_handler(struct pci_dev *pdev,
1695                                       struct iwl_trans_pcie *trans_pcie)
1696 {
1697         int i;
1698
1699         for (i = 0; i < trans_pcie->alloc_vecs; i++) {
1700                 int ret;
1701                 struct msix_entry *msix_entry;
1702                 const char *qname = queue_name(&pdev->dev, trans_pcie, i);
1703
1704                 if (!qname)
1705                         return -ENOMEM;
1706
1707                 msix_entry = &trans_pcie->msix_entries[i];
1708                 ret = devm_request_threaded_irq(&pdev->dev,
1709                                                 msix_entry->vector,
1710                                                 iwl_pcie_msix_isr,
1711                                                 (i == trans_pcie->def_irq) ?
1712                                                 iwl_pcie_irq_msix_handler :
1713                                                 iwl_pcie_irq_rx_msix_handler,
1714                                                 IRQF_SHARED,
1715                                                 qname,
1716                                                 msix_entry);
1717                 if (ret) {
1718                         IWL_ERR(trans_pcie->trans,
1719                                 "Error allocating IRQ %d\n", i);
1720
1721                         return ret;
1722                 }
1723         }
1724         iwl_pcie_irq_set_affinity(trans_pcie->trans);
1725
1726         return 0;
1727 }
1728
1729 static int _iwl_trans_pcie_start_hw(struct iwl_trans *trans, bool low_power)
1730 {
1731         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1732         int err;
1733
1734         lockdep_assert_held(&trans_pcie->mutex);
1735
1736         err = iwl_pcie_prepare_card_hw(trans);
1737         if (err) {
1738                 IWL_ERR(trans, "Error while preparing HW: %d\n", err);
1739                 return err;
1740         }
1741
1742         iwl_trans_pcie_sw_reset(trans);
1743
1744         err = iwl_pcie_apm_init(trans);
1745         if (err)
1746                 return err;
1747
1748         iwl_pcie_init_msix(trans_pcie);
1749
1750         /* From now on, the op_mode will be kept updated about RF kill state */
1751         iwl_enable_rfkill_int(trans);
1752
1753         trans_pcie->opmode_down = false;
1754
1755         /* Set is_down to false here so that...*/
1756         trans_pcie->is_down = false;
1757
1758         /* ...rfkill can call stop_device and set it false if needed */
1759         iwl_pcie_check_hw_rf_kill(trans);
1760
1761         /* Make sure we sync here, because we'll need full access later */
1762         if (low_power)
1763                 pm_runtime_resume(trans->dev);
1764
1765         return 0;
1766 }
1767
1768 static int iwl_trans_pcie_start_hw(struct iwl_trans *trans, bool low_power)
1769 {
1770         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1771         int ret;
1772
1773         mutex_lock(&trans_pcie->mutex);
1774         ret = _iwl_trans_pcie_start_hw(trans, low_power);
1775         mutex_unlock(&trans_pcie->mutex);
1776
1777         return ret;
1778 }
1779
1780 static void iwl_trans_pcie_op_mode_leave(struct iwl_trans *trans)
1781 {
1782         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1783
1784         mutex_lock(&trans_pcie->mutex);
1785
1786         /* disable interrupts - don't enable HW RF kill interrupt */
1787         iwl_disable_interrupts(trans);
1788
1789         iwl_pcie_apm_stop(trans, true);
1790
1791         iwl_disable_interrupts(trans);
1792
1793         iwl_pcie_disable_ict(trans);
1794
1795         mutex_unlock(&trans_pcie->mutex);
1796
1797         iwl_pcie_synchronize_irqs(trans);
1798 }
1799
1800 static void iwl_trans_pcie_write8(struct iwl_trans *trans, u32 ofs, u8 val)
1801 {
1802         writeb(val, IWL_TRANS_GET_PCIE_TRANS(trans)->hw_base + ofs);
1803 }
1804
1805 static void iwl_trans_pcie_write32(struct iwl_trans *trans, u32 ofs, u32 val)
1806 {
1807         writel(val, IWL_TRANS_GET_PCIE_TRANS(trans)->hw_base + ofs);
1808 }
1809
1810 static u32 iwl_trans_pcie_read32(struct iwl_trans *trans, u32 ofs)
1811 {
1812         return readl(IWL_TRANS_GET_PCIE_TRANS(trans)->hw_base + ofs);
1813 }
1814
1815 static u32 iwl_trans_pcie_prph_msk(struct iwl_trans *trans)
1816 {
1817         if (trans->cfg->device_family >= IWL_DEVICE_FAMILY_22560)
1818                 return 0x00FFFFFF;
1819         else
1820                 return 0x000FFFFF;
1821 }
1822
1823 static u32 iwl_trans_pcie_read_prph(struct iwl_trans *trans, u32 reg)
1824 {
1825         u32 mask = iwl_trans_pcie_prph_msk(trans);
1826
1827         iwl_trans_pcie_write32(trans, HBUS_TARG_PRPH_RADDR,
1828                                ((reg & mask) | (3 << 24)));
1829         return iwl_trans_pcie_read32(trans, HBUS_TARG_PRPH_RDAT);
1830 }
1831
1832 static void iwl_trans_pcie_write_prph(struct iwl_trans *trans, u32 addr,
1833                                       u32 val)
1834 {
1835         u32 mask = iwl_trans_pcie_prph_msk(trans);
1836
1837         iwl_trans_pcie_write32(trans, HBUS_TARG_PRPH_WADDR,
1838                                ((addr & mask) | (3 << 24)));
1839         iwl_trans_pcie_write32(trans, HBUS_TARG_PRPH_WDAT, val);
1840 }
1841
1842 static void iwl_trans_pcie_configure(struct iwl_trans *trans,
1843                                      const struct iwl_trans_config *trans_cfg)
1844 {
1845         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1846
1847         trans_pcie->cmd_queue = trans_cfg->cmd_queue;
1848         trans_pcie->cmd_fifo = trans_cfg->cmd_fifo;
1849         trans_pcie->cmd_q_wdg_timeout = trans_cfg->cmd_q_wdg_timeout;
1850         if (WARN_ON(trans_cfg->n_no_reclaim_cmds > MAX_NO_RECLAIM_CMDS))
1851                 trans_pcie->n_no_reclaim_cmds = 0;
1852         else
1853                 trans_pcie->n_no_reclaim_cmds = trans_cfg->n_no_reclaim_cmds;
1854         if (trans_pcie->n_no_reclaim_cmds)
1855                 memcpy(trans_pcie->no_reclaim_cmds, trans_cfg->no_reclaim_cmds,
1856                        trans_pcie->n_no_reclaim_cmds * sizeof(u8));
1857
1858         trans_pcie->rx_buf_size = trans_cfg->rx_buf_size;
1859         trans_pcie->rx_page_order =
1860                 iwl_trans_get_rb_size_order(trans_pcie->rx_buf_size);
1861
1862         trans_pcie->bc_table_dword = trans_cfg->bc_table_dword;
1863         trans_pcie->scd_set_active = trans_cfg->scd_set_active;
1864         trans_pcie->sw_csum_tx = trans_cfg->sw_csum_tx;
1865
1866         trans_pcie->page_offs = trans_cfg->cb_data_offs;
1867         trans_pcie->dev_cmd_offs = trans_cfg->cb_data_offs + sizeof(void *);
1868
1869         trans->command_groups = trans_cfg->command_groups;
1870         trans->command_groups_size = trans_cfg->command_groups_size;
1871
1872         /* Initialize NAPI here - it should be before registering to mac80211
1873          * in the opmode but after the HW struct is allocated.
1874          * As this function may be called again in some corner cases don't
1875          * do anything if NAPI was already initialized.
1876          */
1877         if (trans_pcie->napi_dev.reg_state != NETREG_DUMMY)
1878                 init_dummy_netdev(&trans_pcie->napi_dev);
1879 }
1880
1881 void iwl_trans_pcie_free(struct iwl_trans *trans)
1882 {
1883         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1884         int i;
1885
1886         iwl_pcie_synchronize_irqs(trans);
1887
1888         if (trans->cfg->gen2)
1889                 iwl_pcie_gen2_tx_free(trans);
1890         else
1891                 iwl_pcie_tx_free(trans);
1892         iwl_pcie_rx_free(trans);
1893
1894         if (trans_pcie->rba.alloc_wq) {
1895                 destroy_workqueue(trans_pcie->rba.alloc_wq);
1896                 trans_pcie->rba.alloc_wq = NULL;
1897         }
1898
1899         if (trans_pcie->msix_enabled) {
1900                 for (i = 0; i < trans_pcie->alloc_vecs; i++) {
1901                         irq_set_affinity_hint(
1902                                 trans_pcie->msix_entries[i].vector,
1903                                 NULL);
1904                 }
1905
1906                 trans_pcie->msix_enabled = false;
1907         } else {
1908                 iwl_pcie_free_ict(trans);
1909         }
1910
1911         iwl_pcie_free_fw_monitor(trans);
1912
1913         for_each_possible_cpu(i) {
1914                 struct iwl_tso_hdr_page *p =
1915                         per_cpu_ptr(trans_pcie->tso_hdr_page, i);
1916
1917                 if (p->page)
1918                         __free_page(p->page);
1919         }
1920
1921         free_percpu(trans_pcie->tso_hdr_page);
1922         mutex_destroy(&trans_pcie->mutex);
1923         iwl_trans_free(trans);
1924 }
1925
1926 static void iwl_trans_pcie_set_pmi(struct iwl_trans *trans, bool state)
1927 {
1928         if (state)
1929                 set_bit(STATUS_TPOWER_PMI, &trans->status);
1930         else
1931                 clear_bit(STATUS_TPOWER_PMI, &trans->status);
1932 }
1933
1934 struct iwl_trans_pcie_removal {
1935         struct pci_dev *pdev;
1936         struct work_struct work;
1937 };
1938
1939 static void iwl_trans_pcie_removal_wk(struct work_struct *wk)
1940 {
1941         struct iwl_trans_pcie_removal *removal =
1942                 container_of(wk, struct iwl_trans_pcie_removal, work);
1943         struct pci_dev *pdev = removal->pdev;
1944         char *prop[] = {"EVENT=INACCESSIBLE", NULL};
1945
1946         dev_err(&pdev->dev, "Device gone - attempting removal\n");
1947         kobject_uevent_env(&pdev->dev.kobj, KOBJ_CHANGE, prop);
1948         pci_lock_rescan_remove();
1949         pci_dev_put(pdev);
1950         pci_stop_and_remove_bus_device(pdev);
1951         pci_unlock_rescan_remove();
1952
1953         kfree(removal);
1954         module_put(THIS_MODULE);
1955 }
1956
1957 static bool iwl_trans_pcie_grab_nic_access(struct iwl_trans *trans,
1958                                            unsigned long *flags)
1959 {
1960         int ret;
1961         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
1962
1963         spin_lock_irqsave(&trans_pcie->reg_lock, *flags);
1964
1965         if (trans_pcie->cmd_hold_nic_awake)
1966                 goto out;
1967
1968         /* this bit wakes up the NIC */
1969         __iwl_trans_pcie_set_bit(trans, CSR_GP_CNTRL,
1970                                  BIT(trans->cfg->csr->flag_mac_access_req));
1971         if (trans->cfg->device_family >= IWL_DEVICE_FAMILY_8000)
1972                 udelay(2);
1973
1974         /*
1975          * These bits say the device is running, and should keep running for
1976          * at least a short while (at least as long as MAC_ACCESS_REQ stays 1),
1977          * but they do not indicate that embedded SRAM is restored yet;
1978          * HW with volatile SRAM must save/restore contents to/from
1979          * host DRAM when sleeping/waking for power-saving.
1980          * Each direction takes approximately 1/4 millisecond; with this
1981          * overhead, it's a good idea to grab and hold MAC_ACCESS_REQUEST if a
1982          * series of register accesses are expected (e.g. reading Event Log),
1983          * to keep device from sleeping.
1984          *
1985          * CSR_UCODE_DRV_GP1 register bit MAC_SLEEP == 0 indicates that
1986          * SRAM is okay/restored.  We don't check that here because this call
1987          * is just for hardware register access; but GP1 MAC_SLEEP
1988          * check is a good idea before accessing the SRAM of HW with
1989          * volatile SRAM (e.g. reading Event Log).
1990          *
1991          * 5000 series and later (including 1000 series) have non-volatile SRAM,
1992          * and do not save/restore SRAM when power cycling.
1993          */
1994         ret = iwl_poll_bit(trans, CSR_GP_CNTRL,
1995                            BIT(trans->cfg->csr->flag_val_mac_access_en),
1996                            (BIT(trans->cfg->csr->flag_mac_clock_ready) |
1997                             CSR_GP_CNTRL_REG_FLAG_GOING_TO_SLEEP), 15000);
1998         if (unlikely(ret < 0)) {
1999                 u32 cntrl = iwl_read32(trans, CSR_GP_CNTRL);
2000
2001                 WARN_ONCE(1,
2002                           "Timeout waiting for hardware access (CSR_GP_CNTRL 0x%08x)\n",
2003                           cntrl);
2004
2005                 iwl_trans_pcie_dump_regs(trans);
2006
2007                 if (iwlwifi_mod_params.remove_when_gone && cntrl == ~0U) {
2008                         struct iwl_trans_pcie_removal *removal;
2009
2010                         if (test_bit(STATUS_TRANS_DEAD, &trans->status))
2011                                 goto err;
2012
2013                         IWL_ERR(trans, "Device gone - scheduling removal!\n");
2014
2015                         /*
2016                          * get a module reference to avoid doing this
2017                          * while unloading anyway and to avoid
2018                          * scheduling a work with code that's being
2019                          * removed.
2020                          */
2021                         if (!try_module_get(THIS_MODULE)) {
2022                                 IWL_ERR(trans,
2023                                         "Module is being unloaded - abort\n");
2024                                 goto err;
2025                         }
2026
2027                         removal = kzalloc(sizeof(*removal), GFP_ATOMIC);
2028                         if (!removal) {
2029                                 module_put(THIS_MODULE);
2030                                 goto err;
2031                         }
2032                         /*
2033                          * we don't need to clear this flag, because
2034                          * the trans will be freed and reallocated.
2035                         */
2036                         set_bit(STATUS_TRANS_DEAD, &trans->status);
2037
2038                         removal->pdev = to_pci_dev(trans->dev);
2039                         INIT_WORK(&removal->work, iwl_trans_pcie_removal_wk);
2040                         pci_dev_get(removal->pdev);
2041                         schedule_work(&removal->work);
2042                 } else {
2043                         iwl_write32(trans, CSR_RESET,
2044                                     CSR_RESET_REG_FLAG_FORCE_NMI);
2045                 }
2046
2047 err:
2048                 spin_unlock_irqrestore(&trans_pcie->reg_lock, *flags);
2049                 return false;
2050         }
2051
2052 out:
2053         /*
2054          * Fool sparse by faking we release the lock - sparse will
2055          * track nic_access anyway.
2056          */
2057         __release(&trans_pcie->reg_lock);
2058         return true;
2059 }
2060
2061 static void iwl_trans_pcie_release_nic_access(struct iwl_trans *trans,
2062                                               unsigned long *flags)
2063 {
2064         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2065
2066         lockdep_assert_held(&trans_pcie->reg_lock);
2067
2068         /*
2069          * Fool sparse by faking we acquiring the lock - sparse will
2070          * track nic_access anyway.
2071          */
2072         __acquire(&trans_pcie->reg_lock);
2073
2074         if (trans_pcie->cmd_hold_nic_awake)
2075                 goto out;
2076
2077         __iwl_trans_pcie_clear_bit(trans, CSR_GP_CNTRL,
2078                                    BIT(trans->cfg->csr->flag_mac_access_req));
2079         /*
2080          * Above we read the CSR_GP_CNTRL register, which will flush
2081          * any previous writes, but we need the write that clears the
2082          * MAC_ACCESS_REQ bit to be performed before any other writes
2083          * scheduled on different CPUs (after we drop reg_lock).
2084          */
2085         mmiowb();
2086 out:
2087         spin_unlock_irqrestore(&trans_pcie->reg_lock, *flags);
2088 }
2089
2090 static int iwl_trans_pcie_read_mem(struct iwl_trans *trans, u32 addr,
2091                                    void *buf, int dwords)
2092 {
2093         unsigned long flags;
2094         int offs, ret = 0;
2095         u32 *vals = buf;
2096
2097         if (iwl_trans_grab_nic_access(trans, &flags)) {
2098                 iwl_write32(trans, HBUS_TARG_MEM_RADDR, addr);
2099                 for (offs = 0; offs < dwords; offs++)
2100                         vals[offs] = iwl_read32(trans, HBUS_TARG_MEM_RDAT);
2101                 iwl_trans_release_nic_access(trans, &flags);
2102         } else {
2103                 ret = -EBUSY;
2104         }
2105         return ret;
2106 }
2107
2108 static int iwl_trans_pcie_write_mem(struct iwl_trans *trans, u32 addr,
2109                                     const void *buf, int dwords)
2110 {
2111         unsigned long flags;
2112         int offs, ret = 0;
2113         const u32 *vals = buf;
2114
2115         if (iwl_trans_grab_nic_access(trans, &flags)) {
2116                 iwl_write32(trans, HBUS_TARG_MEM_WADDR, addr);
2117                 for (offs = 0; offs < dwords; offs++)
2118                         iwl_write32(trans, HBUS_TARG_MEM_WDAT,
2119                                     vals ? vals[offs] : 0);
2120                 iwl_trans_release_nic_access(trans, &flags);
2121         } else {
2122                 ret = -EBUSY;
2123         }
2124         return ret;
2125 }
2126
2127 static void iwl_trans_pcie_freeze_txq_timer(struct iwl_trans *trans,
2128                                             unsigned long txqs,
2129                                             bool freeze)
2130 {
2131         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2132         int queue;
2133
2134         for_each_set_bit(queue, &txqs, BITS_PER_LONG) {
2135                 struct iwl_txq *txq = trans_pcie->txq[queue];
2136                 unsigned long now;
2137
2138                 spin_lock_bh(&txq->lock);
2139
2140                 now = jiffies;
2141
2142                 if (txq->frozen == freeze)
2143                         goto next_queue;
2144
2145                 IWL_DEBUG_TX_QUEUES(trans, "%s TXQ %d\n",
2146                                     freeze ? "Freezing" : "Waking", queue);
2147
2148                 txq->frozen = freeze;
2149
2150                 if (txq->read_ptr == txq->write_ptr)
2151                         goto next_queue;
2152
2153                 if (freeze) {
2154                         if (unlikely(time_after(now,
2155                                                 txq->stuck_timer.expires))) {
2156                                 /*
2157                                  * The timer should have fired, maybe it is
2158                                  * spinning right now on the lock.
2159                                  */
2160                                 goto next_queue;
2161                         }
2162                         /* remember how long until the timer fires */
2163                         txq->frozen_expiry_remainder =
2164                                 txq->stuck_timer.expires - now;
2165                         del_timer(&txq->stuck_timer);
2166                         goto next_queue;
2167                 }
2168
2169                 /*
2170                  * Wake a non-empty queue -> arm timer with the
2171                  * remainder before it froze
2172                  */
2173                 mod_timer(&txq->stuck_timer,
2174                           now + txq->frozen_expiry_remainder);
2175
2176 next_queue:
2177                 spin_unlock_bh(&txq->lock);
2178         }
2179 }
2180
2181 static void iwl_trans_pcie_block_txq_ptrs(struct iwl_trans *trans, bool block)
2182 {
2183         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2184         int i;
2185
2186         for (i = 0; i < trans->cfg->base_params->num_of_queues; i++) {
2187                 struct iwl_txq *txq = trans_pcie->txq[i];
2188
2189                 if (i == trans_pcie->cmd_queue)
2190                         continue;
2191
2192                 spin_lock_bh(&txq->lock);
2193
2194                 if (!block && !(WARN_ON_ONCE(!txq->block))) {
2195                         txq->block--;
2196                         if (!txq->block) {
2197                                 iwl_write32(trans, HBUS_TARG_WRPTR,
2198                                             txq->write_ptr | (i << 8));
2199                         }
2200                 } else if (block) {
2201                         txq->block++;
2202                 }
2203
2204                 spin_unlock_bh(&txq->lock);
2205         }
2206 }
2207
2208 #define IWL_FLUSH_WAIT_MS       2000
2209
2210 void iwl_trans_pcie_log_scd_error(struct iwl_trans *trans, struct iwl_txq *txq)
2211 {
2212         u32 txq_id = txq->id;
2213         u32 status;
2214         bool active;
2215         u8 fifo;
2216
2217         if (trans->cfg->use_tfh) {
2218                 IWL_ERR(trans, "Queue %d is stuck %d %d\n", txq_id,
2219                         txq->read_ptr, txq->write_ptr);
2220                 /* TODO: access new SCD registers and dump them */
2221                 return;
2222         }
2223
2224         status = iwl_read_prph(trans, SCD_QUEUE_STATUS_BITS(txq_id));
2225         fifo = (status >> SCD_QUEUE_STTS_REG_POS_TXF) & 0x7;
2226         active = !!(status & BIT(SCD_QUEUE_STTS_REG_POS_ACTIVE));
2227
2228         IWL_ERR(trans,
2229                 "Queue %d is %sactive on fifo %d and stuck for %u ms. SW [%d, %d] HW [%d, %d] FH TRB=0x0%x\n",
2230                 txq_id, active ? "" : "in", fifo,
2231                 jiffies_to_msecs(txq->wd_timeout),
2232                 txq->read_ptr, txq->write_ptr,
2233                 iwl_read_prph(trans, SCD_QUEUE_RDPTR(txq_id)) &
2234                         (trans->cfg->base_params->max_tfd_queue_size - 1),
2235                 iwl_read_prph(trans, SCD_QUEUE_WRPTR(txq_id)) &
2236                         (trans->cfg->base_params->max_tfd_queue_size - 1),
2237                 iwl_read_direct32(trans, FH_TX_TRB_REG(fifo)));
2238 }
2239
2240 static int iwl_trans_pcie_rxq_dma_data(struct iwl_trans *trans, int queue,
2241                                        struct iwl_trans_rxq_dma_data *data)
2242 {
2243         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2244
2245         if (queue >= trans->num_rx_queues || !trans_pcie->rxq)
2246                 return -EINVAL;
2247
2248         data->fr_bd_cb = trans_pcie->rxq[queue].bd_dma;
2249         data->urbd_stts_wrptr = trans_pcie->rxq[queue].rb_stts_dma;
2250         data->ur_bd_cb = trans_pcie->rxq[queue].used_bd_dma;
2251         data->fr_bd_wid = 0;
2252
2253         return 0;
2254 }
2255
2256 static int iwl_trans_pcie_wait_txq_empty(struct iwl_trans *trans, int txq_idx)
2257 {
2258         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2259         struct iwl_txq *txq;
2260         unsigned long now = jiffies;
2261         u8 wr_ptr;
2262
2263         /* Make sure the NIC is still alive in the bus */
2264         if (test_bit(STATUS_TRANS_DEAD, &trans->status))
2265                 return -ENODEV;
2266
2267         if (!test_bit(txq_idx, trans_pcie->queue_used))
2268                 return -EINVAL;
2269
2270         IWL_DEBUG_TX_QUEUES(trans, "Emptying queue %d...\n", txq_idx);
2271         txq = trans_pcie->txq[txq_idx];
2272         wr_ptr = READ_ONCE(txq->write_ptr);
2273
2274         while (txq->read_ptr != READ_ONCE(txq->write_ptr) &&
2275                !time_after(jiffies,
2276                            now + msecs_to_jiffies(IWL_FLUSH_WAIT_MS))) {
2277                 u8 write_ptr = READ_ONCE(txq->write_ptr);
2278
2279                 if (WARN_ONCE(wr_ptr != write_ptr,
2280                               "WR pointer moved while flushing %d -> %d\n",
2281                               wr_ptr, write_ptr))
2282                         return -ETIMEDOUT;
2283                 usleep_range(1000, 2000);
2284         }
2285
2286         if (txq->read_ptr != txq->write_ptr) {
2287                 IWL_ERR(trans,
2288                         "fail to flush all tx fifo queues Q %d\n", txq_idx);
2289                 iwl_trans_pcie_log_scd_error(trans, txq);
2290                 return -ETIMEDOUT;
2291         }
2292
2293         IWL_DEBUG_TX_QUEUES(trans, "Queue %d is now empty.\n", txq_idx);
2294
2295         return 0;
2296 }
2297
2298 static int iwl_trans_pcie_wait_txqs_empty(struct iwl_trans *trans, u32 txq_bm)
2299 {
2300         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2301         int cnt;
2302         int ret = 0;
2303
2304         /* waiting for all the tx frames complete might take a while */
2305         for (cnt = 0; cnt < trans->cfg->base_params->num_of_queues; cnt++) {
2306
2307                 if (cnt == trans_pcie->cmd_queue)
2308                         continue;
2309                 if (!test_bit(cnt, trans_pcie->queue_used))
2310                         continue;
2311                 if (!(BIT(cnt) & txq_bm))
2312                         continue;
2313
2314                 ret = iwl_trans_pcie_wait_txq_empty(trans, cnt);
2315                 if (ret)
2316                         break;
2317         }
2318
2319         return ret;
2320 }
2321
2322 static void iwl_trans_pcie_set_bits_mask(struct iwl_trans *trans, u32 reg,
2323                                          u32 mask, u32 value)
2324 {
2325         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2326         unsigned long flags;
2327
2328         spin_lock_irqsave(&trans_pcie->reg_lock, flags);
2329         __iwl_trans_pcie_set_bits_mask(trans, reg, mask, value);
2330         spin_unlock_irqrestore(&trans_pcie->reg_lock, flags);
2331 }
2332
2333 static void iwl_trans_pcie_ref(struct iwl_trans *trans)
2334 {
2335         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2336
2337         if (iwlwifi_mod_params.d0i3_disable)
2338                 return;
2339
2340         pm_runtime_get(&trans_pcie->pci_dev->dev);
2341
2342 #ifdef CONFIG_PM
2343         IWL_DEBUG_RPM(trans, "runtime usage count: %d\n",
2344                       atomic_read(&trans_pcie->pci_dev->dev.power.usage_count));
2345 #endif /* CONFIG_PM */
2346 }
2347
2348 static void iwl_trans_pcie_unref(struct iwl_trans *trans)
2349 {
2350         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2351
2352         if (iwlwifi_mod_params.d0i3_disable)
2353                 return;
2354
2355         pm_runtime_mark_last_busy(&trans_pcie->pci_dev->dev);
2356         pm_runtime_put_autosuspend(&trans_pcie->pci_dev->dev);
2357
2358 #ifdef CONFIG_PM
2359         IWL_DEBUG_RPM(trans, "runtime usage count: %d\n",
2360                       atomic_read(&trans_pcie->pci_dev->dev.power.usage_count));
2361 #endif /* CONFIG_PM */
2362 }
2363
2364 static const char *get_csr_string(int cmd)
2365 {
2366 #define IWL_CMD(x) case x: return #x
2367         switch (cmd) {
2368         IWL_CMD(CSR_HW_IF_CONFIG_REG);
2369         IWL_CMD(CSR_INT_COALESCING);
2370         IWL_CMD(CSR_INT);
2371         IWL_CMD(CSR_INT_MASK);
2372         IWL_CMD(CSR_FH_INT_STATUS);
2373         IWL_CMD(CSR_GPIO_IN);
2374         IWL_CMD(CSR_RESET);
2375         IWL_CMD(CSR_GP_CNTRL);
2376         IWL_CMD(CSR_HW_REV);
2377         IWL_CMD(CSR_EEPROM_REG);
2378         IWL_CMD(CSR_EEPROM_GP);
2379         IWL_CMD(CSR_OTP_GP_REG);
2380         IWL_CMD(CSR_GIO_REG);
2381         IWL_CMD(CSR_GP_UCODE_REG);
2382         IWL_CMD(CSR_GP_DRIVER_REG);
2383         IWL_CMD(CSR_UCODE_DRV_GP1);
2384         IWL_CMD(CSR_UCODE_DRV_GP2);
2385         IWL_CMD(CSR_LED_REG);
2386         IWL_CMD(CSR_DRAM_INT_TBL_REG);
2387         IWL_CMD(CSR_GIO_CHICKEN_BITS);
2388         IWL_CMD(CSR_ANA_PLL_CFG);
2389         IWL_CMD(CSR_HW_REV_WA_REG);
2390         IWL_CMD(CSR_MONITOR_STATUS_REG);
2391         IWL_CMD(CSR_DBG_HPET_MEM_REG);
2392         default:
2393                 return "UNKNOWN";
2394         }
2395 #undef IWL_CMD
2396 }
2397
2398 void iwl_pcie_dump_csr(struct iwl_trans *trans)
2399 {
2400         int i;
2401         static const u32 csr_tbl[] = {
2402                 CSR_HW_IF_CONFIG_REG,
2403                 CSR_INT_COALESCING,
2404                 CSR_INT,
2405                 CSR_INT_MASK,
2406                 CSR_FH_INT_STATUS,
2407                 CSR_GPIO_IN,
2408                 CSR_RESET,
2409                 CSR_GP_CNTRL,
2410                 CSR_HW_REV,
2411                 CSR_EEPROM_REG,
2412                 CSR_EEPROM_GP,
2413                 CSR_OTP_GP_REG,
2414                 CSR_GIO_REG,
2415                 CSR_GP_UCODE_REG,
2416                 CSR_GP_DRIVER_REG,
2417                 CSR_UCODE_DRV_GP1,
2418                 CSR_UCODE_DRV_GP2,
2419                 CSR_LED_REG,
2420                 CSR_DRAM_INT_TBL_REG,
2421                 CSR_GIO_CHICKEN_BITS,
2422                 CSR_ANA_PLL_CFG,
2423                 CSR_MONITOR_STATUS_REG,
2424                 CSR_HW_REV_WA_REG,
2425                 CSR_DBG_HPET_MEM_REG
2426         };
2427         IWL_ERR(trans, "CSR values:\n");
2428         IWL_ERR(trans, "(2nd byte of CSR_INT_COALESCING is "
2429                 "CSR_INT_PERIODIC_REG)\n");
2430         for (i = 0; i <  ARRAY_SIZE(csr_tbl); i++) {
2431                 IWL_ERR(trans, "  %25s: 0X%08x\n",
2432                         get_csr_string(csr_tbl[i]),
2433                         iwl_read32(trans, csr_tbl[i]));
2434         }
2435 }
2436
2437 #ifdef CONFIG_IWLWIFI_DEBUGFS
2438 /* create and remove of files */
2439 #define DEBUGFS_ADD_FILE(name, parent, mode) do {                       \
2440         if (!debugfs_create_file(#name, mode, parent, trans,            \
2441                                  &iwl_dbgfs_##name##_ops))              \
2442                 goto err;                                               \
2443 } while (0)
2444
2445 /* file operation */
2446 #define DEBUGFS_READ_FILE_OPS(name)                                     \
2447 static const struct file_operations iwl_dbgfs_##name##_ops = {          \
2448         .read = iwl_dbgfs_##name##_read,                                \
2449         .open = simple_open,                                            \
2450         .llseek = generic_file_llseek,                                  \
2451 };
2452
2453 #define DEBUGFS_WRITE_FILE_OPS(name)                                    \
2454 static const struct file_operations iwl_dbgfs_##name##_ops = {          \
2455         .write = iwl_dbgfs_##name##_write,                              \
2456         .open = simple_open,                                            \
2457         .llseek = generic_file_llseek,                                  \
2458 };
2459
2460 #define DEBUGFS_READ_WRITE_FILE_OPS(name)                               \
2461 static const struct file_operations iwl_dbgfs_##name##_ops = {          \
2462         .write = iwl_dbgfs_##name##_write,                              \
2463         .read = iwl_dbgfs_##name##_read,                                \
2464         .open = simple_open,                                            \
2465         .llseek = generic_file_llseek,                                  \
2466 };
2467
2468 static ssize_t iwl_dbgfs_tx_queue_read(struct file *file,
2469                                        char __user *user_buf,
2470                                        size_t count, loff_t *ppos)
2471 {
2472         struct iwl_trans *trans = file->private_data;
2473         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2474         struct iwl_txq *txq;
2475         char *buf;
2476         int pos = 0;
2477         int cnt;
2478         int ret;
2479         size_t bufsz;
2480
2481         bufsz = sizeof(char) * 75 * trans->cfg->base_params->num_of_queues;
2482
2483         if (!trans_pcie->txq_memory)
2484                 return -EAGAIN;
2485
2486         buf = kzalloc(bufsz, GFP_KERNEL);
2487         if (!buf)
2488                 return -ENOMEM;
2489
2490         for (cnt = 0; cnt < trans->cfg->base_params->num_of_queues; cnt++) {
2491                 txq = trans_pcie->txq[cnt];
2492                 pos += scnprintf(buf + pos, bufsz - pos,
2493                                 "hwq %.2d: read=%u write=%u use=%d stop=%d need_update=%d frozen=%d%s\n",
2494                                 cnt, txq->read_ptr, txq->write_ptr,
2495                                 !!test_bit(cnt, trans_pcie->queue_used),
2496                                  !!test_bit(cnt, trans_pcie->queue_stopped),
2497                                  txq->need_update, txq->frozen,
2498                                  (cnt == trans_pcie->cmd_queue ? " HCMD" : ""));
2499         }
2500         ret = simple_read_from_buffer(user_buf, count, ppos, buf, pos);
2501         kfree(buf);
2502         return ret;
2503 }
2504
2505 static ssize_t iwl_dbgfs_rx_queue_read(struct file *file,
2506                                        char __user *user_buf,
2507                                        size_t count, loff_t *ppos)
2508 {
2509         struct iwl_trans *trans = file->private_data;
2510         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2511         char *buf;
2512         int pos = 0, i, ret;
2513         size_t bufsz = sizeof(buf);
2514
2515         bufsz = sizeof(char) * 121 * trans->num_rx_queues;
2516
2517         if (!trans_pcie->rxq)
2518                 return -EAGAIN;
2519
2520         buf = kzalloc(bufsz, GFP_KERNEL);
2521         if (!buf)
2522                 return -ENOMEM;
2523
2524         for (i = 0; i < trans->num_rx_queues && pos < bufsz; i++) {
2525                 struct iwl_rxq *rxq = &trans_pcie->rxq[i];
2526
2527                 pos += scnprintf(buf + pos, bufsz - pos, "queue#: %2d\n",
2528                                  i);
2529                 pos += scnprintf(buf + pos, bufsz - pos, "\tread: %u\n",
2530                                  rxq->read);
2531                 pos += scnprintf(buf + pos, bufsz - pos, "\twrite: %u\n",
2532                                  rxq->write);
2533                 pos += scnprintf(buf + pos, bufsz - pos, "\twrite_actual: %u\n",
2534                                  rxq->write_actual);
2535                 pos += scnprintf(buf + pos, bufsz - pos, "\tneed_update: %2d\n",
2536                                  rxq->need_update);
2537                 pos += scnprintf(buf + pos, bufsz - pos, "\tfree_count: %u\n",
2538                                  rxq->free_count);
2539                 if (rxq->rb_stts) {
2540                         u32 r = __le16_to_cpu(iwl_get_closed_rb_stts(trans,
2541                                                                      rxq));
2542                         pos += scnprintf(buf + pos, bufsz - pos,
2543                                          "\tclosed_rb_num: %u\n",
2544                                          r & 0x0FFF);
2545                 } else {
2546                         pos += scnprintf(buf + pos, bufsz - pos,
2547                                          "\tclosed_rb_num: Not Allocated\n");
2548                 }
2549         }
2550         ret = simple_read_from_buffer(user_buf, count, ppos, buf, pos);
2551         kfree(buf);
2552
2553         return ret;
2554 }
2555
2556 static ssize_t iwl_dbgfs_interrupt_read(struct file *file,
2557                                         char __user *user_buf,
2558                                         size_t count, loff_t *ppos)
2559 {
2560         struct iwl_trans *trans = file->private_data;
2561         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2562         struct isr_statistics *isr_stats = &trans_pcie->isr_stats;
2563
2564         int pos = 0;
2565         char *buf;
2566         int bufsz = 24 * 64; /* 24 items * 64 char per item */
2567         ssize_t ret;
2568
2569         buf = kzalloc(bufsz, GFP_KERNEL);
2570         if (!buf)
2571                 return -ENOMEM;
2572
2573         pos += scnprintf(buf + pos, bufsz - pos,
2574                         "Interrupt Statistics Report:\n");
2575
2576         pos += scnprintf(buf + pos, bufsz - pos, "HW Error:\t\t\t %u\n",
2577                 isr_stats->hw);
2578         pos += scnprintf(buf + pos, bufsz - pos, "SW Error:\t\t\t %u\n",
2579                 isr_stats->sw);
2580         if (isr_stats->sw || isr_stats->hw) {
2581                 pos += scnprintf(buf + pos, bufsz - pos,
2582                         "\tLast Restarting Code:  0x%X\n",
2583                         isr_stats->err_code);
2584         }
2585 #ifdef CONFIG_IWLWIFI_DEBUG
2586         pos += scnprintf(buf + pos, bufsz - pos, "Frame transmitted:\t\t %u\n",
2587                 isr_stats->sch);
2588         pos += scnprintf(buf + pos, bufsz - pos, "Alive interrupt:\t\t %u\n",
2589                 isr_stats->alive);
2590 #endif
2591         pos += scnprintf(buf + pos, bufsz - pos,
2592                 "HW RF KILL switch toggled:\t %u\n", isr_stats->rfkill);
2593
2594         pos += scnprintf(buf + pos, bufsz - pos, "CT KILL:\t\t\t %u\n",
2595                 isr_stats->ctkill);
2596
2597         pos += scnprintf(buf + pos, bufsz - pos, "Wakeup Interrupt:\t\t %u\n",
2598                 isr_stats->wakeup);
2599
2600         pos += scnprintf(buf + pos, bufsz - pos,
2601                 "Rx command responses:\t\t %u\n", isr_stats->rx);
2602
2603         pos += scnprintf(buf + pos, bufsz - pos, "Tx/FH interrupt:\t\t %u\n",
2604                 isr_stats->tx);
2605
2606         pos += scnprintf(buf + pos, bufsz - pos, "Unexpected INTA:\t\t %u\n",
2607                 isr_stats->unhandled);
2608
2609         ret = simple_read_from_buffer(user_buf, count, ppos, buf, pos);
2610         kfree(buf);
2611         return ret;
2612 }
2613
2614 static ssize_t iwl_dbgfs_interrupt_write(struct file *file,
2615                                          const char __user *user_buf,
2616                                          size_t count, loff_t *ppos)
2617 {
2618         struct iwl_trans *trans = file->private_data;
2619         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2620         struct isr_statistics *isr_stats = &trans_pcie->isr_stats;
2621         u32 reset_flag;
2622         int ret;
2623
2624         ret = kstrtou32_from_user(user_buf, count, 16, &reset_flag);
2625         if (ret)
2626                 return ret;
2627         if (reset_flag == 0)
2628                 memset(isr_stats, 0, sizeof(*isr_stats));
2629
2630         return count;
2631 }
2632
2633 static ssize_t iwl_dbgfs_csr_write(struct file *file,
2634                                    const char __user *user_buf,
2635                                    size_t count, loff_t *ppos)
2636 {
2637         struct iwl_trans *trans = file->private_data;
2638
2639         iwl_pcie_dump_csr(trans);
2640
2641         return count;
2642 }
2643
2644 static ssize_t iwl_dbgfs_fh_reg_read(struct file *file,
2645                                      char __user *user_buf,
2646                                      size_t count, loff_t *ppos)
2647 {
2648         struct iwl_trans *trans = file->private_data;
2649         char *buf = NULL;
2650         ssize_t ret;
2651
2652         ret = iwl_dump_fh(trans, &buf);
2653         if (ret < 0)
2654                 return ret;
2655         if (!buf)
2656                 return -EINVAL;
2657         ret = simple_read_from_buffer(user_buf, count, ppos, buf, ret);
2658         kfree(buf);
2659         return ret;
2660 }
2661
2662 static ssize_t iwl_dbgfs_rfkill_read(struct file *file,
2663                                      char __user *user_buf,
2664                                      size_t count, loff_t *ppos)
2665 {
2666         struct iwl_trans *trans = file->private_data;
2667         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2668         char buf[100];
2669         int pos;
2670
2671         pos = scnprintf(buf, sizeof(buf), "debug: %d\nhw: %d\n",
2672                         trans_pcie->debug_rfkill,
2673                         !(iwl_read32(trans, CSR_GP_CNTRL) &
2674                                 CSR_GP_CNTRL_REG_FLAG_HW_RF_KILL_SW));
2675
2676         return simple_read_from_buffer(user_buf, count, ppos, buf, pos);
2677 }
2678
2679 static ssize_t iwl_dbgfs_rfkill_write(struct file *file,
2680                                       const char __user *user_buf,
2681                                       size_t count, loff_t *ppos)
2682 {
2683         struct iwl_trans *trans = file->private_data;
2684         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2685         bool old = trans_pcie->debug_rfkill;
2686         int ret;
2687
2688         ret = kstrtobool_from_user(user_buf, count, &trans_pcie->debug_rfkill);
2689         if (ret)
2690                 return ret;
2691         if (old == trans_pcie->debug_rfkill)
2692                 return count;
2693         IWL_WARN(trans, "changing debug rfkill %d->%d\n",
2694                  old, trans_pcie->debug_rfkill);
2695         iwl_pcie_handle_rfkill_irq(trans);
2696
2697         return count;
2698 }
2699
2700 DEBUGFS_READ_WRITE_FILE_OPS(interrupt);
2701 DEBUGFS_READ_FILE_OPS(fh_reg);
2702 DEBUGFS_READ_FILE_OPS(rx_queue);
2703 DEBUGFS_READ_FILE_OPS(tx_queue);
2704 DEBUGFS_WRITE_FILE_OPS(csr);
2705 DEBUGFS_READ_WRITE_FILE_OPS(rfkill);
2706
2707 /* Create the debugfs files and directories */
2708 int iwl_trans_pcie_dbgfs_register(struct iwl_trans *trans)
2709 {
2710         struct dentry *dir = trans->dbgfs_dir;
2711
2712         DEBUGFS_ADD_FILE(rx_queue, dir, 0400);
2713         DEBUGFS_ADD_FILE(tx_queue, dir, 0400);
2714         DEBUGFS_ADD_FILE(interrupt, dir, 0600);
2715         DEBUGFS_ADD_FILE(csr, dir, 0200);
2716         DEBUGFS_ADD_FILE(fh_reg, dir, 0400);
2717         DEBUGFS_ADD_FILE(rfkill, dir, 0600);
2718         return 0;
2719
2720 err:
2721         IWL_ERR(trans, "failed to create the trans debugfs entry\n");
2722         return -ENOMEM;
2723 }
2724 #endif /*CONFIG_IWLWIFI_DEBUGFS */
2725
2726 static u32 iwl_trans_pcie_get_cmdlen(struct iwl_trans *trans, void *tfd)
2727 {
2728         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2729         u32 cmdlen = 0;
2730         int i;
2731
2732         for (i = 0; i < trans_pcie->max_tbs; i++)
2733                 cmdlen += iwl_pcie_tfd_tb_get_len(trans, tfd, i);
2734
2735         return cmdlen;
2736 }
2737
2738 static u32 iwl_trans_pcie_dump_rbs(struct iwl_trans *trans,
2739                                    struct iwl_fw_error_dump_data **data,
2740                                    int allocated_rb_nums)
2741 {
2742         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2743         int max_len = PAGE_SIZE << trans_pcie->rx_page_order;
2744         /* Dump RBs is supported only for pre-9000 devices (1 queue) */
2745         struct iwl_rxq *rxq = &trans_pcie->rxq[0];
2746         u32 i, r, j, rb_len = 0;
2747
2748         spin_lock(&rxq->lock);
2749
2750         r = le16_to_cpu(iwl_get_closed_rb_stts(trans, rxq)) & 0x0FFF;
2751
2752         for (i = rxq->read, j = 0;
2753              i != r && j < allocated_rb_nums;
2754              i = (i + 1) & RX_QUEUE_MASK, j++) {
2755                 struct iwl_rx_mem_buffer *rxb = rxq->queue[i];
2756                 struct iwl_fw_error_dump_rb *rb;
2757
2758                 dma_unmap_page(trans->dev, rxb->page_dma, max_len,
2759                                DMA_FROM_DEVICE);
2760
2761                 rb_len += sizeof(**data) + sizeof(*rb) + max_len;
2762
2763                 (*data)->type = cpu_to_le32(IWL_FW_ERROR_DUMP_RB);
2764                 (*data)->len = cpu_to_le32(sizeof(*rb) + max_len);
2765                 rb = (void *)(*data)->data;
2766                 rb->index = cpu_to_le32(i);
2767                 memcpy(rb->data, page_address(rxb->page), max_len);
2768                 /* remap the page for the free benefit */
2769                 rxb->page_dma = dma_map_page(trans->dev, rxb->page, 0,
2770                                                      max_len,
2771                                                      DMA_FROM_DEVICE);
2772
2773                 *data = iwl_fw_error_next_data(*data);
2774         }
2775
2776         spin_unlock(&rxq->lock);
2777
2778         return rb_len;
2779 }
2780 #define IWL_CSR_TO_DUMP (0x250)
2781
2782 static u32 iwl_trans_pcie_dump_csr(struct iwl_trans *trans,
2783                                    struct iwl_fw_error_dump_data **data)
2784 {
2785         u32 csr_len = sizeof(**data) + IWL_CSR_TO_DUMP;
2786         __le32 *val;
2787         int i;
2788
2789         (*data)->type = cpu_to_le32(IWL_FW_ERROR_DUMP_CSR);
2790         (*data)->len = cpu_to_le32(IWL_CSR_TO_DUMP);
2791         val = (void *)(*data)->data;
2792
2793         for (i = 0; i < IWL_CSR_TO_DUMP; i += 4)
2794                 *val++ = cpu_to_le32(iwl_trans_pcie_read32(trans, i));
2795
2796         *data = iwl_fw_error_next_data(*data);
2797
2798         return csr_len;
2799 }
2800
2801 static u32 iwl_trans_pcie_fh_regs_dump(struct iwl_trans *trans,
2802                                        struct iwl_fw_error_dump_data **data)
2803 {
2804         u32 fh_regs_len = FH_MEM_UPPER_BOUND - FH_MEM_LOWER_BOUND;
2805         unsigned long flags;
2806         __le32 *val;
2807         int i;
2808
2809         if (!iwl_trans_grab_nic_access(trans, &flags))
2810                 return 0;
2811
2812         (*data)->type = cpu_to_le32(IWL_FW_ERROR_DUMP_FH_REGS);
2813         (*data)->len = cpu_to_le32(fh_regs_len);
2814         val = (void *)(*data)->data;
2815
2816         if (!trans->cfg->gen2)
2817                 for (i = FH_MEM_LOWER_BOUND; i < FH_MEM_UPPER_BOUND;
2818                      i += sizeof(u32))
2819                         *val++ = cpu_to_le32(iwl_trans_pcie_read32(trans, i));
2820         else
2821                 for (i = FH_MEM_LOWER_BOUND_GEN2; i < FH_MEM_UPPER_BOUND_GEN2;
2822                      i += sizeof(u32))
2823                         *val++ = cpu_to_le32(iwl_trans_pcie_read_prph(trans,
2824                                                                       i));
2825
2826         iwl_trans_release_nic_access(trans, &flags);
2827
2828         *data = iwl_fw_error_next_data(*data);
2829
2830         return sizeof(**data) + fh_regs_len;
2831 }
2832
2833 static u32
2834 iwl_trans_pci_dump_marbh_monitor(struct iwl_trans *trans,
2835                                  struct iwl_fw_error_dump_fw_mon *fw_mon_data,
2836                                  u32 monitor_len)
2837 {
2838         u32 buf_size_in_dwords = (monitor_len >> 2);
2839         u32 *buffer = (u32 *)fw_mon_data->data;
2840         unsigned long flags;
2841         u32 i;
2842
2843         if (!iwl_trans_grab_nic_access(trans, &flags))
2844                 return 0;
2845
2846         iwl_write_prph_no_grab(trans, MON_DMARB_RD_CTL_ADDR, 0x1);
2847         for (i = 0; i < buf_size_in_dwords; i++)
2848                 buffer[i] = iwl_read_prph_no_grab(trans,
2849                                 MON_DMARB_RD_DATA_ADDR);
2850         iwl_write_prph_no_grab(trans, MON_DMARB_RD_CTL_ADDR, 0x0);
2851
2852         iwl_trans_release_nic_access(trans, &flags);
2853
2854         return monitor_len;
2855 }
2856
2857 static u32
2858 iwl_trans_pcie_dump_monitor(struct iwl_trans *trans,
2859                             struct iwl_fw_error_dump_data **data,
2860                             u32 monitor_len)
2861 {
2862         u32 len = 0;
2863
2864         if ((trans->num_blocks &&
2865              trans->cfg->device_family == IWL_DEVICE_FAMILY_7000) ||
2866             trans->dbg_dest_tlv) {
2867                 struct iwl_fw_error_dump_fw_mon *fw_mon_data;
2868                 u32 base, write_ptr, wrap_cnt;
2869
2870                 /* If there was a dest TLV - use the values from there */
2871                 if (trans->dbg_dest_tlv) {
2872                         write_ptr =
2873                                 le32_to_cpu(trans->dbg_dest_tlv->write_ptr_reg);
2874                         wrap_cnt = le32_to_cpu(trans->dbg_dest_tlv->wrap_count);
2875                         base = le32_to_cpu(trans->dbg_dest_tlv->base_reg);
2876                 } else {
2877                         base = MON_BUFF_BASE_ADDR;
2878                         write_ptr = MON_BUFF_WRPTR;
2879                         wrap_cnt = MON_BUFF_CYCLE_CNT;
2880                 }
2881
2882                 (*data)->type = cpu_to_le32(IWL_FW_ERROR_DUMP_FW_MONITOR);
2883                 fw_mon_data = (void *)(*data)->data;
2884                 fw_mon_data->fw_mon_wr_ptr =
2885                         cpu_to_le32(iwl_read_prph(trans, write_ptr));
2886                 fw_mon_data->fw_mon_cycle_cnt =
2887                         cpu_to_le32(iwl_read_prph(trans, wrap_cnt));
2888                 fw_mon_data->fw_mon_base_ptr =
2889                         cpu_to_le32(iwl_read_prph(trans, base));
2890
2891                 len += sizeof(**data) + sizeof(*fw_mon_data);
2892                 if (trans->num_blocks) {
2893                         memcpy(fw_mon_data->data,
2894                                trans->fw_mon[0].block,
2895                                trans->fw_mon[0].size);
2896
2897                         monitor_len = trans->fw_mon[0].size;
2898                 } else if (trans->dbg_dest_tlv->monitor_mode == SMEM_MODE) {
2899                         /*
2900                          * Update pointers to reflect actual values after
2901                          * shifting
2902                          */
2903                         if (trans->dbg_dest_tlv->version) {
2904                                 base = (iwl_read_prph(trans, base) &
2905                                         IWL_LDBG_M2S_BUF_BA_MSK) <<
2906                                        trans->dbg_dest_tlv->base_shift;
2907                                 base *= IWL_M2S_UNIT_SIZE;
2908                                 base += trans->cfg->smem_offset;
2909                         } else {
2910                                 base = iwl_read_prph(trans, base) <<
2911                                        trans->dbg_dest_tlv->base_shift;
2912                         }
2913
2914                         iwl_trans_read_mem(trans, base, fw_mon_data->data,
2915                                            monitor_len / sizeof(u32));
2916                 } else if (trans->dbg_dest_tlv->monitor_mode == MARBH_MODE) {
2917                         monitor_len =
2918                                 iwl_trans_pci_dump_marbh_monitor(trans,
2919                                                                  fw_mon_data,
2920                                                                  monitor_len);
2921                 } else {
2922                         /* Didn't match anything - output no monitor data */
2923                         monitor_len = 0;
2924                 }
2925
2926                 len += monitor_len;
2927                 (*data)->len = cpu_to_le32(monitor_len + sizeof(*fw_mon_data));
2928         }
2929
2930         return len;
2931 }
2932
2933 static int iwl_trans_get_fw_monitor_len(struct iwl_trans *trans, int *len)
2934 {
2935         if (trans->num_blocks) {
2936                 *len += sizeof(struct iwl_fw_error_dump_data) +
2937                         sizeof(struct iwl_fw_error_dump_fw_mon) +
2938                         trans->fw_mon[0].size;
2939                 return trans->fw_mon[0].size;
2940         } else if (trans->dbg_dest_tlv) {
2941                 u32 base, end, cfg_reg, monitor_len;
2942
2943                 if (trans->dbg_dest_tlv->version == 1) {
2944                         cfg_reg = le32_to_cpu(trans->dbg_dest_tlv->base_reg);
2945                         cfg_reg = iwl_read_prph(trans, cfg_reg);
2946                         base = (cfg_reg & IWL_LDBG_M2S_BUF_BA_MSK) <<
2947                                 trans->dbg_dest_tlv->base_shift;
2948                         base *= IWL_M2S_UNIT_SIZE;
2949                         base += trans->cfg->smem_offset;
2950
2951                         monitor_len =
2952                                 (cfg_reg & IWL_LDBG_M2S_BUF_SIZE_MSK) >>
2953                                 trans->dbg_dest_tlv->end_shift;
2954                         monitor_len *= IWL_M2S_UNIT_SIZE;
2955                 } else {
2956                         base = le32_to_cpu(trans->dbg_dest_tlv->base_reg);
2957                         end = le32_to_cpu(trans->dbg_dest_tlv->end_reg);
2958
2959                         base = iwl_read_prph(trans, base) <<
2960                                trans->dbg_dest_tlv->base_shift;
2961                         end = iwl_read_prph(trans, end) <<
2962                               trans->dbg_dest_tlv->end_shift;
2963
2964                         /* Make "end" point to the actual end */
2965                         if (trans->cfg->device_family >=
2966                             IWL_DEVICE_FAMILY_8000 ||
2967                             trans->dbg_dest_tlv->monitor_mode == MARBH_MODE)
2968                                 end += (1 << trans->dbg_dest_tlv->end_shift);
2969                         monitor_len = end - base;
2970                 }
2971                 *len += sizeof(struct iwl_fw_error_dump_data) +
2972                         sizeof(struct iwl_fw_error_dump_fw_mon) +
2973                         monitor_len;
2974                 return monitor_len;
2975         }
2976         return 0;
2977 }
2978
2979 static struct iwl_trans_dump_data
2980 *iwl_trans_pcie_dump_data(struct iwl_trans *trans,
2981                           const struct iwl_fw_dbg_trigger_tlv *trigger)
2982 {
2983         struct iwl_trans_pcie *trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
2984         struct iwl_fw_error_dump_data *data;
2985         struct iwl_txq *cmdq = trans_pcie->txq[trans_pcie->cmd_queue];
2986         struct iwl_fw_error_dump_txcmd *txcmd;
2987         struct iwl_trans_dump_data *dump_data;
2988         u32 len, num_rbs = 0;
2989         u32 monitor_len;
2990         int i, ptr;
2991         bool dump_rbs = test_bit(STATUS_FW_ERROR, &trans->status) &&
2992                         !trans->cfg->mq_rx_supported &&
2993                         trans->dbg_dump_mask & BIT(IWL_FW_ERROR_DUMP_RB);
2994
2995         /* transport dump header */
2996         len = sizeof(*dump_data);
2997
2998         /* host commands */
2999         len += sizeof(*data) +
3000                 cmdq->n_window * (sizeof(*txcmd) + TFD_MAX_PAYLOAD_SIZE);
3001
3002         /* FW monitor */
3003         monitor_len = iwl_trans_get_fw_monitor_len(trans, &len);
3004
3005         if (trigger && (trigger->mode & IWL_FW_DBG_TRIGGER_MONITOR_ONLY)) {
3006                 if (!(trans->dbg_dump_mask &
3007                       BIT(IWL_FW_ERROR_DUMP_FW_MONITOR)))
3008                         return NULL;
3009
3010                 dump_data = vzalloc(len);
3011                 if (!dump_data)
3012                         return NULL;
3013
3014                 data = (void *)dump_data->data;
3015                 len = iwl_trans_pcie_dump_monitor(trans, &data, monitor_len);
3016                 dump_data->len = len;
3017
3018                 return dump_data;
3019         }
3020
3021         /* CSR registers */
3022         if (trans->dbg_dump_mask & BIT(IWL_FW_ERROR_DUMP_CSR))
3023                 len += sizeof(*data) + IWL_CSR_TO_DUMP;
3024
3025         /* FH registers */
3026         if (trans->dbg_dump_mask & BIT(IWL_FW_ERROR_DUMP_FH_REGS)) {
3027                 if (trans->cfg->gen2)
3028                         len += sizeof(*data) +
3029                                (FH_MEM_UPPER_BOUND_GEN2 -
3030                                 FH_MEM_LOWER_BOUND_GEN2);
3031                 else
3032                         len += sizeof(*data) +
3033                                (FH_MEM_UPPER_BOUND -
3034                                 FH_MEM_LOWER_BOUND);
3035         }
3036
3037         if (dump_rbs) {
3038                 /* Dump RBs is supported only for pre-9000 devices (1 queue) */
3039                 struct iwl_rxq *rxq = &trans_pcie->rxq[0];
3040                 /* RBs */
3041                 num_rbs =
3042                         le16_to_cpu(iwl_get_closed_rb_stts(trans, rxq))
3043                         & 0x0FFF;
3044                 num_rbs = (num_rbs - rxq->read) & RX_QUEUE_MASK;
3045                 len += num_rbs * (sizeof(*data) +
3046                                   sizeof(struct iwl_fw_error_dump_rb) +
3047                                   (PAGE_SIZE << trans_pcie->rx_page_order));
3048         }
3049
3050         /* Paged memory for gen2 HW */
3051         if (trans->cfg->gen2 &&
3052             trans->dbg_dump_mask & BIT(IWL_FW_ERROR_DUMP_PAGING))
3053                 for (i = 0; i < trans_pcie->init_dram.paging_cnt; i++)
3054                         len += sizeof(*data) +
3055                                sizeof(struct iwl_fw_error_dump_paging) +
3056                                trans_pcie->init_dram.paging[i].size;
3057
3058         dump_data = vzalloc(len);
3059         if (!dump_data)
3060                 return NULL;
3061
3062         len = 0;
3063         data = (void *)dump_data->data;
3064
3065         if (trans->dbg_dump_mask & BIT(IWL_FW_ERROR_DUMP_TXCMD)) {
3066                 u16 tfd_size = trans_pcie->tfd_size;
3067
3068                 data->type = cpu_to_le32(IWL_FW_ERROR_DUMP_TXCMD);
3069                 txcmd = (void *)data->data;
3070                 spin_lock_bh(&cmdq->lock);
3071                 ptr = cmdq->write_ptr;
3072                 for (i = 0; i < cmdq->n_window; i++) {
3073                         u8 idx = iwl_pcie_get_cmd_index(cmdq, ptr);
3074                         u32 caplen, cmdlen;
3075
3076                         cmdlen = iwl_trans_pcie_get_cmdlen(trans,
3077                                                            cmdq->tfds +
3078                                                            tfd_size * ptr);
3079                         caplen = min_t(u32, TFD_MAX_PAYLOAD_SIZE, cmdlen);
3080
3081                         if (cmdlen) {
3082                                 len += sizeof(*txcmd) + caplen;
3083                                 txcmd->cmdlen = cpu_to_le32(cmdlen);
3084                                 txcmd->caplen = cpu_to_le32(caplen);
3085                                 memcpy(txcmd->data, cmdq->entries[idx].cmd,
3086                                        caplen);
3087                                 txcmd = (void *)((u8 *)txcmd->data + caplen);
3088                         }
3089
3090                         ptr = iwl_queue_dec_wrap(trans, ptr);
3091                 }
3092                 spin_unlock_bh(&cmdq->lock);
3093
3094                 data->len = cpu_to_le32(len);
3095                 len += sizeof(*data);
3096                 data = iwl_fw_error_next_data(data);
3097         }
3098
3099         if (trans->dbg_dump_mask & BIT(IWL_FW_ERROR_DUMP_CSR))
3100                 len += iwl_trans_pcie_dump_csr(trans, &data);
3101         if (trans->dbg_dump_mask & BIT(IWL_FW_ERROR_DUMP_FH_REGS))
3102                 len += iwl_trans_pcie_fh_regs_dump(trans, &data);
3103         if (dump_rbs)
3104                 len += iwl_trans_pcie_dump_rbs(trans, &data, num_rbs);
3105
3106         /* Paged memory for gen2 HW */
3107         if (trans->cfg->gen2 &&
3108             trans->dbg_dump_mask & BIT(IWL_FW_ERROR_DUMP_PAGING)) {
3109                 for (i = 0; i < trans_pcie->init_dram.paging_cnt; i++) {
3110                         struct iwl_fw_error_dump_paging *paging;
3111                         dma_addr_t addr =
3112                                 trans_pcie->init_dram.paging[i].physical;
3113                         u32 page_len = trans_pcie->init_dram.paging[i].size;
3114
3115                         data->type = cpu_to_le32(IWL_FW_ERROR_DUMP_PAGING);
3116                         data->len = cpu_to_le32(sizeof(*paging) + page_len);
3117                         paging = (void *)data->data;
3118                         paging->index = cpu_to_le32(i);
3119                         dma_sync_single_for_cpu(trans->dev, addr, page_len,
3120                                                 DMA_BIDIRECTIONAL);
3121                         memcpy(paging->data,
3122                                trans_pcie->init_dram.paging[i].block, page_len);
3123                         data = iwl_fw_error_next_data(data);
3124
3125                         len += sizeof(*data) + sizeof(*paging) + page_len;
3126                 }
3127         }
3128         if (trans->dbg_dump_mask & BIT(IWL_FW_ERROR_DUMP_FW_MONITOR))
3129                 len += iwl_trans_pcie_dump_monitor(trans, &data, monitor_len);
3130
3131         dump_data->len = len;
3132
3133         return dump_data;
3134 }
3135
3136 #ifdef CONFIG_PM_SLEEP
3137 static int iwl_trans_pcie_suspend(struct iwl_trans *trans)
3138 {
3139         if (trans->runtime_pm_mode == IWL_PLAT_PM_MODE_D0I3 &&
3140             (trans->system_pm_mode == IWL_PLAT_PM_MODE_D0I3))
3141                 return iwl_pci_fw_enter_d0i3(trans);
3142
3143         return 0;
3144 }
3145
3146 static void iwl_trans_pcie_resume(struct iwl_trans *trans)
3147 {
3148         if (trans->runtime_pm_mode == IWL_PLAT_PM_MODE_D0I3 &&
3149             (trans->system_pm_mode == IWL_PLAT_PM_MODE_D0I3))
3150                 iwl_pci_fw_exit_d0i3(trans);
3151 }
3152 #endif /* CONFIG_PM_SLEEP */
3153
3154 #define IWL_TRANS_COMMON_OPS                                            \
3155         .op_mode_leave = iwl_trans_pcie_op_mode_leave,                  \
3156         .write8 = iwl_trans_pcie_write8,                                \
3157         .write32 = iwl_trans_pcie_write32,                              \
3158         .read32 = iwl_trans_pcie_read32,                                \
3159         .read_prph = iwl_trans_pcie_read_prph,                          \
3160         .write_prph = iwl_trans_pcie_write_prph,                        \
3161         .read_mem = iwl_trans_pcie_read_mem,                            \
3162         .write_mem = iwl_trans_pcie_write_mem,                          \
3163         .configure = iwl_trans_pcie_configure,                          \
3164         .set_pmi = iwl_trans_pcie_set_pmi,                              \
3165         .sw_reset = iwl_trans_pcie_sw_reset,                            \
3166         .grab_nic_access = iwl_trans_pcie_grab_nic_access,              \
3167         .release_nic_access = iwl_trans_pcie_release_nic_access,        \
3168         .set_bits_mask = iwl_trans_pcie_set_bits_mask,                  \
3169         .ref = iwl_trans_pcie_ref,                                      \
3170         .unref = iwl_trans_pcie_unref,                                  \
3171         .dump_data = iwl_trans_pcie_dump_data,                          \
3172         .d3_suspend = iwl_trans_pcie_d3_suspend,                        \
3173         .d3_resume = iwl_trans_pcie_d3_resume
3174
3175 #ifdef CONFIG_PM_SLEEP
3176 #define IWL_TRANS_PM_OPS                                                \
3177         .suspend = iwl_trans_pcie_suspend,                              \
3178         .resume = iwl_trans_pcie_resume,
3179 #else
3180 #define IWL_TRANS_PM_OPS
3181 #endif /* CONFIG_PM_SLEEP */
3182
3183 static const struct iwl_trans_ops trans_ops_pcie = {
3184         IWL_TRANS_COMMON_OPS,
3185         IWL_TRANS_PM_OPS
3186         .start_hw = iwl_trans_pcie_start_hw,
3187         .fw_alive = iwl_trans_pcie_fw_alive,
3188         .start_fw = iwl_trans_pcie_start_fw,
3189         .stop_device = iwl_trans_pcie_stop_device,
3190
3191         .send_cmd = iwl_trans_pcie_send_hcmd,
3192
3193         .tx = iwl_trans_pcie_tx,
3194         .reclaim = iwl_trans_pcie_reclaim,
3195
3196         .txq_disable = iwl_trans_pcie_txq_disable,
3197         .txq_enable = iwl_trans_pcie_txq_enable,
3198
3199         .txq_set_shared_mode = iwl_trans_pcie_txq_set_shared_mode,
3200
3201         .wait_tx_queues_empty = iwl_trans_pcie_wait_txqs_empty,
3202
3203         .freeze_txq_timer = iwl_trans_pcie_freeze_txq_timer,
3204         .block_txq_ptrs = iwl_trans_pcie_block_txq_ptrs,
3205 };
3206
3207 static const struct iwl_trans_ops trans_ops_pcie_gen2 = {
3208         IWL_TRANS_COMMON_OPS,
3209         IWL_TRANS_PM_OPS
3210         .start_hw = iwl_trans_pcie_start_hw,
3211         .fw_alive = iwl_trans_pcie_gen2_fw_alive,
3212         .start_fw = iwl_trans_pcie_gen2_start_fw,
3213         .stop_device = iwl_trans_pcie_gen2_stop_device,
3214
3215         .send_cmd = iwl_trans_pcie_gen2_send_hcmd,
3216
3217         .tx = iwl_trans_pcie_gen2_tx,
3218         .reclaim = iwl_trans_pcie_reclaim,
3219
3220         .txq_alloc = iwl_trans_pcie_dyn_txq_alloc,
3221         .txq_free = iwl_trans_pcie_dyn_txq_free,
3222         .wait_txq_empty = iwl_trans_pcie_wait_txq_empty,
3223         .rxq_dma_data = iwl_trans_pcie_rxq_dma_data,
3224 };
3225
3226 struct iwl_trans *iwl_trans_pcie_alloc(struct pci_dev *pdev,
3227                                        const struct pci_device_id *ent,
3228                                        const struct iwl_cfg *cfg)
3229 {
3230         struct iwl_trans_pcie *trans_pcie;
3231         struct iwl_trans *trans;
3232         int ret, addr_size;
3233
3234         ret = pcim_enable_device(pdev);
3235         if (ret)
3236                 return ERR_PTR(ret);
3237
3238         if (cfg->gen2)
3239                 trans = iwl_trans_alloc(sizeof(struct iwl_trans_pcie),
3240                                         &pdev->dev, cfg, &trans_ops_pcie_gen2);
3241         else
3242                 trans = iwl_trans_alloc(sizeof(struct iwl_trans_pcie),
3243                                         &pdev->dev, cfg, &trans_ops_pcie);
3244         if (!trans)
3245                 return ERR_PTR(-ENOMEM);
3246
3247         trans_pcie = IWL_TRANS_GET_PCIE_TRANS(trans);
3248
3249         trans_pcie->trans = trans;
3250         trans_pcie->opmode_down = true;
3251         spin_lock_init(&trans_pcie->irq_lock);
3252         spin_lock_init(&trans_pcie->reg_lock);
3253         mutex_init(&trans_pcie->mutex);
3254         init_waitqueue_head(&trans_pcie->ucode_write_waitq);
3255         trans_pcie->tso_hdr_page = alloc_percpu(struct iwl_tso_hdr_page);
3256         if (!trans_pcie->tso_hdr_page) {
3257                 ret = -ENOMEM;
3258                 goto out_no_pci;
3259         }
3260
3261
3262         if (!cfg->base_params->pcie_l1_allowed) {
3263                 /*
3264                  * W/A - seems to solve weird behavior. We need to remove this
3265                  * if we don't want to stay in L1 all the time. This wastes a
3266                  * lot of power.
3267                  */
3268                 pci_disable_link_state(pdev, PCIE_LINK_STATE_L0S |
3269                                        PCIE_LINK_STATE_L1 |
3270                                        PCIE_LINK_STATE_CLKPM);
3271         }
3272
3273         trans_pcie->def_rx_queue = 0;
3274
3275         if (cfg->use_tfh) {
3276                 addr_size = 64;
3277                 trans_pcie->max_tbs = IWL_TFH_NUM_TBS;
3278                 trans_pcie->tfd_size = sizeof(struct iwl_tfh_tfd);
3279         } else {
3280                 addr_size = 36;
3281                 trans_pcie->max_tbs = IWL_NUM_OF_TBS;
3282                 trans_pcie->tfd_size = sizeof(struct iwl_tfd);
3283         }
3284         trans->max_skb_frags = IWL_PCIE_MAX_FRAGS(trans_pcie);
3285
3286         pci_set_master(pdev);
3287
3288         ret = pci_set_dma_mask(pdev, DMA_BIT_MASK(addr_size));
3289         if (!ret)
3290                 ret = pci_set_consistent_dma_mask(pdev,
3291                                                   DMA_BIT_MASK(addr_size));
3292         if (ret) {
3293                 ret = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
3294                 if (!ret)
3295                         ret = pci_set_consistent_dma_mask(pdev,
3296                                                           DMA_BIT_MASK(32));
3297                 /* both attempts failed: */
3298                 if (ret) {
3299                         dev_err(&pdev->dev, "No suitable DMA available\n");
3300                         goto out_no_pci;
3301                 }
3302         }
3303
3304         ret = pcim_iomap_regions_request_all(pdev, BIT(0), DRV_NAME);
3305         if (ret) {
3306                 dev_err(&pdev->dev, "pcim_iomap_regions_request_all failed\n");
3307                 goto out_no_pci;
3308         }
3309
3310         trans_pcie->hw_base = pcim_iomap_table(pdev)[0];
3311         if (!trans_pcie->hw_base) {
3312                 dev_err(&pdev->dev, "pcim_iomap_table failed\n");
3313                 ret = -ENODEV;
3314                 goto out_no_pci;
3315         }
3316
3317         /* We disable the RETRY_TIMEOUT register (0x41) to keep
3318          * PCI Tx retries from interfering with C3 CPU state */
3319         pci_write_config_byte(pdev, PCI_CFG_RETRY_TIMEOUT, 0x00);
3320
3321         trans_pcie->pci_dev = pdev;
3322         iwl_disable_interrupts(trans);
3323
3324         trans->hw_rev = iwl_read32(trans, CSR_HW_REV);
3325         if (trans->hw_rev == 0xffffffff) {
3326                 dev_err(&pdev->dev, "HW_REV=0xFFFFFFFF, PCI issues?\n");
3327                 ret = -EIO;
3328                 goto out_no_pci;
3329         }
3330
3331         /*
3332          * In the 8000 HW family the format of the 4 bytes of CSR_HW_REV have
3333          * changed, and now the revision step also includes bit 0-1 (no more
3334          * "dash" value). To keep hw_rev backwards compatible - we'll store it
3335          * in the old format.
3336          */
3337         if (trans->cfg->device_family >= IWL_DEVICE_FAMILY_8000) {
3338                 unsigned long flags;
3339
3340                 trans->hw_rev = (trans->hw_rev & 0xfff0) |
3341                                 (CSR_HW_REV_STEP(trans->hw_rev << 2) << 2);
3342
3343                 ret = iwl_pcie_prepare_card_hw(trans);
3344                 if (ret) {
3345                         IWL_WARN(trans, "Exit HW not ready\n");
3346                         goto out_no_pci;
3347                 }
3348
3349                 /*
3350                  * in-order to recognize C step driver should read chip version
3351                  * id located at the AUX bus MISC address space.
3352                  */
3353                 iwl_set_bit(trans, CSR_GP_CNTRL,
3354                             BIT(trans->cfg->csr->flag_init_done));
3355                 udelay(2);
3356
3357                 ret = iwl_poll_bit(trans, CSR_GP_CNTRL,
3358                                    BIT(trans->cfg->csr->flag_mac_clock_ready),
3359                                    BIT(trans->cfg->csr->flag_mac_clock_ready),
3360                                    25000);
3361                 if (ret < 0) {
3362                         IWL_DEBUG_INFO(trans, "Failed to wake up the nic\n");
3363                         goto out_no_pci;
3364                 }
3365
3366                 if (iwl_trans_grab_nic_access(trans, &flags)) {
3367                         u32 hw_step;
3368
3369                         hw_step = iwl_read_prph_no_grab(trans, WFPM_CTRL_REG);
3370                         hw_step |= ENABLE_WFPM;
3371                         iwl_write_prph_no_grab(trans, WFPM_CTRL_REG, hw_step);
3372                         hw_step = iwl_read_prph_no_grab(trans, AUX_MISC_REG);
3373                         hw_step = (hw_step >> HW_STEP_LOCATION_BITS) & 0xF;
3374                         if (hw_step == 0x3)
3375                                 trans->hw_rev = (trans->hw_rev & 0xFFFFFFF3) |
3376                                                 (SILICON_C_STEP << 2);
3377                         iwl_trans_release_nic_access(trans, &flags);
3378                 }
3379         }
3380
3381         /*
3382          * 9000-series integrated A-step has a problem with suspend/resume
3383          * and sometimes even causes the whole platform to get stuck. This
3384          * workaround makes the hardware not go into the problematic state.
3385          */
3386         if (trans->cfg->integrated &&
3387             trans->cfg->device_family == IWL_DEVICE_FAMILY_9000 &&
3388             CSR_HW_REV_STEP(trans->hw_rev) == SILICON_A_STEP)
3389                 iwl_set_bit(trans, CSR_HOST_CHICKEN,
3390                             CSR_HOST_CHICKEN_PM_IDLE_SRC_DIS_SB_PME);
3391
3392 #if IS_ENABLED(CONFIG_IWLMVM)
3393         trans->hw_rf_id = iwl_read32(trans, CSR_HW_RF_ID);
3394
3395         if (CSR_HW_RF_ID_TYPE_CHIP_ID(trans->hw_rf_id) ==
3396             CSR_HW_RF_ID_TYPE_CHIP_ID(CSR_HW_RF_ID_TYPE_HR)) {
3397                 u32 hw_status;
3398
3399                 hw_status = iwl_read_prph(trans, UMAG_GEN_HW_STATUS);
3400                 if (CSR_HW_RF_STEP(trans->hw_rf_id) == SILICON_B_STEP)
3401                         /*
3402                         * b step fw is the same for physical card and fpga
3403                         */
3404                         trans->cfg = &iwl22000_2ax_cfg_qnj_hr_b0;
3405                 else if ((hw_status & UMAG_GEN_HW_IS_FPGA) &&
3406                          CSR_HW_RF_STEP(trans->hw_rf_id) == SILICON_A_STEP) {
3407                         trans->cfg = &iwl22000_2ax_cfg_qnj_hr_a0_f0;
3408                 } else {
3409                         /*
3410                         * a step no FPGA
3411                         */
3412                         trans->cfg = &iwl22000_2ac_cfg_hr;
3413                 }
3414         }
3415 #endif
3416
3417         iwl_pcie_set_interrupt_capa(pdev, trans);
3418         trans->hw_id = (pdev->device << 16) + pdev->subsystem_device;
3419         snprintf(trans->hw_id_str, sizeof(trans->hw_id_str),
3420                  "PCI ID: 0x%04X:0x%04X", pdev->device, pdev->subsystem_device);
3421
3422         /* Initialize the wait queue for commands */
3423         init_waitqueue_head(&trans_pcie->wait_command_queue);
3424
3425         init_waitqueue_head(&trans_pcie->d0i3_waitq);
3426
3427         if (trans_pcie->msix_enabled) {
3428                 ret = iwl_pcie_init_msix_handler(pdev, trans_pcie);
3429                 if (ret)
3430                         goto out_no_pci;
3431          } else {
3432                 ret = iwl_pcie_alloc_ict(trans);
3433                 if (ret)
3434                         goto out_no_pci;
3435
3436                 ret = devm_request_threaded_irq(&pdev->dev, pdev->irq,
3437                                                 iwl_pcie_isr,
3438                                                 iwl_pcie_irq_handler,
3439                                                 IRQF_SHARED, DRV_NAME, trans);
3440                 if (ret) {
3441                         IWL_ERR(trans, "Error allocating IRQ %d\n", pdev->irq);
3442                         goto out_free_ict;
3443                 }
3444                 trans_pcie->inta_mask = CSR_INI_SET_MASK;
3445          }
3446
3447         trans_pcie->rba.alloc_wq = alloc_workqueue("rb_allocator",
3448                                                    WQ_HIGHPRI | WQ_UNBOUND, 1);
3449         INIT_WORK(&trans_pcie->rba.rx_alloc, iwl_pcie_rx_allocator_work);
3450
3451 #ifdef CONFIG_IWLWIFI_PCIE_RTPM
3452         trans->runtime_pm_mode = IWL_PLAT_PM_MODE_D0I3;
3453 #else
3454         trans->runtime_pm_mode = IWL_PLAT_PM_MODE_DISABLED;
3455 #endif /* CONFIG_IWLWIFI_PCIE_RTPM */
3456
3457         return trans;
3458
3459 out_free_ict:
3460         iwl_pcie_free_ict(trans);
3461 out_no_pci:
3462         free_percpu(trans_pcie->tso_hdr_page);
3463         iwl_trans_free(trans);
3464         return ERR_PTR(ret);
3465 }