drivers/spi/spi-cadence-xspi.c

   1 // SPDX-License-Identifier: GPL-2.0+
   2 // Cadence XSPI flash controller driver
   3 // Copyright (C) 2020-21 Cadence
   4
   5 #include <linux/acpi.h>
   6 #include <linux/completion.h>
   7 #include <linux/delay.h>
   8 #include <linux/err.h>
   9 #include <linux/errno.h>
  10 #include <linux/interrupt.h>
  11 #include <linux/io.h>
  12 #include <linux/iopoll.h>
  13 #include <linux/kernel.h>
  14 #include <linux/module.h>
  15 #include <linux/of.h>
  16 #include <linux/platform_device.h>
  17 #include <linux/pm_runtime.h>
  18 #include <linux/spi/spi.h>
  19 #include <linux/spi/spi-mem.h>
  20 #include <linux/bitfield.h>
  21 #include <linux/limits.h>
  22 #include <linux/log2.h>
  23 #include <linux/bitrev.h>
  24
  25 #define CDNS_XSPI_MAGIC_NUM_VALUE       0x6522
  26 #define CDNS_XSPI_MAX_BANKS             8
  27 #define CDNS_XSPI_NAME                  "cadence-xspi"
  28
  29 /*
  30  * Note: below are additional auxiliary registers to
  31  * configure XSPI controller pin-strap settings
  32  */
  33
  34 /* PHY DQ timing register */
  35 #define CDNS_XSPI_CCP_PHY_DQ_TIMING             0x0000
  36
  37 /* PHY DQS timing register */
  38 #define CDNS_XSPI_CCP_PHY_DQS_TIMING            0x0004
  39
  40 /* PHY gate loopback control register */
  41 #define CDNS_XSPI_CCP_PHY_GATE_LPBCK_CTRL       0x0008
  42
  43 /* PHY DLL slave control register */
  44 #define CDNS_XSPI_CCP_PHY_DLL_SLAVE_CTRL        0x0010
  45
  46 /* DLL PHY control register */
  47 #define CDNS_XSPI_DLL_PHY_CTRL                  0x1034
  48
  49 /* Command registers */
  50 #define CDNS_XSPI_CMD_REG_0                     0x0000
  51 #define CDNS_XSPI_CMD_REG_1                     0x0004
  52 #define CDNS_XSPI_CMD_REG_2                     0x0008
  53 #define CDNS_XSPI_CMD_REG_3                     0x000C
  54 #define CDNS_XSPI_CMD_REG_4                     0x0010
  55 #define CDNS_XSPI_CMD_REG_5                     0x0014
  56
  57 /* Command status registers */
  58 #define CDNS_XSPI_CMD_STATUS_REG                0x0044
  59
  60 /* Controller status register */
  61 #define CDNS_XSPI_CTRL_STATUS_REG               0x0100
  62 #define CDNS_XSPI_INIT_COMPLETED                BIT(16)
  63 #define CDNS_XSPI_INIT_LEGACY                   BIT(9)
  64 #define CDNS_XSPI_INIT_FAIL                     BIT(8)
  65 #define CDNS_XSPI_CTRL_BUSY                     BIT(7)
  66
  67 /* Controller interrupt status register */
  68 #define CDNS_XSPI_INTR_STATUS_REG               0x0110
  69 #define CDNS_XSPI_STIG_DONE                     BIT(23)
  70 #define CDNS_XSPI_SDMA_ERROR                    BIT(22)
  71 #define CDNS_XSPI_SDMA_TRIGGER                  BIT(21)
  72 #define CDNS_XSPI_CMD_IGNRD_EN                  BIT(20)
  73 #define CDNS_XSPI_DDMA_TERR_EN                  BIT(18)
  74 #define CDNS_XSPI_CDMA_TREE_EN                  BIT(17)
  75 #define CDNS_XSPI_CTRL_IDLE_EN                  BIT(16)
  76
  77 #define CDNS_XSPI_TRD_COMP_INTR_STATUS          0x0120
  78 #define CDNS_XSPI_TRD_ERR_INTR_STATUS           0x0130
  79 #define CDNS_XSPI_TRD_ERR_INTR_EN               0x0134
  80
  81 /* Controller interrupt enable register */
  82 #define CDNS_XSPI_INTR_ENABLE_REG               0x0114
  83 #define CDNS_XSPI_INTR_EN                       BIT(31)
  84 #define CDNS_XSPI_STIG_DONE_EN                  BIT(23)
  85 #define CDNS_XSPI_SDMA_ERROR_EN                 BIT(22)
  86 #define CDNS_XSPI_SDMA_TRIGGER_EN               BIT(21)
  87
  88 #define CDNS_XSPI_INTR_MASK (CDNS_XSPI_INTR_EN | \
  89         CDNS_XSPI_STIG_DONE_EN  | \
  90         CDNS_XSPI_SDMA_ERROR_EN | \
  91         CDNS_XSPI_SDMA_TRIGGER_EN)
  92
  93 /* Controller config register */
  94 #define CDNS_XSPI_CTRL_CONFIG_REG               0x0230
  95 #define CDNS_XSPI_CTRL_WORK_MODE                GENMASK(6, 5)
  96
  97 #define CDNS_XSPI_WORK_MODE_DIRECT              0
  98 #define CDNS_XSPI_WORK_MODE_STIG                1
  99 #define CDNS_XSPI_WORK_MODE_ACMD                3
 100
 101 /* SDMA trigger transaction registers */
 102 #define CDNS_XSPI_SDMA_SIZE_REG                 0x0240
 103 #define CDNS_XSPI_SDMA_TRD_INFO_REG             0x0244
 104 #define CDNS_XSPI_SDMA_DIR                      BIT(8)
 105
 106 /* Controller features register */
 107 #define CDNS_XSPI_CTRL_FEATURES_REG             0x0F04
 108 #define CDNS_XSPI_NUM_BANKS                     GENMASK(25, 24)
 109 #define CDNS_XSPI_DMA_DATA_WIDTH                BIT(21)
 110 #define CDNS_XSPI_NUM_THREADS                   GENMASK(3, 0)
 111
 112 /* Controller version register */
 113 #define CDNS_XSPI_CTRL_VERSION_REG              0x0F00
 114 #define CDNS_XSPI_MAGIC_NUM                     GENMASK(31, 16)
 115 #define CDNS_XSPI_CTRL_REV                      GENMASK(7, 0)
 116
 117 /* STIG Profile 1.0 instruction fields (split into registers) */
 118 #define CDNS_XSPI_CMD_INSTR_TYPE                GENMASK(6, 0)
 119 #define CDNS_XSPI_CMD_P1_R1_ADDR0               GENMASK(31, 24)
 120 #define CDNS_XSPI_CMD_P1_R2_ADDR1               GENMASK(7, 0)
 121 #define CDNS_XSPI_CMD_P1_R2_ADDR2               GENMASK(15, 8)
 122 #define CDNS_XSPI_CMD_P1_R2_ADDR3               GENMASK(23, 16)
 123 #define CDNS_XSPI_CMD_P1_R2_ADDR4               GENMASK(31, 24)
 124 #define CDNS_XSPI_CMD_P1_R3_ADDR5               GENMASK(7, 0)
 125 #define CDNS_XSPI_CMD_P1_R3_CMD                 GENMASK(23, 16)
 126 #define CDNS_XSPI_CMD_P1_R3_NUM_ADDR_BYTES      GENMASK(30, 28)
 127 #define CDNS_XSPI_CMD_P1_R4_ADDR_IOS            GENMASK(1, 0)
 128 #define CDNS_XSPI_CMD_P1_R4_CMD_IOS             GENMASK(9, 8)
 129 #define CDNS_XSPI_CMD_P1_R4_BANK                GENMASK(14, 12)
 130
 131 /* STIG data sequence instruction fields (split into registers) */
 132 #define CDNS_XSPI_CMD_DSEQ_R2_DCNT_L            GENMASK(31, 16)
 133 #define CDNS_XSPI_CMD_DSEQ_R3_DCNT_H            GENMASK(15, 0)
 134 #define CDNS_XSPI_CMD_DSEQ_R3_NUM_OF_DUMMY      GENMASK(25, 20)
 135 #define CDNS_XSPI_CMD_DSEQ_R4_BANK              GENMASK(14, 12)
 136 #define CDNS_XSPI_CMD_DSEQ_R4_DATA_IOS          GENMASK(9, 8)
 137 #define CDNS_XSPI_CMD_DSEQ_R4_DIR               BIT(4)
 138
 139 /* STIG command status fields */
 140 #define CDNS_XSPI_CMD_STATUS_COMPLETED          BIT(15)
 141 #define CDNS_XSPI_CMD_STATUS_FAILED             BIT(14)
 142 #define CDNS_XSPI_CMD_STATUS_DQS_ERROR          BIT(3)
 143 #define CDNS_XSPI_CMD_STATUS_CRC_ERROR          BIT(2)
 144 #define CDNS_XSPI_CMD_STATUS_BUS_ERROR          BIT(1)
 145 #define CDNS_XSPI_CMD_STATUS_INV_SEQ_ERROR      BIT(0)
 146
 147 #define CDNS_XSPI_STIG_DONE_FLAG                BIT(0)
 148 #define CDNS_XSPI_TRD_STATUS                    0x0104
 149
 150 #define MODE_NO_OF_BYTES                        GENMASK(25, 24)
 151 #define MODEBYTES_COUNT                 1
 152
 153 /* Helper macros for filling command registers */
 154 #define CDNS_XSPI_CMD_FLD_P1_INSTR_CMD_1(op, data_phase) ( \
 155         FIELD_PREP(CDNS_XSPI_CMD_INSTR_TYPE, (data_phase) ? \
 156                 CDNS_XSPI_STIG_INSTR_TYPE_1 : CDNS_XSPI_STIG_INSTR_TYPE_0) | \
 157         FIELD_PREP(CDNS_XSPI_CMD_P1_R1_ADDR0, (op)->addr.val & 0xff))
 158
 159 #define CDNS_XSPI_CMD_FLD_P1_INSTR_CMD_2(op) ( \
 160         FIELD_PREP(CDNS_XSPI_CMD_P1_R2_ADDR1, ((op)->addr.val >> 8)  & 0xFF) | \
 161         FIELD_PREP(CDNS_XSPI_CMD_P1_R2_ADDR2, ((op)->addr.val >> 16) & 0xFF) | \
 162         FIELD_PREP(CDNS_XSPI_CMD_P1_R2_ADDR3, ((op)->addr.val >> 24) & 0xFF) | \
 163         FIELD_PREP(CDNS_XSPI_CMD_P1_R2_ADDR4, ((op)->addr.val >> 32) & 0xFF))
 164
 165 #define CDNS_XSPI_CMD_FLD_P1_INSTR_CMD_3(op, modebytes) ( \
 166         FIELD_PREP(CDNS_XSPI_CMD_P1_R3_ADDR5, ((op)->addr.val >> 40) & 0xFF) | \
 167         FIELD_PREP(CDNS_XSPI_CMD_P1_R3_CMD, (op)->cmd.opcode) | \
 168         FIELD_PREP(MODE_NO_OF_BYTES, modebytes) | \
 169         FIELD_PREP(CDNS_XSPI_CMD_P1_R3_NUM_ADDR_BYTES, (op)->addr.nbytes))
 170
 171 #define CDNS_XSPI_CMD_FLD_P1_INSTR_CMD_4(op, chipsel) ( \
 172         FIELD_PREP(CDNS_XSPI_CMD_P1_R4_ADDR_IOS, ilog2((op)->addr.buswidth)) | \
 173         FIELD_PREP(CDNS_XSPI_CMD_P1_R4_CMD_IOS, ilog2((op)->cmd.buswidth)) | \
 174         FIELD_PREP(CDNS_XSPI_CMD_P1_R4_BANK, chipsel))
 175
 176 #define CDNS_XSPI_CMD_FLD_DSEQ_CMD_1(op) \
 177         FIELD_PREP(CDNS_XSPI_CMD_INSTR_TYPE, CDNS_XSPI_STIG_INSTR_TYPE_DATA_SEQ)
 178
 179 #define CDNS_XSPI_CMD_FLD_DSEQ_CMD_2(op) \
 180         FIELD_PREP(CDNS_XSPI_CMD_DSEQ_R2_DCNT_L, (op)->data.nbytes & 0xFFFF)
 181
 182 #define CDNS_XSPI_CMD_FLD_DSEQ_CMD_3(op, dummybytes) ( \
 183         FIELD_PREP(CDNS_XSPI_CMD_DSEQ_R3_DCNT_H, \
 184                 ((op)->data.nbytes >> 16) & 0xffff) | \
 185         FIELD_PREP(CDNS_XSPI_CMD_DSEQ_R3_NUM_OF_DUMMY, \
 186                   (op)->dummy.buswidth != 0 ? \
 187                   (((dummybytes) * 8) / (op)->dummy.buswidth) : \
 188                   0))
 189
 190 #define CDNS_XSPI_CMD_FLD_DSEQ_CMD_4(op, chipsel) ( \
 191         FIELD_PREP(CDNS_XSPI_CMD_DSEQ_R4_BANK, chipsel) | \
 192         FIELD_PREP(CDNS_XSPI_CMD_DSEQ_R4_DATA_IOS, \
 193                 ilog2((op)->data.buswidth)) | \
 194         FIELD_PREP(CDNS_XSPI_CMD_DSEQ_R4_DIR, \
 195                 ((op)->data.dir == SPI_MEM_DATA_IN) ? \
 196                 CDNS_XSPI_STIG_CMD_DIR_READ : CDNS_XSPI_STIG_CMD_DIR_WRITE))
 197
 198 /* Helper macros for GENERIC and GENERIC-DSEQ instruction type */
 199 #define CMD_REG_LEN (6*4)
 200 #define INSTRUCTION_TYPE_GENERIC 96
 201 #define CDNS_XSPI_CMD_FLD_P1_GENERIC_CMD (\
 202         FIELD_PREP(CDNS_XSPI_CMD_INSTR_TYPE, INSTRUCTION_TYPE_GENERIC))
 203
 204 #define GENERIC_NUM_OF_BYTES GENMASK(27, 24)
 205 #define CDNS_XSPI_CMD_FLD_P3_GENERIC_CMD(len) (\
 206         FIELD_PREP(GENERIC_NUM_OF_BYTES, len))
 207
 208 #define GENERIC_BANK_NUM GENMASK(14, 12)
 209 #define GENERIC_GLUE_CMD BIT(28)
 210 #define CDNS_XSPI_CMD_FLD_P4_GENERIC_CMD(cs, glue) (\
 211         FIELD_PREP(GENERIC_BANK_NUM, cs) | FIELD_PREP(GENERIC_GLUE_CMD, glue))
 212
 213 #define CDNS_XSPI_CMD_FLD_GENERIC_DSEQ_CMD_1 (\
 214         FIELD_PREP(CDNS_XSPI_CMD_INSTR_TYPE, CDNS_XSPI_STIG_INSTR_TYPE_DATA_SEQ))
 215
 216 #define CDNS_XSPI_CMD_FLD_GENERIC_DSEQ_CMD_2(nbytes) (\
 217         FIELD_PREP(CDNS_XSPI_CMD_DSEQ_R2_DCNT_L, nbytes & 0xffff))
 218
 219 #define CDNS_XSPI_CMD_FLD_GENERIC_DSEQ_CMD_3(nbytes) ( \
 220         FIELD_PREP(CDNS_XSPI_CMD_DSEQ_R3_DCNT_H, (nbytes >> 16) & 0xffff))
 221
 222 #define CDNS_XSPI_CMD_FLD_GENERIC_DSEQ_CMD_4(dir, chipsel) ( \
 223         FIELD_PREP(CDNS_XSPI_CMD_DSEQ_R4_BANK, chipsel) | \
 224         FIELD_PREP(CDNS_XSPI_CMD_DSEQ_R4_DIR, dir))
 225
 226 /* Marvell PHY default values */
 227 #define MARVELL_REGS_DLL_PHY_CTRL               0x00000707
 228 #define MARVELL_CTB_RFILE_PHY_CTRL              0x00004000
 229 #define MARVELL_RFILE_PHY_TSEL                  0x00000000
 230 #define MARVELL_RFILE_PHY_DQ_TIMING             0x00000101
 231 #define MARVELL_RFILE_PHY_DQS_TIMING            0x00700404
 232 #define MARVELL_RFILE_PHY_GATE_LPBK_CTRL        0x00200030
 233 #define MARVELL_RFILE_PHY_DLL_MASTER_CTRL       0x00800000
 234 #define MARVELL_RFILE_PHY_DLL_SLAVE_CTRL        0x0000ff01
 235
 236 /* PHY config registers */
 237 #define CDNS_XSPI_RF_MINICTRL_REGS_DLL_PHY_CTRL                 0x1034
 238 #define CDNS_XSPI_PHY_CTB_RFILE_PHY_CTRL                        0x0080
 239 #define CDNS_XSPI_PHY_CTB_RFILE_PHY_TSEL                        0x0084
 240 #define CDNS_XSPI_PHY_DATASLICE_RFILE_PHY_DQ_TIMING             0x0000
 241 #define CDNS_XSPI_PHY_DATASLICE_RFILE_PHY_DQS_TIMING            0x0004
 242 #define CDNS_XSPI_PHY_DATASLICE_RFILE_PHY_GATE_LPBK_CTRL        0x0008
 243 #define CDNS_XSPI_PHY_DATASLICE_RFILE_PHY_DLL_MASTER_CTRL       0x000c
 244 #define CDNS_XSPI_PHY_DATASLICE_RFILE_PHY_DLL_SLAVE_CTRL        0x0010
 245 #define CDNS_XSPI_DATASLICE_RFILE_PHY_DLL_OBS_REG_0             0x001c
 246
 247 #define CDNS_XSPI_DLL_RST_N BIT(24)
 248 #define CDNS_XSPI_DLL_LOCK  BIT(0)
 249
 250 /* Marvell overlay registers - clock */
 251 #define MRVL_XSPI_CLK_CTRL_AUX_REG   0x2020
 252 #define MRVL_XSPI_CLK_ENABLE         BIT(0)
 253 #define MRVL_XSPI_CLK_DIV            GENMASK(4, 1)
 254 #define MRVL_XSPI_IRQ_ENABLE         BIT(6)
 255 #define MRVL_XSPI_CLOCK_IO_HZ        800000000
 256 #define MRVL_XSPI_CLOCK_DIVIDED(div) ((MRVL_XSPI_CLOCK_IO_HZ) / (div))
 257 #define MRVL_DEFAULT_CLK             25000000
 258
 259 /* Marvell overlay registers - xfer */
 260 #define MRVL_XFER_FUNC_CTRL              0x210
 261 #define MRVL_XFER_FUNC_CTRL_READ_DATA(i) (0x000 + 8 * (i))
 262 #define MRVL_XFER_SOFT_RESET             BIT(11)
 263 #define MRVL_XFER_CS_N_HOLD              GENMASK(9, 6)
 264 #define MRVL_XFER_RECEIVE_ENABLE         BIT(4)
 265 #define MRVL_XFER_FUNC_ENABLE            BIT(3)
 266 #define MRVL_XFER_CLK_CAPTURE_POL        BIT(2)
 267 #define MRVL_XFER_CLK_DRIVE_POL          BIT(1)
 268 #define MRVL_XFER_FUNC_START             BIT(0)
 269 #define MRVL_XFER_QWORD_COUNT            32
 270 #define MRVL_XFER_QWORD_BYTECOUNT        8
 271
 272 #define MRVL_XSPI_POLL_TIMEOUT_US       1000
 273 #define MRVL_XSPI_POLL_DELAY_US         10
 274
 275 /* Macros for calculating data bits in generic command
 276  * Up to 10 bytes can be fit into cmd_registers
 277  * least significant is placed in cmd_reg[1]
 278  * Other bits are inserted after it in cmd_reg[1,2,3] register
 279  */
 280 #define GENERIC_CMD_DATA_REG_3_COUNT(len)       (len >= 10 ? 2 : len - 8)
 281 #define GENERIC_CMD_DATA_REG_2_COUNT(len)       (len >= 7 ? 3 : len - 4)
 282 #define GENERIC_CMD_DATA_REG_1_COUNT(len)       (len >= 3 ? 2 : len - 1)
 283 #define GENERIC_CMD_DATA_3_OFFSET(position)     (8*(position))
 284 #define GENERIC_CMD_DATA_2_OFFSET(position)     (8*(position))
 285 #define GENERIC_CMD_DATA_1_OFFSET(position)     (8 + 8*(position))
 286 #define GENERIC_CMD_DATA_INSERT(data, pos)      ((data) << (pos))
 287 #define GENERIC_CMD_REG_3_NEEDED(len)           (len > 7)
 288 #define GENERIC_CMD_REG_2_NEEDED(len)           (len > 3)
 289
 290 enum cdns_xspi_stig_instr_type {
 291         CDNS_XSPI_STIG_INSTR_TYPE_0,
 292         CDNS_XSPI_STIG_INSTR_TYPE_1,
 293         CDNS_XSPI_STIG_INSTR_TYPE_DATA_SEQ = 127,
 294 };
 295
 296 enum cdns_xspi_sdma_dir {
 297         CDNS_XSPI_SDMA_DIR_READ,
 298         CDNS_XSPI_SDMA_DIR_WRITE,
 299 };
 300
 301 enum cdns_xspi_stig_cmd_dir {
 302         CDNS_XSPI_STIG_CMD_DIR_READ,
 303         CDNS_XSPI_STIG_CMD_DIR_WRITE,
 304 };
 305
 306 struct cdns_xspi_driver_data {
 307         bool mrvl_hw_overlay;
 308         u32 dll_phy_ctrl;
 309         u32 ctb_rfile_phy_ctrl;
 310         u32 rfile_phy_tsel;
 311         u32 rfile_phy_dq_timing;
 312         u32 rfile_phy_dqs_timing;
 313         u32 rfile_phy_gate_lpbk_ctrl;
 314         u32 rfile_phy_dll_master_ctrl;
 315         u32 rfile_phy_dll_slave_ctrl;
 316 };
 317
 318 static struct cdns_xspi_driver_data marvell_driver_data = {
 319         .mrvl_hw_overlay = true,
 320         .dll_phy_ctrl = MARVELL_REGS_DLL_PHY_CTRL,
 321         .ctb_rfile_phy_ctrl = MARVELL_CTB_RFILE_PHY_CTRL,
 322         .rfile_phy_tsel = MARVELL_RFILE_PHY_TSEL,
 323         .rfile_phy_dq_timing = MARVELL_RFILE_PHY_DQ_TIMING,
 324         .rfile_phy_dqs_timing = MARVELL_RFILE_PHY_DQS_TIMING,
 325         .rfile_phy_gate_lpbk_ctrl = MARVELL_RFILE_PHY_GATE_LPBK_CTRL,
 326         .rfile_phy_dll_master_ctrl = MARVELL_RFILE_PHY_DLL_MASTER_CTRL,
 327         .rfile_phy_dll_slave_ctrl = MARVELL_RFILE_PHY_DLL_SLAVE_CTRL,
 328 };
 329
 330 static struct cdns_xspi_driver_data cdns_driver_data = {
 331         .mrvl_hw_overlay = false,
 332 };
 333
 334 static const int cdns_mrvl_xspi_clk_div_list[] = {
 335         4,      //0x0 = Divide by 4.   SPI clock is 200 MHz.
 336         6,      //0x1 = Divide by 6.   SPI clock is 133.33 MHz.
 337         8,      //0x2 = Divide by 8.   SPI clock is 100 MHz.
 338         10,     //0x3 = Divide by 10.  SPI clock is 80 MHz.
 339         12,     //0x4 = Divide by 12.  SPI clock is 66.666 MHz.
 340         16,     //0x5 = Divide by 16.  SPI clock is 50 MHz.
 341         18,     //0x6 = Divide by 18.  SPI clock is 44.44 MHz.
 342         20,     //0x7 = Divide by 20.  SPI clock is 40 MHz.
 343         24,     //0x8 = Divide by 24.  SPI clock is 33.33 MHz.
 344         32,     //0x9 = Divide by 32.  SPI clock is 25 MHz.
 345         40,     //0xA = Divide by 40.  SPI clock is 20 MHz.
 346         50,     //0xB = Divide by 50.  SPI clock is 16 MHz.
 347         64,     //0xC = Divide by 64.  SPI clock is 12.5 MHz.
 348         128     //0xD = Divide by 128. SPI clock is 6.25 MHz.
 349 };
 350
 351 struct cdns_xspi_dev {
 352         struct platform_device *pdev;
 353         struct device *dev;
 354
 355         void __iomem *iobase;
 356         void __iomem *auxbase;
 357         void __iomem *sdmabase;
 358         void __iomem *xferbase;
 359
 360         int irq;
 361         int cur_cs;
 362         unsigned int sdmasize;
 363
 364         struct completion cmd_complete;
 365         struct completion auto_cmd_complete;
 366         struct completion sdma_complete;
 367         bool sdma_error;
 368
 369         void *in_buffer;
 370         const void *out_buffer;
 371
 372         u8 hw_num_banks;
 373
 374         const struct cdns_xspi_driver_data *driver_data;
 375         void (*sdma_handler)(struct cdns_xspi_dev *cdns_xspi);
 376         void (*set_interrupts_handler)(struct cdns_xspi_dev *cdns_xspi, bool enabled);
 377
 378         bool xfer_in_progress;
 379         int current_xfer_qword;
 380 };
 381
 382 static void cdns_xspi_reset_dll(struct cdns_xspi_dev *cdns_xspi)
 383 {
 384         u32 dll_cntrl = readl(cdns_xspi->iobase +
 385                               CDNS_XSPI_RF_MINICTRL_REGS_DLL_PHY_CTRL);
 386
 387         /* Reset DLL */
 388         dll_cntrl |= CDNS_XSPI_DLL_RST_N;
 389         writel(dll_cntrl, cdns_xspi->iobase +
 390                           CDNS_XSPI_RF_MINICTRL_REGS_DLL_PHY_CTRL);
 391 }
 392
 393 static bool cdns_xspi_is_dll_locked(struct cdns_xspi_dev *cdns_xspi)
 394 {
 395         u32 dll_lock;
 396
 397         return !readl_relaxed_poll_timeout(cdns_xspi->iobase +
 398                 CDNS_XSPI_INTR_STATUS_REG,
 399                 dll_lock, ((dll_lock & CDNS_XSPI_DLL_LOCK) == 1), 10, 10000);
 400 }
 401
 402 /* Static configuration of PHY */
 403 static bool cdns_xspi_configure_phy(struct cdns_xspi_dev *cdns_xspi)
 404 {
 405         writel(cdns_xspi->driver_data->dll_phy_ctrl,
 406                cdns_xspi->iobase + CDNS_XSPI_RF_MINICTRL_REGS_DLL_PHY_CTRL);
 407         writel(cdns_xspi->driver_data->ctb_rfile_phy_ctrl,
 408                cdns_xspi->auxbase + CDNS_XSPI_PHY_CTB_RFILE_PHY_CTRL);
 409         writel(cdns_xspi->driver_data->rfile_phy_tsel,
 410                cdns_xspi->auxbase + CDNS_XSPI_PHY_CTB_RFILE_PHY_TSEL);
 411         writel(cdns_xspi->driver_data->rfile_phy_dq_timing,
 412                cdns_xspi->auxbase + CDNS_XSPI_PHY_DATASLICE_RFILE_PHY_DQ_TIMING);
 413         writel(cdns_xspi->driver_data->rfile_phy_dqs_timing,
 414                cdns_xspi->auxbase + CDNS_XSPI_PHY_DATASLICE_RFILE_PHY_DQS_TIMING);
 415         writel(cdns_xspi->driver_data->rfile_phy_gate_lpbk_ctrl,
 416                cdns_xspi->auxbase + CDNS_XSPI_PHY_DATASLICE_RFILE_PHY_GATE_LPBK_CTRL);
 417         writel(cdns_xspi->driver_data->rfile_phy_dll_master_ctrl,
 418                cdns_xspi->auxbase + CDNS_XSPI_PHY_DATASLICE_RFILE_PHY_DLL_MASTER_CTRL);
 419         writel(cdns_xspi->driver_data->rfile_phy_dll_slave_ctrl,
 420                cdns_xspi->auxbase + CDNS_XSPI_PHY_DATASLICE_RFILE_PHY_DLL_SLAVE_CTRL);
 421
 422         cdns_xspi_reset_dll(cdns_xspi);
 423
 424         return cdns_xspi_is_dll_locked(cdns_xspi);
 425 }
 426
 427 static bool cdns_mrvl_xspi_setup_clock(struct cdns_xspi_dev *cdns_xspi,
 428                                        int requested_clk)
 429 {
 430         int i = 0;
 431         int clk_val;
 432         u32 clk_reg;
 433         bool update_clk = false;
 434
 435         while (i < ARRAY_SIZE(cdns_mrvl_xspi_clk_div_list)) {
 436                 clk_val = MRVL_XSPI_CLOCK_DIVIDED(
 437                                 cdns_mrvl_xspi_clk_div_list[i]);
 438                 if (clk_val <= requested_clk)
 439                         break;
 440                 i++;
 441         }
 442
 443         dev_dbg(cdns_xspi->dev, "Found clk div: %d, clk val: %d\n",
 444                 cdns_mrvl_xspi_clk_div_list[i],
 445                 MRVL_XSPI_CLOCK_DIVIDED(
 446                 cdns_mrvl_xspi_clk_div_list[i]));
 447
 448         clk_reg = readl(cdns_xspi->auxbase + MRVL_XSPI_CLK_CTRL_AUX_REG);
 449
 450         if (FIELD_GET(MRVL_XSPI_CLK_DIV, clk_reg) != i) {
 451                 clk_reg &= ~MRVL_XSPI_CLK_ENABLE;
 452                 writel(clk_reg,
 453                        cdns_xspi->auxbase + MRVL_XSPI_CLK_CTRL_AUX_REG);
 454                 clk_reg = FIELD_PREP(MRVL_XSPI_CLK_DIV, i);
 455                 clk_reg &= ~MRVL_XSPI_CLK_DIV;
 456                 clk_reg |= FIELD_PREP(MRVL_XSPI_CLK_DIV, i);
 457                 clk_reg |= MRVL_XSPI_CLK_ENABLE;
 458                 clk_reg |= MRVL_XSPI_IRQ_ENABLE;
 459                 update_clk = true;
 460         }
 461
 462         if (update_clk)
 463                 writel(clk_reg,
 464                        cdns_xspi->auxbase + MRVL_XSPI_CLK_CTRL_AUX_REG);
 465
 466         return update_clk;
 467 }
 468
 469 static int cdns_xspi_wait_for_controller_idle(struct cdns_xspi_dev *cdns_xspi)
 470 {
 471         u32 ctrl_stat;
 472
 473         return readl_relaxed_poll_timeout(cdns_xspi->iobase +
 474                                           CDNS_XSPI_CTRL_STATUS_REG,
 475                                           ctrl_stat,
 476                                           ((ctrl_stat &
 477                                             CDNS_XSPI_CTRL_BUSY) == 0),
 478                                           100, 1000);
 479 }
 480
 481 static void cdns_xspi_trigger_command(struct cdns_xspi_dev *cdns_xspi,
 482                                       u32 cmd_regs[6])
 483 {
 484         writel(cmd_regs[5], cdns_xspi->iobase + CDNS_XSPI_CMD_REG_5);
 485         writel(cmd_regs[4], cdns_xspi->iobase + CDNS_XSPI_CMD_REG_4);
 486         writel(cmd_regs[3], cdns_xspi->iobase + CDNS_XSPI_CMD_REG_3);
 487         writel(cmd_regs[2], cdns_xspi->iobase + CDNS_XSPI_CMD_REG_2);
 488         writel(cmd_regs[1], cdns_xspi->iobase + CDNS_XSPI_CMD_REG_1);
 489         writel(cmd_regs[0], cdns_xspi->iobase + CDNS_XSPI_CMD_REG_0);
 490 }
 491
 492 static int cdns_xspi_check_command_status(struct cdns_xspi_dev *cdns_xspi)
 493 {
 494         int ret = 0;
 495         u32 cmd_status = readl(cdns_xspi->iobase + CDNS_XSPI_CMD_STATUS_REG);
 496
 497         if (cmd_status & CDNS_XSPI_CMD_STATUS_COMPLETED) {
 498                 if ((cmd_status & CDNS_XSPI_CMD_STATUS_FAILED) != 0) {
 499                         if (cmd_status & CDNS_XSPI_CMD_STATUS_DQS_ERROR) {
 500                                 dev_err(cdns_xspi->dev,
 501                                         "Incorrect DQS pulses detected\n");
 502                                 ret = -EPROTO;
 503                         }
 504                         if (cmd_status & CDNS_XSPI_CMD_STATUS_CRC_ERROR) {
 505                                 dev_err(cdns_xspi->dev,
 506                                         "CRC error received\n");
 507                                 ret = -EPROTO;
 508                         }
 509                         if (cmd_status & CDNS_XSPI_CMD_STATUS_BUS_ERROR) {
 510                                 dev_err(cdns_xspi->dev,
 511                                         "Error resp on system DMA interface\n");
 512                                 ret = -EPROTO;
 513                         }
 514                         if (cmd_status & CDNS_XSPI_CMD_STATUS_INV_SEQ_ERROR) {
 515                                 dev_err(cdns_xspi->dev,
 516                                         "Invalid command sequence detected\n");
 517                                 ret = -EPROTO;
 518                         }
 519                 }
 520         } else {
 521                 dev_err(cdns_xspi->dev, "Fatal err - command not completed\n");
 522                 ret = -EPROTO;
 523         }
 524
 525         return ret;
 526 }
 527
 528 static void cdns_xspi_set_interrupts(struct cdns_xspi_dev *cdns_xspi,
 529                                      bool enabled)
 530 {
 531         u32 intr_enable;
 532
 533         intr_enable = readl(cdns_xspi->iobase + CDNS_XSPI_INTR_ENABLE_REG);
 534         if (enabled)
 535                 intr_enable |= CDNS_XSPI_INTR_MASK;
 536         else
 537                 intr_enable &= ~CDNS_XSPI_INTR_MASK;
 538         writel(intr_enable, cdns_xspi->iobase + CDNS_XSPI_INTR_ENABLE_REG);
 539 }
 540
 541 static void marvell_xspi_set_interrupts(struct cdns_xspi_dev *cdns_xspi,
 542                                      bool enabled)
 543 {
 544         u32 intr_enable;
 545         u32 irq_status;
 546
 547         irq_status = readl(cdns_xspi->iobase + CDNS_XSPI_INTR_STATUS_REG);
 548         writel(irq_status, cdns_xspi->iobase + CDNS_XSPI_INTR_STATUS_REG);
 549
 550         intr_enable = readl(cdns_xspi->iobase + CDNS_XSPI_INTR_ENABLE_REG);
 551         if (enabled)
 552                 intr_enable |= CDNS_XSPI_INTR_MASK;
 553         else
 554                 intr_enable &= ~CDNS_XSPI_INTR_MASK;
 555         writel(intr_enable, cdns_xspi->iobase + CDNS_XSPI_INTR_ENABLE_REG);
 556 }
 557
 558 static int cdns_xspi_controller_init(struct cdns_xspi_dev *cdns_xspi)
 559 {
 560         u32 ctrl_ver;
 561         u32 ctrl_features;
 562         u16 hw_magic_num;
 563
 564         ctrl_ver = readl(cdns_xspi->iobase + CDNS_XSPI_CTRL_VERSION_REG);
 565         hw_magic_num = FIELD_GET(CDNS_XSPI_MAGIC_NUM, ctrl_ver);
 566         if (hw_magic_num != CDNS_XSPI_MAGIC_NUM_VALUE) {
 567                 dev_err(cdns_xspi->dev,
 568                         "Incorrect XSPI magic number: %x, expected: %x\n",
 569                         hw_magic_num, CDNS_XSPI_MAGIC_NUM_VALUE);
 570                 return -EIO;
 571         }
 572
 573         ctrl_features = readl(cdns_xspi->iobase + CDNS_XSPI_CTRL_FEATURES_REG);
 574         cdns_xspi->hw_num_banks = FIELD_GET(CDNS_XSPI_NUM_BANKS, ctrl_features);
 575         cdns_xspi->set_interrupts_handler(cdns_xspi, false);
 576
 577         return 0;
 578 }
 579
 580 static void cdns_xspi_sdma_handle(struct cdns_xspi_dev *cdns_xspi)
 581 {
 582         u32 sdma_size, sdma_trd_info;
 583         u8 sdma_dir;
 584
 585         sdma_size = readl(cdns_xspi->iobase + CDNS_XSPI_SDMA_SIZE_REG);
 586         sdma_trd_info = readl(cdns_xspi->iobase + CDNS_XSPI_SDMA_TRD_INFO_REG);
 587         sdma_dir = FIELD_GET(CDNS_XSPI_SDMA_DIR, sdma_trd_info);
 588
 589         switch (sdma_dir) {
 590         case CDNS_XSPI_SDMA_DIR_READ:
 591                 ioread8_rep(cdns_xspi->sdmabase,
 592                             cdns_xspi->in_buffer, sdma_size);
 593                 break;
 594
 595         case CDNS_XSPI_SDMA_DIR_WRITE:
 596                 iowrite8_rep(cdns_xspi->sdmabase,
 597                              cdns_xspi->out_buffer, sdma_size);
 598                 break;
 599         }
 600 }
 601
 602 static void m_ioreadq(void __iomem  *addr, void *buf, int len)
 603 {
 604         if (IS_ALIGNED((long)buf, 8) && len >= 8) {
 605                 u64 full_ops = len / 8;
 606                 u64 *buffer = buf;
 607
 608                 len -= full_ops * 8;
 609                 buf += full_ops * 8;
 610
 611                 do {
 612                         u64 b = readq(addr);
 613                         *buffer++ = b;
 614                 } while (--full_ops);
 615         }
 616
 617
 618         while (len) {
 619                 u64 tmp_buf;
 620
 621                 tmp_buf = readq(addr);
 622                 memcpy(buf, &tmp_buf, min(len, 8));
 623                 len = len > 8 ? len - 8 : 0;
 624                 buf += 8;
 625         }
 626 }
 627
 628 static void m_iowriteq(void __iomem *addr, const void *buf, int len)
 629 {
 630         if (IS_ALIGNED((long)buf, 8) && len >= 8) {
 631                 u64 full_ops = len / 8;
 632                 const u64 *buffer = buf;
 633
 634                 len -= full_ops * 8;
 635                 buf += full_ops * 8;
 636
 637                 do {
 638                         writeq(*buffer++, addr);
 639                 } while (--full_ops);
 640         }
 641
 642         while (len) {
 643                 u64 tmp_buf;
 644
 645                 memcpy(&tmp_buf, buf, min(len, 8));
 646                 writeq(tmp_buf, addr);
 647                 len = len > 8 ? len - 8 : 0;
 648                 buf += 8;
 649         }
 650 }
 651
 652 static void marvell_xspi_sdma_handle(struct cdns_xspi_dev *cdns_xspi)
 653 {
 654         u32 sdma_size, sdma_trd_info;
 655         u8 sdma_dir;
 656
 657         sdma_size = readl(cdns_xspi->iobase + CDNS_XSPI_SDMA_SIZE_REG);
 658         sdma_trd_info = readl(cdns_xspi->iobase + CDNS_XSPI_SDMA_TRD_INFO_REG);
 659         sdma_dir = FIELD_GET(CDNS_XSPI_SDMA_DIR, sdma_trd_info);
 660
 661         switch (sdma_dir) {
 662         case CDNS_XSPI_SDMA_DIR_READ:
 663                 m_ioreadq(cdns_xspi->sdmabase,
 664                             cdns_xspi->in_buffer, sdma_size);
 665                 break;
 666
 667         case CDNS_XSPI_SDMA_DIR_WRITE:
 668                 m_iowriteq(cdns_xspi->sdmabase,
 669                              cdns_xspi->out_buffer, sdma_size);
 670                 break;
 671         }
 672 }
 673
 674 static int cdns_xspi_send_stig_command(struct cdns_xspi_dev *cdns_xspi,
 675                                        const struct spi_mem_op *op,
 676                                        bool data_phase)
 677 {
 678         u32 cmd_regs[6];
 679         u32 cmd_status;
 680         int ret;
 681         int dummybytes = op->dummy.nbytes;
 682
 683         ret = cdns_xspi_wait_for_controller_idle(cdns_xspi);
 684         if (ret < 0)
 685                 return -EIO;
 686
 687         writel(FIELD_PREP(CDNS_XSPI_CTRL_WORK_MODE, CDNS_XSPI_WORK_MODE_STIG),
 688                cdns_xspi->iobase + CDNS_XSPI_CTRL_CONFIG_REG);
 689
 690         cdns_xspi->set_interrupts_handler(cdns_xspi, true);
 691         cdns_xspi->sdma_error = false;
 692
 693         memset(cmd_regs, 0, sizeof(cmd_regs));
 694         cmd_regs[1] = CDNS_XSPI_CMD_FLD_P1_INSTR_CMD_1(op, data_phase);
 695         cmd_regs[2] = CDNS_XSPI_CMD_FLD_P1_INSTR_CMD_2(op);
 696         if (dummybytes != 0) {
 697                 cmd_regs[3] = CDNS_XSPI_CMD_FLD_P1_INSTR_CMD_3(op, 1);
 698                 dummybytes--;
 699         } else {
 700                 cmd_regs[3] = CDNS_XSPI_CMD_FLD_P1_INSTR_CMD_3(op, 0);
 701         }
 702         cmd_regs[4] = CDNS_XSPI_CMD_FLD_P1_INSTR_CMD_4(op,
 703                                                        cdns_xspi->cur_cs);
 704
 705         cdns_xspi_trigger_command(cdns_xspi, cmd_regs);
 706
 707         if (data_phase) {
 708                 cmd_regs[0] = CDNS_XSPI_STIG_DONE_FLAG;
 709                 cmd_regs[1] = CDNS_XSPI_CMD_FLD_DSEQ_CMD_1(op);
 710                 cmd_regs[2] = CDNS_XSPI_CMD_FLD_DSEQ_CMD_2(op);
 711                 cmd_regs[3] = CDNS_XSPI_CMD_FLD_DSEQ_CMD_3(op, dummybytes);
 712                 cmd_regs[4] = CDNS_XSPI_CMD_FLD_DSEQ_CMD_4(op,
 713                                                            cdns_xspi->cur_cs);
 714
 715                 cdns_xspi->in_buffer = op->data.buf.in;
 716                 cdns_xspi->out_buffer = op->data.buf.out;
 717
 718                 cdns_xspi_trigger_command(cdns_xspi, cmd_regs);
 719
 720                 wait_for_completion(&cdns_xspi->sdma_complete);
 721                 if (cdns_xspi->sdma_error) {
 722                         cdns_xspi->set_interrupts_handler(cdns_xspi, false);
 723                         return -EIO;
 724                 }
 725                 cdns_xspi->sdma_handler(cdns_xspi);
 726         }
 727
 728         wait_for_completion(&cdns_xspi->cmd_complete);
 729         cdns_xspi->set_interrupts_handler(cdns_xspi, false);
 730
 731         cmd_status = cdns_xspi_check_command_status(cdns_xspi);
 732         if (cmd_status)
 733                 return -EPROTO;
 734
 735         return 0;
 736 }
 737
 738 static int cdns_xspi_mem_op(struct cdns_xspi_dev *cdns_xspi,
 739                             struct spi_mem *mem,
 740                             const struct spi_mem_op *op)
 741 {
 742         enum spi_mem_data_dir dir = op->data.dir;
 743
 744         if (cdns_xspi->cur_cs != spi_get_chipselect(mem->spi, 0))
 745                 cdns_xspi->cur_cs = spi_get_chipselect(mem->spi, 0);
 746
 747         return cdns_xspi_send_stig_command(cdns_xspi, op,
 748                                            (dir != SPI_MEM_NO_DATA));
 749 }
 750
 751 static int cdns_xspi_mem_op_execute(struct spi_mem *mem,
 752                                     const struct spi_mem_op *op)
 753 {
 754         struct cdns_xspi_dev *cdns_xspi =
 755                 spi_controller_get_devdata(mem->spi->controller);
 756         int ret = 0;
 757
 758         ret = cdns_xspi_mem_op(cdns_xspi, mem, op);
 759
 760         return ret;
 761 }
 762
 763 static int marvell_xspi_mem_op_execute(struct spi_mem *mem,
 764                                     const struct spi_mem_op *op)
 765 {
 766         struct cdns_xspi_dev *cdns_xspi =
 767                 spi_controller_get_devdata(mem->spi->controller);
 768         int ret = 0;
 769
 770         cdns_mrvl_xspi_setup_clock(cdns_xspi, mem->spi->max_speed_hz);
 771
 772         ret = cdns_xspi_mem_op(cdns_xspi, mem, op);
 773
 774         return ret;
 775 }
 776
 777 #ifdef CONFIG_ACPI
 778 static bool cdns_xspi_supports_op(struct spi_mem *mem,
 779                                   const struct spi_mem_op *op)
 780 {
 781         struct spi_device *spi = mem->spi;
 782         const union acpi_object *obj;
 783         struct acpi_device *adev;
 784
 785         adev = ACPI_COMPANION(&spi->dev);
 786
 787         if (!acpi_dev_get_property(adev, "spi-tx-bus-width", ACPI_TYPE_INTEGER,
 788                                    &obj)) {
 789                 switch (obj->integer.value) {
 790                 case 1:
 791                         break;
 792                 case 2:
 793                         spi->mode |= SPI_TX_DUAL;
 794                         break;
 795                 case 4:
 796                         spi->mode |= SPI_TX_QUAD;
 797                         break;
 798                 case 8:
 799                         spi->mode |= SPI_TX_OCTAL;
 800                         break;
 801                 default:
 802                         dev_warn(&spi->dev,
 803                                  "spi-tx-bus-width %lld not supported\n",
 804                                  obj->integer.value);
 805                         break;
 806                 }
 807         }
 808
 809         if (!acpi_dev_get_property(adev, "spi-rx-bus-width", ACPI_TYPE_INTEGER,
 810                                    &obj)) {
 811                 switch (obj->integer.value) {
 812                 case 1:
 813                         break;
 814                 case 2:
 815                         spi->mode |= SPI_RX_DUAL;
 816                         break;
 817                 case 4:
 818                         spi->mode |= SPI_RX_QUAD;
 819                         break;
 820                 case 8:
 821                         spi->mode |= SPI_RX_OCTAL;
 822                         break;
 823                 default:
 824                         dev_warn(&spi->dev,
 825                                  "spi-rx-bus-width %lld not supported\n",
 826                                  obj->integer.value);
 827                         break;
 828                 }
 829         }
 830
 831         if (!spi_mem_default_supports_op(mem, op))
 832                 return false;
 833
 834         return true;
 835 }
 836 #endif
 837
 838 static int cdns_xspi_adjust_mem_op_size(struct spi_mem *mem, struct spi_mem_op *op)
 839 {
 840         struct cdns_xspi_dev *cdns_xspi =
 841                 spi_controller_get_devdata(mem->spi->controller);
 842
 843         op->data.nbytes = clamp_val(op->data.nbytes, 0, cdns_xspi->sdmasize);
 844
 845         return 0;
 846 }
 847
 848 static const struct spi_controller_mem_ops cadence_xspi_mem_ops = {
 849 #ifdef CONFIG_ACPI
 850         .supports_op = cdns_xspi_supports_op,
 851 #endif
 852         .exec_op = cdns_xspi_mem_op_execute,
 853         .adjust_op_size = cdns_xspi_adjust_mem_op_size,
 854 };
 855
 856 static const struct spi_controller_mem_ops marvell_xspi_mem_ops = {
 857 #ifdef CONFIG_ACPI
 858         .supports_op = cdns_xspi_supports_op,
 859 #endif
 860         .exec_op = marvell_xspi_mem_op_execute,
 861         .adjust_op_size = cdns_xspi_adjust_mem_op_size,
 862 };
 863
 864 static irqreturn_t cdns_xspi_irq_handler(int this_irq, void *dev)
 865 {
 866         struct cdns_xspi_dev *cdns_xspi = dev;
 867         u32 irq_status;
 868         irqreturn_t result = IRQ_NONE;
 869
 870         irq_status = readl(cdns_xspi->iobase + CDNS_XSPI_INTR_STATUS_REG);
 871         writel(irq_status, cdns_xspi->iobase + CDNS_XSPI_INTR_STATUS_REG);
 872
 873         if (irq_status &
 874             (CDNS_XSPI_SDMA_ERROR | CDNS_XSPI_SDMA_TRIGGER |
 875              CDNS_XSPI_STIG_DONE)) {
 876                 if (irq_status & CDNS_XSPI_SDMA_ERROR) {
 877                         dev_err(cdns_xspi->dev,
 878                                 "Slave DMA transaction error\n");
 879                         cdns_xspi->sdma_error = true;
 880                         complete(&cdns_xspi->sdma_complete);
 881                 }
 882
 883                 if (irq_status & CDNS_XSPI_SDMA_TRIGGER)
 884                         complete(&cdns_xspi->sdma_complete);
 885
 886                 if (irq_status & CDNS_XSPI_STIG_DONE)
 887                         complete(&cdns_xspi->cmd_complete);
 888
 889                 result = IRQ_HANDLED;
 890         }
 891
 892         irq_status = readl(cdns_xspi->iobase + CDNS_XSPI_TRD_COMP_INTR_STATUS);
 893         if (irq_status) {
 894                 writel(irq_status,
 895                        cdns_xspi->iobase + CDNS_XSPI_TRD_COMP_INTR_STATUS);
 896
 897                 complete(&cdns_xspi->auto_cmd_complete);
 898
 899                 result = IRQ_HANDLED;
 900         }
 901
 902         return result;
 903 }
 904
 905 static int cdns_xspi_of_get_plat_data(struct platform_device *pdev)
 906 {
 907         struct fwnode_handle *fwnode_child;
 908         unsigned int cs;
 909
 910         device_for_each_child_node(&pdev->dev, fwnode_child) {
 911                 if (!fwnode_device_is_available(fwnode_child))
 912                         continue;
 913
 914                 if (fwnode_property_read_u32(fwnode_child, "reg", &cs)) {
 915                         dev_err(&pdev->dev, "Couldn't get memory chip select\n");
 916                         fwnode_handle_put(fwnode_child);
 917                         return -ENXIO;
 918                 } else if (cs >= CDNS_XSPI_MAX_BANKS) {
 919                         dev_err(&pdev->dev, "reg (cs) parameter value too large\n");
 920                         fwnode_handle_put(fwnode_child);
 921                         return -ENXIO;
 922                 }
 923         }
 924
 925         return 0;
 926 }
 927
 928 static void cdns_xspi_print_phy_config(struct cdns_xspi_dev *cdns_xspi)
 929 {
 930         struct device *dev = cdns_xspi->dev;
 931
 932         dev_info(dev, "PHY configuration\n");
 933         dev_info(dev, "   * xspi_dll_phy_ctrl: %08x\n",
 934                  readl(cdns_xspi->iobase + CDNS_XSPI_DLL_PHY_CTRL));
 935         dev_info(dev, "   * phy_dq_timing: %08x\n",
 936                  readl(cdns_xspi->auxbase + CDNS_XSPI_CCP_PHY_DQ_TIMING));
 937         dev_info(dev, "   * phy_dqs_timing: %08x\n",
 938                  readl(cdns_xspi->auxbase + CDNS_XSPI_CCP_PHY_DQS_TIMING));
 939         dev_info(dev, "   * phy_gate_loopback_ctrl: %08x\n",
 940                  readl(cdns_xspi->auxbase + CDNS_XSPI_CCP_PHY_GATE_LPBCK_CTRL));
 941         dev_info(dev, "   * phy_dll_slave_ctrl: %08x\n",
 942                  readl(cdns_xspi->auxbase + CDNS_XSPI_CCP_PHY_DLL_SLAVE_CTRL));
 943 }
 944
 945 static int cdns_xspi_prepare_generic(int cs, const void *dout, int len, int glue, u32 *cmd_regs)
 946 {
 947         u8 *data = (u8 *)dout;
 948         int i;
 949         int data_counter = 0;
 950
 951         memset(cmd_regs, 0x00, CMD_REG_LEN);
 952
 953         if (GENERIC_CMD_REG_3_NEEDED(len)) {
 954                 for (i = GENERIC_CMD_DATA_REG_3_COUNT(len); i >= 0 ; i--)
 955                         cmd_regs[3] |= GENERIC_CMD_DATA_INSERT(data[data_counter++],
 956                                                                GENERIC_CMD_DATA_3_OFFSET(i));
 957         }
 958         if (GENERIC_CMD_REG_2_NEEDED(len)) {
 959                 for (i = GENERIC_CMD_DATA_REG_2_COUNT(len); i >= 0; i--)
 960                         cmd_regs[2] |= GENERIC_CMD_DATA_INSERT(data[data_counter++],
 961                                                                GENERIC_CMD_DATA_2_OFFSET(i));
 962         }
 963         for (i = GENERIC_CMD_DATA_REG_1_COUNT(len); i >= 0 ; i--)
 964                 cmd_regs[1] |= GENERIC_CMD_DATA_INSERT(data[data_counter++],
 965                                                        GENERIC_CMD_DATA_1_OFFSET(i));
 966
 967         cmd_regs[1] |= CDNS_XSPI_CMD_FLD_P1_GENERIC_CMD;
 968         cmd_regs[3] |= CDNS_XSPI_CMD_FLD_P3_GENERIC_CMD(len);
 969         cmd_regs[4] |= CDNS_XSPI_CMD_FLD_P4_GENERIC_CMD(cs, glue);
 970
 971         return 0;
 972 }
 973
 974 static void marvell_xspi_read_single_qword(struct cdns_xspi_dev *cdns_xspi, u8 **buffer)
 975 {
 976         u64 d = readq(cdns_xspi->xferbase +
 977                       MRVL_XFER_FUNC_CTRL_READ_DATA(cdns_xspi->current_xfer_qword));
 978         u8 *ptr = (u8 *)&d;
 979         int k;
 980
 981         for (k = 0; k < 8; k++) {
 982                 u8 val = bitrev8((ptr[k]));
 983                 **buffer = val;
 984                 *buffer = *buffer + 1;
 985         }
 986
 987         cdns_xspi->current_xfer_qword++;
 988         cdns_xspi->current_xfer_qword %= MRVL_XFER_QWORD_COUNT;
 989 }
 990
 991 static void cdns_xspi_finish_read(struct cdns_xspi_dev *cdns_xspi, u8 **buffer, u32 data_count)
 992 {
 993         u64 d = readq(cdns_xspi->xferbase +
 994                       MRVL_XFER_FUNC_CTRL_READ_DATA(cdns_xspi->current_xfer_qword));
 995         u8 *ptr = (u8 *)&d;
 996         int k;
 997
 998         for (k = 0; k < data_count % MRVL_XFER_QWORD_BYTECOUNT; k++) {
 999                 u8 val = bitrev8((ptr[k]));
1000                 **buffer = val;
1001                 *buffer = *buffer + 1;
1002         }
1003
1004         cdns_xspi->current_xfer_qword++;
1005         cdns_xspi->current_xfer_qword %= MRVL_XFER_QWORD_COUNT;
1006 }
1007
1008 static int cdns_xspi_prepare_transfer(int cs, int dir, int len, u32 *cmd_regs)
1009 {
1010         memset(cmd_regs, 0x00, CMD_REG_LEN);
1011
1012         cmd_regs[1] |= CDNS_XSPI_CMD_FLD_GENERIC_DSEQ_CMD_1;
1013         cmd_regs[2] |= CDNS_XSPI_CMD_FLD_GENERIC_DSEQ_CMD_2(len);
1014         cmd_regs[4] |= CDNS_XSPI_CMD_FLD_GENERIC_DSEQ_CMD_4(dir, cs);
1015
1016         return 0;
1017 }
1018
1019 static bool cdns_xspi_is_stig_ready(struct cdns_xspi_dev *cdns_xspi, bool sleep)
1020 {
1021         u32 ctrl_stat;
1022
1023         return !readl_relaxed_poll_timeout
1024                 (cdns_xspi->iobase + CDNS_XSPI_CTRL_STATUS_REG,
1025                 ctrl_stat,
1026                 ((ctrl_stat & BIT(3)) == 0),
1027                 sleep ? MRVL_XSPI_POLL_DELAY_US : 0,
1028                 sleep ? MRVL_XSPI_POLL_TIMEOUT_US : 0);
1029 }
1030
1031 static bool cdns_xspi_is_sdma_ready(struct cdns_xspi_dev *cdns_xspi, bool sleep)
1032 {
1033         u32 ctrl_stat;
1034
1035         return !readl_relaxed_poll_timeout
1036                 (cdns_xspi->iobase + CDNS_XSPI_INTR_STATUS_REG,
1037                 ctrl_stat,
1038                 (ctrl_stat & CDNS_XSPI_SDMA_TRIGGER),
1039                 sleep ? MRVL_XSPI_POLL_DELAY_US : 0,
1040                 sleep ? MRVL_XSPI_POLL_TIMEOUT_US : 0);
1041 }
1042
1043 static int cdns_xspi_transfer_one_message_b0(struct spi_controller *controller,
1044                                            struct spi_message *m)
1045 {
1046         struct cdns_xspi_dev *cdns_xspi = spi_controller_get_devdata(controller);
1047         struct spi_device *spi = m->spi;
1048         struct spi_transfer *t = NULL;
1049
1050         const unsigned int max_len = MRVL_XFER_QWORD_BYTECOUNT * MRVL_XFER_QWORD_COUNT;
1051         int current_transfer_len;
1052         int cs = spi_get_chipselect(spi, 0);
1053         int cs_change = 0;
1054
1055         /* Enable xfer state machine */
1056         if (!cdns_xspi->xfer_in_progress) {
1057                 u32 xfer_control = readl(cdns_xspi->xferbase + MRVL_XFER_FUNC_CTRL);
1058
1059                 cdns_xspi->current_xfer_qword = 0;
1060                 cdns_xspi->xfer_in_progress = true;
1061                 xfer_control |= (MRVL_XFER_RECEIVE_ENABLE |
1062                                  MRVL_XFER_CLK_CAPTURE_POL |
1063                                  MRVL_XFER_FUNC_START |
1064                                  MRVL_XFER_SOFT_RESET |
1065                                  FIELD_PREP(MRVL_XFER_CS_N_HOLD, (1 << cs)));
1066                 xfer_control &= ~(MRVL_XFER_FUNC_ENABLE | MRVL_XFER_CLK_DRIVE_POL);
1067                 writel(xfer_control, cdns_xspi->xferbase + MRVL_XFER_FUNC_CTRL);
1068         }
1069
1070         list_for_each_entry(t, &m->transfers, transfer_list) {
1071                 u8 *txd = (u8 *) t->tx_buf;
1072                 u8 *rxd = (u8 *) t->rx_buf;
1073                 u8 data[10];
1074                 u32 cmd_regs[6];
1075
1076                 if (!txd)
1077                         txd = data;
1078
1079                 cdns_xspi->in_buffer = txd + 1;
1080                 cdns_xspi->out_buffer = txd + 1;
1081
1082                 while (t->len) {
1083
1084                         current_transfer_len = min(max_len, t->len);
1085
1086                         if (current_transfer_len < 10) {
1087                                 cdns_xspi_prepare_generic(cs, txd, current_transfer_len,
1088                                                           false, cmd_regs);
1089                                 cdns_xspi_trigger_command(cdns_xspi, cmd_regs);
1090                                 if (!cdns_xspi_is_stig_ready(cdns_xspi, true))
1091                                         return -EIO;
1092                         } else {
1093                                 cdns_xspi_prepare_generic(cs, txd, 1, true, cmd_regs);
1094                                 cdns_xspi_trigger_command(cdns_xspi, cmd_regs);
1095                                 cdns_xspi_prepare_transfer(cs, 1, current_transfer_len - 1,
1096                                                            cmd_regs);
1097                                 cdns_xspi_trigger_command(cdns_xspi, cmd_regs);
1098                                 if (!cdns_xspi_is_sdma_ready(cdns_xspi, true))
1099                                         return -EIO;
1100                                 cdns_xspi->sdma_handler(cdns_xspi);
1101                                 if (!cdns_xspi_is_stig_ready(cdns_xspi, true))
1102                                         return -EIO;
1103
1104                                 cdns_xspi->in_buffer += current_transfer_len;
1105                                 cdns_xspi->out_buffer += current_transfer_len;
1106                         }
1107
1108                         if (rxd) {
1109                                 int j;
1110
1111                                 for (j = 0; j < current_transfer_len / 8; j++)
1112                                         marvell_xspi_read_single_qword(cdns_xspi, &rxd);
1113                                 cdns_xspi_finish_read(cdns_xspi, &rxd, current_transfer_len);
1114                         } else {
1115                                 cdns_xspi->current_xfer_qword += current_transfer_len /
1116                                                                  MRVL_XFER_QWORD_BYTECOUNT;
1117                                 if (current_transfer_len % MRVL_XFER_QWORD_BYTECOUNT)
1118                                         cdns_xspi->current_xfer_qword++;
1119
1120                                 cdns_xspi->current_xfer_qword %= MRVL_XFER_QWORD_COUNT;
1121                         }
1122                         cs_change = t->cs_change;
1123                         t->len -= current_transfer_len;
1124                 }
1125                 spi_transfer_delay_exec(t);
1126         }
1127
1128         if (!cs_change) {
1129                 u32 xfer_control = readl(cdns_xspi->xferbase + MRVL_XFER_FUNC_CTRL);
1130
1131                 xfer_control &= ~(MRVL_XFER_RECEIVE_ENABLE |
1132                                   MRVL_XFER_SOFT_RESET);
1133                 writel(xfer_control, cdns_xspi->xferbase + MRVL_XFER_FUNC_CTRL);
1134                 cdns_xspi->xfer_in_progress = false;
1135         }
1136
1137         m->status = 0;
1138         spi_finalize_current_message(controller);
1139
1140         return 0;
1141 }
1142
1143 static int cdns_xspi_probe(struct platform_device *pdev)
1144 {
1145         struct device *dev = &pdev->dev;
1146         struct spi_controller *host = NULL;
1147         struct cdns_xspi_dev *cdns_xspi = NULL;
1148         struct resource *res;
1149         int ret;
1150
1151         host = devm_spi_alloc_host(dev, sizeof(*cdns_xspi));
1152         if (!host)
1153                 return -ENOMEM;
1154
1155         host->mode_bits = SPI_3WIRE | SPI_TX_DUAL  | SPI_TX_QUAD  |
1156                 SPI_RX_DUAL | SPI_RX_QUAD | SPI_TX_OCTAL | SPI_RX_OCTAL |
1157                 SPI_MODE_0  | SPI_MODE_3;
1158
1159         cdns_xspi = spi_controller_get_devdata(host);
1160         cdns_xspi->driver_data = of_device_get_match_data(dev);
1161         if (!cdns_xspi->driver_data) {
1162                 cdns_xspi->driver_data = acpi_device_get_match_data(dev);
1163                 if (!cdns_xspi->driver_data)
1164                         return -ENODEV;
1165         }
1166
1167         if (cdns_xspi->driver_data->mrvl_hw_overlay) {
1168                 host->mem_ops = &marvell_xspi_mem_ops;
1169                 host->transfer_one_message = cdns_xspi_transfer_one_message_b0;
1170                 cdns_xspi->sdma_handler = &marvell_xspi_sdma_handle;
1171                 cdns_xspi->set_interrupts_handler = &marvell_xspi_set_interrupts;
1172         } else {
1173                 host->mem_ops = &cadence_xspi_mem_ops;
1174                 cdns_xspi->sdma_handler = &cdns_xspi_sdma_handle;
1175                 cdns_xspi->set_interrupts_handler = &cdns_xspi_set_interrupts;
1176         }
1177         host->dev.of_node = pdev->dev.of_node;
1178         host->bus_num = -1;
1179
1180         platform_set_drvdata(pdev, host);
1181
1182         cdns_xspi->pdev = pdev;
1183         cdns_xspi->dev = &pdev->dev;
1184         cdns_xspi->cur_cs = 0;
1185
1186         init_completion(&cdns_xspi->cmd_complete);
1187         init_completion(&cdns_xspi->auto_cmd_complete);
1188         init_completion(&cdns_xspi->sdma_complete);
1189
1190         ret = cdns_xspi_of_get_plat_data(pdev);
1191         if (ret)
1192                 return -ENODEV;
1193
1194         cdns_xspi->iobase = devm_platform_ioremap_resource_byname(pdev, "io");
1195         if (IS_ERR(cdns_xspi->iobase)) {
1196                 cdns_xspi->iobase = devm_platform_ioremap_resource(pdev, 0);
1197                 if (IS_ERR(cdns_xspi->iobase)) {
1198                         dev_err(dev, "Failed to remap controller base address\n");
1199                         return PTR_ERR(cdns_xspi->iobase);
1200                 }
1201         }
1202
1203         res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "sdma");
1204         cdns_xspi->sdmabase = devm_ioremap_resource(dev, res);
1205         if (IS_ERR(cdns_xspi->sdmabase)) {
1206                 res = platform_get_resource(pdev, IORESOURCE_MEM, 1);
1207                 cdns_xspi->sdmabase = devm_ioremap_resource(dev, res);
1208                 if (IS_ERR(cdns_xspi->sdmabase))
1209                         return PTR_ERR(cdns_xspi->sdmabase);
1210         }
1211         cdns_xspi->sdmasize = resource_size(res);
1212
1213         cdns_xspi->auxbase = devm_platform_ioremap_resource_byname(pdev, "aux");
1214         if (IS_ERR(cdns_xspi->auxbase)) {
1215                 cdns_xspi->auxbase = devm_platform_ioremap_resource(pdev, 2);
1216                 if (IS_ERR(cdns_xspi->auxbase)) {
1217                         dev_err(dev, "Failed to remap AUX address\n");
1218                         return PTR_ERR(cdns_xspi->auxbase);
1219                 }
1220         }
1221
1222         if (cdns_xspi->driver_data->mrvl_hw_overlay) {
1223                 cdns_xspi->xferbase = devm_platform_ioremap_resource_byname(pdev, "xfer");
1224                 if (IS_ERR(cdns_xspi->xferbase)) {
1225                         cdns_xspi->xferbase = devm_platform_ioremap_resource(pdev, 3);
1226                         if (IS_ERR(cdns_xspi->xferbase)) {
1227                                 dev_info(dev, "XFER register base not found, set it\n");
1228                                 // For compatibility with older firmware
1229                                 cdns_xspi->xferbase = cdns_xspi->iobase + 0x8000;
1230                         }
1231                 }
1232         }
1233
1234         cdns_xspi->irq = platform_get_irq(pdev, 0);
1235         if (cdns_xspi->irq < 0)
1236                 return -ENXIO;
1237
1238         ret = devm_request_irq(dev, cdns_xspi->irq, cdns_xspi_irq_handler,
1239                                IRQF_SHARED, pdev->name, cdns_xspi);
1240         if (ret) {
1241                 dev_err(dev, "Failed to request IRQ: %d\n", cdns_xspi->irq);
1242                 return ret;
1243         }
1244
1245         if (cdns_xspi->driver_data->mrvl_hw_overlay) {
1246                 cdns_mrvl_xspi_setup_clock(cdns_xspi, MRVL_DEFAULT_CLK);
1247                 cdns_xspi_configure_phy(cdns_xspi);
1248         }
1249
1250         cdns_xspi_print_phy_config(cdns_xspi);
1251
1252         ret = cdns_xspi_controller_init(cdns_xspi);
1253         if (ret) {
1254                 dev_err(dev, "Failed to initialize controller\n");
1255                 return ret;
1256         }
1257
1258         host->num_chipselect = 1 << cdns_xspi->hw_num_banks;
1259
1260         ret = devm_spi_register_controller(dev, host);
1261         if (ret) {
1262                 dev_err(dev, "Failed to register SPI host\n");
1263                 return ret;
1264         }
1265
1266         dev_info(dev, "Successfully registered SPI host\n");
1267
1268         return 0;
1269 }
1270
1271 static const struct of_device_id cdns_xspi_of_match[] = {
1272         {
1273                 .compatible = "cdns,xspi-nor",
1274                 .data = &cdns_driver_data,
1275         },
1276         {
1277                 .compatible = "marvell,cn10-xspi-nor",
1278                 .data = &marvell_driver_data,
1279         },
1280         { /* end of table */}
1281 };
1282 MODULE_DEVICE_TABLE(of, cdns_xspi_of_match);
1283
1284 static struct platform_driver cdns_xspi_platform_driver = {
1285         .probe          = cdns_xspi_probe,
1286         .driver = {
1287                 .name = CDNS_XSPI_NAME,
1288                 .of_match_table = cdns_xspi_of_match,
1289         },
1290 };
1291
1292 module_platform_driver(cdns_xspi_platform_driver);
1293
1294 MODULE_DESCRIPTION("Cadence XSPI Controller Driver");
1295 MODULE_LICENSE("GPL v2");
1296 MODULE_ALIAS("platform:" CDNS_XSPI_NAME);
1297 MODULE_AUTHOR("Konrad Kociolek <konrad@cadence.com>");
1298 MODULE_AUTHOR("Jayshri Pawar <jpawar@cadence.com>");
1299 MODULE_AUTHOR("Parshuram Thombare <pthombar@cadence.com>");