drivers/net/ethernet/freescale/fman/fman.c

   1 /*
   2  * Copyright 2008-2015 Freescale Semiconductor Inc.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions are met:
   6  *     * Redistributions of source code must retain the above copyright
   7  *       notice, this list of conditions and the following disclaimer.
   8  *     * Redistributions in binary form must reproduce the above copyright
   9  *       notice, this list of conditions and the following disclaimer in the
  10  *       documentation and/or other materials provided with the distribution.
  11  *     * Neither the name of Freescale Semiconductor nor the
  12  *       names of its contributors may be used to endorse or promote products
  13  *       derived from this software without specific prior written permission.
  14  *
  15  *
  16  * ALTERNATIVELY, this software may be distributed under the terms of the
  17  * GNU General Public License ("GPL") as published by the Free Software
  18  * Foundation, either version 2 of that License or (at your option) any
  19  * later version.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
  22  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  23  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  24  * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
  25  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  26  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  27  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
  28  * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  29  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  30  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  31  */
  32
  33 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  34
  35 #include "fman.h"
  36 #include "fman_muram.h"
  37
  38 #include <linux/fsl/guts.h>
  39 #include <linux/slab.h>
  40 #include <linux/delay.h>
  41 #include <linux/module.h>
  42 #include <linux/of_platform.h>
  43 #include <linux/clk.h>
  44 #include <linux/of_address.h>
  45 #include <linux/of_irq.h>
  46 #include <linux/interrupt.h>
  47 #include <linux/libfdt_env.h>
  48
  49 /* General defines */
  50 #define FMAN_LIODN_TBL                  64      /* size of LIODN table */
  51 #define MAX_NUM_OF_MACS                 10
  52 #define FM_NUM_OF_FMAN_CTRL_EVENT_REGS  4
  53 #define BASE_RX_PORTID                  0x08
  54 #define BASE_TX_PORTID                  0x28
  55
  56 /* Modules registers offsets */
  57 #define BMI_OFFSET              0x00080000
  58 #define QMI_OFFSET              0x00080400
  59 #define DMA_OFFSET              0x000C2000
  60 #define FPM_OFFSET              0x000C3000
  61 #define IMEM_OFFSET             0x000C4000
  62 #define CGP_OFFSET              0x000DB000
  63
  64 /* Exceptions bit map */
  65 #define EX_DMA_BUS_ERROR                0x80000000
  66 #define EX_DMA_READ_ECC                 0x40000000
  67 #define EX_DMA_SYSTEM_WRITE_ECC 0x20000000
  68 #define EX_DMA_FM_WRITE_ECC             0x10000000
  69 #define EX_FPM_STALL_ON_TASKS           0x08000000
  70 #define EX_FPM_SINGLE_ECC               0x04000000
  71 #define EX_FPM_DOUBLE_ECC               0x02000000
  72 #define EX_QMI_SINGLE_ECC               0x01000000
  73 #define EX_QMI_DEQ_FROM_UNKNOWN_PORTID  0x00800000
  74 #define EX_QMI_DOUBLE_ECC               0x00400000
  75 #define EX_BMI_LIST_RAM_ECC             0x00200000
  76 #define EX_BMI_STORAGE_PROFILE_ECC      0x00100000
  77 #define EX_BMI_STATISTICS_RAM_ECC       0x00080000
  78 #define EX_IRAM_ECC                     0x00040000
  79 #define EX_MURAM_ECC                    0x00020000
  80 #define EX_BMI_DISPATCH_RAM_ECC 0x00010000
  81 #define EX_DMA_SINGLE_PORT_ECC          0x00008000
  82
  83 /* DMA defines */
  84 /* masks */
  85 #define DMA_MODE_BER                    0x00200000
  86 #define DMA_MODE_ECC                    0x00000020
  87 #define DMA_MODE_SECURE_PROT            0x00000800
  88 #define DMA_MODE_AXI_DBG_MASK           0x0F000000
  89
  90 #define DMA_TRANSFER_PORTID_MASK        0xFF000000
  91 #define DMA_TRANSFER_TNUM_MASK          0x00FF0000
  92 #define DMA_TRANSFER_LIODN_MASK 0x00000FFF
  93
  94 #define DMA_STATUS_BUS_ERR              0x08000000
  95 #define DMA_STATUS_READ_ECC             0x04000000
  96 #define DMA_STATUS_SYSTEM_WRITE_ECC     0x02000000
  97 #define DMA_STATUS_FM_WRITE_ECC 0x01000000
  98 #define DMA_STATUS_FM_SPDAT_ECC 0x00080000
  99
 100 #define DMA_MODE_CACHE_OR_SHIFT         30
 101 #define DMA_MODE_AXI_DBG_SHIFT                  24
 102 #define DMA_MODE_CEN_SHIFT                      13
 103 #define DMA_MODE_CEN_MASK                       0x00000007
 104 #define DMA_MODE_DBG_SHIFT                      7
 105 #define DMA_MODE_AID_MODE_SHIFT         4
 106
 107 #define DMA_THRESH_COMMQ_SHIFT                  24
 108 #define DMA_THRESH_READ_INT_BUF_SHIFT           16
 109 #define DMA_THRESH_READ_INT_BUF_MASK            0x0000003f
 110 #define DMA_THRESH_WRITE_INT_BUF_MASK           0x0000003f
 111
 112 #define DMA_TRANSFER_PORTID_SHIFT               24
 113 #define DMA_TRANSFER_TNUM_SHIFT         16
 114
 115 #define DMA_CAM_SIZEOF_ENTRY                    0x40
 116 #define DMA_CAM_UNITS                           8
 117
 118 #define DMA_LIODN_SHIFT         16
 119 #define DMA_LIODN_BASE_MASK     0x00000FFF
 120
 121 /* FPM defines */
 122 #define FPM_EV_MASK_DOUBLE_ECC          0x80000000
 123 #define FPM_EV_MASK_STALL               0x40000000
 124 #define FPM_EV_MASK_SINGLE_ECC          0x20000000
 125 #define FPM_EV_MASK_RELEASE_FM          0x00010000
 126 #define FPM_EV_MASK_DOUBLE_ECC_EN       0x00008000
 127 #define FPM_EV_MASK_STALL_EN            0x00004000
 128 #define FPM_EV_MASK_SINGLE_ECC_EN       0x00002000
 129 #define FPM_EV_MASK_EXTERNAL_HALT       0x00000008
 130 #define FPM_EV_MASK_ECC_ERR_HALT        0x00000004
 131
 132 #define FPM_RAM_MURAM_ECC               0x00008000
 133 #define FPM_RAM_IRAM_ECC                0x00004000
 134 #define FPM_IRAM_ECC_ERR_EX_EN          0x00020000
 135 #define FPM_MURAM_ECC_ERR_EX_EN 0x00040000
 136 #define FPM_RAM_IRAM_ECC_EN             0x40000000
 137 #define FPM_RAM_RAMS_ECC_EN             0x80000000
 138 #define FPM_RAM_RAMS_ECC_EN_SRC_SEL     0x08000000
 139
 140 #define FPM_REV1_MAJOR_MASK             0x0000FF00
 141 #define FPM_REV1_MINOR_MASK             0x000000FF
 142
 143 #define FPM_DISP_LIMIT_SHIFT            24
 144
 145 #define FPM_PRT_FM_CTL1                 0x00000001
 146 #define FPM_PRT_FM_CTL2                 0x00000002
 147 #define FPM_PORT_FM_CTL_PORTID_SHIFT    24
 148 #define FPM_PRC_ORA_FM_CTL_SEL_SHIFT    16
 149
 150 #define FPM_THR1_PRS_SHIFT              24
 151 #define FPM_THR1_KG_SHIFT               16
 152 #define FPM_THR1_PLCR_SHIFT             8
 153 #define FPM_THR1_BMI_SHIFT              0
 154
 155 #define FPM_THR2_QMI_ENQ_SHIFT          24
 156 #define FPM_THR2_QMI_DEQ_SHIFT          0
 157 #define FPM_THR2_FM_CTL1_SHIFT          16
 158 #define FPM_THR2_FM_CTL2_SHIFT          8
 159
 160 #define FPM_EV_MASK_CAT_ERR_SHIFT       1
 161 #define FPM_EV_MASK_DMA_ERR_SHIFT       0
 162
 163 #define FPM_REV1_MAJOR_SHIFT            8
 164
 165 #define FPM_RSTC_FM_RESET               0x80000000
 166 #define FPM_RSTC_MAC0_RESET             0x40000000
 167 #define FPM_RSTC_MAC1_RESET             0x20000000
 168 #define FPM_RSTC_MAC2_RESET             0x10000000
 169 #define FPM_RSTC_MAC3_RESET             0x08000000
 170 #define FPM_RSTC_MAC8_RESET             0x04000000
 171 #define FPM_RSTC_MAC4_RESET             0x02000000
 172 #define FPM_RSTC_MAC5_RESET             0x01000000
 173 #define FPM_RSTC_MAC6_RESET             0x00800000
 174 #define FPM_RSTC_MAC7_RESET             0x00400000
 175 #define FPM_RSTC_MAC9_RESET             0x00200000
 176
 177 #define FPM_TS_INT_SHIFT                16
 178 #define FPM_TS_CTL_EN                   0x80000000
 179
 180 /* BMI defines */
 181 #define BMI_INIT_START                          0x80000000
 182 #define BMI_ERR_INTR_EN_STORAGE_PROFILE_ECC     0x80000000
 183 #define BMI_ERR_INTR_EN_LIST_RAM_ECC            0x40000000
 184 #define BMI_ERR_INTR_EN_STATISTICS_RAM_ECC      0x20000000
 185 #define BMI_ERR_INTR_EN_DISPATCH_RAM_ECC        0x10000000
 186 #define BMI_NUM_OF_TASKS_MASK                   0x3F000000
 187 #define BMI_NUM_OF_EXTRA_TASKS_MASK             0x000F0000
 188 #define BMI_NUM_OF_DMAS_MASK                    0x00000F00
 189 #define BMI_NUM_OF_EXTRA_DMAS_MASK              0x0000000F
 190 #define BMI_FIFO_SIZE_MASK                      0x000003FF
 191 #define BMI_EXTRA_FIFO_SIZE_MASK                0x03FF0000
 192 #define BMI_CFG2_DMAS_MASK                      0x0000003F
 193 #define BMI_CFG2_TASKS_MASK                     0x0000003F
 194
 195 #define BMI_CFG2_TASKS_SHIFT            16
 196 #define BMI_CFG2_DMAS_SHIFT             0
 197 #define BMI_CFG1_FIFO_SIZE_SHIFT        16
 198 #define BMI_NUM_OF_TASKS_SHIFT          24
 199 #define BMI_EXTRA_NUM_OF_TASKS_SHIFT    16
 200 #define BMI_NUM_OF_DMAS_SHIFT           8
 201 #define BMI_EXTRA_NUM_OF_DMAS_SHIFT     0
 202
 203 #define BMI_FIFO_ALIGN                  0x100
 204
 205 #define BMI_EXTRA_FIFO_SIZE_SHIFT       16
 206
 207 /* QMI defines */
 208 #define QMI_CFG_ENQ_EN                  0x80000000
 209 #define QMI_CFG_DEQ_EN                  0x40000000
 210 #define QMI_CFG_EN_COUNTERS             0x10000000
 211 #define QMI_CFG_DEQ_MASK                0x0000003F
 212 #define QMI_CFG_ENQ_MASK                0x00003F00
 213 #define QMI_CFG_ENQ_SHIFT               8
 214
 215 #define QMI_ERR_INTR_EN_DOUBLE_ECC      0x80000000
 216 #define QMI_ERR_INTR_EN_DEQ_FROM_DEF    0x40000000
 217 #define QMI_INTR_EN_SINGLE_ECC          0x80000000
 218
 219 #define QMI_GS_HALT_NOT_BUSY            0x00000002
 220
 221 /* IRAM defines */
 222 #define IRAM_IADD_AIE                   0x80000000
 223 #define IRAM_READY                      0x80000000
 224
 225 /* Default values */
 226 #define DEFAULT_CATASTROPHIC_ERR                0
 227 #define DEFAULT_DMA_ERR                         0
 228 #define DEFAULT_AID_MODE                        FMAN_DMA_AID_OUT_TNUM
 229 #define DEFAULT_DMA_COMM_Q_LOW                  0x2A
 230 #define DEFAULT_DMA_COMM_Q_HIGH         0x3F
 231 #define DEFAULT_CACHE_OVERRIDE                  0
 232 #define DEFAULT_DMA_CAM_NUM_OF_ENTRIES          64
 233 #define DEFAULT_DMA_DBG_CNT_MODE                0
 234 #define DEFAULT_DMA_SOS_EMERGENCY               0
 235 #define DEFAULT_DMA_WATCHDOG                    0
 236 #define DEFAULT_DISP_LIMIT                      0
 237 #define DEFAULT_PRS_DISP_TH                     16
 238 #define DEFAULT_PLCR_DISP_TH                    16
 239 #define DEFAULT_KG_DISP_TH                      16
 240 #define DEFAULT_BMI_DISP_TH                     16
 241 #define DEFAULT_QMI_ENQ_DISP_TH         16
 242 #define DEFAULT_QMI_DEQ_DISP_TH         16
 243 #define DEFAULT_FM_CTL1_DISP_TH         16
 244 #define DEFAULT_FM_CTL2_DISP_TH         16
 245
 246 #define DFLT_AXI_DBG_NUM_OF_BEATS               1
 247
 248 #define DFLT_DMA_READ_INT_BUF_LOW(dma_thresh_max_buf)   \
 249         ((dma_thresh_max_buf + 1) / 2)
 250 #define DFLT_DMA_READ_INT_BUF_HIGH(dma_thresh_max_buf)  \
 251         ((dma_thresh_max_buf + 1) * 3 / 4)
 252 #define DFLT_DMA_WRITE_INT_BUF_LOW(dma_thresh_max_buf)  \
 253         ((dma_thresh_max_buf + 1) / 2)
 254 #define DFLT_DMA_WRITE_INT_BUF_HIGH(dma_thresh_max_buf)\
 255         ((dma_thresh_max_buf + 1) * 3 / 4)
 256
 257 #define DMA_COMM_Q_LOW_FMAN_V3          0x2A
 258 #define DMA_COMM_Q_LOW_FMAN_V2(dma_thresh_max_commq)            \
 259         ((dma_thresh_max_commq + 1) / 2)
 260 #define DFLT_DMA_COMM_Q_LOW(major, dma_thresh_max_commq)        \
 261         ((major == 6) ? DMA_COMM_Q_LOW_FMAN_V3 :                \
 262         DMA_COMM_Q_LOW_FMAN_V2(dma_thresh_max_commq))
 263
 264 #define DMA_COMM_Q_HIGH_FMAN_V3 0x3f
 265 #define DMA_COMM_Q_HIGH_FMAN_V2(dma_thresh_max_commq)           \
 266         ((dma_thresh_max_commq + 1) * 3 / 4)
 267 #define DFLT_DMA_COMM_Q_HIGH(major, dma_thresh_max_commq)       \
 268         ((major == 6) ? DMA_COMM_Q_HIGH_FMAN_V3 :               \
 269         DMA_COMM_Q_HIGH_FMAN_V2(dma_thresh_max_commq))
 270
 271 #define TOTAL_NUM_OF_TASKS_FMAN_V3L     59
 272 #define TOTAL_NUM_OF_TASKS_FMAN_V3H     124
 273 #define DFLT_TOTAL_NUM_OF_TASKS(major, minor, bmi_max_num_of_tasks)     \
 274         ((major == 6) ? ((minor == 1 || minor == 4) ?                   \
 275         TOTAL_NUM_OF_TASKS_FMAN_V3L : TOTAL_NUM_OF_TASKS_FMAN_V3H) :    \
 276         bmi_max_num_of_tasks)
 277
 278 #define DMA_CAM_NUM_OF_ENTRIES_FMAN_V3          64
 279 #define DMA_CAM_NUM_OF_ENTRIES_FMAN_V2          32
 280 #define DFLT_DMA_CAM_NUM_OF_ENTRIES(major)                      \
 281         (major == 6 ? DMA_CAM_NUM_OF_ENTRIES_FMAN_V3 :          \
 282         DMA_CAM_NUM_OF_ENTRIES_FMAN_V2)
 283
 284 #define FM_TIMESTAMP_1_USEC_BIT             8
 285
 286 /* Defines used for enabling/disabling FMan interrupts */
 287 #define ERR_INTR_EN_DMA         0x00010000
 288 #define ERR_INTR_EN_FPM         0x80000000
 289 #define ERR_INTR_EN_BMI         0x00800000
 290 #define ERR_INTR_EN_QMI         0x00400000
 291 #define ERR_INTR_EN_MURAM       0x00040000
 292 #define ERR_INTR_EN_MAC0        0x00004000
 293 #define ERR_INTR_EN_MAC1        0x00002000
 294 #define ERR_INTR_EN_MAC2        0x00001000
 295 #define ERR_INTR_EN_MAC3        0x00000800
 296 #define ERR_INTR_EN_MAC4        0x00000400
 297 #define ERR_INTR_EN_MAC5        0x00000200
 298 #define ERR_INTR_EN_MAC6        0x00000100
 299 #define ERR_INTR_EN_MAC7        0x00000080
 300 #define ERR_INTR_EN_MAC8        0x00008000
 301 #define ERR_INTR_EN_MAC9        0x00000040
 302
 303 #define INTR_EN_QMI             0x40000000
 304 #define INTR_EN_MAC0            0x00080000
 305 #define INTR_EN_MAC1            0x00040000
 306 #define INTR_EN_MAC2            0x00020000
 307 #define INTR_EN_MAC3            0x00010000
 308 #define INTR_EN_MAC4            0x00000040
 309 #define INTR_EN_MAC5            0x00000020
 310 #define INTR_EN_MAC6            0x00000008
 311 #define INTR_EN_MAC7            0x00000002
 312 #define INTR_EN_MAC8            0x00200000
 313 #define INTR_EN_MAC9            0x00100000
 314 #define INTR_EN_REV0            0x00008000
 315 #define INTR_EN_REV1            0x00004000
 316 #define INTR_EN_REV2            0x00002000
 317 #define INTR_EN_REV3            0x00001000
 318 #define INTR_EN_TMR             0x01000000
 319
 320 enum fman_dma_aid_mode {
 321         FMAN_DMA_AID_OUT_PORT_ID = 0,             /* 4 LSB of PORT_ID */
 322         FMAN_DMA_AID_OUT_TNUM                     /* 4 LSB of TNUM */
 323 };
 324
 325 struct fman_iram_regs {
 326         u32 iadd;       /* FM IRAM instruction address register */
 327         u32 idata;      /* FM IRAM instruction data register */
 328         u32 itcfg;      /* FM IRAM timing config register */
 329         u32 iready;     /* FM IRAM ready register */
 330 };
 331
 332 struct fman_fpm_regs {
 333         u32 fmfp_tnc;           /* FPM TNUM Control 0x00 */
 334         u32 fmfp_prc;           /* FPM Port_ID FmCtl Association 0x04 */
 335         u32 fmfp_brkc;          /* FPM Breakpoint Control 0x08 */
 336         u32 fmfp_mxd;           /* FPM Flush Control 0x0c */
 337         u32 fmfp_dist1;         /* FPM Dispatch Thresholds1 0x10 */
 338         u32 fmfp_dist2;         /* FPM Dispatch Thresholds2 0x14 */
 339         u32 fm_epi;             /* FM Error Pending Interrupts 0x18 */
 340         u32 fm_rie;             /* FM Error Interrupt Enable 0x1c */
 341         u32 fmfp_fcev[4];       /* FPM FMan-Controller Event 1-4 0x20-0x2f */
 342         u32 res0030[4];         /* res 0x30 - 0x3f */
 343         u32 fmfp_cee[4];        /* PM FMan-Controller Event 1-4 0x40-0x4f */
 344         u32 res0050[4];         /* res 0x50-0x5f */
 345         u32 fmfp_tsc1;          /* FPM TimeStamp Control1 0x60 */
 346         u32 fmfp_tsc2;          /* FPM TimeStamp Control2 0x64 */
 347         u32 fmfp_tsp;           /* FPM Time Stamp 0x68 */
 348         u32 fmfp_tsf;           /* FPM Time Stamp Fraction 0x6c */
 349         u32 fm_rcr;             /* FM Rams Control 0x70 */
 350         u32 fmfp_extc;          /* FPM External Requests Control 0x74 */
 351         u32 fmfp_ext1;          /* FPM External Requests Config1 0x78 */
 352         u32 fmfp_ext2;          /* FPM External Requests Config2 0x7c */
 353         u32 fmfp_drd[16];       /* FPM Data_Ram Data 0-15 0x80 - 0xbf */
 354         u32 fmfp_dra;           /* FPM Data Ram Access 0xc0 */
 355         u32 fm_ip_rev_1;        /* FM IP Block Revision 1 0xc4 */
 356         u32 fm_ip_rev_2;        /* FM IP Block Revision 2 0xc8 */
 357         u32 fm_rstc;            /* FM Reset Command 0xcc */
 358         u32 fm_cld;             /* FM Classifier Debug 0xd0 */
 359         u32 fm_npi;             /* FM Normal Pending Interrupts 0xd4 */
 360         u32 fmfp_exte;          /* FPM External Requests Enable 0xd8 */
 361         u32 fmfp_ee;            /* FPM Event&Mask 0xdc */
 362         u32 fmfp_cev[4];        /* FPM CPU Event 1-4 0xe0-0xef */
 363         u32 res00f0[4];         /* res 0xf0-0xff */
 364         u32 fmfp_ps[50];        /* FPM Port Status 0x100-0x1c7 */
 365         u32 res01c8[14];        /* res 0x1c8-0x1ff */
 366         u32 fmfp_clfabc;        /* FPM CLFABC 0x200 */
 367         u32 fmfp_clfcc;         /* FPM CLFCC 0x204 */
 368         u32 fmfp_clfaval;       /* FPM CLFAVAL 0x208 */
 369         u32 fmfp_clfbval;       /* FPM CLFBVAL 0x20c */
 370         u32 fmfp_clfcval;       /* FPM CLFCVAL 0x210 */
 371         u32 fmfp_clfamsk;       /* FPM CLFAMSK 0x214 */
 372         u32 fmfp_clfbmsk;       /* FPM CLFBMSK 0x218 */
 373         u32 fmfp_clfcmsk;       /* FPM CLFCMSK 0x21c */
 374         u32 fmfp_clfamc;        /* FPM CLFAMC 0x220 */
 375         u32 fmfp_clfbmc;        /* FPM CLFBMC 0x224 */
 376         u32 fmfp_clfcmc;        /* FPM CLFCMC 0x228 */
 377         u32 fmfp_decceh;        /* FPM DECCEH 0x22c */
 378         u32 res0230[116];       /* res 0x230 - 0x3ff */
 379         u32 fmfp_ts[128];       /* 0x400: FPM Task Status 0x400 - 0x5ff */
 380         u32 res0600[0x400 - 384];
 381 };
 382
 383 struct fman_bmi_regs {
 384         u32 fmbm_init;          /* BMI Initialization 0x00 */
 385         u32 fmbm_cfg1;          /* BMI Configuration 1 0x04 */
 386         u32 fmbm_cfg2;          /* BMI Configuration 2 0x08 */
 387         u32 res000c[5];         /* 0x0c - 0x1f */
 388         u32 fmbm_ievr;          /* Interrupt Event Register 0x20 */
 389         u32 fmbm_ier;           /* Interrupt Enable Register 0x24 */
 390         u32 fmbm_ifr;           /* Interrupt Force Register 0x28 */
 391         u32 res002c[5];         /* 0x2c - 0x3f */
 392         u32 fmbm_arb[8];        /* BMI Arbitration 0x40 - 0x5f */
 393         u32 res0060[12];        /* 0x60 - 0x8f */
 394         u32 fmbm_dtc[3];        /* Debug Trap Counter 0x90 - 0x9b */
 395         u32 res009c;            /* 0x9c */
 396         u32 fmbm_dcv[3][4];     /* Debug Compare val 0xa0-0xcf */
 397         u32 fmbm_dcm[3][4];     /* Debug Compare Mask 0xd0-0xff */
 398         u32 fmbm_gde;           /* BMI Global Debug Enable 0x100 */
 399         u32 fmbm_pp[63];        /* BMI Port Parameters 0x104 - 0x1ff */
 400         u32 res0200;            /* 0x200 */
 401         u32 fmbm_pfs[63];       /* BMI Port FIFO Size 0x204 - 0x2ff */
 402         u32 res0300;            /* 0x300 */
 403         u32 fmbm_spliodn[63];   /* Port Partition ID 0x304 - 0x3ff */
 404 };
 405
 406 struct fman_qmi_regs {
 407         u32 fmqm_gc;            /* General Configuration Register 0x00 */
 408         u32 res0004;            /* 0x04 */
 409         u32 fmqm_eie;           /* Error Interrupt Event Register 0x08 */
 410         u32 fmqm_eien;          /* Error Interrupt Enable Register 0x0c */
 411         u32 fmqm_eif;           /* Error Interrupt Force Register 0x10 */
 412         u32 fmqm_ie;            /* Interrupt Event Register 0x14 */
 413         u32 fmqm_ien;           /* Interrupt Enable Register 0x18 */
 414         u32 fmqm_if;            /* Interrupt Force Register 0x1c */
 415         u32 fmqm_gs;            /* Global Status Register 0x20 */
 416         u32 fmqm_ts;            /* Task Status Register 0x24 */
 417         u32 fmqm_etfc;          /* Enqueue Total Frame Counter 0x28 */
 418         u32 fmqm_dtfc;          /* Dequeue Total Frame Counter 0x2c */
 419         u32 fmqm_dc0;           /* Dequeue Counter 0 0x30 */
 420         u32 fmqm_dc1;           /* Dequeue Counter 1 0x34 */
 421         u32 fmqm_dc2;           /* Dequeue Counter 2 0x38 */
 422         u32 fmqm_dc3;           /* Dequeue Counter 3 0x3c */
 423         u32 fmqm_dfdc;          /* Dequeue FQID from Default Counter 0x40 */
 424         u32 fmqm_dfcc;          /* Dequeue FQID from Context Counter 0x44 */
 425         u32 fmqm_dffc;          /* Dequeue FQID from FD Counter 0x48 */
 426         u32 fmqm_dcc;           /* Dequeue Confirm Counter 0x4c */
 427         u32 res0050[7];         /* 0x50 - 0x6b */
 428         u32 fmqm_tapc;          /* Tnum Aging Period Control 0x6c */
 429         u32 fmqm_dmcvc;         /* Dequeue MAC Command Valid Counter 0x70 */
 430         u32 fmqm_difdcc;        /* Dequeue Invalid FD Command Counter 0x74 */
 431         u32 fmqm_da1v;          /* Dequeue A1 Valid Counter 0x78 */
 432         u32 res007c;            /* 0x7c */
 433         u32 fmqm_dtc;           /* 0x80 Debug Trap Counter 0x80 */
 434         u32 fmqm_efddd;         /* 0x84 Enqueue Frame desc Dynamic dbg 0x84 */
 435         u32 res0088[2];         /* 0x88 - 0x8f */
 436         struct {
 437                 u32 fmqm_dtcfg1;        /* 0x90 dbg trap cfg 1 Register 0x00 */
 438                 u32 fmqm_dtval1;        /* Debug Trap Value 1 Register 0x04 */
 439                 u32 fmqm_dtm1;          /* Debug Trap Mask 1 Register 0x08 */
 440                 u32 fmqm_dtc1;          /* Debug Trap Counter 1 Register 0x0c */
 441                 u32 fmqm_dtcfg2;        /* dbg Trap cfg 2 Register 0x10 */
 442                 u32 fmqm_dtval2;        /* Debug Trap Value 2 Register 0x14 */
 443                 u32 fmqm_dtm2;          /* Debug Trap Mask 2 Register 0x18 */
 444                 u32 res001c;            /* 0x1c */
 445         } dbg_traps[3];                 /* 0x90 - 0xef */
 446         u8 res00f0[0x400 - 0xf0];       /* 0xf0 - 0x3ff */
 447 };
 448
 449 struct fman_dma_regs {
 450         u32 fmdmsr;     /* FM DMA status register 0x00 */
 451         u32 fmdmmr;     /* FM DMA mode register 0x04 */
 452         u32 fmdmtr;     /* FM DMA bus threshold register 0x08 */
 453         u32 fmdmhy;     /* FM DMA bus hysteresis register 0x0c */
 454         u32 fmdmsetr;   /* FM DMA SOS emergency Threshold Register 0x10 */
 455         u32 fmdmtah;    /* FM DMA transfer bus address high reg 0x14 */
 456         u32 fmdmtal;    /* FM DMA transfer bus address low reg 0x18 */
 457         u32 fmdmtcid;   /* FM DMA transfer bus communication ID reg 0x1c */
 458         u32 fmdmra;     /* FM DMA bus internal ram address register 0x20 */
 459         u32 fmdmrd;     /* FM DMA bus internal ram data register 0x24 */
 460         u32 fmdmwcr;    /* FM DMA CAM watchdog counter value 0x28 */
 461         u32 fmdmebcr;   /* FM DMA CAM base in MURAM register 0x2c */
 462         u32 fmdmccqdr;  /* FM DMA CAM and CMD Queue Debug reg 0x30 */
 463         u32 fmdmccqvr1; /* FM DMA CAM and CMD Queue Value reg #1 0x34 */
 464         u32 fmdmccqvr2; /* FM DMA CAM and CMD Queue Value reg #2 0x38 */
 465         u32 fmdmcqvr3;  /* FM DMA CMD Queue Value register #3 0x3c */
 466         u32 fmdmcqvr4;  /* FM DMA CMD Queue Value register #4 0x40 */
 467         u32 fmdmcqvr5;  /* FM DMA CMD Queue Value register #5 0x44 */
 468         u32 fmdmsefrc;  /* FM DMA Semaphore Entry Full Reject Cntr 0x48 */
 469         u32 fmdmsqfrc;  /* FM DMA Semaphore Queue Full Reject Cntr 0x4c */
 470         u32 fmdmssrc;   /* FM DMA Semaphore SYNC Reject Counter 0x50 */
 471         u32 fmdmdcr;    /* FM DMA Debug Counter 0x54 */
 472         u32 fmdmemsr;   /* FM DMA Emergency Smoother Register 0x58 */
 473         u32 res005c;    /* 0x5c */
 474         u32 fmdmplr[FMAN_LIODN_TBL / 2];        /* DMA LIODN regs 0x60-0xdf */
 475         u32 res00e0[0x400 - 56];
 476 };
 477
 478 /* Structure that holds current FMan state.
 479  * Used for saving run time information.
 480  */
 481 struct fman_state_struct {
 482         u8 fm_id;
 483         u16 fm_clk_freq;
 484         struct fman_rev_info rev_info;
 485         bool enabled_time_stamp;
 486         u8 count1_micro_bit;
 487         u8 total_num_of_tasks;
 488         u8 accumulated_num_of_tasks;
 489         u32 accumulated_fifo_size;
 490         u8 accumulated_num_of_open_dmas;
 491         u8 accumulated_num_of_deq_tnums;
 492         u32 exceptions;
 493         u32 extra_fifo_pool_size;
 494         u8 extra_tasks_pool_size;
 495         u8 extra_open_dmas_pool_size;
 496         u16 port_mfl[MAX_NUM_OF_MACS];
 497         u16 mac_mfl[MAX_NUM_OF_MACS];
 498
 499         /* SOC specific */
 500         u32 fm_iram_size;
 501         /* DMA */
 502         u32 dma_thresh_max_commq;
 503         u32 dma_thresh_max_buf;
 504         u32 max_num_of_open_dmas;
 505         /* QMI */
 506         u32 qmi_max_num_of_tnums;
 507         u32 qmi_def_tnums_thresh;
 508         /* BMI */
 509         u32 bmi_max_num_of_tasks;
 510         u32 bmi_max_fifo_size;
 511         /* General */
 512         u32 fm_port_num_of_cg;
 513         u32 num_of_rx_ports;
 514         u32 total_fifo_size;
 515
 516         u32 qman_channel_base;
 517         u32 num_of_qman_channels;
 518
 519         struct resource *res;
 520 };
 521
 522 /* Structure that holds FMan initial configuration */
 523 struct fman_cfg {
 524         u8 disp_limit_tsh;
 525         u8 prs_disp_tsh;
 526         u8 plcr_disp_tsh;
 527         u8 kg_disp_tsh;
 528         u8 bmi_disp_tsh;
 529         u8 qmi_enq_disp_tsh;
 530         u8 qmi_deq_disp_tsh;
 531         u8 fm_ctl1_disp_tsh;
 532         u8 fm_ctl2_disp_tsh;
 533         int dma_cache_override;
 534         enum fman_dma_aid_mode dma_aid_mode;
 535         u32 dma_axi_dbg_num_of_beats;
 536         u32 dma_cam_num_of_entries;
 537         u32 dma_watchdog;
 538         u8 dma_comm_qtsh_asrt_emer;
 539         u32 dma_write_buf_tsh_asrt_emer;
 540         u32 dma_read_buf_tsh_asrt_emer;
 541         u8 dma_comm_qtsh_clr_emer;
 542         u32 dma_write_buf_tsh_clr_emer;
 543         u32 dma_read_buf_tsh_clr_emer;
 544         u32 dma_sos_emergency;
 545         int dma_dbg_cnt_mode;
 546         int catastrophic_err;
 547         int dma_err;
 548         u32 exceptions;
 549         u16 clk_freq;
 550         u32 cam_base_addr;
 551         u32 fifo_base_addr;
 552         u32 total_fifo_size;
 553         u32 total_num_of_tasks;
 554         u32 qmi_def_tnums_thresh;
 555 };
 556
 557 /* Structure that holds information received from device tree */
 558 struct fman_dts_params {
 559         void __iomem *base_addr;                /* FMan virtual address */
 560         struct resource *res;                   /* FMan memory resource */
 561         u8 id;                                  /* FMan ID */
 562
 563         int err_irq;                            /* FMan Error IRQ */
 564
 565         u16 clk_freq;                           /* FMan clock freq (In Mhz) */
 566
 567         u32 qman_channel_base;                  /* QMan channels base */
 568         u32 num_of_qman_channels;               /* Number of QMan channels */
 569
 570         struct resource muram_res;              /* MURAM resource */
 571 };
 572
 573 /** fman_exceptions_cb
 574  * fman         - Pointer to FMan
 575  * exception    - The exception.
 576  *
 577  * Exceptions user callback routine, will be called upon an exception
 578  * passing the exception identification.
 579  *
 580  * Return: irq status
 581  */
 582 typedef irqreturn_t (fman_exceptions_cb)(struct fman *fman,
 583                                          enum fman_exceptions exception);
 584
 585 /** fman_bus_error_cb
 586  * fman         - Pointer to FMan
 587  * port_id      - Port id
 588  * addr         - Address that caused the error
 589  * tnum         - Owner of error
 590  * liodn        - Logical IO device number
 591  *
 592  * Bus error user callback routine, will be called upon bus error,
 593  * passing parameters describing the errors and the owner.
 594  *
 595  * Return: IRQ status
 596  */
 597 typedef irqreturn_t (fman_bus_error_cb)(struct fman *fman, u8 port_id,
 598                                         u64 addr, u8 tnum, u16 liodn);
 599
 600 struct fman {
 601         struct device *dev;
 602         void __iomem *base_addr;
 603         struct fman_intr_src intr_mng[FMAN_EV_CNT];
 604
 605         struct fman_fpm_regs __iomem *fpm_regs;
 606         struct fman_bmi_regs __iomem *bmi_regs;
 607         struct fman_qmi_regs __iomem *qmi_regs;
 608         struct fman_dma_regs __iomem *dma_regs;
 609         fman_exceptions_cb *exception_cb;
 610         fman_bus_error_cb *bus_error_cb;
 611         /* Spinlock for FMan use */
 612         spinlock_t spinlock;
 613         struct fman_state_struct *state;
 614
 615         struct fman_cfg *cfg;
 616         struct muram_info *muram;
 617         /* cam section in muram */
 618         unsigned long cam_offset;
 619         size_t cam_size;
 620         /* Fifo in MURAM */
 621         int fifo_offset;
 622         size_t fifo_size;
 623
 624         u32 liodn_base[64];
 625         u32 liodn_offset[64];
 626
 627         struct fman_dts_params dts_params;
 628 };
 629
 630 static irqreturn_t fman_exceptions(struct fman *fman,
 631                                    enum fman_exceptions exception)
 632 {
 633         dev_dbg(fman->dev, "%s: FMan[%d] exception %d\n",
 634                 __func__, fman->state->fm_id, exception);
 635
 636         return IRQ_HANDLED;
 637 }
 638
 639 static irqreturn_t fman_bus_error(struct fman *fman, u8 __maybe_unused port_id,
 640                                   u64 __maybe_unused addr,
 641                                   u8 __maybe_unused tnum,
 642                                   u16 __maybe_unused liodn)
 643 {
 644         dev_dbg(fman->dev, "%s: FMan[%d] bus error: port_id[%d]\n",
 645                 __func__, fman->state->fm_id, port_id);
 646
 647         return IRQ_HANDLED;
 648 }
 649
 650 static inline irqreturn_t call_mac_isr(struct fman *fman, u8 id)
 651 {
 652         if (fman->intr_mng[id].isr_cb) {
 653                 fman->intr_mng[id].isr_cb(fman->intr_mng[id].src_handle);
 654
 655                 return IRQ_HANDLED;
 656         }
 657
 658         return IRQ_NONE;
 659 }
 660
 661 static inline u8 hw_port_id_to_sw_port_id(u8 major, u8 hw_port_id)
 662 {
 663         u8 sw_port_id = 0;
 664
 665         if (hw_port_id >= BASE_TX_PORTID)
 666                 sw_port_id = hw_port_id - BASE_TX_PORTID;
 667         else if (hw_port_id >= BASE_RX_PORTID)
 668                 sw_port_id = hw_port_id - BASE_RX_PORTID;
 669         else
 670                 sw_port_id = 0;
 671
 672         return sw_port_id;
 673 }
 674
 675 static void set_port_order_restoration(struct fman_fpm_regs __iomem *fpm_rg,
 676                                        u8 port_id)
 677 {
 678         u32 tmp = 0;
 679
 680         tmp = port_id << FPM_PORT_FM_CTL_PORTID_SHIFT;
 681
 682         tmp |= FPM_PRT_FM_CTL2 | FPM_PRT_FM_CTL1;
 683
 684         /* order restoration */
 685         if (port_id % 2)
 686                 tmp |= FPM_PRT_FM_CTL1 << FPM_PRC_ORA_FM_CTL_SEL_SHIFT;
 687         else
 688                 tmp |= FPM_PRT_FM_CTL2 << FPM_PRC_ORA_FM_CTL_SEL_SHIFT;
 689
 690         iowrite32be(tmp, &fpm_rg->fmfp_prc);
 691 }
 692
 693 static void set_port_liodn(struct fman *fman, u8 port_id,
 694                            u32 liodn_base, u32 liodn_ofst)
 695 {
 696         u32 tmp;
 697
 698         /* set LIODN base for this port */
 699         tmp = ioread32be(&fman->dma_regs->fmdmplr[port_id / 2]);
 700         if (port_id % 2) {
 701                 tmp &= ~DMA_LIODN_BASE_MASK;
 702                 tmp |= liodn_base;
 703         } else {
 704                 tmp &= ~(DMA_LIODN_BASE_MASK << DMA_LIODN_SHIFT);
 705                 tmp |= liodn_base << DMA_LIODN_SHIFT;
 706         }
 707         iowrite32be(tmp, &fman->dma_regs->fmdmplr[port_id / 2]);
 708         iowrite32be(liodn_ofst, &fman->bmi_regs->fmbm_spliodn[port_id - 1]);
 709 }
 710
 711 static void enable_rams_ecc(struct fman_fpm_regs __iomem *fpm_rg)
 712 {
 713         u32 tmp;
 714
 715         tmp = ioread32be(&fpm_rg->fm_rcr);
 716         if (tmp & FPM_RAM_RAMS_ECC_EN_SRC_SEL)
 717                 iowrite32be(tmp | FPM_RAM_IRAM_ECC_EN, &fpm_rg->fm_rcr);
 718         else
 719                 iowrite32be(tmp | FPM_RAM_RAMS_ECC_EN |
 720                             FPM_RAM_IRAM_ECC_EN, &fpm_rg->fm_rcr);
 721 }
 722
 723 static void disable_rams_ecc(struct fman_fpm_regs __iomem *fpm_rg)
 724 {
 725         u32 tmp;
 726
 727         tmp = ioread32be(&fpm_rg->fm_rcr);
 728         if (tmp & FPM_RAM_RAMS_ECC_EN_SRC_SEL)
 729                 iowrite32be(tmp & ~FPM_RAM_IRAM_ECC_EN, &fpm_rg->fm_rcr);
 730         else
 731                 iowrite32be(tmp & ~(FPM_RAM_RAMS_ECC_EN | FPM_RAM_IRAM_ECC_EN),
 732                             &fpm_rg->fm_rcr);
 733 }
 734
 735 static void fman_defconfig(struct fman_cfg *cfg)
 736 {
 737         memset(cfg, 0, sizeof(struct fman_cfg));
 738
 739         cfg->catastrophic_err = DEFAULT_CATASTROPHIC_ERR;
 740         cfg->dma_err = DEFAULT_DMA_ERR;
 741         cfg->dma_aid_mode = DEFAULT_AID_MODE;
 742         cfg->dma_comm_qtsh_clr_emer = DEFAULT_DMA_COMM_Q_LOW;
 743         cfg->dma_comm_qtsh_asrt_emer = DEFAULT_DMA_COMM_Q_HIGH;
 744         cfg->dma_cache_override = DEFAULT_CACHE_OVERRIDE;
 745         cfg->dma_cam_num_of_entries = DEFAULT_DMA_CAM_NUM_OF_ENTRIES;
 746         cfg->dma_dbg_cnt_mode = DEFAULT_DMA_DBG_CNT_MODE;
 747         cfg->dma_sos_emergency = DEFAULT_DMA_SOS_EMERGENCY;
 748         cfg->dma_watchdog = DEFAULT_DMA_WATCHDOG;
 749         cfg->disp_limit_tsh = DEFAULT_DISP_LIMIT;
 750         cfg->prs_disp_tsh = DEFAULT_PRS_DISP_TH;
 751         cfg->plcr_disp_tsh = DEFAULT_PLCR_DISP_TH;
 752         cfg->kg_disp_tsh = DEFAULT_KG_DISP_TH;
 753         cfg->bmi_disp_tsh = DEFAULT_BMI_DISP_TH;
 754         cfg->qmi_enq_disp_tsh = DEFAULT_QMI_ENQ_DISP_TH;
 755         cfg->qmi_deq_disp_tsh = DEFAULT_QMI_DEQ_DISP_TH;
 756         cfg->fm_ctl1_disp_tsh = DEFAULT_FM_CTL1_DISP_TH;
 757         cfg->fm_ctl2_disp_tsh = DEFAULT_FM_CTL2_DISP_TH;
 758 }
 759
 760 static int dma_init(struct fman *fman)
 761 {
 762         struct fman_dma_regs __iomem *dma_rg = fman->dma_regs;
 763         struct fman_cfg *cfg = fman->cfg;
 764         u32 tmp_reg;
 765
 766         /* Init DMA Registers */
 767
 768         /* clear status reg events */
 769         tmp_reg = (DMA_STATUS_BUS_ERR | DMA_STATUS_READ_ECC |
 770                    DMA_STATUS_SYSTEM_WRITE_ECC | DMA_STATUS_FM_WRITE_ECC);
 771         iowrite32be(ioread32be(&dma_rg->fmdmsr) | tmp_reg, &dma_rg->fmdmsr);
 772
 773         /* configure mode register */
 774         tmp_reg = 0;
 775         tmp_reg |= cfg->dma_cache_override << DMA_MODE_CACHE_OR_SHIFT;
 776         if (cfg->exceptions & EX_DMA_BUS_ERROR)
 777                 tmp_reg |= DMA_MODE_BER;
 778         if ((cfg->exceptions & EX_DMA_SYSTEM_WRITE_ECC) |
 779             (cfg->exceptions & EX_DMA_READ_ECC) |
 780             (cfg->exceptions & EX_DMA_FM_WRITE_ECC))
 781                 tmp_reg |= DMA_MODE_ECC;
 782         if (cfg->dma_axi_dbg_num_of_beats)
 783                 tmp_reg |= (DMA_MODE_AXI_DBG_MASK &
 784                         ((cfg->dma_axi_dbg_num_of_beats - 1)
 785                         << DMA_MODE_AXI_DBG_SHIFT));
 786
 787         tmp_reg |= (((cfg->dma_cam_num_of_entries / DMA_CAM_UNITS) - 1) &
 788                 DMA_MODE_CEN_MASK) << DMA_MODE_CEN_SHIFT;
 789         tmp_reg |= DMA_MODE_SECURE_PROT;
 790         tmp_reg |= cfg->dma_dbg_cnt_mode << DMA_MODE_DBG_SHIFT;
 791         tmp_reg |= cfg->dma_aid_mode << DMA_MODE_AID_MODE_SHIFT;
 792
 793         iowrite32be(tmp_reg, &dma_rg->fmdmmr);
 794
 795         /* configure thresholds register */
 796         tmp_reg = ((u32)cfg->dma_comm_qtsh_asrt_emer <<
 797                 DMA_THRESH_COMMQ_SHIFT);
 798         tmp_reg |= (cfg->dma_read_buf_tsh_asrt_emer &
 799                 DMA_THRESH_READ_INT_BUF_MASK) << DMA_THRESH_READ_INT_BUF_SHIFT;
 800         tmp_reg |= cfg->dma_write_buf_tsh_asrt_emer &
 801                 DMA_THRESH_WRITE_INT_BUF_MASK;
 802
 803         iowrite32be(tmp_reg, &dma_rg->fmdmtr);
 804
 805         /* configure hysteresis register */
 806         tmp_reg = ((u32)cfg->dma_comm_qtsh_clr_emer <<
 807                 DMA_THRESH_COMMQ_SHIFT);
 808         tmp_reg |= (cfg->dma_read_buf_tsh_clr_emer &
 809                 DMA_THRESH_READ_INT_BUF_MASK) << DMA_THRESH_READ_INT_BUF_SHIFT;
 810         tmp_reg |= cfg->dma_write_buf_tsh_clr_emer &
 811                 DMA_THRESH_WRITE_INT_BUF_MASK;
 812
 813         iowrite32be(tmp_reg, &dma_rg->fmdmhy);
 814
 815         /* configure emergency threshold */
 816         iowrite32be(cfg->dma_sos_emergency, &dma_rg->fmdmsetr);
 817
 818         /* configure Watchdog */
 819         iowrite32be((cfg->dma_watchdog * cfg->clk_freq), &dma_rg->fmdmwcr);
 820
 821         iowrite32be(cfg->cam_base_addr, &dma_rg->fmdmebcr);
 822
 823         /* Allocate MURAM for CAM */
 824         fman->cam_size =
 825                 (u32)(fman->cfg->dma_cam_num_of_entries * DMA_CAM_SIZEOF_ENTRY);
 826         fman->cam_offset = fman_muram_alloc(fman->muram, fman->cam_size);
 827         if (IS_ERR_VALUE(fman->cam_offset)) {
 828                 dev_err(fman->dev, "%s: MURAM alloc for DMA CAM failed\n",
 829                         __func__);
 830                 return -ENOMEM;
 831         }
 832
 833         if (fman->state->rev_info.major == 2) {
 834                 u32 __iomem *cam_base_addr;
 835
 836                 fman_muram_free_mem(fman->muram, fman->cam_offset,
 837                                     fman->cam_size);
 838
 839                 fman->cam_size = fman->cfg->dma_cam_num_of_entries * 72 + 128;
 840                 fman->cam_offset = fman_muram_alloc(fman->muram,
 841                                                     fman->cam_size);
 842                 if (IS_ERR_VALUE(fman->cam_offset)) {
 843                         dev_err(fman->dev, "%s: MURAM alloc for DMA CAM failed\n",
 844                                 __func__);
 845                         return -ENOMEM;
 846                 }
 847
 848                 if (fman->cfg->dma_cam_num_of_entries % 8 ||
 849                     fman->cfg->dma_cam_num_of_entries > 32) {
 850                         dev_err(fman->dev, "%s: wrong dma_cam_num_of_entries\n",
 851                                 __func__);
 852                         return -EINVAL;
 853                 }
 854
 855                 cam_base_addr = (u32 __iomem *)
 856                         fman_muram_offset_to_vbase(fman->muram,
 857                                                    fman->cam_offset);
 858                 iowrite32be(~((1 <<
 859                             (32 - fman->cfg->dma_cam_num_of_entries)) - 1),
 860                             cam_base_addr);
 861         }
 862
 863         fman->cfg->cam_base_addr = fman->cam_offset;
 864
 865         return 0;
 866 }
 867
 868 static void fpm_init(struct fman_fpm_regs __iomem *fpm_rg, struct fman_cfg *cfg)
 869 {
 870         u32 tmp_reg;
 871         int i;
 872
 873         /* Init FPM Registers */
 874
 875         tmp_reg = (u32)(cfg->disp_limit_tsh << FPM_DISP_LIMIT_SHIFT);
 876         iowrite32be(tmp_reg, &fpm_rg->fmfp_mxd);
 877
 878         tmp_reg = (((u32)cfg->prs_disp_tsh << FPM_THR1_PRS_SHIFT) |
 879                    ((u32)cfg->kg_disp_tsh << FPM_THR1_KG_SHIFT) |
 880                    ((u32)cfg->plcr_disp_tsh << FPM_THR1_PLCR_SHIFT) |
 881                    ((u32)cfg->bmi_disp_tsh << FPM_THR1_BMI_SHIFT));
 882         iowrite32be(tmp_reg, &fpm_rg->fmfp_dist1);
 883
 884         tmp_reg =
 885                 (((u32)cfg->qmi_enq_disp_tsh << FPM_THR2_QMI_ENQ_SHIFT) |
 886                  ((u32)cfg->qmi_deq_disp_tsh << FPM_THR2_QMI_DEQ_SHIFT) |
 887                  ((u32)cfg->fm_ctl1_disp_tsh << FPM_THR2_FM_CTL1_SHIFT) |
 888                  ((u32)cfg->fm_ctl2_disp_tsh << FPM_THR2_FM_CTL2_SHIFT));
 889         iowrite32be(tmp_reg, &fpm_rg->fmfp_dist2);
 890
 891         /* define exceptions and error behavior */
 892         tmp_reg = 0;
 893         /* Clear events */
 894         tmp_reg |= (FPM_EV_MASK_STALL | FPM_EV_MASK_DOUBLE_ECC |
 895                     FPM_EV_MASK_SINGLE_ECC);
 896         /* enable interrupts */
 897         if (cfg->exceptions & EX_FPM_STALL_ON_TASKS)
 898                 tmp_reg |= FPM_EV_MASK_STALL_EN;
 899         if (cfg->exceptions & EX_FPM_SINGLE_ECC)
 900                 tmp_reg |= FPM_EV_MASK_SINGLE_ECC_EN;
 901         if (cfg->exceptions & EX_FPM_DOUBLE_ECC)
 902                 tmp_reg |= FPM_EV_MASK_DOUBLE_ECC_EN;
 903         tmp_reg |= (cfg->catastrophic_err << FPM_EV_MASK_CAT_ERR_SHIFT);
 904         tmp_reg |= (cfg->dma_err << FPM_EV_MASK_DMA_ERR_SHIFT);
 905         /* FMan is not halted upon external halt activation */
 906         tmp_reg |= FPM_EV_MASK_EXTERNAL_HALT;
 907         /* Man is not halted upon  Unrecoverable ECC error behavior */
 908         tmp_reg |= FPM_EV_MASK_ECC_ERR_HALT;
 909         iowrite32be(tmp_reg, &fpm_rg->fmfp_ee);
 910
 911         /* clear all fmCtls event registers */
 912         for (i = 0; i < FM_NUM_OF_FMAN_CTRL_EVENT_REGS; i++)
 913                 iowrite32be(0xFFFFFFFF, &fpm_rg->fmfp_cev[i]);
 914
 915         /* RAM ECC -  enable and clear events */
 916         /* first we need to clear all parser memory,
 917          * as it is uninitialized and may cause ECC errors
 918          */
 919         /* event bits */
 920         tmp_reg = (FPM_RAM_MURAM_ECC | FPM_RAM_IRAM_ECC);
 921
 922         iowrite32be(tmp_reg, &fpm_rg->fm_rcr);
 923
 924         tmp_reg = 0;
 925         if (cfg->exceptions & EX_IRAM_ECC) {
 926                 tmp_reg |= FPM_IRAM_ECC_ERR_EX_EN;
 927                 enable_rams_ecc(fpm_rg);
 928         }
 929         if (cfg->exceptions & EX_MURAM_ECC) {
 930                 tmp_reg |= FPM_MURAM_ECC_ERR_EX_EN;
 931                 enable_rams_ecc(fpm_rg);
 932         }
 933         iowrite32be(tmp_reg, &fpm_rg->fm_rie);
 934 }
 935
 936 static void bmi_init(struct fman_bmi_regs __iomem *bmi_rg,
 937                      struct fman_cfg *cfg)
 938 {
 939         u32 tmp_reg;
 940
 941         /* Init BMI Registers */
 942
 943         /* define common resources */
 944         tmp_reg = cfg->fifo_base_addr;
 945         tmp_reg = tmp_reg / BMI_FIFO_ALIGN;
 946
 947         tmp_reg |= ((cfg->total_fifo_size / FMAN_BMI_FIFO_UNITS - 1) <<
 948                     BMI_CFG1_FIFO_SIZE_SHIFT);
 949         iowrite32be(tmp_reg, &bmi_rg->fmbm_cfg1);
 950
 951         tmp_reg = ((cfg->total_num_of_tasks - 1) & BMI_CFG2_TASKS_MASK) <<
 952                    BMI_CFG2_TASKS_SHIFT;
 953         /* num of DMA's will be dynamically updated when each port is set */
 954         iowrite32be(tmp_reg, &bmi_rg->fmbm_cfg2);
 955
 956         /* define unmaskable exceptions, enable and clear events */
 957         tmp_reg = 0;
 958         iowrite32be(BMI_ERR_INTR_EN_LIST_RAM_ECC |
 959                     BMI_ERR_INTR_EN_STORAGE_PROFILE_ECC |
 960                     BMI_ERR_INTR_EN_STATISTICS_RAM_ECC |
 961                     BMI_ERR_INTR_EN_DISPATCH_RAM_ECC, &bmi_rg->fmbm_ievr);
 962
 963         if (cfg->exceptions & EX_BMI_LIST_RAM_ECC)
 964                 tmp_reg |= BMI_ERR_INTR_EN_LIST_RAM_ECC;
 965         if (cfg->exceptions & EX_BMI_STORAGE_PROFILE_ECC)
 966                 tmp_reg |= BMI_ERR_INTR_EN_STORAGE_PROFILE_ECC;
 967         if (cfg->exceptions & EX_BMI_STATISTICS_RAM_ECC)
 968                 tmp_reg |= BMI_ERR_INTR_EN_STATISTICS_RAM_ECC;
 969         if (cfg->exceptions & EX_BMI_DISPATCH_RAM_ECC)
 970                 tmp_reg |= BMI_ERR_INTR_EN_DISPATCH_RAM_ECC;
 971         iowrite32be(tmp_reg, &bmi_rg->fmbm_ier);
 972 }
 973
 974 static void qmi_init(struct fman_qmi_regs __iomem *qmi_rg,
 975                      struct fman_cfg *cfg)
 976 {
 977         u32 tmp_reg;
 978
 979         /* Init QMI Registers */
 980
 981         /* Clear error interrupt events */
 982
 983         iowrite32be(QMI_ERR_INTR_EN_DOUBLE_ECC | QMI_ERR_INTR_EN_DEQ_FROM_DEF,
 984                     &qmi_rg->fmqm_eie);
 985         tmp_reg = 0;
 986         if (cfg->exceptions & EX_QMI_DEQ_FROM_UNKNOWN_PORTID)
 987                 tmp_reg |= QMI_ERR_INTR_EN_DEQ_FROM_DEF;
 988         if (cfg->exceptions & EX_QMI_DOUBLE_ECC)
 989                 tmp_reg |= QMI_ERR_INTR_EN_DOUBLE_ECC;
 990         /* enable events */
 991         iowrite32be(tmp_reg, &qmi_rg->fmqm_eien);
 992
 993         tmp_reg = 0;
 994         /* Clear interrupt events */
 995         iowrite32be(QMI_INTR_EN_SINGLE_ECC, &qmi_rg->fmqm_ie);
 996         if (cfg->exceptions & EX_QMI_SINGLE_ECC)
 997                 tmp_reg |= QMI_INTR_EN_SINGLE_ECC;
 998         /* enable events */
 999         iowrite32be(tmp_reg, &qmi_rg->fmqm_ien);
1000 }
1001
1002 static int enable(struct fman *fman, struct fman_cfg *cfg)
1003 {
1004         u32 cfg_reg = 0;
1005
1006         /* Enable all modules */
1007
1008         /* clear&enable global counters - calculate reg and save for later,
1009          * because it's the same reg for QMI enable
1010          */
1011         cfg_reg = QMI_CFG_EN_COUNTERS;
1012
1013         /* Set enqueue and dequeue thresholds */
1014         cfg_reg |= (cfg->qmi_def_tnums_thresh << 8) | cfg->qmi_def_tnums_thresh;
1015
1016         iowrite32be(BMI_INIT_START, &fman->bmi_regs->fmbm_init);
1017         iowrite32be(cfg_reg | QMI_CFG_ENQ_EN | QMI_CFG_DEQ_EN,
1018                     &fman->qmi_regs->fmqm_gc);
1019
1020         return 0;
1021 }
1022
1023 static int set_exception(struct fman *fman,
1024                          enum fman_exceptions exception, bool enable)
1025 {
1026         u32 tmp;
1027
1028         switch (exception) {
1029         case FMAN_EX_DMA_BUS_ERROR:
1030                 tmp = ioread32be(&fman->dma_regs->fmdmmr);
1031                 if (enable)
1032                         tmp |= DMA_MODE_BER;
1033                 else
1034                         tmp &= ~DMA_MODE_BER;
1035                 /* disable bus error */
1036                 iowrite32be(tmp, &fman->dma_regs->fmdmmr);
1037                 break;
1038         case FMAN_EX_DMA_READ_ECC:
1039         case FMAN_EX_DMA_SYSTEM_WRITE_ECC:
1040         case FMAN_EX_DMA_FM_WRITE_ECC:
1041                 tmp = ioread32be(&fman->dma_regs->fmdmmr);
1042                 if (enable)
1043                         tmp |= DMA_MODE_ECC;
1044                 else
1045                         tmp &= ~DMA_MODE_ECC;
1046                 iowrite32be(tmp, &fman->dma_regs->fmdmmr);
1047                 break;
1048         case FMAN_EX_FPM_STALL_ON_TASKS:
1049                 tmp = ioread32be(&fman->fpm_regs->fmfp_ee);
1050                 if (enable)
1051                         tmp |= FPM_EV_MASK_STALL_EN;
1052                 else
1053                         tmp &= ~FPM_EV_MASK_STALL_EN;
1054                 iowrite32be(tmp, &fman->fpm_regs->fmfp_ee);
1055                 break;
1056         case FMAN_EX_FPM_SINGLE_ECC:
1057                 tmp = ioread32be(&fman->fpm_regs->fmfp_ee);
1058                 if (enable)
1059                         tmp |= FPM_EV_MASK_SINGLE_ECC_EN;
1060                 else
1061                         tmp &= ~FPM_EV_MASK_SINGLE_ECC_EN;
1062                 iowrite32be(tmp, &fman->fpm_regs->fmfp_ee);
1063                 break;
1064         case FMAN_EX_FPM_DOUBLE_ECC:
1065                 tmp = ioread32be(&fman->fpm_regs->fmfp_ee);
1066                 if (enable)
1067                         tmp |= FPM_EV_MASK_DOUBLE_ECC_EN;
1068                 else
1069                         tmp &= ~FPM_EV_MASK_DOUBLE_ECC_EN;
1070                 iowrite32be(tmp, &fman->fpm_regs->fmfp_ee);
1071                 break;
1072         case FMAN_EX_QMI_SINGLE_ECC:
1073                 tmp = ioread32be(&fman->qmi_regs->fmqm_ien);
1074                 if (enable)
1075                         tmp |= QMI_INTR_EN_SINGLE_ECC;
1076                 else
1077                         tmp &= ~QMI_INTR_EN_SINGLE_ECC;
1078                 iowrite32be(tmp, &fman->qmi_regs->fmqm_ien);
1079                 break;
1080         case FMAN_EX_QMI_DOUBLE_ECC:
1081                 tmp = ioread32be(&fman->qmi_regs->fmqm_eien);
1082                 if (enable)
1083                         tmp |= QMI_ERR_INTR_EN_DOUBLE_ECC;
1084                 else
1085                         tmp &= ~QMI_ERR_INTR_EN_DOUBLE_ECC;
1086                 iowrite32be(tmp, &fman->qmi_regs->fmqm_eien);
1087                 break;
1088         case FMAN_EX_QMI_DEQ_FROM_UNKNOWN_PORTID:
1089                 tmp = ioread32be(&fman->qmi_regs->fmqm_eien);
1090                 if (enable)
1091                         tmp |= QMI_ERR_INTR_EN_DEQ_FROM_DEF;
1092                 else
1093                         tmp &= ~QMI_ERR_INTR_EN_DEQ_FROM_DEF;
1094                 iowrite32be(tmp, &fman->qmi_regs->fmqm_eien);
1095                 break;
1096         case FMAN_EX_BMI_LIST_RAM_ECC:
1097                 tmp = ioread32be(&fman->bmi_regs->fmbm_ier);
1098                 if (enable)
1099                         tmp |= BMI_ERR_INTR_EN_LIST_RAM_ECC;
1100                 else
1101                         tmp &= ~BMI_ERR_INTR_EN_LIST_RAM_ECC;
1102                 iowrite32be(tmp, &fman->bmi_regs->fmbm_ier);
1103                 break;
1104         case FMAN_EX_BMI_STORAGE_PROFILE_ECC:
1105                 tmp = ioread32be(&fman->bmi_regs->fmbm_ier);
1106                 if (enable)
1107                         tmp |= BMI_ERR_INTR_EN_STORAGE_PROFILE_ECC;
1108                 else
1109                         tmp &= ~BMI_ERR_INTR_EN_STORAGE_PROFILE_ECC;
1110                 iowrite32be(tmp, &fman->bmi_regs->fmbm_ier);
1111                 break;
1112         case FMAN_EX_BMI_STATISTICS_RAM_ECC:
1113                 tmp = ioread32be(&fman->bmi_regs->fmbm_ier);
1114                 if (enable)
1115                         tmp |= BMI_ERR_INTR_EN_STATISTICS_RAM_ECC;
1116                 else
1117                         tmp &= ~BMI_ERR_INTR_EN_STATISTICS_RAM_ECC;
1118                 iowrite32be(tmp, &fman->bmi_regs->fmbm_ier);
1119                 break;
1120         case FMAN_EX_BMI_DISPATCH_RAM_ECC:
1121                 tmp = ioread32be(&fman->bmi_regs->fmbm_ier);
1122                 if (enable)
1123                         tmp |= BMI_ERR_INTR_EN_DISPATCH_RAM_ECC;
1124                 else
1125                         tmp &= ~BMI_ERR_INTR_EN_DISPATCH_RAM_ECC;
1126                 iowrite32be(tmp, &fman->bmi_regs->fmbm_ier);
1127                 break;
1128         case FMAN_EX_IRAM_ECC:
1129                 tmp = ioread32be(&fman->fpm_regs->fm_rie);
1130                 if (enable) {
1131                         /* enable ECC if not enabled */
1132                         enable_rams_ecc(fman->fpm_regs);
1133                         /* enable ECC interrupts */
1134                         tmp |= FPM_IRAM_ECC_ERR_EX_EN;
1135                 } else {
1136                         /* ECC mechanism may be disabled,
1137                          * depending on driver status
1138                          */
1139                         disable_rams_ecc(fman->fpm_regs);
1140                         tmp &= ~FPM_IRAM_ECC_ERR_EX_EN;
1141                 }
1142                 iowrite32be(tmp, &fman->fpm_regs->fm_rie);
1143                 break;
1144         case FMAN_EX_MURAM_ECC:
1145                 tmp = ioread32be(&fman->fpm_regs->fm_rie);
1146                 if (enable) {
1147                         /* enable ECC if not enabled */
1148                         enable_rams_ecc(fman->fpm_regs);
1149                         /* enable ECC interrupts */
1150                         tmp |= FPM_MURAM_ECC_ERR_EX_EN;
1151                 } else {
1152                         /* ECC mechanism may be disabled,
1153                          * depending on driver status
1154                          */
1155                         disable_rams_ecc(fman->fpm_regs);
1156                         tmp &= ~FPM_MURAM_ECC_ERR_EX_EN;
1157                 }
1158                 iowrite32be(tmp, &fman->fpm_regs->fm_rie);
1159                 break;
1160         default:
1161                 return -EINVAL;
1162         }
1163         return 0;
1164 }
1165
1166 static void resume(struct fman_fpm_regs __iomem *fpm_rg)
1167 {
1168         u32 tmp;
1169
1170         tmp = ioread32be(&fpm_rg->fmfp_ee);
1171         /* clear tmp_reg event bits in order not to clear standing events */
1172         tmp &= ~(FPM_EV_MASK_DOUBLE_ECC |
1173                  FPM_EV_MASK_STALL | FPM_EV_MASK_SINGLE_ECC);
1174         tmp |= FPM_EV_MASK_RELEASE_FM;
1175
1176         iowrite32be(tmp, &fpm_rg->fmfp_ee);
1177 }
1178
1179 static int fill_soc_specific_params(struct fman_state_struct *state)
1180 {
1181         u8 minor = state->rev_info.minor;
1182         /* P4080 - Major 2
1183          * P2041/P3041/P5020/P5040 - Major 3
1184          * Tx/Bx - Major 6
1185          */
1186         switch (state->rev_info.major) {
1187         case 3:
1188                 state->bmi_max_fifo_size        = 160 * 1024;
1189                 state->fm_iram_size             = 64 * 1024;
1190                 state->dma_thresh_max_commq     = 31;
1191                 state->dma_thresh_max_buf       = 127;
1192                 state->qmi_max_num_of_tnums     = 64;
1193                 state->qmi_def_tnums_thresh     = 48;
1194                 state->bmi_max_num_of_tasks     = 128;
1195                 state->max_num_of_open_dmas     = 32;
1196                 state->fm_port_num_of_cg        = 256;
1197                 state->num_of_rx_ports  = 6;
1198                 state->total_fifo_size  = 122 * 1024;
1199                 break;
1200
1201         case 2:
1202                 state->bmi_max_fifo_size        = 160 * 1024;
1203                 state->fm_iram_size             = 64 * 1024;
1204                 state->dma_thresh_max_commq     = 31;
1205                 state->dma_thresh_max_buf       = 127;
1206                 state->qmi_max_num_of_tnums     = 64;
1207                 state->qmi_def_tnums_thresh     = 48;
1208                 state->bmi_max_num_of_tasks     = 128;
1209                 state->max_num_of_open_dmas     = 32;
1210                 state->fm_port_num_of_cg        = 256;
1211                 state->num_of_rx_ports  = 5;
1212                 state->total_fifo_size  = 100 * 1024;
1213                 break;
1214
1215         case 6:
1216                 state->dma_thresh_max_commq     = 83;
1217                 state->dma_thresh_max_buf       = 127;
1218                 state->qmi_max_num_of_tnums     = 64;
1219                 state->qmi_def_tnums_thresh     = 32;
1220                 state->fm_port_num_of_cg        = 256;
1221
1222                 /* FManV3L */
1223                 if (minor == 1 || minor == 4) {
1224                         state->bmi_max_fifo_size        = 192 * 1024;
1225                         state->bmi_max_num_of_tasks     = 64;
1226                         state->max_num_of_open_dmas     = 32;
1227                         state->num_of_rx_ports          = 5;
1228                         if (minor == 1)
1229                                 state->fm_iram_size     = 32 * 1024;
1230                         else
1231                                 state->fm_iram_size     = 64 * 1024;
1232                         state->total_fifo_size          = 156 * 1024;
1233                 }
1234                 /* FManV3H */
1235                 else if (minor == 0 || minor == 2 || minor == 3) {
1236                         state->bmi_max_fifo_size        = 384 * 1024;
1237                         state->fm_iram_size             = 64 * 1024;
1238                         state->bmi_max_num_of_tasks     = 128;
1239                         state->max_num_of_open_dmas     = 84;
1240                         state->num_of_rx_ports          = 8;
1241                         state->total_fifo_size          = 295 * 1024;
1242                 } else {
1243                         pr_err("Unsupported FManv3 version\n");
1244                         return -EINVAL;
1245                 }
1246
1247                 break;
1248         default:
1249                 pr_err("Unsupported FMan version\n");
1250                 return -EINVAL;
1251         }
1252
1253         return 0;
1254 }
1255
1256 static bool is_init_done(struct fman_cfg *cfg)
1257 {
1258         /* Checks if FMan driver parameters were initialized */
1259         if (!cfg)
1260                 return true;
1261
1262         return false;
1263 }
1264
1265 static void free_init_resources(struct fman *fman)
1266 {
1267         if (fman->cam_offset)
1268                 fman_muram_free_mem(fman->muram, fman->cam_offset,
1269                                     fman->cam_size);
1270         if (fman->fifo_offset)
1271                 fman_muram_free_mem(fman->muram, fman->fifo_offset,
1272                                     fman->fifo_size);
1273 }
1274
1275 static irqreturn_t bmi_err_event(struct fman *fman)
1276 {
1277         u32 event, mask, force;
1278         struct fman_bmi_regs __iomem *bmi_rg = fman->bmi_regs;
1279         irqreturn_t ret = IRQ_NONE;
1280
1281         event = ioread32be(&bmi_rg->fmbm_ievr);
1282         mask = ioread32be(&bmi_rg->fmbm_ier);
1283         event &= mask;
1284         /* clear the forced events */
1285         force = ioread32be(&bmi_rg->fmbm_ifr);
1286         if (force & event)
1287                 iowrite32be(force & ~event, &bmi_rg->fmbm_ifr);
1288         /* clear the acknowledged events */
1289         iowrite32be(event, &bmi_rg->fmbm_ievr);
1290
1291         if (event & BMI_ERR_INTR_EN_STORAGE_PROFILE_ECC)
1292                 ret = fman->exception_cb(fman, FMAN_EX_BMI_STORAGE_PROFILE_ECC);
1293         if (event & BMI_ERR_INTR_EN_LIST_RAM_ECC)
1294                 ret = fman->exception_cb(fman, FMAN_EX_BMI_LIST_RAM_ECC);
1295         if (event & BMI_ERR_INTR_EN_STATISTICS_RAM_ECC)
1296                 ret = fman->exception_cb(fman, FMAN_EX_BMI_STATISTICS_RAM_ECC);
1297         if (event & BMI_ERR_INTR_EN_DISPATCH_RAM_ECC)
1298                 ret = fman->exception_cb(fman, FMAN_EX_BMI_DISPATCH_RAM_ECC);
1299
1300         return ret;
1301 }
1302
1303 static irqreturn_t qmi_err_event(struct fman *fman)
1304 {
1305         u32 event, mask, force;
1306         struct fman_qmi_regs __iomem *qmi_rg = fman->qmi_regs;
1307         irqreturn_t ret = IRQ_NONE;
1308
1309         event = ioread32be(&qmi_rg->fmqm_eie);
1310         mask = ioread32be(&qmi_rg->fmqm_eien);
1311         event &= mask;
1312
1313         /* clear the forced events */
1314         force = ioread32be(&qmi_rg->fmqm_eif);
1315         if (force & event)
1316                 iowrite32be(force & ~event, &qmi_rg->fmqm_eif);
1317         /* clear the acknowledged events */
1318         iowrite32be(event, &qmi_rg->fmqm_eie);
1319
1320         if (event & QMI_ERR_INTR_EN_DOUBLE_ECC)
1321                 ret = fman->exception_cb(fman, FMAN_EX_QMI_DOUBLE_ECC);
1322         if (event & QMI_ERR_INTR_EN_DEQ_FROM_DEF)
1323                 ret = fman->exception_cb(fman,
1324                                          FMAN_EX_QMI_DEQ_FROM_UNKNOWN_PORTID);
1325
1326         return ret;
1327 }
1328
1329 static irqreturn_t dma_err_event(struct fman *fman)
1330 {
1331         u32 status, mask, com_id;
1332         u8 tnum, port_id, relative_port_id;
1333         u16 liodn;
1334         struct fman_dma_regs __iomem *dma_rg = fman->dma_regs;
1335         irqreturn_t ret = IRQ_NONE;
1336
1337         status = ioread32be(&dma_rg->fmdmsr);
1338         mask = ioread32be(&dma_rg->fmdmmr);
1339
1340         /* clear DMA_STATUS_BUS_ERR if mask has no DMA_MODE_BER */
1341         if ((mask & DMA_MODE_BER) != DMA_MODE_BER)
1342                 status &= ~DMA_STATUS_BUS_ERR;
1343
1344         /* clear relevant bits if mask has no DMA_MODE_ECC */
1345         if ((mask & DMA_MODE_ECC) != DMA_MODE_ECC)
1346                 status &= ~(DMA_STATUS_FM_SPDAT_ECC |
1347                             DMA_STATUS_READ_ECC |
1348                             DMA_STATUS_SYSTEM_WRITE_ECC |
1349                             DMA_STATUS_FM_WRITE_ECC);
1350
1351         /* clear set events */
1352         iowrite32be(status, &dma_rg->fmdmsr);
1353
1354         if (status & DMA_STATUS_BUS_ERR) {
1355                 u64 addr;
1356
1357                 addr = (u64)ioread32be(&dma_rg->fmdmtal);
1358                 addr |= ((u64)(ioread32be(&dma_rg->fmdmtah)) << 32);
1359
1360                 com_id = ioread32be(&dma_rg->fmdmtcid);
1361                 port_id = (u8)(((com_id & DMA_TRANSFER_PORTID_MASK) >>
1362                                DMA_TRANSFER_PORTID_SHIFT));
1363                 relative_port_id =
1364                 hw_port_id_to_sw_port_id(fman->state->rev_info.major, port_id);
1365                 tnum = (u8)((com_id & DMA_TRANSFER_TNUM_MASK) >>
1366                             DMA_TRANSFER_TNUM_SHIFT);
1367                 liodn = (u16)(com_id & DMA_TRANSFER_LIODN_MASK);
1368                 ret = fman->bus_error_cb(fman, relative_port_id, addr, tnum,
1369                                          liodn);
1370         }
1371         if (status & DMA_STATUS_FM_SPDAT_ECC)
1372                 ret = fman->exception_cb(fman, FMAN_EX_DMA_SINGLE_PORT_ECC);
1373         if (status & DMA_STATUS_READ_ECC)
1374                 ret = fman->exception_cb(fman, FMAN_EX_DMA_READ_ECC);
1375         if (status & DMA_STATUS_SYSTEM_WRITE_ECC)
1376                 ret = fman->exception_cb(fman, FMAN_EX_DMA_SYSTEM_WRITE_ECC);
1377         if (status & DMA_STATUS_FM_WRITE_ECC)
1378                 ret = fman->exception_cb(fman, FMAN_EX_DMA_FM_WRITE_ECC);
1379
1380         return ret;
1381 }
1382
1383 static irqreturn_t fpm_err_event(struct fman *fman)
1384 {
1385         u32 event;
1386         struct fman_fpm_regs __iomem *fpm_rg = fman->fpm_regs;
1387         irqreturn_t ret = IRQ_NONE;
1388
1389         event = ioread32be(&fpm_rg->fmfp_ee);
1390         /* clear the all occurred events */
1391         iowrite32be(event, &fpm_rg->fmfp_ee);
1392
1393         if ((event & FPM_EV_MASK_DOUBLE_ECC) &&
1394             (event & FPM_EV_MASK_DOUBLE_ECC_EN))
1395                 ret = fman->exception_cb(fman, FMAN_EX_FPM_DOUBLE_ECC);
1396         if ((event & FPM_EV_MASK_STALL) && (event & FPM_EV_MASK_STALL_EN))
1397                 ret = fman->exception_cb(fman, FMAN_EX_FPM_STALL_ON_TASKS);
1398         if ((event & FPM_EV_MASK_SINGLE_ECC) &&
1399             (event & FPM_EV_MASK_SINGLE_ECC_EN))
1400                 ret = fman->exception_cb(fman, FMAN_EX_FPM_SINGLE_ECC);
1401
1402         return ret;
1403 }
1404
1405 static irqreturn_t muram_err_intr(struct fman *fman)
1406 {
1407         u32 event, mask;
1408         struct fman_fpm_regs __iomem *fpm_rg = fman->fpm_regs;
1409         irqreturn_t ret = IRQ_NONE;
1410
1411         event = ioread32be(&fpm_rg->fm_rcr);
1412         mask = ioread32be(&fpm_rg->fm_rie);
1413
1414         /* clear MURAM event bit (do not clear IRAM event) */
1415         iowrite32be(event & ~FPM_RAM_IRAM_ECC, &fpm_rg->fm_rcr);
1416
1417         if ((mask & FPM_MURAM_ECC_ERR_EX_EN) && (event & FPM_RAM_MURAM_ECC))
1418                 ret = fman->exception_cb(fman, FMAN_EX_MURAM_ECC);
1419
1420         return ret;
1421 }
1422
1423 static irqreturn_t qmi_event(struct fman *fman)
1424 {
1425         u32 event, mask, force;
1426         struct fman_qmi_regs __iomem *qmi_rg = fman->qmi_regs;
1427         irqreturn_t ret = IRQ_NONE;
1428
1429         event = ioread32be(&qmi_rg->fmqm_ie);
1430         mask = ioread32be(&qmi_rg->fmqm_ien);
1431         event &= mask;
1432         /* clear the forced events */
1433         force = ioread32be(&qmi_rg->fmqm_if);
1434         if (force & event)
1435                 iowrite32be(force & ~event, &qmi_rg->fmqm_if);
1436         /* clear the acknowledged events */
1437         iowrite32be(event, &qmi_rg->fmqm_ie);
1438
1439         if (event & QMI_INTR_EN_SINGLE_ECC)
1440                 ret = fman->exception_cb(fman, FMAN_EX_QMI_SINGLE_ECC);
1441
1442         return ret;
1443 }
1444
1445 static void enable_time_stamp(struct fman *fman)
1446 {
1447         struct fman_fpm_regs __iomem *fpm_rg = fman->fpm_regs;
1448         u16 fm_clk_freq = fman->state->fm_clk_freq;
1449         u32 tmp, intgr, ts_freq;
1450         u64 frac;
1451
1452         ts_freq = (u32)(1 << fman->state->count1_micro_bit);
1453         /* configure timestamp so that bit 8 will count 1 microsecond
1454          * Find effective count rate at TIMESTAMP least significant bits:
1455          * Effective_Count_Rate = 1MHz x 2^8 = 256MHz
1456          * Find frequency ratio between effective count rate and the clock:
1457          * Effective_Count_Rate / CLK e.g. for 600 MHz clock:
1458          * 256/600 = 0.4266666...
1459          */
1460
1461         intgr = ts_freq / fm_clk_freq;
1462         /* we multiply by 2^16 to keep the fraction of the division
1463          * we do not div back, since we write this value as a fraction
1464          * see spec
1465          */
1466
1467         frac = ((ts_freq << 16) - (intgr << 16) * fm_clk_freq) / fm_clk_freq;
1468         /* we check remainder of the division in order to round up if not int */
1469         if (((ts_freq << 16) - (intgr << 16) * fm_clk_freq) % fm_clk_freq)
1470                 frac++;
1471
1472         tmp = (intgr << FPM_TS_INT_SHIFT) | (u16)frac;
1473         iowrite32be(tmp, &fpm_rg->fmfp_tsc2);
1474
1475         /* enable timestamp with original clock */
1476         iowrite32be(FPM_TS_CTL_EN, &fpm_rg->fmfp_tsc1);
1477         fman->state->enabled_time_stamp = true;
1478 }
1479
1480 static int clear_iram(struct fman *fman)
1481 {
1482         struct fman_iram_regs __iomem *iram;
1483         int i, count;
1484
1485         iram = fman->base_addr + IMEM_OFFSET;
1486
1487         /* Enable the auto-increment */
1488         iowrite32be(IRAM_IADD_AIE, &iram->iadd);
1489         count = 100;
1490         do {
1491                 udelay(1);
1492         } while ((ioread32be(&iram->iadd) != IRAM_IADD_AIE) && --count);
1493         if (count == 0)
1494                 return -EBUSY;
1495
1496         for (i = 0; i < (fman->state->fm_iram_size / 4); i++)
1497                 iowrite32be(0xffffffff, &iram->idata);
1498
1499         iowrite32be(fman->state->fm_iram_size - 4, &iram->iadd);
1500         count = 100;
1501         do {
1502                 udelay(1);
1503         } while ((ioread32be(&iram->idata) != 0xffffffff) && --count);
1504         if (count == 0)
1505                 return -EBUSY;
1506
1507         return 0;
1508 }
1509
1510 static u32 get_exception_flag(enum fman_exceptions exception)
1511 {
1512         u32 bit_mask;
1513
1514         switch (exception) {
1515         case FMAN_EX_DMA_BUS_ERROR:
1516                 bit_mask = EX_DMA_BUS_ERROR;
1517                 break;
1518         case FMAN_EX_DMA_SINGLE_PORT_ECC:
1519                 bit_mask = EX_DMA_SINGLE_PORT_ECC;
1520                 break;
1521         case FMAN_EX_DMA_READ_ECC:
1522                 bit_mask = EX_DMA_READ_ECC;
1523                 break;
1524         case FMAN_EX_DMA_SYSTEM_WRITE_ECC:
1525                 bit_mask = EX_DMA_SYSTEM_WRITE_ECC;
1526                 break;
1527         case FMAN_EX_DMA_FM_WRITE_ECC:
1528                 bit_mask = EX_DMA_FM_WRITE_ECC;
1529                 break;
1530         case FMAN_EX_FPM_STALL_ON_TASKS:
1531                 bit_mask = EX_FPM_STALL_ON_TASKS;
1532                 break;
1533         case FMAN_EX_FPM_SINGLE_ECC:
1534                 bit_mask = EX_FPM_SINGLE_ECC;
1535                 break;
1536         case FMAN_EX_FPM_DOUBLE_ECC:
1537                 bit_mask = EX_FPM_DOUBLE_ECC;
1538                 break;
1539         case FMAN_EX_QMI_SINGLE_ECC:
1540                 bit_mask = EX_QMI_SINGLE_ECC;
1541                 break;
1542         case FMAN_EX_QMI_DOUBLE_ECC:
1543                 bit_mask = EX_QMI_DOUBLE_ECC;
1544                 break;
1545         case FMAN_EX_QMI_DEQ_FROM_UNKNOWN_PORTID:
1546                 bit_mask = EX_QMI_DEQ_FROM_UNKNOWN_PORTID;
1547                 break;
1548         case FMAN_EX_BMI_LIST_RAM_ECC:
1549                 bit_mask = EX_BMI_LIST_RAM_ECC;
1550                 break;
1551         case FMAN_EX_BMI_STORAGE_PROFILE_ECC:
1552                 bit_mask = EX_BMI_STORAGE_PROFILE_ECC;
1553                 break;
1554         case FMAN_EX_BMI_STATISTICS_RAM_ECC:
1555                 bit_mask = EX_BMI_STATISTICS_RAM_ECC;
1556                 break;
1557         case FMAN_EX_BMI_DISPATCH_RAM_ECC:
1558                 bit_mask = EX_BMI_DISPATCH_RAM_ECC;
1559                 break;
1560         case FMAN_EX_MURAM_ECC:
1561                 bit_mask = EX_MURAM_ECC;
1562                 break;
1563         default:
1564                 bit_mask = 0;
1565                 break;
1566         }
1567
1568         return bit_mask;
1569 }
1570
1571 static int get_module_event(enum fman_event_modules module, u8 mod_id,
1572                             enum fman_intr_type intr_type)
1573 {
1574         int event;
1575
1576         switch (module) {
1577         case FMAN_MOD_MAC:
1578                 if (intr_type == FMAN_INTR_TYPE_ERR)
1579                         event = FMAN_EV_ERR_MAC0 + mod_id;
1580                 else
1581                         event = FMAN_EV_MAC0 + mod_id;
1582                 break;
1583         case FMAN_MOD_FMAN_CTRL:
1584                 if (intr_type == FMAN_INTR_TYPE_ERR)
1585                         event = FMAN_EV_CNT;
1586                 else
1587                         event = (FMAN_EV_FMAN_CTRL_0 + mod_id);
1588                 break;
1589         case FMAN_MOD_DUMMY_LAST:
1590                 event = FMAN_EV_CNT;
1591                 break;
1592         default:
1593                 event = FMAN_EV_CNT;
1594                 break;
1595         }
1596
1597         return event;
1598 }
1599
1600 static int set_size_of_fifo(struct fman *fman, u8 port_id, u32 *size_of_fifo,
1601                             u32 *extra_size_of_fifo)
1602 {
1603         struct fman_bmi_regs __iomem *bmi_rg = fman->bmi_regs;
1604         u32 fifo = *size_of_fifo;
1605         u32 extra_fifo = *extra_size_of_fifo;
1606         u32 tmp;
1607
1608         /* if this is the first time a port requires extra_fifo_pool_size,
1609          * the total extra_fifo_pool_size must be initialized to 1 buffer per
1610          * port
1611          */
1612         if (extra_fifo && !fman->state->extra_fifo_pool_size)
1613                 fman->state->extra_fifo_pool_size =
1614                         fman->state->num_of_rx_ports * FMAN_BMI_FIFO_UNITS;
1615
1616         fman->state->extra_fifo_pool_size =
1617                 max(fman->state->extra_fifo_pool_size, extra_fifo);
1618
1619         /* check that there are enough uncommitted fifo size */
1620         if ((fman->state->accumulated_fifo_size + fifo) >
1621             (fman->state->total_fifo_size -
1622             fman->state->extra_fifo_pool_size)) {
1623                 dev_err(fman->dev, "%s: Requested fifo size and extra size exceed total FIFO size.\n",
1624                         __func__);
1625                 return -EAGAIN;
1626         }
1627
1628         /* Read, modify and write to HW */
1629         tmp = (fifo / FMAN_BMI_FIFO_UNITS - 1) |
1630                ((extra_fifo / FMAN_BMI_FIFO_UNITS) <<
1631                BMI_EXTRA_FIFO_SIZE_SHIFT);
1632         iowrite32be(tmp, &bmi_rg->fmbm_pfs[port_id - 1]);
1633
1634         /* update accumulated */
1635         fman->state->accumulated_fifo_size += fifo;
1636
1637         return 0;
1638 }
1639
1640 static int set_num_of_tasks(struct fman *fman, u8 port_id, u8 *num_of_tasks,
1641                             u8 *num_of_extra_tasks)
1642 {
1643         struct fman_bmi_regs __iomem *bmi_rg = fman->bmi_regs;
1644         u8 tasks = *num_of_tasks;
1645         u8 extra_tasks = *num_of_extra_tasks;
1646         u32 tmp;
1647
1648         if (extra_tasks)
1649                 fman->state->extra_tasks_pool_size =
1650                 max(fman->state->extra_tasks_pool_size, extra_tasks);
1651
1652         /* check that there are enough uncommitted tasks */
1653         if ((fman->state->accumulated_num_of_tasks + tasks) >
1654             (fman->state->total_num_of_tasks -
1655              fman->state->extra_tasks_pool_size)) {
1656                 dev_err(fman->dev, "%s: Requested num_of_tasks and extra tasks pool for fm%d exceed total num_of_tasks.\n",
1657                         __func__, fman->state->fm_id);
1658                 return -EAGAIN;
1659         }
1660         /* update accumulated */
1661         fman->state->accumulated_num_of_tasks += tasks;
1662
1663         /* Write to HW */
1664         tmp = ioread32be(&bmi_rg->fmbm_pp[port_id - 1]) &
1665             ~(BMI_NUM_OF_TASKS_MASK | BMI_NUM_OF_EXTRA_TASKS_MASK);
1666         tmp |= ((u32)((tasks - 1) << BMI_NUM_OF_TASKS_SHIFT) |
1667                 (u32)(extra_tasks << BMI_EXTRA_NUM_OF_TASKS_SHIFT));
1668         iowrite32be(tmp, &bmi_rg->fmbm_pp[port_id - 1]);
1669
1670         return 0;
1671 }
1672
1673 static int set_num_of_open_dmas(struct fman *fman, u8 port_id,
1674                                 u8 *num_of_open_dmas,
1675                                 u8 *num_of_extra_open_dmas)
1676 {
1677         struct fman_bmi_regs __iomem *bmi_rg = fman->bmi_regs;
1678         u8 open_dmas = *num_of_open_dmas;
1679         u8 extra_open_dmas = *num_of_extra_open_dmas;
1680         u8 total_num_dmas = 0, current_val = 0, current_extra_val = 0;
1681         u32 tmp;
1682
1683         if (!open_dmas) {
1684                 /* Configuration according to values in the HW.
1685                  * read the current number of open Dma's
1686                  */
1687                 tmp = ioread32be(&bmi_rg->fmbm_pp[port_id - 1]);
1688                 current_extra_val = (u8)((tmp & BMI_NUM_OF_EXTRA_DMAS_MASK) >>
1689                                          BMI_EXTRA_NUM_OF_DMAS_SHIFT);
1690
1691                 tmp = ioread32be(&bmi_rg->fmbm_pp[port_id - 1]);
1692                 current_val = (u8)(((tmp & BMI_NUM_OF_DMAS_MASK) >>
1693                                    BMI_NUM_OF_DMAS_SHIFT) + 1);
1694
1695                 /* This is the first configuration and user did not
1696                  * specify value (!open_dmas), reset values will be used
1697                  * and we just save these values for resource management
1698                  */
1699                 fman->state->extra_open_dmas_pool_size =
1700                         (u8)max(fman->state->extra_open_dmas_pool_size,
1701                                 current_extra_val);
1702                 fman->state->accumulated_num_of_open_dmas += current_val;
1703                 *num_of_open_dmas = current_val;
1704                 *num_of_extra_open_dmas = current_extra_val;
1705                 return 0;
1706         }
1707
1708         if (extra_open_dmas > current_extra_val)
1709                 fman->state->extra_open_dmas_pool_size =
1710                     (u8)max(fman->state->extra_open_dmas_pool_size,
1711                             extra_open_dmas);
1712
1713         if ((fman->state->rev_info.major < 6) &&
1714             (fman->state->accumulated_num_of_open_dmas - current_val +
1715              open_dmas > fman->state->max_num_of_open_dmas)) {
1716                 dev_err(fman->dev, "%s: Requested num_of_open_dmas for fm%d exceeds total num_of_open_dmas.\n",
1717                         __func__, fman->state->fm_id);
1718                 return -EAGAIN;
1719         } else if ((fman->state->rev_info.major >= 6) &&
1720                    !((fman->state->rev_info.major == 6) &&
1721                    (fman->state->rev_info.minor == 0)) &&
1722                    (fman->state->accumulated_num_of_open_dmas -
1723                    current_val + open_dmas >
1724                    fman->state->dma_thresh_max_commq + 1)) {
1725                 dev_err(fman->dev, "%s: Requested num_of_open_dmas for fm%d exceeds DMA Command queue (%d)\n",
1726                         __func__, fman->state->fm_id,
1727                        fman->state->dma_thresh_max_commq + 1);
1728                 return -EAGAIN;
1729         }
1730
1731         WARN_ON(fman->state->accumulated_num_of_open_dmas < current_val);
1732         /* update acummulated */
1733         fman->state->accumulated_num_of_open_dmas -= current_val;
1734         fman->state->accumulated_num_of_open_dmas += open_dmas;
1735
1736         if (fman->state->rev_info.major < 6)
1737                 total_num_dmas =
1738                     (u8)(fman->state->accumulated_num_of_open_dmas +
1739                     fman->state->extra_open_dmas_pool_size);
1740
1741         /* calculate reg */
1742         tmp = ioread32be(&bmi_rg->fmbm_pp[port_id - 1]) &
1743             ~(BMI_NUM_OF_DMAS_MASK | BMI_NUM_OF_EXTRA_DMAS_MASK);
1744         tmp |= (u32)(((open_dmas - 1) << BMI_NUM_OF_DMAS_SHIFT) |
1745                            (extra_open_dmas << BMI_EXTRA_NUM_OF_DMAS_SHIFT));
1746         iowrite32be(tmp, &bmi_rg->fmbm_pp[port_id - 1]);
1747
1748         /* update total num of DMA's with committed number of open DMAS,
1749          * and max uncommitted pool.
1750          */
1751         if (total_num_dmas) {
1752                 tmp = ioread32be(&bmi_rg->fmbm_cfg2) & ~BMI_CFG2_DMAS_MASK;
1753                 tmp |= (u32)(total_num_dmas - 1) << BMI_CFG2_DMAS_SHIFT;
1754                 iowrite32be(tmp, &bmi_rg->fmbm_cfg2);
1755         }
1756
1757         return 0;
1758 }
1759
1760 static int fman_config(struct fman *fman)
1761 {
1762         void __iomem *base_addr;
1763         int err;
1764
1765         base_addr = fman->dts_params.base_addr;
1766
1767         fman->state = kzalloc(sizeof(*fman->state), GFP_KERNEL);
1768         if (!fman->state)
1769                 goto err_fm_state;
1770
1771         /* Allocate the FM driver's parameters structure */
1772         fman->cfg = kzalloc(sizeof(*fman->cfg), GFP_KERNEL);
1773         if (!fman->cfg)
1774                 goto err_fm_drv;
1775
1776         /* Initialize MURAM block */
1777         fman->muram =
1778                 fman_muram_init(fman->dts_params.muram_res.start,
1779                                 resource_size(&fman->dts_params.muram_res));
1780         if (!fman->muram)
1781                 goto err_fm_soc_specific;
1782
1783         /* Initialize FM parameters which will be kept by the driver */
1784         fman->state->fm_id = fman->dts_params.id;
1785         fman->state->fm_clk_freq = fman->dts_params.clk_freq;
1786         fman->state->qman_channel_base = fman->dts_params.qman_channel_base;
1787         fman->state->num_of_qman_channels =
1788                 fman->dts_params.num_of_qman_channels;
1789         fman->state->res = fman->dts_params.res;
1790         fman->exception_cb = fman_exceptions;
1791         fman->bus_error_cb = fman_bus_error;
1792         fman->fpm_regs = base_addr + FPM_OFFSET;
1793         fman->bmi_regs = base_addr + BMI_OFFSET;
1794         fman->qmi_regs = base_addr + QMI_OFFSET;
1795         fman->dma_regs = base_addr + DMA_OFFSET;
1796         fman->base_addr = base_addr;
1797
1798         spin_lock_init(&fman->spinlock);
1799         fman_defconfig(fman->cfg);
1800
1801         fman->state->extra_fifo_pool_size = 0;
1802         fman->state->exceptions = (EX_DMA_BUS_ERROR                 |
1803                                         EX_DMA_READ_ECC              |
1804                                         EX_DMA_SYSTEM_WRITE_ECC      |
1805                                         EX_DMA_FM_WRITE_ECC          |
1806                                         EX_FPM_STALL_ON_TASKS        |
1807                                         EX_FPM_SINGLE_ECC            |
1808                                         EX_FPM_DOUBLE_ECC            |
1809                                         EX_QMI_DEQ_FROM_UNKNOWN_PORTID |
1810                                         EX_BMI_LIST_RAM_ECC          |
1811                                         EX_BMI_STORAGE_PROFILE_ECC   |
1812                                         EX_BMI_STATISTICS_RAM_ECC    |
1813                                         EX_MURAM_ECC                 |
1814                                         EX_BMI_DISPATCH_RAM_ECC      |
1815                                         EX_QMI_DOUBLE_ECC            |
1816                                         EX_QMI_SINGLE_ECC);
1817
1818         /* Read FMan revision for future use*/
1819         fman_get_revision(fman, &fman->state->rev_info);
1820
1821         err = fill_soc_specific_params(fman->state);
1822         if (err)
1823                 goto err_fm_soc_specific;
1824
1825         /* FM_AID_MODE_NO_TNUM_SW005 Errata workaround */
1826         if (fman->state->rev_info.major >= 6)
1827                 fman->cfg->dma_aid_mode = FMAN_DMA_AID_OUT_PORT_ID;
1828
1829         fman->cfg->qmi_def_tnums_thresh = fman->state->qmi_def_tnums_thresh;
1830
1831         fman->state->total_num_of_tasks =
1832         (u8)DFLT_TOTAL_NUM_OF_TASKS(fman->state->rev_info.major,
1833                                     fman->state->rev_info.minor,
1834                                     fman->state->bmi_max_num_of_tasks);
1835
1836         if (fman->state->rev_info.major < 6) {
1837                 fman->cfg->dma_comm_qtsh_clr_emer =
1838                 (u8)DFLT_DMA_COMM_Q_LOW(fman->state->rev_info.major,
1839                                         fman->state->dma_thresh_max_commq);
1840
1841                 fman->cfg->dma_comm_qtsh_asrt_emer =
1842                 (u8)DFLT_DMA_COMM_Q_HIGH(fman->state->rev_info.major,
1843                                          fman->state->dma_thresh_max_commq);
1844
1845                 fman->cfg->dma_cam_num_of_entries =
1846                 DFLT_DMA_CAM_NUM_OF_ENTRIES(fman->state->rev_info.major);
1847
1848                 fman->cfg->dma_read_buf_tsh_clr_emer =
1849                 DFLT_DMA_READ_INT_BUF_LOW(fman->state->dma_thresh_max_buf);
1850
1851                 fman->cfg->dma_read_buf_tsh_asrt_emer =
1852                 DFLT_DMA_READ_INT_BUF_HIGH(fman->state->dma_thresh_max_buf);
1853
1854                 fman->cfg->dma_write_buf_tsh_clr_emer =
1855                 DFLT_DMA_WRITE_INT_BUF_LOW(fman->state->dma_thresh_max_buf);
1856
1857                 fman->cfg->dma_write_buf_tsh_asrt_emer =
1858                 DFLT_DMA_WRITE_INT_BUF_HIGH(fman->state->dma_thresh_max_buf);
1859
1860                 fman->cfg->dma_axi_dbg_num_of_beats =
1861                 DFLT_AXI_DBG_NUM_OF_BEATS;
1862         }
1863
1864         return 0;
1865
1866 err_fm_soc_specific:
1867         kfree(fman->cfg);
1868 err_fm_drv:
1869         kfree(fman->state);
1870 err_fm_state:
1871         kfree(fman);
1872         return -EINVAL;
1873 }
1874
1875 static int fman_reset(struct fman *fman)
1876 {
1877         u32 count;
1878         int err = 0;
1879
1880         if (fman->state->rev_info.major < 6) {
1881                 iowrite32be(FPM_RSTC_FM_RESET, &fman->fpm_regs->fm_rstc);
1882                 /* Wait for reset completion */
1883                 count = 100;
1884                 do {
1885                         udelay(1);
1886                 } while (((ioread32be(&fman->fpm_regs->fm_rstc)) &
1887                          FPM_RSTC_FM_RESET) && --count);
1888                 if (count == 0)
1889                         err = -EBUSY;
1890
1891                 goto _return;
1892         } else {
1893                 struct device_node *guts_node;
1894                 struct ccsr_guts __iomem *guts_regs;
1895                 u32 devdisr2, reg;
1896
1897                 /* Errata A007273 */
1898                 guts_node =
1899                         of_find_compatible_node(NULL, NULL,
1900                                                 "fsl,qoriq-device-config-2.0");
1901                 if (!guts_node) {
1902                         dev_err(fman->dev, "%s: Couldn't find guts node\n",
1903                                 __func__);
1904                         goto guts_node;
1905                 }
1906
1907                 guts_regs = of_iomap(guts_node, 0);
1908                 if (!guts_regs) {
1909                         dev_err(fman->dev, "%s: Couldn't map %s regs\n",
1910                                 __func__, guts_node->full_name);
1911                         goto guts_regs;
1912                 }
1913 #define FMAN1_ALL_MACS_MASK     0xFCC00000
1914 #define FMAN2_ALL_MACS_MASK     0x000FCC00
1915                 /* Read current state */
1916                 devdisr2 = ioread32be(&guts_regs->devdisr2);
1917                 if (fman->dts_params.id == 0)
1918                         reg = devdisr2 & ~FMAN1_ALL_MACS_MASK;
1919                 else
1920                         reg = devdisr2 & ~FMAN2_ALL_MACS_MASK;
1921
1922                 /* Enable all MACs */
1923                 iowrite32be(reg, &guts_regs->devdisr2);
1924
1925                 /* Perform FMan reset */
1926                 iowrite32be(FPM_RSTC_FM_RESET, &fman->fpm_regs->fm_rstc);
1927
1928                 /* Wait for reset completion */
1929                 count = 100;
1930                 do {
1931                         udelay(1);
1932                 } while (((ioread32be(&fman->fpm_regs->fm_rstc)) &
1933                          FPM_RSTC_FM_RESET) && --count);
1934                 if (count == 0) {
1935                         iounmap(guts_regs);
1936                         of_node_put(guts_node);
1937                         err = -EBUSY;
1938                         goto _return;
1939                 }
1940
1941                 /* Restore devdisr2 value */
1942                 iowrite32be(devdisr2, &guts_regs->devdisr2);
1943
1944                 iounmap(guts_regs);
1945                 of_node_put(guts_node);
1946
1947                 goto _return;
1948
1949 guts_regs:
1950                 of_node_put(guts_node);
1951 guts_node:
1952                 dev_dbg(fman->dev, "%s: Didn't perform FManV3 reset due to Errata A007273!\n",
1953                         __func__);
1954         }
1955 _return:
1956         return err;
1957 }
1958
1959 static int fman_init(struct fman *fman)
1960 {
1961         struct fman_cfg *cfg = NULL;
1962         int err = 0, i, count;
1963
1964         if (is_init_done(fman->cfg))
1965                 return -EINVAL;
1966
1967         fman->state->count1_micro_bit = FM_TIMESTAMP_1_USEC_BIT;
1968
1969         cfg = fman->cfg;
1970
1971         /* clear revision-dependent non existing exception */
1972         if (fman->state->rev_info.major < 6)
1973                 fman->state->exceptions &= ~FMAN_EX_BMI_DISPATCH_RAM_ECC;
1974
1975         if (fman->state->rev_info.major >= 6)
1976                 fman->state->exceptions &= ~FMAN_EX_QMI_SINGLE_ECC;
1977
1978         /* clear CPG */
1979         memset_io((void __iomem *)(fman->base_addr + CGP_OFFSET), 0,
1980                   fman->state->fm_port_num_of_cg);
1981
1982         /* Save LIODN info before FMan reset
1983          * Skipping non-existent port 0 (i = 1)
1984          */
1985         for (i = 1; i < FMAN_LIODN_TBL; i++) {
1986                 u32 liodn_base;
1987
1988                 fman->liodn_offset[i] =
1989                         ioread32be(&fman->bmi_regs->fmbm_spliodn[i - 1]);
1990                 liodn_base = ioread32be(&fman->dma_regs->fmdmplr[i / 2]);
1991                 if (i % 2) {
1992                         /* FMDM_PLR LSB holds LIODN base for odd ports */
1993                         liodn_base &= DMA_LIODN_BASE_MASK;
1994                 } else {
1995                         /* FMDM_PLR MSB holds LIODN base for even ports */
1996                         liodn_base >>= DMA_LIODN_SHIFT;
1997                         liodn_base &= DMA_LIODN_BASE_MASK;
1998                 }
1999                 fman->liodn_base[i] = liodn_base;
2000         }
2001
2002         err = fman_reset(fman);
2003         if (err)
2004                 return err;
2005
2006         if (ioread32be(&fman->qmi_regs->fmqm_gs) & QMI_GS_HALT_NOT_BUSY) {
2007                 resume(fman->fpm_regs);
2008                 /* Wait until QMI is not in halt not busy state */
2009                 count = 100;
2010                 do {
2011                         udelay(1);
2012                 } while (((ioread32be(&fman->qmi_regs->fmqm_gs)) &
2013                          QMI_GS_HALT_NOT_BUSY) && --count);
2014                 if (count == 0)
2015                         dev_warn(fman->dev, "%s: QMI is in halt not busy state\n",
2016                                  __func__);
2017         }
2018
2019         if (clear_iram(fman) != 0)
2020                 return -EINVAL;
2021
2022         cfg->exceptions = fman->state->exceptions;
2023
2024         /* Init DMA Registers */
2025
2026         err = dma_init(fman);
2027         if (err != 0) {
2028                 free_init_resources(fman);
2029                 return err;
2030         }
2031
2032         /* Init FPM Registers */
2033         fpm_init(fman->fpm_regs, fman->cfg);
2034
2035         /* define common resources */
2036         /* allocate MURAM for FIFO according to total size */
2037         fman->fifo_offset = fman_muram_alloc(fman->muram,
2038                                              fman->state->total_fifo_size);
2039         if (IS_ERR_VALUE(fman->fifo_offset)) {
2040                 free_init_resources(fman);
2041                 dev_err(fman->dev, "%s: MURAM alloc for BMI FIFO failed\n",
2042                         __func__);
2043                 return -ENOMEM;
2044         }
2045
2046         cfg->fifo_base_addr = fman->fifo_offset;
2047         cfg->total_fifo_size = fman->state->total_fifo_size;
2048         cfg->total_num_of_tasks = fman->state->total_num_of_tasks;
2049         cfg->clk_freq = fman->state->fm_clk_freq;
2050
2051         /* Init BMI Registers */
2052         bmi_init(fman->bmi_regs, fman->cfg);
2053
2054         /* Init QMI Registers */
2055         qmi_init(fman->qmi_regs, fman->cfg);
2056
2057         err = enable(fman, cfg);
2058         if (err != 0)
2059                 return err;
2060
2061         enable_time_stamp(fman);
2062
2063         kfree(fman->cfg);
2064         fman->cfg = NULL;
2065
2066         return 0;
2067 }
2068
2069 static int fman_set_exception(struct fman *fman,
2070                               enum fman_exceptions exception, bool enable)
2071 {
2072         u32 bit_mask = 0;
2073
2074         if (!is_init_done(fman->cfg))
2075                 return -EINVAL;
2076
2077         bit_mask = get_exception_flag(exception);
2078         if (bit_mask) {
2079                 if (enable)
2080                         fman->state->exceptions |= bit_mask;
2081                 else
2082                         fman->state->exceptions &= ~bit_mask;
2083         } else {
2084                 dev_err(fman->dev, "%s: Undefined exception (%d)\n",
2085                         __func__, exception);
2086                 return -EINVAL;
2087         }
2088
2089         return set_exception(fman, exception, enable);
2090 }
2091
2092 /**
2093  * fman_register_intr
2094  * @fman:       A Pointer to FMan device
2095  * @mod:        Calling module
2096  * @mod_id:     Module id (if more than 1 exists, '0' if not)
2097  * @intr_type:  Interrupt type (error/normal) selection.
2098  * @f_isr:      The interrupt service routine.
2099  * @h_src_arg:  Argument to be passed to f_isr.
2100  *
2101  * Used to register an event handler to be processed by FMan
2102  *
2103  * Return: 0 on success; Error code otherwise.
2104  */
2105 void fman_register_intr(struct fman *fman, enum fman_event_modules module,
2106                         u8 mod_id, enum fman_intr_type intr_type,
2107                         void (*isr_cb)(void *src_arg), void *src_arg)
2108 {
2109         int event = 0;
2110
2111         event = get_module_event(module, mod_id, intr_type);
2112         WARN_ON(event >= FMAN_EV_CNT);
2113
2114         /* register in local FM structure */
2115         fman->intr_mng[event].isr_cb = isr_cb;
2116         fman->intr_mng[event].src_handle = src_arg;
2117 }
2118
2119 /**
2120  * fman_unregister_intr
2121  * @fman:       A Pointer to FMan device
2122  * @mod:        Calling module
2123  * @mod_id:     Module id (if more than 1 exists, '0' if not)
2124  * @intr_type:  Interrupt type (error/normal) selection.
2125  *
2126  * Used to unregister an event handler to be processed by FMan
2127  *
2128  * Return: 0 on success; Error code otherwise.
2129  */
2130 void fman_unregister_intr(struct fman *fman, enum fman_event_modules module,
2131                           u8 mod_id, enum fman_intr_type intr_type)
2132 {
2133         int event = 0;
2134
2135         event = get_module_event(module, mod_id, intr_type);
2136         WARN_ON(event >= FMAN_EV_CNT);
2137
2138         fman->intr_mng[event].isr_cb = NULL;
2139         fman->intr_mng[event].src_handle = NULL;
2140 }
2141
2142 /**
2143  * fman_set_port_params
2144  * @fman:               A Pointer to FMan device
2145  * @port_params:        Port parameters
2146  *
2147  * Used by FMan Port to pass parameters to the FMan
2148  *
2149  * Return: 0 on success; Error code otherwise.
2150  */
2151 int fman_set_port_params(struct fman *fman,
2152                          struct fman_port_init_params *port_params)
2153 {
2154         int err;
2155         unsigned long flags;
2156         u8 port_id = port_params->port_id, mac_id;
2157
2158         spin_lock_irqsave(&fman->spinlock, flags);
2159
2160         err = set_num_of_tasks(fman, port_params->port_id,
2161                                &port_params->num_of_tasks,
2162                                &port_params->num_of_extra_tasks);
2163         if (err)
2164                 goto return_err;
2165
2166         /* TX Ports */
2167         if (port_params->port_type != FMAN_PORT_TYPE_RX) {
2168                 u32 enq_th, deq_th, reg;
2169
2170                 /* update qmi ENQ/DEQ threshold */
2171                 fman->state->accumulated_num_of_deq_tnums +=
2172                         port_params->deq_pipeline_depth;
2173                 enq_th = (ioread32be(&fman->qmi_regs->fmqm_gc) &
2174                           QMI_CFG_ENQ_MASK) >> QMI_CFG_ENQ_SHIFT;
2175                 /* if enq_th is too big, we reduce it to the max value
2176                  * that is still 0
2177                  */
2178                 if (enq_th >= (fman->state->qmi_max_num_of_tnums -
2179                     fman->state->accumulated_num_of_deq_tnums)) {
2180                         enq_th =
2181                         fman->state->qmi_max_num_of_tnums -
2182                         fman->state->accumulated_num_of_deq_tnums - 1;
2183
2184                         reg = ioread32be(&fman->qmi_regs->fmqm_gc);
2185                         reg &= ~QMI_CFG_ENQ_MASK;
2186                         reg |= (enq_th << QMI_CFG_ENQ_SHIFT);
2187                         iowrite32be(reg, &fman->qmi_regs->fmqm_gc);
2188                 }
2189
2190                 deq_th = ioread32be(&fman->qmi_regs->fmqm_gc) &
2191                                     QMI_CFG_DEQ_MASK;
2192                 /* if deq_th is too small, we enlarge it to the min
2193                  * value that is still 0.
2194                  * depTh may not be larger than 63
2195                  * (fman->state->qmi_max_num_of_tnums-1).
2196                  */
2197                 if ((deq_th <= fman->state->accumulated_num_of_deq_tnums) &&
2198                     (deq_th < fman->state->qmi_max_num_of_tnums - 1)) {
2199                         deq_th = fman->state->accumulated_num_of_deq_tnums + 1;
2200                         reg = ioread32be(&fman->qmi_regs->fmqm_gc);
2201                         reg &= ~QMI_CFG_DEQ_MASK;
2202                         reg |= deq_th;
2203                         iowrite32be(reg, &fman->qmi_regs->fmqm_gc);
2204                 }
2205         }
2206
2207         err = set_size_of_fifo(fman, port_params->port_id,
2208                                &port_params->size_of_fifo,
2209                                &port_params->extra_size_of_fifo);
2210         if (err)
2211                 goto return_err;
2212
2213         err = set_num_of_open_dmas(fman, port_params->port_id,
2214                                    &port_params->num_of_open_dmas,
2215                                    &port_params->num_of_extra_open_dmas);
2216         if (err)
2217                 goto return_err;
2218
2219         set_port_liodn(fman, port_id, fman->liodn_base[port_id],
2220                        fman->liodn_offset[port_id]);
2221
2222         if (fman->state->rev_info.major < 6)
2223                 set_port_order_restoration(fman->fpm_regs, port_id);
2224
2225         mac_id = hw_port_id_to_sw_port_id(fman->state->rev_info.major, port_id);
2226
2227         if (port_params->max_frame_length >= fman->state->mac_mfl[mac_id]) {
2228                 fman->state->port_mfl[mac_id] = port_params->max_frame_length;
2229         } else {
2230                 dev_warn(fman->dev, "%s: Port (%d) max_frame_length is smaller than MAC (%d) current MTU\n",
2231                          __func__, port_id, mac_id);
2232                 err = -EINVAL;
2233                 goto return_err;
2234         }
2235
2236         spin_unlock_irqrestore(&fman->spinlock, flags);
2237
2238         return 0;
2239
2240 return_err:
2241         spin_unlock_irqrestore(&fman->spinlock, flags);
2242         return err;
2243 }
2244
2245 /**
2246  * fman_reset_mac
2247  * @fman:       A Pointer to FMan device
2248  * @mac_id:     MAC id to be reset
2249  *
2250  * Reset a specific MAC
2251  *
2252  * Return: 0 on success; Error code otherwise.
2253  */
2254 int fman_reset_mac(struct fman *fman, u8 mac_id)
2255 {
2256         struct fman_fpm_regs __iomem *fpm_rg = fman->fpm_regs;
2257         u32 msk, timeout = 100;
2258
2259         if (fman->state->rev_info.major >= 6) {
2260                 dev_err(fman->dev, "%s: FMan MAC reset no available for FMan V3!\n",
2261                         __func__);
2262                 return -EINVAL;
2263         }
2264
2265         /* Get the relevant bit mask */
2266         switch (mac_id) {
2267         case 0:
2268                 msk = FPM_RSTC_MAC0_RESET;
2269                 break;
2270         case 1:
2271                 msk = FPM_RSTC_MAC1_RESET;
2272                 break;
2273         case 2:
2274                 msk = FPM_RSTC_MAC2_RESET;
2275                 break;
2276         case 3:
2277                 msk = FPM_RSTC_MAC3_RESET;
2278                 break;
2279         case 4:
2280                 msk = FPM_RSTC_MAC4_RESET;
2281                 break;
2282         case 5:
2283                 msk = FPM_RSTC_MAC5_RESET;
2284                 break;
2285         case 6:
2286                 msk = FPM_RSTC_MAC6_RESET;
2287                 break;
2288         case 7:
2289                 msk = FPM_RSTC_MAC7_RESET;
2290                 break;
2291         case 8:
2292                 msk = FPM_RSTC_MAC8_RESET;
2293                 break;
2294         case 9:
2295                 msk = FPM_RSTC_MAC9_RESET;
2296                 break;
2297         default:
2298                 dev_warn(fman->dev, "%s: Illegal MAC Id [%d]\n",
2299                          __func__, mac_id);
2300                 return -EINVAL;
2301         }
2302
2303         /* reset */
2304         iowrite32be(msk, &fpm_rg->fm_rstc);
2305         while ((ioread32be(&fpm_rg->fm_rstc) & msk) && --timeout)
2306                 udelay(10);
2307
2308         if (!timeout)
2309                 return -EIO;
2310
2311         return 0;
2312 }
2313
2314 /**
2315  * fman_set_mac_max_frame
2316  * @fman:       A Pointer to FMan device
2317  * @mac_id:     MAC id
2318  * @mfl:        Maximum frame length
2319  *
2320  * Set maximum frame length of specific MAC in FMan driver
2321  *
2322  * Return: 0 on success; Error code otherwise.
2323  */
2324 int fman_set_mac_max_frame(struct fman *fman, u8 mac_id, u16 mfl)
2325 {
2326         /* if port is already initialized, check that MaxFrameLength is smaller
2327          * or equal to the port's max
2328          */
2329         if ((!fman->state->port_mfl[mac_id]) ||
2330             (fman->state->port_mfl[mac_id] &&
2331             (mfl <= fman->state->port_mfl[mac_id]))) {
2332                 fman->state->mac_mfl[mac_id] = mfl;
2333         } else {
2334                 dev_warn(fman->dev, "%s: MAC max_frame_length is larger than Port max_frame_length\n",
2335                          __func__);
2336                 return -EINVAL;
2337         }
2338         return 0;
2339 }
2340
2341 /**
2342  * fman_get_clock_freq
2343  * @fman:       A Pointer to FMan device
2344  *
2345  * Get FMan clock frequency
2346  *
2347  * Return: FMan clock frequency
2348  */
2349 u16 fman_get_clock_freq(struct fman *fman)
2350 {
2351         return fman->state->fm_clk_freq;
2352 }
2353
2354 /**
2355  * fman_get_bmi_max_fifo_size
2356  * @fman:       A Pointer to FMan device
2357  *
2358  * Get FMan maximum FIFO size
2359  *
2360  * Return: FMan Maximum FIFO size
2361  */
2362 u32 fman_get_bmi_max_fifo_size(struct fman *fman)
2363 {
2364         return fman->state->bmi_max_fifo_size;
2365 }
2366
2367 /**
2368  * fman_get_revision
2369  * @fman                - Pointer to the FMan module
2370  * @rev_info            - A structure of revision information parameters.
2371  *
2372  * Returns the FM revision
2373  *
2374  * Allowed only following fman_init().
2375  *
2376  * Return: 0 on success; Error code otherwise.
2377  */
2378 void fman_get_revision(struct fman *fman, struct fman_rev_info *rev_info)
2379 {
2380         u32 tmp;
2381
2382         tmp = ioread32be(&fman->fpm_regs->fm_ip_rev_1);
2383         rev_info->major = (u8)((tmp & FPM_REV1_MAJOR_MASK) >>
2384                                 FPM_REV1_MAJOR_SHIFT);
2385         rev_info->minor = tmp & FPM_REV1_MINOR_MASK;
2386 }
2387
2388 /**
2389  * fman_get_qman_channel_id
2390  * @fman:       A Pointer to FMan device
2391  * @port_id:    Port id
2392  *
2393  * Get QMan channel ID associated to the Port id
2394  *
2395  * Return: QMan channel ID
2396  */
2397 u32 fman_get_qman_channel_id(struct fman *fman, u32 port_id)
2398 {
2399         int i;
2400
2401         if (fman->state->rev_info.major >= 6) {
2402                 u32 port_ids[] = {0x30, 0x31, 0x28, 0x29, 0x2a, 0x2b,
2403                                   0x2c, 0x2d, 0x2, 0x3, 0x4, 0x5, 0x7, 0x7};
2404                 for (i = 0; i < fman->state->num_of_qman_channels; i++) {
2405                         if (port_ids[i] == port_id)
2406                                 break;
2407                 }
2408         } else {
2409                 u32 port_ids[] = {0x30, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x1,
2410                                   0x2, 0x3, 0x4, 0x5, 0x7, 0x7};
2411                 for (i = 0; i < fman->state->num_of_qman_channels; i++) {
2412                         if (port_ids[i] == port_id)
2413                                 break;
2414                 }
2415         }
2416
2417         if (i == fman->state->num_of_qman_channels)
2418                 return 0;
2419
2420         return fman->state->qman_channel_base + i;
2421 }
2422
2423 /**
2424  * fman_get_mem_region
2425  * @fman:       A Pointer to FMan device
2426  *
2427  * Get FMan memory region
2428  *
2429  * Return: A structure with FMan memory region information
2430  */
2431 struct resource *fman_get_mem_region(struct fman *fman)
2432 {
2433         return fman->state->res;
2434 }
2435
2436 /* Bootargs defines */
2437 /* Extra headroom for RX buffers - Default, min and max */
2438 #define FSL_FM_RX_EXTRA_HEADROOM        64
2439 #define FSL_FM_RX_EXTRA_HEADROOM_MIN    16
2440 #define FSL_FM_RX_EXTRA_HEADROOM_MAX    384
2441
2442 /* Maximum frame length */
2443 #define FSL_FM_MAX_FRAME_SIZE                   1522
2444 #define FSL_FM_MAX_POSSIBLE_FRAME_SIZE          9600
2445 #define FSL_FM_MIN_POSSIBLE_FRAME_SIZE          64
2446
2447 /* Extra headroom for Rx buffers.
2448  * FMan is instructed to allocate, on the Rx path, this amount of
2449  * space at the beginning of a data buffer, beside the DPA private
2450  * data area and the IC fields.
2451  * Does not impact Tx buffer layout.
2452  * Configurable from bootargs. 64 by default, it's needed on
2453  * particular forwarding scenarios that add extra headers to the
2454  * forwarded frame.
2455  */
2456 int fsl_fm_rx_extra_headroom = FSL_FM_RX_EXTRA_HEADROOM;
2457 module_param(fsl_fm_rx_extra_headroom, int, 0);
2458 MODULE_PARM_DESC(fsl_fm_rx_extra_headroom, "Extra headroom for Rx buffers");
2459
2460 /* Max frame size, across all interfaces.
2461  * Configurable from bootargs, to avoid allocating oversized (socket)
2462  * buffers when not using jumbo frames.
2463  * Must be large enough to accommodate the network MTU, but small enough
2464  * to avoid wasting skb memory.
2465  *
2466  * Could be overridden once, at boot-time, via the
2467  * fm_set_max_frm() callback.
2468  */
2469 int fsl_fm_max_frm = FSL_FM_MAX_FRAME_SIZE;
2470 module_param(fsl_fm_max_frm, int, 0);
2471 MODULE_PARM_DESC(fsl_fm_max_frm, "Maximum frame size, across all interfaces");
2472
2473 /**
2474  * fman_get_max_frm
2475  *
2476  * Return: Max frame length configured in the FM driver
2477  */
2478 u16 fman_get_max_frm(void)
2479 {
2480         static bool fm_check_mfl;
2481
2482         if (!fm_check_mfl) {
2483                 if (fsl_fm_max_frm > FSL_FM_MAX_POSSIBLE_FRAME_SIZE ||
2484                     fsl_fm_max_frm < FSL_FM_MIN_POSSIBLE_FRAME_SIZE) {
2485                         pr_warn("Invalid fsl_fm_max_frm value (%d) in bootargs, valid range is %d-%d. Falling back to the default (%d)\n",
2486                                 fsl_fm_max_frm,
2487                                 FSL_FM_MIN_POSSIBLE_FRAME_SIZE,
2488                                 FSL_FM_MAX_POSSIBLE_FRAME_SIZE,
2489                                 FSL_FM_MAX_FRAME_SIZE);
2490                         fsl_fm_max_frm = FSL_FM_MAX_FRAME_SIZE;
2491                 }
2492                 fm_check_mfl = true;
2493         }
2494
2495         return fsl_fm_max_frm;
2496 }
2497 EXPORT_SYMBOL(fman_get_max_frm);
2498
2499 /**
2500  * fman_get_rx_extra_headroom
2501  *
2502  * Return: Extra headroom size configured in the FM driver
2503  */
2504 int fman_get_rx_extra_headroom(void)
2505 {
2506         static bool fm_check_rx_extra_headroom;
2507
2508         if (!fm_check_rx_extra_headroom) {
2509                 if (fsl_fm_rx_extra_headroom > FSL_FM_RX_EXTRA_HEADROOM_MAX ||
2510                     fsl_fm_rx_extra_headroom < FSL_FM_RX_EXTRA_HEADROOM_MIN) {
2511                         pr_warn("Invalid fsl_fm_rx_extra_headroom value (%d) in bootargs, valid range is %d-%d. Falling back to the default (%d)\n",
2512                                 fsl_fm_rx_extra_headroom,
2513                                 FSL_FM_RX_EXTRA_HEADROOM_MIN,
2514                                 FSL_FM_RX_EXTRA_HEADROOM_MAX,
2515                                 FSL_FM_RX_EXTRA_HEADROOM);
2516                         fsl_fm_rx_extra_headroom = FSL_FM_RX_EXTRA_HEADROOM;
2517                 }
2518
2519                 fm_check_rx_extra_headroom = true;
2520                 fsl_fm_rx_extra_headroom = ALIGN(fsl_fm_rx_extra_headroom, 16);
2521         }
2522
2523         return fsl_fm_rx_extra_headroom;
2524 }
2525 EXPORT_SYMBOL(fman_get_rx_extra_headroom);
2526
2527 /**
2528  * fman_bind
2529  * @dev:        FMan OF device pointer
2530  *
2531  * Bind to a specific FMan device.
2532  *
2533  * Allowed only after the port was created.
2534  *
2535  * Return: A pointer to the FMan device
2536  */
2537 struct fman *fman_bind(struct device *fm_dev)
2538 {
2539         return (struct fman *)(dev_get_drvdata(get_device(fm_dev)));
2540 }
2541
2542 static irqreturn_t fman_err_irq(int irq, void *handle)
2543 {
2544         struct fman *fman = (struct fman *)handle;
2545         u32 pending;
2546         struct fman_fpm_regs __iomem *fpm_rg;
2547         irqreturn_t single_ret, ret = IRQ_NONE;
2548
2549         if (!is_init_done(fman->cfg))
2550                 return IRQ_NONE;
2551
2552         fpm_rg = fman->fpm_regs;
2553
2554         /* error interrupts */
2555         pending = ioread32be(&fpm_rg->fm_epi);
2556         if (!pending)
2557                 return IRQ_NONE;
2558
2559         if (pending & ERR_INTR_EN_BMI) {
2560                 single_ret = bmi_err_event(fman);
2561                 if (single_ret == IRQ_HANDLED)
2562                         ret = IRQ_HANDLED;
2563         }
2564         if (pending & ERR_INTR_EN_QMI) {
2565                 single_ret = qmi_err_event(fman);
2566                 if (single_ret == IRQ_HANDLED)
2567                         ret = IRQ_HANDLED;
2568         }
2569         if (pending & ERR_INTR_EN_FPM) {
2570                 single_ret = fpm_err_event(fman);
2571                 if (single_ret == IRQ_HANDLED)
2572                         ret = IRQ_HANDLED;
2573         }
2574         if (pending & ERR_INTR_EN_DMA) {
2575                 single_ret = dma_err_event(fman);
2576                 if (single_ret == IRQ_HANDLED)
2577                         ret = IRQ_HANDLED;
2578         }
2579         if (pending & ERR_INTR_EN_MURAM) {
2580                 single_ret = muram_err_intr(fman);
2581                 if (single_ret == IRQ_HANDLED)
2582                         ret = IRQ_HANDLED;
2583         }
2584
2585         /* MAC error interrupts */
2586         if (pending & ERR_INTR_EN_MAC0) {
2587                 single_ret = call_mac_isr(fman, FMAN_EV_ERR_MAC0 + 0);
2588                 if (single_ret == IRQ_HANDLED)
2589                         ret = IRQ_HANDLED;
2590         }
2591         if (pending & ERR_INTR_EN_MAC1) {
2592                 single_ret = call_mac_isr(fman, FMAN_EV_ERR_MAC0 + 1);
2593                 if (single_ret == IRQ_HANDLED)
2594                         ret = IRQ_HANDLED;
2595         }
2596         if (pending & ERR_INTR_EN_MAC2) {
2597                 single_ret = call_mac_isr(fman, FMAN_EV_ERR_MAC0 + 2);
2598                 if (single_ret == IRQ_HANDLED)
2599                         ret = IRQ_HANDLED;
2600         }
2601         if (pending & ERR_INTR_EN_MAC3) {
2602                 single_ret = call_mac_isr(fman, FMAN_EV_ERR_MAC0 + 3);
2603                 if (single_ret == IRQ_HANDLED)
2604                         ret = IRQ_HANDLED;
2605         }
2606         if (pending & ERR_INTR_EN_MAC4) {
2607                 single_ret = call_mac_isr(fman, FMAN_EV_ERR_MAC0 + 4);
2608                 if (single_ret == IRQ_HANDLED)
2609                         ret = IRQ_HANDLED;
2610         }
2611         if (pending & ERR_INTR_EN_MAC5) {
2612                 single_ret = call_mac_isr(fman, FMAN_EV_ERR_MAC0 + 5);
2613                 if (single_ret == IRQ_HANDLED)
2614                         ret = IRQ_HANDLED;
2615         }
2616         if (pending & ERR_INTR_EN_MAC6) {
2617                 single_ret = call_mac_isr(fman, FMAN_EV_ERR_MAC0 + 6);
2618                 if (single_ret == IRQ_HANDLED)
2619                         ret = IRQ_HANDLED;
2620         }
2621         if (pending & ERR_INTR_EN_MAC7) {
2622                 single_ret = call_mac_isr(fman, FMAN_EV_ERR_MAC0 + 7);
2623                 if (single_ret == IRQ_HANDLED)
2624                         ret = IRQ_HANDLED;
2625         }
2626         if (pending & ERR_INTR_EN_MAC8) {
2627                 single_ret = call_mac_isr(fman, FMAN_EV_ERR_MAC0 + 8);
2628                 if (single_ret == IRQ_HANDLED)
2629                         ret = IRQ_HANDLED;
2630         }
2631         if (pending & ERR_INTR_EN_MAC9) {
2632                 single_ret = call_mac_isr(fman, FMAN_EV_ERR_MAC0 + 9);
2633                 if (single_ret == IRQ_HANDLED)
2634                         ret = IRQ_HANDLED;
2635         }
2636
2637         return ret;
2638 }
2639
2640 static irqreturn_t fman_irq(int irq, void *handle)
2641 {
2642         struct fman *fman = (struct fman *)handle;
2643         u32 pending;
2644         struct fman_fpm_regs __iomem *fpm_rg;
2645         irqreturn_t single_ret, ret = IRQ_NONE;
2646
2647         if (!is_init_done(fman->cfg))
2648                 return IRQ_NONE;
2649
2650         fpm_rg = fman->fpm_regs;
2651
2652         /* normal interrupts */
2653         pending = ioread32be(&fpm_rg->fm_npi);
2654         if (!pending)
2655                 return IRQ_NONE;
2656
2657         if (pending & INTR_EN_QMI) {
2658                 single_ret = qmi_event(fman);
2659                 if (single_ret == IRQ_HANDLED)
2660                         ret = IRQ_HANDLED;
2661         }
2662
2663         /* MAC interrupts */
2664         if (pending & INTR_EN_MAC0) {
2665                 single_ret = call_mac_isr(fman, FMAN_EV_MAC0 + 0);
2666                 if (single_ret == IRQ_HANDLED)
2667                         ret = IRQ_HANDLED;
2668         }
2669         if (pending & INTR_EN_MAC1) {
2670                 single_ret = call_mac_isr(fman, FMAN_EV_MAC0 + 1);
2671                 if (single_ret == IRQ_HANDLED)
2672                         ret = IRQ_HANDLED;
2673         }
2674         if (pending & INTR_EN_MAC2) {
2675                 single_ret = call_mac_isr(fman, FMAN_EV_MAC0 + 2);
2676                 if (single_ret == IRQ_HANDLED)
2677                         ret = IRQ_HANDLED;
2678         }
2679         if (pending & INTR_EN_MAC3) {
2680                 single_ret = call_mac_isr(fman, FMAN_EV_MAC0 + 3);
2681                 if (single_ret == IRQ_HANDLED)
2682                         ret = IRQ_HANDLED;
2683         }
2684         if (pending & INTR_EN_MAC4) {
2685                 single_ret = call_mac_isr(fman, FMAN_EV_MAC0 + 4);
2686                 if (single_ret == IRQ_HANDLED)
2687                         ret = IRQ_HANDLED;
2688         }
2689         if (pending & INTR_EN_MAC5) {
2690                 single_ret = call_mac_isr(fman, FMAN_EV_MAC0 + 5);
2691                 if (single_ret == IRQ_HANDLED)
2692                         ret = IRQ_HANDLED;
2693         }
2694         if (pending & INTR_EN_MAC6) {
2695                 single_ret = call_mac_isr(fman, FMAN_EV_MAC0 + 6);
2696                 if (single_ret == IRQ_HANDLED)
2697                         ret = IRQ_HANDLED;
2698         }
2699         if (pending & INTR_EN_MAC7) {
2700                 single_ret = call_mac_isr(fman, FMAN_EV_MAC0 + 7);
2701                 if (single_ret == IRQ_HANDLED)
2702                         ret = IRQ_HANDLED;
2703         }
2704         if (pending & INTR_EN_MAC8) {
2705                 single_ret = call_mac_isr(fman, FMAN_EV_MAC0 + 8);
2706                 if (single_ret == IRQ_HANDLED)
2707                         ret = IRQ_HANDLED;
2708         }
2709         if (pending & INTR_EN_MAC9) {
2710                 single_ret = call_mac_isr(fman, FMAN_EV_MAC0 + 9);
2711                 if (single_ret == IRQ_HANDLED)
2712                         ret = IRQ_HANDLED;
2713         }
2714
2715         return ret;
2716 }
2717
2718 static const struct of_device_id fman_muram_match[] = {
2719         {
2720                 .compatible = "fsl,fman-muram"},
2721         {}
2722 };
2723 MODULE_DEVICE_TABLE(of, fman_muram_match);
2724
2725 static struct fman *read_dts_node(struct platform_device *of_dev)
2726 {
2727         struct fman *fman;
2728         struct device_node *fm_node, *muram_node;
2729         struct resource *res;
2730         const u32 *u32_prop;
2731         int lenp, err, irq;
2732         struct clk *clk;
2733         u32 clk_rate;
2734         phys_addr_t phys_base_addr;
2735         resource_size_t mem_size;
2736
2737         fman = kzalloc(sizeof(*fman), GFP_KERNEL);
2738         if (!fman)
2739                 return NULL;
2740
2741         fm_node = of_node_get(of_dev->dev.of_node);
2742
2743         u32_prop = (const u32 *)of_get_property(fm_node, "cell-index", &lenp);
2744         if (!u32_prop) {
2745                 dev_err(&of_dev->dev, "%s: of_get_property(%s, cell-index) failed\n",
2746                         __func__, fm_node->full_name);
2747                 goto fman_node_put;
2748         }
2749         if (WARN_ON(lenp != sizeof(u32)))
2750                 goto fman_node_put;
2751
2752         fman->dts_params.id = (u8)fdt32_to_cpu(u32_prop[0]);
2753
2754         /* Get the FM interrupt */
2755         res = platform_get_resource(of_dev, IORESOURCE_IRQ, 0);
2756         if (!res) {
2757                 dev_err(&of_dev->dev, "%s: Can't get FMan IRQ resource\n",
2758                         __func__);
2759                 goto fman_node_put;
2760         }
2761         irq = res->start;
2762
2763         /* Get the FM error interrupt */
2764         res = platform_get_resource(of_dev, IORESOURCE_IRQ, 1);
2765         if (!res) {
2766                 dev_err(&of_dev->dev, "%s: Can't get FMan Error IRQ resource\n",
2767                         __func__);
2768                 goto fman_node_put;
2769         }
2770         fman->dts_params.err_irq = res->start;
2771
2772         /* Get the FM address */
2773         res = platform_get_resource(of_dev, IORESOURCE_MEM, 0);
2774         if (!res) {
2775                 dev_err(&of_dev->dev, "%s: Can't get FMan memory resource\n",
2776                         __func__);
2777                 goto fman_node_put;
2778         }
2779
2780         phys_base_addr = res->start;
2781         mem_size = resource_size(res);
2782
2783         clk = of_clk_get(fm_node, 0);
2784         if (IS_ERR(clk)) {
2785                 dev_err(&of_dev->dev, "%s: Failed to get FM%d clock structure\n",
2786                         __func__, fman->dts_params.id);
2787                 goto fman_node_put;
2788         }
2789
2790         clk_rate = clk_get_rate(clk);
2791         if (!clk_rate) {
2792                 dev_err(&of_dev->dev, "%s: Failed to determine FM%d clock rate\n",
2793                         __func__, fman->dts_params.id);
2794                 goto fman_node_put;
2795         }
2796         /* Rounding to MHz */
2797         fman->dts_params.clk_freq = DIV_ROUND_UP(clk_rate, 1000000);
2798
2799         u32_prop = (const u32 *)of_get_property(fm_node,
2800                                                 "fsl,qman-channel-range",
2801                                                 &lenp);
2802         if (!u32_prop) {
2803                 dev_err(&of_dev->dev, "%s: of_get_property(%s, fsl,qman-channel-range) failed\n",
2804                         __func__, fm_node->full_name);
2805                 goto fman_node_put;
2806         }
2807         if (WARN_ON(lenp != sizeof(u32) * 2))
2808                 goto fman_node_put;
2809         fman->dts_params.qman_channel_base = fdt32_to_cpu(u32_prop[0]);
2810         fman->dts_params.num_of_qman_channels = fdt32_to_cpu(u32_prop[1]);
2811
2812         /* Get the MURAM base address and size */
2813         muram_node = of_find_matching_node(fm_node, fman_muram_match);
2814         if (!muram_node) {
2815                 dev_err(&of_dev->dev, "%s: could not find MURAM node\n",
2816                         __func__);
2817                 goto fman_node_put;
2818         }
2819
2820         err = of_address_to_resource(muram_node, 0,
2821                                      &fman->dts_params.muram_res);
2822         if (err) {
2823                 of_node_put(muram_node);
2824                 dev_err(&of_dev->dev, "%s: of_address_to_resource() = %d\n",
2825                         __func__, err);
2826                 goto fman_node_put;
2827         }
2828
2829         of_node_put(muram_node);
2830         of_node_put(fm_node);
2831
2832         err = devm_request_irq(&of_dev->dev, irq, fman_irq, 0, "fman", fman);
2833         if (err < 0) {
2834                 dev_err(&of_dev->dev, "%s: irq %d allocation failed (error = %d)\n",
2835                         __func__, irq, err);
2836                 goto fman_free;
2837         }
2838
2839         if (fman->dts_params.err_irq != 0) {
2840                 err = devm_request_irq(&of_dev->dev, fman->dts_params.err_irq,
2841                                        fman_err_irq, IRQF_SHARED,
2842                                        "fman-err", fman);
2843                 if (err < 0) {
2844                         dev_err(&of_dev->dev, "%s: irq %d allocation failed (error = %d)\n",
2845                                 __func__, fman->dts_params.err_irq, err);
2846                         goto fman_free;
2847                 }
2848         }
2849
2850         fman->dts_params.res =
2851                 devm_request_mem_region(&of_dev->dev, phys_base_addr,
2852                                         mem_size, "fman");
2853         if (!fman->dts_params.res) {
2854                 dev_err(&of_dev->dev, "%s: request_mem_region() failed\n",
2855                         __func__);
2856                 goto fman_free;
2857         }
2858
2859         fman->dts_params.base_addr =
2860                 devm_ioremap(&of_dev->dev, phys_base_addr, mem_size);
2861         if (fman->dts_params.base_addr == 0) {
2862                 dev_err(&of_dev->dev, "%s: devm_ioremap() failed\n", __func__);
2863                 goto fman_free;
2864         }
2865
2866         fman->dev = &of_dev->dev;
2867
2868         return fman;
2869
2870 fman_node_put:
2871         of_node_put(fm_node);
2872 fman_free:
2873         kfree(fman);
2874         return NULL;
2875 }
2876
2877 static int fman_probe(struct platform_device *of_dev)
2878 {
2879         struct fman *fman;
2880         struct device *dev;
2881         int err;
2882
2883         dev = &of_dev->dev;
2884
2885         fman = read_dts_node(of_dev);
2886         if (!fman)
2887                 return -EIO;
2888
2889         err = fman_config(fman);
2890         if (err) {
2891                 dev_err(dev, "%s: FMan config failed\n", __func__);
2892                 return -EINVAL;
2893         }
2894
2895         if (fman_init(fman) != 0) {
2896                 dev_err(dev, "%s: FMan init failed\n", __func__);
2897                 return -EINVAL;
2898         }
2899
2900         if (fman->dts_params.err_irq == 0) {
2901                 fman_set_exception(fman, FMAN_EX_DMA_BUS_ERROR, false);
2902                 fman_set_exception(fman, FMAN_EX_DMA_READ_ECC, false);
2903                 fman_set_exception(fman, FMAN_EX_DMA_SYSTEM_WRITE_ECC, false);
2904                 fman_set_exception(fman, FMAN_EX_DMA_FM_WRITE_ECC, false);
2905                 fman_set_exception(fman, FMAN_EX_DMA_SINGLE_PORT_ECC, false);
2906                 fman_set_exception(fman, FMAN_EX_FPM_STALL_ON_TASKS, false);
2907                 fman_set_exception(fman, FMAN_EX_FPM_SINGLE_ECC, false);
2908                 fman_set_exception(fman, FMAN_EX_FPM_DOUBLE_ECC, false);
2909                 fman_set_exception(fman, FMAN_EX_QMI_SINGLE_ECC, false);
2910                 fman_set_exception(fman, FMAN_EX_QMI_DOUBLE_ECC, false);
2911                 fman_set_exception(fman,
2912                                    FMAN_EX_QMI_DEQ_FROM_UNKNOWN_PORTID, false);
2913                 fman_set_exception(fman, FMAN_EX_BMI_LIST_RAM_ECC, false);
2914                 fman_set_exception(fman, FMAN_EX_BMI_STORAGE_PROFILE_ECC,
2915                                    false);
2916                 fman_set_exception(fman, FMAN_EX_BMI_STATISTICS_RAM_ECC, false);
2917                 fman_set_exception(fman, FMAN_EX_BMI_DISPATCH_RAM_ECC, false);
2918         }
2919
2920         dev_set_drvdata(dev, fman);
2921
2922         dev_dbg(dev, "FMan%d probed\n", fman->dts_params.id);
2923
2924         return 0;
2925 }
2926
2927 static const struct of_device_id fman_match[] = {
2928         {
2929                 .compatible = "fsl,fman"},
2930         {}
2931 };
2932
2933 MODULE_DEVICE_TABLE(of, fm_match);
2934
2935 static struct platform_driver fman_driver = {
2936         .driver = {
2937                 .name = "fsl-fman",
2938                 .of_match_table = fman_match,
2939         },
2940         .probe = fman_probe,
2941 };
2942
2943 builtin_platform_driver(fman_driver);