drivers/net/ethernet/freescale/fman/fman.c

   1 /*
   2  * Copyright 2008-2015 Freescale Semiconductor Inc.
   3  *
   4  * Redistribution and use in source and binary forms, with or without
   5  * modification, are permitted provided that the following conditions are met:
   6  *     * Redistributions of source code must retain the above copyright
   7  *       notice, this list of conditions and the following disclaimer.
   8  *     * Redistributions in binary form must reproduce the above copyright
   9  *       notice, this list of conditions and the following disclaimer in the
  10  *       documentation and/or other materials provided with the distribution.
  11  *     * Neither the name of Freescale Semiconductor nor the
  12  *       names of its contributors may be used to endorse or promote products
  13  *       derived from this software without specific prior written permission.
  14  *
  15  *
  16  * ALTERNATIVELY, this software may be distributed under the terms of the
  17  * GNU General Public License ("GPL") as published by the Free Software
  18  * Foundation, either version 2 of that License or (at your option) any
  19  * later version.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY
  22  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
  23  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
  24  * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY
  25  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
  26  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  27  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
  28  * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  29  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  30  * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  31  */
  32
  33 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  34
  35 #include "fman.h"
  36 #include "fman_muram.h"
  37
  38 #include <linux/fsl/guts.h>
  39 #include <linux/slab.h>
  40 #include <linux/delay.h>
  41 #include <linux/module.h>
  42 #include <linux/of_platform.h>
  43 #include <linux/clk.h>
  44 #include <linux/of_address.h>
  45 #include <linux/of_irq.h>
  46 #include <linux/interrupt.h>
  47 #include <linux/libfdt_env.h>
  48
  49 /* General defines */
  50 #define FMAN_LIODN_TBL                  64      /* size of LIODN table */
  51 #define MAX_NUM_OF_MACS                 10
  52 #define FM_NUM_OF_FMAN_CTRL_EVENT_REGS  4
  53 #define BASE_RX_PORTID                  0x08
  54 #define BASE_TX_PORTID                  0x28
  55
  56 /* Modules registers offsets */
  57 #define BMI_OFFSET              0x00080000
  58 #define QMI_OFFSET              0x00080400
  59 #define DMA_OFFSET              0x000C2000
  60 #define FPM_OFFSET              0x000C3000
  61 #define IMEM_OFFSET             0x000C4000
  62 #define HWP_OFFSET              0x000C7000
  63 #define CGP_OFFSET              0x000DB000
  64
  65 /* Exceptions bit map */
  66 #define EX_DMA_BUS_ERROR                0x80000000
  67 #define EX_DMA_READ_ECC                 0x40000000
  68 #define EX_DMA_SYSTEM_WRITE_ECC 0x20000000
  69 #define EX_DMA_FM_WRITE_ECC             0x10000000
  70 #define EX_FPM_STALL_ON_TASKS           0x08000000
  71 #define EX_FPM_SINGLE_ECC               0x04000000
  72 #define EX_FPM_DOUBLE_ECC               0x02000000
  73 #define EX_QMI_SINGLE_ECC               0x01000000
  74 #define EX_QMI_DEQ_FROM_UNKNOWN_PORTID  0x00800000
  75 #define EX_QMI_DOUBLE_ECC               0x00400000
  76 #define EX_BMI_LIST_RAM_ECC             0x00200000
  77 #define EX_BMI_STORAGE_PROFILE_ECC      0x00100000
  78 #define EX_BMI_STATISTICS_RAM_ECC       0x00080000
  79 #define EX_IRAM_ECC                     0x00040000
  80 #define EX_MURAM_ECC                    0x00020000
  81 #define EX_BMI_DISPATCH_RAM_ECC 0x00010000
  82 #define EX_DMA_SINGLE_PORT_ECC          0x00008000
  83
  84 /* DMA defines */
  85 /* masks */
  86 #define DMA_MODE_BER                    0x00200000
  87 #define DMA_MODE_ECC                    0x00000020
  88 #define DMA_MODE_SECURE_PROT            0x00000800
  89 #define DMA_MODE_AXI_DBG_MASK           0x0F000000
  90
  91 #define DMA_TRANSFER_PORTID_MASK        0xFF000000
  92 #define DMA_TRANSFER_TNUM_MASK          0x00FF0000
  93 #define DMA_TRANSFER_LIODN_MASK 0x00000FFF
  94
  95 #define DMA_STATUS_BUS_ERR              0x08000000
  96 #define DMA_STATUS_READ_ECC             0x04000000
  97 #define DMA_STATUS_SYSTEM_WRITE_ECC     0x02000000
  98 #define DMA_STATUS_FM_WRITE_ECC 0x01000000
  99 #define DMA_STATUS_FM_SPDAT_ECC 0x00080000
 100
 101 #define DMA_MODE_CACHE_OR_SHIFT         30
 102 #define DMA_MODE_AXI_DBG_SHIFT                  24
 103 #define DMA_MODE_CEN_SHIFT                      13
 104 #define DMA_MODE_CEN_MASK                       0x00000007
 105 #define DMA_MODE_DBG_SHIFT                      7
 106 #define DMA_MODE_AID_MODE_SHIFT         4
 107
 108 #define DMA_THRESH_COMMQ_SHIFT                  24
 109 #define DMA_THRESH_READ_INT_BUF_SHIFT           16
 110 #define DMA_THRESH_READ_INT_BUF_MASK            0x0000003f
 111 #define DMA_THRESH_WRITE_INT_BUF_MASK           0x0000003f
 112
 113 #define DMA_TRANSFER_PORTID_SHIFT               24
 114 #define DMA_TRANSFER_TNUM_SHIFT         16
 115
 116 #define DMA_CAM_SIZEOF_ENTRY                    0x40
 117 #define DMA_CAM_UNITS                           8
 118
 119 #define DMA_LIODN_SHIFT         16
 120 #define DMA_LIODN_BASE_MASK     0x00000FFF
 121
 122 /* FPM defines */
 123 #define FPM_EV_MASK_DOUBLE_ECC          0x80000000
 124 #define FPM_EV_MASK_STALL               0x40000000
 125 #define FPM_EV_MASK_SINGLE_ECC          0x20000000
 126 #define FPM_EV_MASK_RELEASE_FM          0x00010000
 127 #define FPM_EV_MASK_DOUBLE_ECC_EN       0x00008000
 128 #define FPM_EV_MASK_STALL_EN            0x00004000
 129 #define FPM_EV_MASK_SINGLE_ECC_EN       0x00002000
 130 #define FPM_EV_MASK_EXTERNAL_HALT       0x00000008
 131 #define FPM_EV_MASK_ECC_ERR_HALT        0x00000004
 132
 133 #define FPM_RAM_MURAM_ECC               0x00008000
 134 #define FPM_RAM_IRAM_ECC                0x00004000
 135 #define FPM_IRAM_ECC_ERR_EX_EN          0x00020000
 136 #define FPM_MURAM_ECC_ERR_EX_EN 0x00040000
 137 #define FPM_RAM_IRAM_ECC_EN             0x40000000
 138 #define FPM_RAM_RAMS_ECC_EN             0x80000000
 139 #define FPM_RAM_RAMS_ECC_EN_SRC_SEL     0x08000000
 140
 141 #define FPM_REV1_MAJOR_MASK             0x0000FF00
 142 #define FPM_REV1_MINOR_MASK             0x000000FF
 143
 144 #define FPM_DISP_LIMIT_SHIFT            24
 145
 146 #define FPM_PRT_FM_CTL1                 0x00000001
 147 #define FPM_PRT_FM_CTL2                 0x00000002
 148 #define FPM_PORT_FM_CTL_PORTID_SHIFT    24
 149 #define FPM_PRC_ORA_FM_CTL_SEL_SHIFT    16
 150
 151 #define FPM_THR1_PRS_SHIFT              24
 152 #define FPM_THR1_KG_SHIFT               16
 153 #define FPM_THR1_PLCR_SHIFT             8
 154 #define FPM_THR1_BMI_SHIFT              0
 155
 156 #define FPM_THR2_QMI_ENQ_SHIFT          24
 157 #define FPM_THR2_QMI_DEQ_SHIFT          0
 158 #define FPM_THR2_FM_CTL1_SHIFT          16
 159 #define FPM_THR2_FM_CTL2_SHIFT          8
 160
 161 #define FPM_EV_MASK_CAT_ERR_SHIFT       1
 162 #define FPM_EV_MASK_DMA_ERR_SHIFT       0
 163
 164 #define FPM_REV1_MAJOR_SHIFT            8
 165
 166 #define FPM_RSTC_FM_RESET               0x80000000
 167 #define FPM_RSTC_MAC0_RESET             0x40000000
 168 #define FPM_RSTC_MAC1_RESET             0x20000000
 169 #define FPM_RSTC_MAC2_RESET             0x10000000
 170 #define FPM_RSTC_MAC3_RESET             0x08000000
 171 #define FPM_RSTC_MAC8_RESET             0x04000000
 172 #define FPM_RSTC_MAC4_RESET             0x02000000
 173 #define FPM_RSTC_MAC5_RESET             0x01000000
 174 #define FPM_RSTC_MAC6_RESET             0x00800000
 175 #define FPM_RSTC_MAC7_RESET             0x00400000
 176 #define FPM_RSTC_MAC9_RESET             0x00200000
 177
 178 #define FPM_TS_INT_SHIFT                16
 179 #define FPM_TS_CTL_EN                   0x80000000
 180
 181 /* BMI defines */
 182 #define BMI_INIT_START                          0x80000000
 183 #define BMI_ERR_INTR_EN_STORAGE_PROFILE_ECC     0x80000000
 184 #define BMI_ERR_INTR_EN_LIST_RAM_ECC            0x40000000
 185 #define BMI_ERR_INTR_EN_STATISTICS_RAM_ECC      0x20000000
 186 #define BMI_ERR_INTR_EN_DISPATCH_RAM_ECC        0x10000000
 187 #define BMI_NUM_OF_TASKS_MASK                   0x3F000000
 188 #define BMI_NUM_OF_EXTRA_TASKS_MASK             0x000F0000
 189 #define BMI_NUM_OF_DMAS_MASK                    0x00000F00
 190 #define BMI_NUM_OF_EXTRA_DMAS_MASK              0x0000000F
 191 #define BMI_FIFO_SIZE_MASK                      0x000003FF
 192 #define BMI_EXTRA_FIFO_SIZE_MASK                0x03FF0000
 193 #define BMI_CFG2_DMAS_MASK                      0x0000003F
 194 #define BMI_CFG2_TASKS_MASK                     0x0000003F
 195
 196 #define BMI_CFG2_TASKS_SHIFT            16
 197 #define BMI_CFG2_DMAS_SHIFT             0
 198 #define BMI_CFG1_FIFO_SIZE_SHIFT        16
 199 #define BMI_NUM_OF_TASKS_SHIFT          24
 200 #define BMI_EXTRA_NUM_OF_TASKS_SHIFT    16
 201 #define BMI_NUM_OF_DMAS_SHIFT           8
 202 #define BMI_EXTRA_NUM_OF_DMAS_SHIFT     0
 203
 204 #define BMI_FIFO_ALIGN                  0x100
 205
 206 #define BMI_EXTRA_FIFO_SIZE_SHIFT       16
 207
 208 /* QMI defines */
 209 #define QMI_CFG_ENQ_EN                  0x80000000
 210 #define QMI_CFG_DEQ_EN                  0x40000000
 211 #define QMI_CFG_EN_COUNTERS             0x10000000
 212 #define QMI_CFG_DEQ_MASK                0x0000003F
 213 #define QMI_CFG_ENQ_MASK                0x00003F00
 214 #define QMI_CFG_ENQ_SHIFT               8
 215
 216 #define QMI_ERR_INTR_EN_DOUBLE_ECC      0x80000000
 217 #define QMI_ERR_INTR_EN_DEQ_FROM_DEF    0x40000000
 218 #define QMI_INTR_EN_SINGLE_ECC          0x80000000
 219
 220 #define QMI_GS_HALT_NOT_BUSY            0x00000002
 221
 222 /* HWP defines */
 223 #define HWP_RPIMAC_PEN                  0x00000001
 224
 225 /* IRAM defines */
 226 #define IRAM_IADD_AIE                   0x80000000
 227 #define IRAM_READY                      0x80000000
 228
 229 /* Default values */
 230 #define DEFAULT_CATASTROPHIC_ERR                0
 231 #define DEFAULT_DMA_ERR                         0
 232 #define DEFAULT_AID_MODE                        FMAN_DMA_AID_OUT_TNUM
 233 #define DEFAULT_DMA_COMM_Q_LOW                  0x2A
 234 #define DEFAULT_DMA_COMM_Q_HIGH         0x3F
 235 #define DEFAULT_CACHE_OVERRIDE                  0
 236 #define DEFAULT_DMA_CAM_NUM_OF_ENTRIES          64
 237 #define DEFAULT_DMA_DBG_CNT_MODE                0
 238 #define DEFAULT_DMA_SOS_EMERGENCY               0
 239 #define DEFAULT_DMA_WATCHDOG                    0
 240 #define DEFAULT_DISP_LIMIT                      0
 241 #define DEFAULT_PRS_DISP_TH                     16
 242 #define DEFAULT_PLCR_DISP_TH                    16
 243 #define DEFAULT_KG_DISP_TH                      16
 244 #define DEFAULT_BMI_DISP_TH                     16
 245 #define DEFAULT_QMI_ENQ_DISP_TH         16
 246 #define DEFAULT_QMI_DEQ_DISP_TH         16
 247 #define DEFAULT_FM_CTL1_DISP_TH         16
 248 #define DEFAULT_FM_CTL2_DISP_TH         16
 249
 250 #define DFLT_AXI_DBG_NUM_OF_BEATS               1
 251
 252 #define DFLT_DMA_READ_INT_BUF_LOW(dma_thresh_max_buf)   \
 253         ((dma_thresh_max_buf + 1) / 2)
 254 #define DFLT_DMA_READ_INT_BUF_HIGH(dma_thresh_max_buf)  \
 255         ((dma_thresh_max_buf + 1) * 3 / 4)
 256 #define DFLT_DMA_WRITE_INT_BUF_LOW(dma_thresh_max_buf)  \
 257         ((dma_thresh_max_buf + 1) / 2)
 258 #define DFLT_DMA_WRITE_INT_BUF_HIGH(dma_thresh_max_buf)\
 259         ((dma_thresh_max_buf + 1) * 3 / 4)
 260
 261 #define DMA_COMM_Q_LOW_FMAN_V3          0x2A
 262 #define DMA_COMM_Q_LOW_FMAN_V2(dma_thresh_max_commq)            \
 263         ((dma_thresh_max_commq + 1) / 2)
 264 #define DFLT_DMA_COMM_Q_LOW(major, dma_thresh_max_commq)        \
 265         ((major == 6) ? DMA_COMM_Q_LOW_FMAN_V3 :                \
 266         DMA_COMM_Q_LOW_FMAN_V2(dma_thresh_max_commq))
 267
 268 #define DMA_COMM_Q_HIGH_FMAN_V3 0x3f
 269 #define DMA_COMM_Q_HIGH_FMAN_V2(dma_thresh_max_commq)           \
 270         ((dma_thresh_max_commq + 1) * 3 / 4)
 271 #define DFLT_DMA_COMM_Q_HIGH(major, dma_thresh_max_commq)       \
 272         ((major == 6) ? DMA_COMM_Q_HIGH_FMAN_V3 :               \
 273         DMA_COMM_Q_HIGH_FMAN_V2(dma_thresh_max_commq))
 274
 275 #define TOTAL_NUM_OF_TASKS_FMAN_V3L     59
 276 #define TOTAL_NUM_OF_TASKS_FMAN_V3H     124
 277 #define DFLT_TOTAL_NUM_OF_TASKS(major, minor, bmi_max_num_of_tasks)     \
 278         ((major == 6) ? ((minor == 1 || minor == 4) ?                   \
 279         TOTAL_NUM_OF_TASKS_FMAN_V3L : TOTAL_NUM_OF_TASKS_FMAN_V3H) :    \
 280         bmi_max_num_of_tasks)
 281
 282 #define DMA_CAM_NUM_OF_ENTRIES_FMAN_V3          64
 283 #define DMA_CAM_NUM_OF_ENTRIES_FMAN_V2          32
 284 #define DFLT_DMA_CAM_NUM_OF_ENTRIES(major)                      \
 285         (major == 6 ? DMA_CAM_NUM_OF_ENTRIES_FMAN_V3 :          \
 286         DMA_CAM_NUM_OF_ENTRIES_FMAN_V2)
 287
 288 #define FM_TIMESTAMP_1_USEC_BIT             8
 289
 290 /* Defines used for enabling/disabling FMan interrupts */
 291 #define ERR_INTR_EN_DMA         0x00010000
 292 #define ERR_INTR_EN_FPM         0x80000000
 293 #define ERR_INTR_EN_BMI         0x00800000
 294 #define ERR_INTR_EN_QMI         0x00400000
 295 #define ERR_INTR_EN_MURAM       0x00040000
 296 #define ERR_INTR_EN_MAC0        0x00004000
 297 #define ERR_INTR_EN_MAC1        0x00002000
 298 #define ERR_INTR_EN_MAC2        0x00001000
 299 #define ERR_INTR_EN_MAC3        0x00000800
 300 #define ERR_INTR_EN_MAC4        0x00000400
 301 #define ERR_INTR_EN_MAC5        0x00000200
 302 #define ERR_INTR_EN_MAC6        0x00000100
 303 #define ERR_INTR_EN_MAC7        0x00000080
 304 #define ERR_INTR_EN_MAC8        0x00008000
 305 #define ERR_INTR_EN_MAC9        0x00000040
 306
 307 #define INTR_EN_QMI             0x40000000
 308 #define INTR_EN_MAC0            0x00080000
 309 #define INTR_EN_MAC1            0x00040000
 310 #define INTR_EN_MAC2            0x00020000
 311 #define INTR_EN_MAC3            0x00010000
 312 #define INTR_EN_MAC4            0x00000040
 313 #define INTR_EN_MAC5            0x00000020
 314 #define INTR_EN_MAC6            0x00000008
 315 #define INTR_EN_MAC7            0x00000002
 316 #define INTR_EN_MAC8            0x00200000
 317 #define INTR_EN_MAC9            0x00100000
 318 #define INTR_EN_REV0            0x00008000
 319 #define INTR_EN_REV1            0x00004000
 320 #define INTR_EN_REV2            0x00002000
 321 #define INTR_EN_REV3            0x00001000
 322 #define INTR_EN_TMR             0x01000000
 323
 324 enum fman_dma_aid_mode {
 325         FMAN_DMA_AID_OUT_PORT_ID = 0,             /* 4 LSB of PORT_ID */
 326         FMAN_DMA_AID_OUT_TNUM                     /* 4 LSB of TNUM */
 327 };
 328
 329 struct fman_iram_regs {
 330         u32 iadd;       /* FM IRAM instruction address register */
 331         u32 idata;      /* FM IRAM instruction data register */
 332         u32 itcfg;      /* FM IRAM timing config register */
 333         u32 iready;     /* FM IRAM ready register */
 334 };
 335
 336 struct fman_fpm_regs {
 337         u32 fmfp_tnc;           /* FPM TNUM Control 0x00 */
 338         u32 fmfp_prc;           /* FPM Port_ID FmCtl Association 0x04 */
 339         u32 fmfp_brkc;          /* FPM Breakpoint Control 0x08 */
 340         u32 fmfp_mxd;           /* FPM Flush Control 0x0c */
 341         u32 fmfp_dist1;         /* FPM Dispatch Thresholds1 0x10 */
 342         u32 fmfp_dist2;         /* FPM Dispatch Thresholds2 0x14 */
 343         u32 fm_epi;             /* FM Error Pending Interrupts 0x18 */
 344         u32 fm_rie;             /* FM Error Interrupt Enable 0x1c */
 345         u32 fmfp_fcev[4];       /* FPM FMan-Controller Event 1-4 0x20-0x2f */
 346         u32 res0030[4];         /* res 0x30 - 0x3f */
 347         u32 fmfp_cee[4];        /* PM FMan-Controller Event 1-4 0x40-0x4f */
 348         u32 res0050[4];         /* res 0x50-0x5f */
 349         u32 fmfp_tsc1;          /* FPM TimeStamp Control1 0x60 */
 350         u32 fmfp_tsc2;          /* FPM TimeStamp Control2 0x64 */
 351         u32 fmfp_tsp;           /* FPM Time Stamp 0x68 */
 352         u32 fmfp_tsf;           /* FPM Time Stamp Fraction 0x6c */
 353         u32 fm_rcr;             /* FM Rams Control 0x70 */
 354         u32 fmfp_extc;          /* FPM External Requests Control 0x74 */
 355         u32 fmfp_ext1;          /* FPM External Requests Config1 0x78 */
 356         u32 fmfp_ext2;          /* FPM External Requests Config2 0x7c */
 357         u32 fmfp_drd[16];       /* FPM Data_Ram Data 0-15 0x80 - 0xbf */
 358         u32 fmfp_dra;           /* FPM Data Ram Access 0xc0 */
 359         u32 fm_ip_rev_1;        /* FM IP Block Revision 1 0xc4 */
 360         u32 fm_ip_rev_2;        /* FM IP Block Revision 2 0xc8 */
 361         u32 fm_rstc;            /* FM Reset Command 0xcc */
 362         u32 fm_cld;             /* FM Classifier Debug 0xd0 */
 363         u32 fm_npi;             /* FM Normal Pending Interrupts 0xd4 */
 364         u32 fmfp_exte;          /* FPM External Requests Enable 0xd8 */
 365         u32 fmfp_ee;            /* FPM Event&Mask 0xdc */
 366         u32 fmfp_cev[4];        /* FPM CPU Event 1-4 0xe0-0xef */
 367         u32 res00f0[4];         /* res 0xf0-0xff */
 368         u32 fmfp_ps[50];        /* FPM Port Status 0x100-0x1c7 */
 369         u32 res01c8[14];        /* res 0x1c8-0x1ff */
 370         u32 fmfp_clfabc;        /* FPM CLFABC 0x200 */
 371         u32 fmfp_clfcc;         /* FPM CLFCC 0x204 */
 372         u32 fmfp_clfaval;       /* FPM CLFAVAL 0x208 */
 373         u32 fmfp_clfbval;       /* FPM CLFBVAL 0x20c */
 374         u32 fmfp_clfcval;       /* FPM CLFCVAL 0x210 */
 375         u32 fmfp_clfamsk;       /* FPM CLFAMSK 0x214 */
 376         u32 fmfp_clfbmsk;       /* FPM CLFBMSK 0x218 */
 377         u32 fmfp_clfcmsk;       /* FPM CLFCMSK 0x21c */
 378         u32 fmfp_clfamc;        /* FPM CLFAMC 0x220 */
 379         u32 fmfp_clfbmc;        /* FPM CLFBMC 0x224 */
 380         u32 fmfp_clfcmc;        /* FPM CLFCMC 0x228 */
 381         u32 fmfp_decceh;        /* FPM DECCEH 0x22c */
 382         u32 res0230[116];       /* res 0x230 - 0x3ff */
 383         u32 fmfp_ts[128];       /* 0x400: FPM Task Status 0x400 - 0x5ff */
 384         u32 res0600[0x400 - 384];
 385 };
 386
 387 struct fman_bmi_regs {
 388         u32 fmbm_init;          /* BMI Initialization 0x00 */
 389         u32 fmbm_cfg1;          /* BMI Configuration 1 0x04 */
 390         u32 fmbm_cfg2;          /* BMI Configuration 2 0x08 */
 391         u32 res000c[5];         /* 0x0c - 0x1f */
 392         u32 fmbm_ievr;          /* Interrupt Event Register 0x20 */
 393         u32 fmbm_ier;           /* Interrupt Enable Register 0x24 */
 394         u32 fmbm_ifr;           /* Interrupt Force Register 0x28 */
 395         u32 res002c[5];         /* 0x2c - 0x3f */
 396         u32 fmbm_arb[8];        /* BMI Arbitration 0x40 - 0x5f */
 397         u32 res0060[12];        /* 0x60 - 0x8f */
 398         u32 fmbm_dtc[3];        /* Debug Trap Counter 0x90 - 0x9b */
 399         u32 res009c;            /* 0x9c */
 400         u32 fmbm_dcv[3][4];     /* Debug Compare val 0xa0-0xcf */
 401         u32 fmbm_dcm[3][4];     /* Debug Compare Mask 0xd0-0xff */
 402         u32 fmbm_gde;           /* BMI Global Debug Enable 0x100 */
 403         u32 fmbm_pp[63];        /* BMI Port Parameters 0x104 - 0x1ff */
 404         u32 res0200;            /* 0x200 */
 405         u32 fmbm_pfs[63];       /* BMI Port FIFO Size 0x204 - 0x2ff */
 406         u32 res0300;            /* 0x300 */
 407         u32 fmbm_spliodn[63];   /* Port Partition ID 0x304 - 0x3ff */
 408 };
 409
 410 struct fman_qmi_regs {
 411         u32 fmqm_gc;            /* General Configuration Register 0x00 */
 412         u32 res0004;            /* 0x04 */
 413         u32 fmqm_eie;           /* Error Interrupt Event Register 0x08 */
 414         u32 fmqm_eien;          /* Error Interrupt Enable Register 0x0c */
 415         u32 fmqm_eif;           /* Error Interrupt Force Register 0x10 */
 416         u32 fmqm_ie;            /* Interrupt Event Register 0x14 */
 417         u32 fmqm_ien;           /* Interrupt Enable Register 0x18 */
 418         u32 fmqm_if;            /* Interrupt Force Register 0x1c */
 419         u32 fmqm_gs;            /* Global Status Register 0x20 */
 420         u32 fmqm_ts;            /* Task Status Register 0x24 */
 421         u32 fmqm_etfc;          /* Enqueue Total Frame Counter 0x28 */
 422         u32 fmqm_dtfc;          /* Dequeue Total Frame Counter 0x2c */
 423         u32 fmqm_dc0;           /* Dequeue Counter 0 0x30 */
 424         u32 fmqm_dc1;           /* Dequeue Counter 1 0x34 */
 425         u32 fmqm_dc2;           /* Dequeue Counter 2 0x38 */
 426         u32 fmqm_dc3;           /* Dequeue Counter 3 0x3c */
 427         u32 fmqm_dfdc;          /* Dequeue FQID from Default Counter 0x40 */
 428         u32 fmqm_dfcc;          /* Dequeue FQID from Context Counter 0x44 */
 429         u32 fmqm_dffc;          /* Dequeue FQID from FD Counter 0x48 */
 430         u32 fmqm_dcc;           /* Dequeue Confirm Counter 0x4c */
 431         u32 res0050[7];         /* 0x50 - 0x6b */
 432         u32 fmqm_tapc;          /* Tnum Aging Period Control 0x6c */
 433         u32 fmqm_dmcvc;         /* Dequeue MAC Command Valid Counter 0x70 */
 434         u32 fmqm_difdcc;        /* Dequeue Invalid FD Command Counter 0x74 */
 435         u32 fmqm_da1v;          /* Dequeue A1 Valid Counter 0x78 */
 436         u32 res007c;            /* 0x7c */
 437         u32 fmqm_dtc;           /* 0x80 Debug Trap Counter 0x80 */
 438         u32 fmqm_efddd;         /* 0x84 Enqueue Frame desc Dynamic dbg 0x84 */
 439         u32 res0088[2];         /* 0x88 - 0x8f */
 440         struct {
 441                 u32 fmqm_dtcfg1;        /* 0x90 dbg trap cfg 1 Register 0x00 */
 442                 u32 fmqm_dtval1;        /* Debug Trap Value 1 Register 0x04 */
 443                 u32 fmqm_dtm1;          /* Debug Trap Mask 1 Register 0x08 */
 444                 u32 fmqm_dtc1;          /* Debug Trap Counter 1 Register 0x0c */
 445                 u32 fmqm_dtcfg2;        /* dbg Trap cfg 2 Register 0x10 */
 446                 u32 fmqm_dtval2;        /* Debug Trap Value 2 Register 0x14 */
 447                 u32 fmqm_dtm2;          /* Debug Trap Mask 2 Register 0x18 */
 448                 u32 res001c;            /* 0x1c */
 449         } dbg_traps[3];                 /* 0x90 - 0xef */
 450         u8 res00f0[0x400 - 0xf0];       /* 0xf0 - 0x3ff */
 451 };
 452
 453 struct fman_dma_regs {
 454         u32 fmdmsr;     /* FM DMA status register 0x00 */
 455         u32 fmdmmr;     /* FM DMA mode register 0x04 */
 456         u32 fmdmtr;     /* FM DMA bus threshold register 0x08 */
 457         u32 fmdmhy;     /* FM DMA bus hysteresis register 0x0c */
 458         u32 fmdmsetr;   /* FM DMA SOS emergency Threshold Register 0x10 */
 459         u32 fmdmtah;    /* FM DMA transfer bus address high reg 0x14 */
 460         u32 fmdmtal;    /* FM DMA transfer bus address low reg 0x18 */
 461         u32 fmdmtcid;   /* FM DMA transfer bus communication ID reg 0x1c */
 462         u32 fmdmra;     /* FM DMA bus internal ram address register 0x20 */
 463         u32 fmdmrd;     /* FM DMA bus internal ram data register 0x24 */
 464         u32 fmdmwcr;    /* FM DMA CAM watchdog counter value 0x28 */
 465         u32 fmdmebcr;   /* FM DMA CAM base in MURAM register 0x2c */
 466         u32 fmdmccqdr;  /* FM DMA CAM and CMD Queue Debug reg 0x30 */
 467         u32 fmdmccqvr1; /* FM DMA CAM and CMD Queue Value reg #1 0x34 */
 468         u32 fmdmccqvr2; /* FM DMA CAM and CMD Queue Value reg #2 0x38 */
 469         u32 fmdmcqvr3;  /* FM DMA CMD Queue Value register #3 0x3c */
 470         u32 fmdmcqvr4;  /* FM DMA CMD Queue Value register #4 0x40 */
 471         u32 fmdmcqvr5;  /* FM DMA CMD Queue Value register #5 0x44 */
 472         u32 fmdmsefrc;  /* FM DMA Semaphore Entry Full Reject Cntr 0x48 */
 473         u32 fmdmsqfrc;  /* FM DMA Semaphore Queue Full Reject Cntr 0x4c */
 474         u32 fmdmssrc;   /* FM DMA Semaphore SYNC Reject Counter 0x50 */
 475         u32 fmdmdcr;    /* FM DMA Debug Counter 0x54 */
 476         u32 fmdmemsr;   /* FM DMA Emergency Smoother Register 0x58 */
 477         u32 res005c;    /* 0x5c */
 478         u32 fmdmplr[FMAN_LIODN_TBL / 2];        /* DMA LIODN regs 0x60-0xdf */
 479         u32 res00e0[0x400 - 56];
 480 };
 481
 482 struct fman_hwp_regs {
 483         u32 res0000[0x844 / 4];         /* 0x000..0x843 */
 484         u32 fmprrpimac; /* FM Parser Internal memory access control */
 485         u32 res[(0x1000 - 0x848) / 4];  /* 0x848..0xFFF */
 486 };
 487
 488 /* Structure that holds current FMan state.
 489  * Used for saving run time information.
 490  */
 491 struct fman_state_struct {
 492         u8 fm_id;
 493         u16 fm_clk_freq;
 494         struct fman_rev_info rev_info;
 495         bool enabled_time_stamp;
 496         u8 count1_micro_bit;
 497         u8 total_num_of_tasks;
 498         u8 accumulated_num_of_tasks;
 499         u32 accumulated_fifo_size;
 500         u8 accumulated_num_of_open_dmas;
 501         u8 accumulated_num_of_deq_tnums;
 502         u32 exceptions;
 503         u32 extra_fifo_pool_size;
 504         u8 extra_tasks_pool_size;
 505         u8 extra_open_dmas_pool_size;
 506         u16 port_mfl[MAX_NUM_OF_MACS];
 507         u16 mac_mfl[MAX_NUM_OF_MACS];
 508
 509         /* SOC specific */
 510         u32 fm_iram_size;
 511         /* DMA */
 512         u32 dma_thresh_max_commq;
 513         u32 dma_thresh_max_buf;
 514         u32 max_num_of_open_dmas;
 515         /* QMI */
 516         u32 qmi_max_num_of_tnums;
 517         u32 qmi_def_tnums_thresh;
 518         /* BMI */
 519         u32 bmi_max_num_of_tasks;
 520         u32 bmi_max_fifo_size;
 521         /* General */
 522         u32 fm_port_num_of_cg;
 523         u32 num_of_rx_ports;
 524         u32 total_fifo_size;
 525
 526         u32 qman_channel_base;
 527         u32 num_of_qman_channels;
 528
 529         struct resource *res;
 530 };
 531
 532 /* Structure that holds FMan initial configuration */
 533 struct fman_cfg {
 534         u8 disp_limit_tsh;
 535         u8 prs_disp_tsh;
 536         u8 plcr_disp_tsh;
 537         u8 kg_disp_tsh;
 538         u8 bmi_disp_tsh;
 539         u8 qmi_enq_disp_tsh;
 540         u8 qmi_deq_disp_tsh;
 541         u8 fm_ctl1_disp_tsh;
 542         u8 fm_ctl2_disp_tsh;
 543         int dma_cache_override;
 544         enum fman_dma_aid_mode dma_aid_mode;
 545         u32 dma_axi_dbg_num_of_beats;
 546         u32 dma_cam_num_of_entries;
 547         u32 dma_watchdog;
 548         u8 dma_comm_qtsh_asrt_emer;
 549         u32 dma_write_buf_tsh_asrt_emer;
 550         u32 dma_read_buf_tsh_asrt_emer;
 551         u8 dma_comm_qtsh_clr_emer;
 552         u32 dma_write_buf_tsh_clr_emer;
 553         u32 dma_read_buf_tsh_clr_emer;
 554         u32 dma_sos_emergency;
 555         int dma_dbg_cnt_mode;
 556         int catastrophic_err;
 557         int dma_err;
 558         u32 exceptions;
 559         u16 clk_freq;
 560         u32 cam_base_addr;
 561         u32 fifo_base_addr;
 562         u32 total_fifo_size;
 563         u32 total_num_of_tasks;
 564         u32 qmi_def_tnums_thresh;
 565 };
 566
 567 /* Structure that holds information received from device tree */
 568 struct fman_dts_params {
 569         void __iomem *base_addr;                /* FMan virtual address */
 570         struct resource *res;                   /* FMan memory resource */
 571         u8 id;                                  /* FMan ID */
 572
 573         int err_irq;                            /* FMan Error IRQ */
 574
 575         u16 clk_freq;                           /* FMan clock freq (In Mhz) */
 576
 577         u32 qman_channel_base;                  /* QMan channels base */
 578         u32 num_of_qman_channels;               /* Number of QMan channels */
 579
 580         struct resource muram_res;              /* MURAM resource */
 581 };
 582
 583 /** fman_exceptions_cb
 584  * fman         - Pointer to FMan
 585  * exception    - The exception.
 586  *
 587  * Exceptions user callback routine, will be called upon an exception
 588  * passing the exception identification.
 589  *
 590  * Return: irq status
 591  */
 592 typedef irqreturn_t (fman_exceptions_cb)(struct fman *fman,
 593                                          enum fman_exceptions exception);
 594
 595 /** fman_bus_error_cb
 596  * fman         - Pointer to FMan
 597  * port_id      - Port id
 598  * addr         - Address that caused the error
 599  * tnum         - Owner of error
 600  * liodn        - Logical IO device number
 601  *
 602  * Bus error user callback routine, will be called upon bus error,
 603  * passing parameters describing the errors and the owner.
 604  *
 605  * Return: IRQ status
 606  */
 607 typedef irqreturn_t (fman_bus_error_cb)(struct fman *fman, u8 port_id,
 608                                         u64 addr, u8 tnum, u16 liodn);
 609
 610 struct fman {
 611         struct device *dev;
 612         void __iomem *base_addr;
 613         struct fman_intr_src intr_mng[FMAN_EV_CNT];
 614
 615         struct fman_fpm_regs __iomem *fpm_regs;
 616         struct fman_bmi_regs __iomem *bmi_regs;
 617         struct fman_qmi_regs __iomem *qmi_regs;
 618         struct fman_dma_regs __iomem *dma_regs;
 619         struct fman_hwp_regs __iomem *hwp_regs;
 620         fman_exceptions_cb *exception_cb;
 621         fman_bus_error_cb *bus_error_cb;
 622         /* Spinlock for FMan use */
 623         spinlock_t spinlock;
 624         struct fman_state_struct *state;
 625
 626         struct fman_cfg *cfg;
 627         struct muram_info *muram;
 628         /* cam section in muram */
 629         unsigned long cam_offset;
 630         size_t cam_size;
 631         /* Fifo in MURAM */
 632         unsigned long fifo_offset;
 633         size_t fifo_size;
 634
 635         u32 liodn_base[64];
 636         u32 liodn_offset[64];
 637
 638         struct fman_dts_params dts_params;
 639 };
 640
 641 static irqreturn_t fman_exceptions(struct fman *fman,
 642                                    enum fman_exceptions exception)
 643 {
 644         dev_dbg(fman->dev, "%s: FMan[%d] exception %d\n",
 645                 __func__, fman->state->fm_id, exception);
 646
 647         return IRQ_HANDLED;
 648 }
 649
 650 static irqreturn_t fman_bus_error(struct fman *fman, u8 __maybe_unused port_id,
 651                                   u64 __maybe_unused addr,
 652                                   u8 __maybe_unused tnum,
 653                                   u16 __maybe_unused liodn)
 654 {
 655         dev_dbg(fman->dev, "%s: FMan[%d] bus error: port_id[%d]\n",
 656                 __func__, fman->state->fm_id, port_id);
 657
 658         return IRQ_HANDLED;
 659 }
 660
 661 static inline irqreturn_t call_mac_isr(struct fman *fman, u8 id)
 662 {
 663         if (fman->intr_mng[id].isr_cb) {
 664                 fman->intr_mng[id].isr_cb(fman->intr_mng[id].src_handle);
 665
 666                 return IRQ_HANDLED;
 667         }
 668
 669         return IRQ_NONE;
 670 }
 671
 672 static inline u8 hw_port_id_to_sw_port_id(u8 major, u8 hw_port_id)
 673 {
 674         u8 sw_port_id = 0;
 675
 676         if (hw_port_id >= BASE_TX_PORTID)
 677                 sw_port_id = hw_port_id - BASE_TX_PORTID;
 678         else if (hw_port_id >= BASE_RX_PORTID)
 679                 sw_port_id = hw_port_id - BASE_RX_PORTID;
 680         else
 681                 sw_port_id = 0;
 682
 683         return sw_port_id;
 684 }
 685
 686 static void set_port_order_restoration(struct fman_fpm_regs __iomem *fpm_rg,
 687                                        u8 port_id)
 688 {
 689         u32 tmp = 0;
 690
 691         tmp = port_id << FPM_PORT_FM_CTL_PORTID_SHIFT;
 692
 693         tmp |= FPM_PRT_FM_CTL2 | FPM_PRT_FM_CTL1;
 694
 695         /* order restoration */
 696         if (port_id % 2)
 697                 tmp |= FPM_PRT_FM_CTL1 << FPM_PRC_ORA_FM_CTL_SEL_SHIFT;
 698         else
 699                 tmp |= FPM_PRT_FM_CTL2 << FPM_PRC_ORA_FM_CTL_SEL_SHIFT;
 700
 701         iowrite32be(tmp, &fpm_rg->fmfp_prc);
 702 }
 703
 704 static void set_port_liodn(struct fman *fman, u8 port_id,
 705                            u32 liodn_base, u32 liodn_ofst)
 706 {
 707         u32 tmp;
 708
 709         /* set LIODN base for this port */
 710         tmp = ioread32be(&fman->dma_regs->fmdmplr[port_id / 2]);
 711         if (port_id % 2) {
 712                 tmp &= ~DMA_LIODN_BASE_MASK;
 713                 tmp |= liodn_base;
 714         } else {
 715                 tmp &= ~(DMA_LIODN_BASE_MASK << DMA_LIODN_SHIFT);
 716                 tmp |= liodn_base << DMA_LIODN_SHIFT;
 717         }
 718         iowrite32be(tmp, &fman->dma_regs->fmdmplr[port_id / 2]);
 719         iowrite32be(liodn_ofst, &fman->bmi_regs->fmbm_spliodn[port_id - 1]);
 720 }
 721
 722 static void enable_rams_ecc(struct fman_fpm_regs __iomem *fpm_rg)
 723 {
 724         u32 tmp;
 725
 726         tmp = ioread32be(&fpm_rg->fm_rcr);
 727         if (tmp & FPM_RAM_RAMS_ECC_EN_SRC_SEL)
 728                 iowrite32be(tmp | FPM_RAM_IRAM_ECC_EN, &fpm_rg->fm_rcr);
 729         else
 730                 iowrite32be(tmp | FPM_RAM_RAMS_ECC_EN |
 731                             FPM_RAM_IRAM_ECC_EN, &fpm_rg->fm_rcr);
 732 }
 733
 734 static void disable_rams_ecc(struct fman_fpm_regs __iomem *fpm_rg)
 735 {
 736         u32 tmp;
 737
 738         tmp = ioread32be(&fpm_rg->fm_rcr);
 739         if (tmp & FPM_RAM_RAMS_ECC_EN_SRC_SEL)
 740                 iowrite32be(tmp & ~FPM_RAM_IRAM_ECC_EN, &fpm_rg->fm_rcr);
 741         else
 742                 iowrite32be(tmp & ~(FPM_RAM_RAMS_ECC_EN | FPM_RAM_IRAM_ECC_EN),
 743                             &fpm_rg->fm_rcr);
 744 }
 745
 746 static void fman_defconfig(struct fman_cfg *cfg)
 747 {
 748         memset(cfg, 0, sizeof(struct fman_cfg));
 749
 750         cfg->catastrophic_err = DEFAULT_CATASTROPHIC_ERR;
 751         cfg->dma_err = DEFAULT_DMA_ERR;
 752         cfg->dma_aid_mode = DEFAULT_AID_MODE;
 753         cfg->dma_comm_qtsh_clr_emer = DEFAULT_DMA_COMM_Q_LOW;
 754         cfg->dma_comm_qtsh_asrt_emer = DEFAULT_DMA_COMM_Q_HIGH;
 755         cfg->dma_cache_override = DEFAULT_CACHE_OVERRIDE;
 756         cfg->dma_cam_num_of_entries = DEFAULT_DMA_CAM_NUM_OF_ENTRIES;
 757         cfg->dma_dbg_cnt_mode = DEFAULT_DMA_DBG_CNT_MODE;
 758         cfg->dma_sos_emergency = DEFAULT_DMA_SOS_EMERGENCY;
 759         cfg->dma_watchdog = DEFAULT_DMA_WATCHDOG;
 760         cfg->disp_limit_tsh = DEFAULT_DISP_LIMIT;
 761         cfg->prs_disp_tsh = DEFAULT_PRS_DISP_TH;
 762         cfg->plcr_disp_tsh = DEFAULT_PLCR_DISP_TH;
 763         cfg->kg_disp_tsh = DEFAULT_KG_DISP_TH;
 764         cfg->bmi_disp_tsh = DEFAULT_BMI_DISP_TH;
 765         cfg->qmi_enq_disp_tsh = DEFAULT_QMI_ENQ_DISP_TH;
 766         cfg->qmi_deq_disp_tsh = DEFAULT_QMI_DEQ_DISP_TH;
 767         cfg->fm_ctl1_disp_tsh = DEFAULT_FM_CTL1_DISP_TH;
 768         cfg->fm_ctl2_disp_tsh = DEFAULT_FM_CTL2_DISP_TH;
 769 }
 770
 771 static int dma_init(struct fman *fman)
 772 {
 773         struct fman_dma_regs __iomem *dma_rg = fman->dma_regs;
 774         struct fman_cfg *cfg = fman->cfg;
 775         u32 tmp_reg;
 776
 777         /* Init DMA Registers */
 778
 779         /* clear status reg events */
 780         tmp_reg = (DMA_STATUS_BUS_ERR | DMA_STATUS_READ_ECC |
 781                    DMA_STATUS_SYSTEM_WRITE_ECC | DMA_STATUS_FM_WRITE_ECC);
 782         iowrite32be(ioread32be(&dma_rg->fmdmsr) | tmp_reg, &dma_rg->fmdmsr);
 783
 784         /* configure mode register */
 785         tmp_reg = 0;
 786         tmp_reg |= cfg->dma_cache_override << DMA_MODE_CACHE_OR_SHIFT;
 787         if (cfg->exceptions & EX_DMA_BUS_ERROR)
 788                 tmp_reg |= DMA_MODE_BER;
 789         if ((cfg->exceptions & EX_DMA_SYSTEM_WRITE_ECC) |
 790             (cfg->exceptions & EX_DMA_READ_ECC) |
 791             (cfg->exceptions & EX_DMA_FM_WRITE_ECC))
 792                 tmp_reg |= DMA_MODE_ECC;
 793         if (cfg->dma_axi_dbg_num_of_beats)
 794                 tmp_reg |= (DMA_MODE_AXI_DBG_MASK &
 795                         ((cfg->dma_axi_dbg_num_of_beats - 1)
 796                         << DMA_MODE_AXI_DBG_SHIFT));
 797
 798         tmp_reg |= (((cfg->dma_cam_num_of_entries / DMA_CAM_UNITS) - 1) &
 799                 DMA_MODE_CEN_MASK) << DMA_MODE_CEN_SHIFT;
 800         tmp_reg |= DMA_MODE_SECURE_PROT;
 801         tmp_reg |= cfg->dma_dbg_cnt_mode << DMA_MODE_DBG_SHIFT;
 802         tmp_reg |= cfg->dma_aid_mode << DMA_MODE_AID_MODE_SHIFT;
 803
 804         iowrite32be(tmp_reg, &dma_rg->fmdmmr);
 805
 806         /* configure thresholds register */
 807         tmp_reg = ((u32)cfg->dma_comm_qtsh_asrt_emer <<
 808                 DMA_THRESH_COMMQ_SHIFT);
 809         tmp_reg |= (cfg->dma_read_buf_tsh_asrt_emer &
 810                 DMA_THRESH_READ_INT_BUF_MASK) << DMA_THRESH_READ_INT_BUF_SHIFT;
 811         tmp_reg |= cfg->dma_write_buf_tsh_asrt_emer &
 812                 DMA_THRESH_WRITE_INT_BUF_MASK;
 813
 814         iowrite32be(tmp_reg, &dma_rg->fmdmtr);
 815
 816         /* configure hysteresis register */
 817         tmp_reg = ((u32)cfg->dma_comm_qtsh_clr_emer <<
 818                 DMA_THRESH_COMMQ_SHIFT);
 819         tmp_reg |= (cfg->dma_read_buf_tsh_clr_emer &
 820                 DMA_THRESH_READ_INT_BUF_MASK) << DMA_THRESH_READ_INT_BUF_SHIFT;
 821         tmp_reg |= cfg->dma_write_buf_tsh_clr_emer &
 822                 DMA_THRESH_WRITE_INT_BUF_MASK;
 823
 824         iowrite32be(tmp_reg, &dma_rg->fmdmhy);
 825
 826         /* configure emergency threshold */
 827         iowrite32be(cfg->dma_sos_emergency, &dma_rg->fmdmsetr);
 828
 829         /* configure Watchdog */
 830         iowrite32be((cfg->dma_watchdog * cfg->clk_freq), &dma_rg->fmdmwcr);
 831
 832         iowrite32be(cfg->cam_base_addr, &dma_rg->fmdmebcr);
 833
 834         /* Allocate MURAM for CAM */
 835         fman->cam_size =
 836                 (u32)(fman->cfg->dma_cam_num_of_entries * DMA_CAM_SIZEOF_ENTRY);
 837         fman->cam_offset = fman_muram_alloc(fman->muram, fman->cam_size);
 838         if (IS_ERR_VALUE(fman->cam_offset)) {
 839                 dev_err(fman->dev, "%s: MURAM alloc for DMA CAM failed\n",
 840                         __func__);
 841                 return -ENOMEM;
 842         }
 843
 844         if (fman->state->rev_info.major == 2) {
 845                 u32 __iomem *cam_base_addr;
 846
 847                 fman_muram_free_mem(fman->muram, fman->cam_offset,
 848                                     fman->cam_size);
 849
 850                 fman->cam_size = fman->cfg->dma_cam_num_of_entries * 72 + 128;
 851                 fman->cam_offset = fman_muram_alloc(fman->muram,
 852                                                     fman->cam_size);
 853                 if (IS_ERR_VALUE(fman->cam_offset)) {
 854                         dev_err(fman->dev, "%s: MURAM alloc for DMA CAM failed\n",
 855                                 __func__);
 856                         return -ENOMEM;
 857                 }
 858
 859                 if (fman->cfg->dma_cam_num_of_entries % 8 ||
 860                     fman->cfg->dma_cam_num_of_entries > 32) {
 861                         dev_err(fman->dev, "%s: wrong dma_cam_num_of_entries\n",
 862                                 __func__);
 863                         return -EINVAL;
 864                 }
 865
 866                 cam_base_addr = (u32 __iomem *)
 867                         fman_muram_offset_to_vbase(fman->muram,
 868                                                    fman->cam_offset);
 869                 iowrite32be(~((1 <<
 870                             (32 - fman->cfg->dma_cam_num_of_entries)) - 1),
 871                             cam_base_addr);
 872         }
 873
 874         fman->cfg->cam_base_addr = fman->cam_offset;
 875
 876         return 0;
 877 }
 878
 879 static void fpm_init(struct fman_fpm_regs __iomem *fpm_rg, struct fman_cfg *cfg)
 880 {
 881         u32 tmp_reg;
 882         int i;
 883
 884         /* Init FPM Registers */
 885
 886         tmp_reg = (u32)(cfg->disp_limit_tsh << FPM_DISP_LIMIT_SHIFT);
 887         iowrite32be(tmp_reg, &fpm_rg->fmfp_mxd);
 888
 889         tmp_reg = (((u32)cfg->prs_disp_tsh << FPM_THR1_PRS_SHIFT) |
 890                    ((u32)cfg->kg_disp_tsh << FPM_THR1_KG_SHIFT) |
 891                    ((u32)cfg->plcr_disp_tsh << FPM_THR1_PLCR_SHIFT) |
 892                    ((u32)cfg->bmi_disp_tsh << FPM_THR1_BMI_SHIFT));
 893         iowrite32be(tmp_reg, &fpm_rg->fmfp_dist1);
 894
 895         tmp_reg =
 896                 (((u32)cfg->qmi_enq_disp_tsh << FPM_THR2_QMI_ENQ_SHIFT) |
 897                  ((u32)cfg->qmi_deq_disp_tsh << FPM_THR2_QMI_DEQ_SHIFT) |
 898                  ((u32)cfg->fm_ctl1_disp_tsh << FPM_THR2_FM_CTL1_SHIFT) |
 899                  ((u32)cfg->fm_ctl2_disp_tsh << FPM_THR2_FM_CTL2_SHIFT));
 900         iowrite32be(tmp_reg, &fpm_rg->fmfp_dist2);
 901
 902         /* define exceptions and error behavior */
 903         tmp_reg = 0;
 904         /* Clear events */
 905         tmp_reg |= (FPM_EV_MASK_STALL | FPM_EV_MASK_DOUBLE_ECC |
 906                     FPM_EV_MASK_SINGLE_ECC);
 907         /* enable interrupts */
 908         if (cfg->exceptions & EX_FPM_STALL_ON_TASKS)
 909                 tmp_reg |= FPM_EV_MASK_STALL_EN;
 910         if (cfg->exceptions & EX_FPM_SINGLE_ECC)
 911                 tmp_reg |= FPM_EV_MASK_SINGLE_ECC_EN;
 912         if (cfg->exceptions & EX_FPM_DOUBLE_ECC)
 913                 tmp_reg |= FPM_EV_MASK_DOUBLE_ECC_EN;
 914         tmp_reg |= (cfg->catastrophic_err << FPM_EV_MASK_CAT_ERR_SHIFT);
 915         tmp_reg |= (cfg->dma_err << FPM_EV_MASK_DMA_ERR_SHIFT);
 916         /* FMan is not halted upon external halt activation */
 917         tmp_reg |= FPM_EV_MASK_EXTERNAL_HALT;
 918         /* Man is not halted upon  Unrecoverable ECC error behavior */
 919         tmp_reg |= FPM_EV_MASK_ECC_ERR_HALT;
 920         iowrite32be(tmp_reg, &fpm_rg->fmfp_ee);
 921
 922         /* clear all fmCtls event registers */
 923         for (i = 0; i < FM_NUM_OF_FMAN_CTRL_EVENT_REGS; i++)
 924                 iowrite32be(0xFFFFFFFF, &fpm_rg->fmfp_cev[i]);
 925
 926         /* RAM ECC -  enable and clear events */
 927         /* first we need to clear all parser memory,
 928          * as it is uninitialized and may cause ECC errors
 929          */
 930         /* event bits */
 931         tmp_reg = (FPM_RAM_MURAM_ECC | FPM_RAM_IRAM_ECC);
 932
 933         iowrite32be(tmp_reg, &fpm_rg->fm_rcr);
 934
 935         tmp_reg = 0;
 936         if (cfg->exceptions & EX_IRAM_ECC) {
 937                 tmp_reg |= FPM_IRAM_ECC_ERR_EX_EN;
 938                 enable_rams_ecc(fpm_rg);
 939         }
 940         if (cfg->exceptions & EX_MURAM_ECC) {
 941                 tmp_reg |= FPM_MURAM_ECC_ERR_EX_EN;
 942                 enable_rams_ecc(fpm_rg);
 943         }
 944         iowrite32be(tmp_reg, &fpm_rg->fm_rie);
 945 }
 946
 947 static void bmi_init(struct fman_bmi_regs __iomem *bmi_rg,
 948                      struct fman_cfg *cfg)
 949 {
 950         u32 tmp_reg;
 951
 952         /* Init BMI Registers */
 953
 954         /* define common resources */
 955         tmp_reg = cfg->fifo_base_addr;
 956         tmp_reg = tmp_reg / BMI_FIFO_ALIGN;
 957
 958         tmp_reg |= ((cfg->total_fifo_size / FMAN_BMI_FIFO_UNITS - 1) <<
 959                     BMI_CFG1_FIFO_SIZE_SHIFT);
 960         iowrite32be(tmp_reg, &bmi_rg->fmbm_cfg1);
 961
 962         tmp_reg = ((cfg->total_num_of_tasks - 1) & BMI_CFG2_TASKS_MASK) <<
 963                    BMI_CFG2_TASKS_SHIFT;
 964         /* num of DMA's will be dynamically updated when each port is set */
 965         iowrite32be(tmp_reg, &bmi_rg->fmbm_cfg2);
 966
 967         /* define unmaskable exceptions, enable and clear events */
 968         tmp_reg = 0;
 969         iowrite32be(BMI_ERR_INTR_EN_LIST_RAM_ECC |
 970                     BMI_ERR_INTR_EN_STORAGE_PROFILE_ECC |
 971                     BMI_ERR_INTR_EN_STATISTICS_RAM_ECC |
 972                     BMI_ERR_INTR_EN_DISPATCH_RAM_ECC, &bmi_rg->fmbm_ievr);
 973
 974         if (cfg->exceptions & EX_BMI_LIST_RAM_ECC)
 975                 tmp_reg |= BMI_ERR_INTR_EN_LIST_RAM_ECC;
 976         if (cfg->exceptions & EX_BMI_STORAGE_PROFILE_ECC)
 977                 tmp_reg |= BMI_ERR_INTR_EN_STORAGE_PROFILE_ECC;
 978         if (cfg->exceptions & EX_BMI_STATISTICS_RAM_ECC)
 979                 tmp_reg |= BMI_ERR_INTR_EN_STATISTICS_RAM_ECC;
 980         if (cfg->exceptions & EX_BMI_DISPATCH_RAM_ECC)
 981                 tmp_reg |= BMI_ERR_INTR_EN_DISPATCH_RAM_ECC;
 982         iowrite32be(tmp_reg, &bmi_rg->fmbm_ier);
 983 }
 984
 985 static void qmi_init(struct fman_qmi_regs __iomem *qmi_rg,
 986                      struct fman_cfg *cfg)
 987 {
 988         u32 tmp_reg;
 989
 990         /* Init QMI Registers */
 991
 992         /* Clear error interrupt events */
 993
 994         iowrite32be(QMI_ERR_INTR_EN_DOUBLE_ECC | QMI_ERR_INTR_EN_DEQ_FROM_DEF,
 995                     &qmi_rg->fmqm_eie);
 996         tmp_reg = 0;
 997         if (cfg->exceptions & EX_QMI_DEQ_FROM_UNKNOWN_PORTID)
 998                 tmp_reg |= QMI_ERR_INTR_EN_DEQ_FROM_DEF;
 999         if (cfg->exceptions & EX_QMI_DOUBLE_ECC)
1000                 tmp_reg |= QMI_ERR_INTR_EN_DOUBLE_ECC;
1001         /* enable events */
1002         iowrite32be(tmp_reg, &qmi_rg->fmqm_eien);
1003
1004         tmp_reg = 0;
1005         /* Clear interrupt events */
1006         iowrite32be(QMI_INTR_EN_SINGLE_ECC, &qmi_rg->fmqm_ie);
1007         if (cfg->exceptions & EX_QMI_SINGLE_ECC)
1008                 tmp_reg |= QMI_INTR_EN_SINGLE_ECC;
1009         /* enable events */
1010         iowrite32be(tmp_reg, &qmi_rg->fmqm_ien);
1011 }
1012
1013 static void hwp_init(struct fman_hwp_regs __iomem *hwp_rg)
1014 {
1015         /* enable HW Parser */
1016         iowrite32be(HWP_RPIMAC_PEN, &hwp_rg->fmprrpimac);
1017 }
1018
1019 static int enable(struct fman *fman, struct fman_cfg *cfg)
1020 {
1021         u32 cfg_reg = 0;
1022
1023         /* Enable all modules */
1024
1025         /* clear&enable global counters - calculate reg and save for later,
1026          * because it's the same reg for QMI enable
1027          */
1028         cfg_reg = QMI_CFG_EN_COUNTERS;
1029
1030         /* Set enqueue and dequeue thresholds */
1031         cfg_reg |= (cfg->qmi_def_tnums_thresh << 8) | cfg->qmi_def_tnums_thresh;
1032
1033         iowrite32be(BMI_INIT_START, &fman->bmi_regs->fmbm_init);
1034         iowrite32be(cfg_reg | QMI_CFG_ENQ_EN | QMI_CFG_DEQ_EN,
1035                     &fman->qmi_regs->fmqm_gc);
1036
1037         return 0;
1038 }
1039
1040 static int set_exception(struct fman *fman,
1041                          enum fman_exceptions exception, bool enable)
1042 {
1043         u32 tmp;
1044
1045         switch (exception) {
1046         case FMAN_EX_DMA_BUS_ERROR:
1047                 tmp = ioread32be(&fman->dma_regs->fmdmmr);
1048                 if (enable)
1049                         tmp |= DMA_MODE_BER;
1050                 else
1051                         tmp &= ~DMA_MODE_BER;
1052                 /* disable bus error */
1053                 iowrite32be(tmp, &fman->dma_regs->fmdmmr);
1054                 break;
1055         case FMAN_EX_DMA_READ_ECC:
1056         case FMAN_EX_DMA_SYSTEM_WRITE_ECC:
1057         case FMAN_EX_DMA_FM_WRITE_ECC:
1058                 tmp = ioread32be(&fman->dma_regs->fmdmmr);
1059                 if (enable)
1060                         tmp |= DMA_MODE_ECC;
1061                 else
1062                         tmp &= ~DMA_MODE_ECC;
1063                 iowrite32be(tmp, &fman->dma_regs->fmdmmr);
1064                 break;
1065         case FMAN_EX_FPM_STALL_ON_TASKS:
1066                 tmp = ioread32be(&fman->fpm_regs->fmfp_ee);
1067                 if (enable)
1068                         tmp |= FPM_EV_MASK_STALL_EN;
1069                 else
1070                         tmp &= ~FPM_EV_MASK_STALL_EN;
1071                 iowrite32be(tmp, &fman->fpm_regs->fmfp_ee);
1072                 break;
1073         case FMAN_EX_FPM_SINGLE_ECC:
1074                 tmp = ioread32be(&fman->fpm_regs->fmfp_ee);
1075                 if (enable)
1076                         tmp |= FPM_EV_MASK_SINGLE_ECC_EN;
1077                 else
1078                         tmp &= ~FPM_EV_MASK_SINGLE_ECC_EN;
1079                 iowrite32be(tmp, &fman->fpm_regs->fmfp_ee);
1080                 break;
1081         case FMAN_EX_FPM_DOUBLE_ECC:
1082                 tmp = ioread32be(&fman->fpm_regs->fmfp_ee);
1083                 if (enable)
1084                         tmp |= FPM_EV_MASK_DOUBLE_ECC_EN;
1085                 else
1086                         tmp &= ~FPM_EV_MASK_DOUBLE_ECC_EN;
1087                 iowrite32be(tmp, &fman->fpm_regs->fmfp_ee);
1088                 break;
1089         case FMAN_EX_QMI_SINGLE_ECC:
1090                 tmp = ioread32be(&fman->qmi_regs->fmqm_ien);
1091                 if (enable)
1092                         tmp |= QMI_INTR_EN_SINGLE_ECC;
1093                 else
1094                         tmp &= ~QMI_INTR_EN_SINGLE_ECC;
1095                 iowrite32be(tmp, &fman->qmi_regs->fmqm_ien);
1096                 break;
1097         case FMAN_EX_QMI_DOUBLE_ECC:
1098                 tmp = ioread32be(&fman->qmi_regs->fmqm_eien);
1099                 if (enable)
1100                         tmp |= QMI_ERR_INTR_EN_DOUBLE_ECC;
1101                 else
1102                         tmp &= ~QMI_ERR_INTR_EN_DOUBLE_ECC;
1103                 iowrite32be(tmp, &fman->qmi_regs->fmqm_eien);
1104                 break;
1105         case FMAN_EX_QMI_DEQ_FROM_UNKNOWN_PORTID:
1106                 tmp = ioread32be(&fman->qmi_regs->fmqm_eien);
1107                 if (enable)
1108                         tmp |= QMI_ERR_INTR_EN_DEQ_FROM_DEF;
1109                 else
1110                         tmp &= ~QMI_ERR_INTR_EN_DEQ_FROM_DEF;
1111                 iowrite32be(tmp, &fman->qmi_regs->fmqm_eien);
1112                 break;
1113         case FMAN_EX_BMI_LIST_RAM_ECC:
1114                 tmp = ioread32be(&fman->bmi_regs->fmbm_ier);
1115                 if (enable)
1116                         tmp |= BMI_ERR_INTR_EN_LIST_RAM_ECC;
1117                 else
1118                         tmp &= ~BMI_ERR_INTR_EN_LIST_RAM_ECC;
1119                 iowrite32be(tmp, &fman->bmi_regs->fmbm_ier);
1120                 break;
1121         case FMAN_EX_BMI_STORAGE_PROFILE_ECC:
1122                 tmp = ioread32be(&fman->bmi_regs->fmbm_ier);
1123                 if (enable)
1124                         tmp |= BMI_ERR_INTR_EN_STORAGE_PROFILE_ECC;
1125                 else
1126                         tmp &= ~BMI_ERR_INTR_EN_STORAGE_PROFILE_ECC;
1127                 iowrite32be(tmp, &fman->bmi_regs->fmbm_ier);
1128                 break;
1129         case FMAN_EX_BMI_STATISTICS_RAM_ECC:
1130                 tmp = ioread32be(&fman->bmi_regs->fmbm_ier);
1131                 if (enable)
1132                         tmp |= BMI_ERR_INTR_EN_STATISTICS_RAM_ECC;
1133                 else
1134                         tmp &= ~BMI_ERR_INTR_EN_STATISTICS_RAM_ECC;
1135                 iowrite32be(tmp, &fman->bmi_regs->fmbm_ier);
1136                 break;
1137         case FMAN_EX_BMI_DISPATCH_RAM_ECC:
1138                 tmp = ioread32be(&fman->bmi_regs->fmbm_ier);
1139                 if (enable)
1140                         tmp |= BMI_ERR_INTR_EN_DISPATCH_RAM_ECC;
1141                 else
1142                         tmp &= ~BMI_ERR_INTR_EN_DISPATCH_RAM_ECC;
1143                 iowrite32be(tmp, &fman->bmi_regs->fmbm_ier);
1144                 break;
1145         case FMAN_EX_IRAM_ECC:
1146                 tmp = ioread32be(&fman->fpm_regs->fm_rie);
1147                 if (enable) {
1148                         /* enable ECC if not enabled */
1149                         enable_rams_ecc(fman->fpm_regs);
1150                         /* enable ECC interrupts */
1151                         tmp |= FPM_IRAM_ECC_ERR_EX_EN;
1152                 } else {
1153                         /* ECC mechanism may be disabled,
1154                          * depending on driver status
1155                          */
1156                         disable_rams_ecc(fman->fpm_regs);
1157                         tmp &= ~FPM_IRAM_ECC_ERR_EX_EN;
1158                 }
1159                 iowrite32be(tmp, &fman->fpm_regs->fm_rie);
1160                 break;
1161         case FMAN_EX_MURAM_ECC:
1162                 tmp = ioread32be(&fman->fpm_regs->fm_rie);
1163                 if (enable) {
1164                         /* enable ECC if not enabled */
1165                         enable_rams_ecc(fman->fpm_regs);
1166                         /* enable ECC interrupts */
1167                         tmp |= FPM_MURAM_ECC_ERR_EX_EN;
1168                 } else {
1169                         /* ECC mechanism may be disabled,
1170                          * depending on driver status
1171                          */
1172                         disable_rams_ecc(fman->fpm_regs);
1173                         tmp &= ~FPM_MURAM_ECC_ERR_EX_EN;
1174                 }
1175                 iowrite32be(tmp, &fman->fpm_regs->fm_rie);
1176                 break;
1177         default:
1178                 return -EINVAL;
1179         }
1180         return 0;
1181 }
1182
1183 static void resume(struct fman_fpm_regs __iomem *fpm_rg)
1184 {
1185         u32 tmp;
1186
1187         tmp = ioread32be(&fpm_rg->fmfp_ee);
1188         /* clear tmp_reg event bits in order not to clear standing events */
1189         tmp &= ~(FPM_EV_MASK_DOUBLE_ECC |
1190                  FPM_EV_MASK_STALL | FPM_EV_MASK_SINGLE_ECC);
1191         tmp |= FPM_EV_MASK_RELEASE_FM;
1192
1193         iowrite32be(tmp, &fpm_rg->fmfp_ee);
1194 }
1195
1196 static int fill_soc_specific_params(struct fman_state_struct *state)
1197 {
1198         u8 minor = state->rev_info.minor;
1199         /* P4080 - Major 2
1200          * P2041/P3041/P5020/P5040 - Major 3
1201          * Tx/Bx - Major 6
1202          */
1203         switch (state->rev_info.major) {
1204         case 3:
1205                 state->bmi_max_fifo_size        = 160 * 1024;
1206                 state->fm_iram_size             = 64 * 1024;
1207                 state->dma_thresh_max_commq     = 31;
1208                 state->dma_thresh_max_buf       = 127;
1209                 state->qmi_max_num_of_tnums     = 64;
1210                 state->qmi_def_tnums_thresh     = 48;
1211                 state->bmi_max_num_of_tasks     = 128;
1212                 state->max_num_of_open_dmas     = 32;
1213                 state->fm_port_num_of_cg        = 256;
1214                 state->num_of_rx_ports  = 6;
1215                 state->total_fifo_size  = 136 * 1024;
1216                 break;
1217
1218         case 2:
1219                 state->bmi_max_fifo_size        = 160 * 1024;
1220                 state->fm_iram_size             = 64 * 1024;
1221                 state->dma_thresh_max_commq     = 31;
1222                 state->dma_thresh_max_buf       = 127;
1223                 state->qmi_max_num_of_tnums     = 64;
1224                 state->qmi_def_tnums_thresh     = 48;
1225                 state->bmi_max_num_of_tasks     = 128;
1226                 state->max_num_of_open_dmas     = 32;
1227                 state->fm_port_num_of_cg        = 256;
1228                 state->num_of_rx_ports  = 5;
1229                 state->total_fifo_size  = 100 * 1024;
1230                 break;
1231
1232         case 6:
1233                 state->dma_thresh_max_commq     = 83;
1234                 state->dma_thresh_max_buf       = 127;
1235                 state->qmi_max_num_of_tnums     = 64;
1236                 state->qmi_def_tnums_thresh     = 32;
1237                 state->fm_port_num_of_cg        = 256;
1238
1239                 /* FManV3L */
1240                 if (minor == 1 || minor == 4) {
1241                         state->bmi_max_fifo_size        = 192 * 1024;
1242                         state->bmi_max_num_of_tasks     = 64;
1243                         state->max_num_of_open_dmas     = 32;
1244                         state->num_of_rx_ports          = 5;
1245                         if (minor == 1)
1246                                 state->fm_iram_size     = 32 * 1024;
1247                         else
1248                                 state->fm_iram_size     = 64 * 1024;
1249                         state->total_fifo_size          = 156 * 1024;
1250                 }
1251                 /* FManV3H */
1252                 else if (minor == 0 || minor == 2 || minor == 3) {
1253                         state->bmi_max_fifo_size        = 384 * 1024;
1254                         state->fm_iram_size             = 64 * 1024;
1255                         state->bmi_max_num_of_tasks     = 128;
1256                         state->max_num_of_open_dmas     = 84;
1257                         state->num_of_rx_ports          = 8;
1258                         state->total_fifo_size          = 295 * 1024;
1259                 } else {
1260                         pr_err("Unsupported FManv3 version\n");
1261                         return -EINVAL;
1262                 }
1263
1264                 break;
1265         default:
1266                 pr_err("Unsupported FMan version\n");
1267                 return -EINVAL;
1268         }
1269
1270         return 0;
1271 }
1272
1273 static bool is_init_done(struct fman_cfg *cfg)
1274 {
1275         /* Checks if FMan driver parameters were initialized */
1276         if (!cfg)
1277                 return true;
1278
1279         return false;
1280 }
1281
1282 static void free_init_resources(struct fman *fman)
1283 {
1284         if (fman->cam_offset)
1285                 fman_muram_free_mem(fman->muram, fman->cam_offset,
1286                                     fman->cam_size);
1287         if (fman->fifo_offset)
1288                 fman_muram_free_mem(fman->muram, fman->fifo_offset,
1289                                     fman->fifo_size);
1290 }
1291
1292 static irqreturn_t bmi_err_event(struct fman *fman)
1293 {
1294         u32 event, mask, force;
1295         struct fman_bmi_regs __iomem *bmi_rg = fman->bmi_regs;
1296         irqreturn_t ret = IRQ_NONE;
1297
1298         event = ioread32be(&bmi_rg->fmbm_ievr);
1299         mask = ioread32be(&bmi_rg->fmbm_ier);
1300         event &= mask;
1301         /* clear the forced events */
1302         force = ioread32be(&bmi_rg->fmbm_ifr);
1303         if (force & event)
1304                 iowrite32be(force & ~event, &bmi_rg->fmbm_ifr);
1305         /* clear the acknowledged events */
1306         iowrite32be(event, &bmi_rg->fmbm_ievr);
1307
1308         if (event & BMI_ERR_INTR_EN_STORAGE_PROFILE_ECC)
1309                 ret = fman->exception_cb(fman, FMAN_EX_BMI_STORAGE_PROFILE_ECC);
1310         if (event & BMI_ERR_INTR_EN_LIST_RAM_ECC)
1311                 ret = fman->exception_cb(fman, FMAN_EX_BMI_LIST_RAM_ECC);
1312         if (event & BMI_ERR_INTR_EN_STATISTICS_RAM_ECC)
1313                 ret = fman->exception_cb(fman, FMAN_EX_BMI_STATISTICS_RAM_ECC);
1314         if (event & BMI_ERR_INTR_EN_DISPATCH_RAM_ECC)
1315                 ret = fman->exception_cb(fman, FMAN_EX_BMI_DISPATCH_RAM_ECC);
1316
1317         return ret;
1318 }
1319
1320 static irqreturn_t qmi_err_event(struct fman *fman)
1321 {
1322         u32 event, mask, force;
1323         struct fman_qmi_regs __iomem *qmi_rg = fman->qmi_regs;
1324         irqreturn_t ret = IRQ_NONE;
1325
1326         event = ioread32be(&qmi_rg->fmqm_eie);
1327         mask = ioread32be(&qmi_rg->fmqm_eien);
1328         event &= mask;
1329
1330         /* clear the forced events */
1331         force = ioread32be(&qmi_rg->fmqm_eif);
1332         if (force & event)
1333                 iowrite32be(force & ~event, &qmi_rg->fmqm_eif);
1334         /* clear the acknowledged events */
1335         iowrite32be(event, &qmi_rg->fmqm_eie);
1336
1337         if (event & QMI_ERR_INTR_EN_DOUBLE_ECC)
1338                 ret = fman->exception_cb(fman, FMAN_EX_QMI_DOUBLE_ECC);
1339         if (event & QMI_ERR_INTR_EN_DEQ_FROM_DEF)
1340                 ret = fman->exception_cb(fman,
1341                                          FMAN_EX_QMI_DEQ_FROM_UNKNOWN_PORTID);
1342
1343         return ret;
1344 }
1345
1346 static irqreturn_t dma_err_event(struct fman *fman)
1347 {
1348         u32 status, mask, com_id;
1349         u8 tnum, port_id, relative_port_id;
1350         u16 liodn;
1351         struct fman_dma_regs __iomem *dma_rg = fman->dma_regs;
1352         irqreturn_t ret = IRQ_NONE;
1353
1354         status = ioread32be(&dma_rg->fmdmsr);
1355         mask = ioread32be(&dma_rg->fmdmmr);
1356
1357         /* clear DMA_STATUS_BUS_ERR if mask has no DMA_MODE_BER */
1358         if ((mask & DMA_MODE_BER) != DMA_MODE_BER)
1359                 status &= ~DMA_STATUS_BUS_ERR;
1360
1361         /* clear relevant bits if mask has no DMA_MODE_ECC */
1362         if ((mask & DMA_MODE_ECC) != DMA_MODE_ECC)
1363                 status &= ~(DMA_STATUS_FM_SPDAT_ECC |
1364                             DMA_STATUS_READ_ECC |
1365                             DMA_STATUS_SYSTEM_WRITE_ECC |
1366                             DMA_STATUS_FM_WRITE_ECC);
1367
1368         /* clear set events */
1369         iowrite32be(status, &dma_rg->fmdmsr);
1370
1371         if (status & DMA_STATUS_BUS_ERR) {
1372                 u64 addr;
1373
1374                 addr = (u64)ioread32be(&dma_rg->fmdmtal);
1375                 addr |= ((u64)(ioread32be(&dma_rg->fmdmtah)) << 32);
1376
1377                 com_id = ioread32be(&dma_rg->fmdmtcid);
1378                 port_id = (u8)(((com_id & DMA_TRANSFER_PORTID_MASK) >>
1379                                DMA_TRANSFER_PORTID_SHIFT));
1380                 relative_port_id =
1381                 hw_port_id_to_sw_port_id(fman->state->rev_info.major, port_id);
1382                 tnum = (u8)((com_id & DMA_TRANSFER_TNUM_MASK) >>
1383                             DMA_TRANSFER_TNUM_SHIFT);
1384                 liodn = (u16)(com_id & DMA_TRANSFER_LIODN_MASK);
1385                 ret = fman->bus_error_cb(fman, relative_port_id, addr, tnum,
1386                                          liodn);
1387         }
1388         if (status & DMA_STATUS_FM_SPDAT_ECC)
1389                 ret = fman->exception_cb(fman, FMAN_EX_DMA_SINGLE_PORT_ECC);
1390         if (status & DMA_STATUS_READ_ECC)
1391                 ret = fman->exception_cb(fman, FMAN_EX_DMA_READ_ECC);
1392         if (status & DMA_STATUS_SYSTEM_WRITE_ECC)
1393                 ret = fman->exception_cb(fman, FMAN_EX_DMA_SYSTEM_WRITE_ECC);
1394         if (status & DMA_STATUS_FM_WRITE_ECC)
1395                 ret = fman->exception_cb(fman, FMAN_EX_DMA_FM_WRITE_ECC);
1396
1397         return ret;
1398 }
1399
1400 static irqreturn_t fpm_err_event(struct fman *fman)
1401 {
1402         u32 event;
1403         struct fman_fpm_regs __iomem *fpm_rg = fman->fpm_regs;
1404         irqreturn_t ret = IRQ_NONE;
1405
1406         event = ioread32be(&fpm_rg->fmfp_ee);
1407         /* clear the all occurred events */
1408         iowrite32be(event, &fpm_rg->fmfp_ee);
1409
1410         if ((event & FPM_EV_MASK_DOUBLE_ECC) &&
1411             (event & FPM_EV_MASK_DOUBLE_ECC_EN))
1412                 ret = fman->exception_cb(fman, FMAN_EX_FPM_DOUBLE_ECC);
1413         if ((event & FPM_EV_MASK_STALL) && (event & FPM_EV_MASK_STALL_EN))
1414                 ret = fman->exception_cb(fman, FMAN_EX_FPM_STALL_ON_TASKS);
1415         if ((event & FPM_EV_MASK_SINGLE_ECC) &&
1416             (event & FPM_EV_MASK_SINGLE_ECC_EN))
1417                 ret = fman->exception_cb(fman, FMAN_EX_FPM_SINGLE_ECC);
1418
1419         return ret;
1420 }
1421
1422 static irqreturn_t muram_err_intr(struct fman *fman)
1423 {
1424         u32 event, mask;
1425         struct fman_fpm_regs __iomem *fpm_rg = fman->fpm_regs;
1426         irqreturn_t ret = IRQ_NONE;
1427
1428         event = ioread32be(&fpm_rg->fm_rcr);
1429         mask = ioread32be(&fpm_rg->fm_rie);
1430
1431         /* clear MURAM event bit (do not clear IRAM event) */
1432         iowrite32be(event & ~FPM_RAM_IRAM_ECC, &fpm_rg->fm_rcr);
1433
1434         if ((mask & FPM_MURAM_ECC_ERR_EX_EN) && (event & FPM_RAM_MURAM_ECC))
1435                 ret = fman->exception_cb(fman, FMAN_EX_MURAM_ECC);
1436
1437         return ret;
1438 }
1439
1440 static irqreturn_t qmi_event(struct fman *fman)
1441 {
1442         u32 event, mask, force;
1443         struct fman_qmi_regs __iomem *qmi_rg = fman->qmi_regs;
1444         irqreturn_t ret = IRQ_NONE;
1445
1446         event = ioread32be(&qmi_rg->fmqm_ie);
1447         mask = ioread32be(&qmi_rg->fmqm_ien);
1448         event &= mask;
1449         /* clear the forced events */
1450         force = ioread32be(&qmi_rg->fmqm_if);
1451         if (force & event)
1452                 iowrite32be(force & ~event, &qmi_rg->fmqm_if);
1453         /* clear the acknowledged events */
1454         iowrite32be(event, &qmi_rg->fmqm_ie);
1455
1456         if (event & QMI_INTR_EN_SINGLE_ECC)
1457                 ret = fman->exception_cb(fman, FMAN_EX_QMI_SINGLE_ECC);
1458
1459         return ret;
1460 }
1461
1462 static void enable_time_stamp(struct fman *fman)
1463 {
1464         struct fman_fpm_regs __iomem *fpm_rg = fman->fpm_regs;
1465         u16 fm_clk_freq = fman->state->fm_clk_freq;
1466         u32 tmp, intgr, ts_freq;
1467         u64 frac;
1468
1469         ts_freq = (u32)(1 << fman->state->count1_micro_bit);
1470         /* configure timestamp so that bit 8 will count 1 microsecond
1471          * Find effective count rate at TIMESTAMP least significant bits:
1472          * Effective_Count_Rate = 1MHz x 2^8 = 256MHz
1473          * Find frequency ratio between effective count rate and the clock:
1474          * Effective_Count_Rate / CLK e.g. for 600 MHz clock:
1475          * 256/600 = 0.4266666...
1476          */
1477
1478         intgr = ts_freq / fm_clk_freq;
1479         /* we multiply by 2^16 to keep the fraction of the division
1480          * we do not div back, since we write this value as a fraction
1481          * see spec
1482          */
1483
1484         frac = ((ts_freq << 16) - (intgr << 16) * fm_clk_freq) / fm_clk_freq;
1485         /* we check remainder of the division in order to round up if not int */
1486         if (((ts_freq << 16) - (intgr << 16) * fm_clk_freq) % fm_clk_freq)
1487                 frac++;
1488
1489         tmp = (intgr << FPM_TS_INT_SHIFT) | (u16)frac;
1490         iowrite32be(tmp, &fpm_rg->fmfp_tsc2);
1491
1492         /* enable timestamp with original clock */
1493         iowrite32be(FPM_TS_CTL_EN, &fpm_rg->fmfp_tsc1);
1494         fman->state->enabled_time_stamp = true;
1495 }
1496
1497 static int clear_iram(struct fman *fman)
1498 {
1499         struct fman_iram_regs __iomem *iram;
1500         int i, count;
1501
1502         iram = fman->base_addr + IMEM_OFFSET;
1503
1504         /* Enable the auto-increment */
1505         iowrite32be(IRAM_IADD_AIE, &iram->iadd);
1506         count = 100;
1507         do {
1508                 udelay(1);
1509         } while ((ioread32be(&iram->iadd) != IRAM_IADD_AIE) && --count);
1510         if (count == 0)
1511                 return -EBUSY;
1512
1513         for (i = 0; i < (fman->state->fm_iram_size / 4); i++)
1514                 iowrite32be(0xffffffff, &iram->idata);
1515
1516         iowrite32be(fman->state->fm_iram_size - 4, &iram->iadd);
1517         count = 100;
1518         do {
1519                 udelay(1);
1520         } while ((ioread32be(&iram->idata) != 0xffffffff) && --count);
1521         if (count == 0)
1522                 return -EBUSY;
1523
1524         return 0;
1525 }
1526
1527 static u32 get_exception_flag(enum fman_exceptions exception)
1528 {
1529         u32 bit_mask;
1530
1531         switch (exception) {
1532         case FMAN_EX_DMA_BUS_ERROR:
1533                 bit_mask = EX_DMA_BUS_ERROR;
1534                 break;
1535         case FMAN_EX_DMA_SINGLE_PORT_ECC:
1536                 bit_mask = EX_DMA_SINGLE_PORT_ECC;
1537                 break;
1538         case FMAN_EX_DMA_READ_ECC:
1539                 bit_mask = EX_DMA_READ_ECC;
1540                 break;
1541         case FMAN_EX_DMA_SYSTEM_WRITE_ECC:
1542                 bit_mask = EX_DMA_SYSTEM_WRITE_ECC;
1543                 break;
1544         case FMAN_EX_DMA_FM_WRITE_ECC:
1545                 bit_mask = EX_DMA_FM_WRITE_ECC;
1546                 break;
1547         case FMAN_EX_FPM_STALL_ON_TASKS:
1548                 bit_mask = EX_FPM_STALL_ON_TASKS;
1549                 break;
1550         case FMAN_EX_FPM_SINGLE_ECC:
1551                 bit_mask = EX_FPM_SINGLE_ECC;
1552                 break;
1553         case FMAN_EX_FPM_DOUBLE_ECC:
1554                 bit_mask = EX_FPM_DOUBLE_ECC;
1555                 break;
1556         case FMAN_EX_QMI_SINGLE_ECC:
1557                 bit_mask = EX_QMI_SINGLE_ECC;
1558                 break;
1559         case FMAN_EX_QMI_DOUBLE_ECC:
1560                 bit_mask = EX_QMI_DOUBLE_ECC;
1561                 break;
1562         case FMAN_EX_QMI_DEQ_FROM_UNKNOWN_PORTID:
1563                 bit_mask = EX_QMI_DEQ_FROM_UNKNOWN_PORTID;
1564                 break;
1565         case FMAN_EX_BMI_LIST_RAM_ECC:
1566                 bit_mask = EX_BMI_LIST_RAM_ECC;
1567                 break;
1568         case FMAN_EX_BMI_STORAGE_PROFILE_ECC:
1569                 bit_mask = EX_BMI_STORAGE_PROFILE_ECC;
1570                 break;
1571         case FMAN_EX_BMI_STATISTICS_RAM_ECC:
1572                 bit_mask = EX_BMI_STATISTICS_RAM_ECC;
1573                 break;
1574         case FMAN_EX_BMI_DISPATCH_RAM_ECC:
1575                 bit_mask = EX_BMI_DISPATCH_RAM_ECC;
1576                 break;
1577         case FMAN_EX_MURAM_ECC:
1578                 bit_mask = EX_MURAM_ECC;
1579                 break;
1580         default:
1581                 bit_mask = 0;
1582                 break;
1583         }
1584
1585         return bit_mask;
1586 }
1587
1588 static int get_module_event(enum fman_event_modules module, u8 mod_id,
1589                             enum fman_intr_type intr_type)
1590 {
1591         int event;
1592
1593         switch (module) {
1594         case FMAN_MOD_MAC:
1595                 if (intr_type == FMAN_INTR_TYPE_ERR)
1596                         event = FMAN_EV_ERR_MAC0 + mod_id;
1597                 else
1598                         event = FMAN_EV_MAC0 + mod_id;
1599                 break;
1600         case FMAN_MOD_FMAN_CTRL:
1601                 if (intr_type == FMAN_INTR_TYPE_ERR)
1602                         event = FMAN_EV_CNT;
1603                 else
1604                         event = (FMAN_EV_FMAN_CTRL_0 + mod_id);
1605                 break;
1606         case FMAN_MOD_DUMMY_LAST:
1607                 event = FMAN_EV_CNT;
1608                 break;
1609         default:
1610                 event = FMAN_EV_CNT;
1611                 break;
1612         }
1613
1614         return event;
1615 }
1616
1617 static int set_size_of_fifo(struct fman *fman, u8 port_id, u32 *size_of_fifo,
1618                             u32 *extra_size_of_fifo)
1619 {
1620         struct fman_bmi_regs __iomem *bmi_rg = fman->bmi_regs;
1621         u32 fifo = *size_of_fifo;
1622         u32 extra_fifo = *extra_size_of_fifo;
1623         u32 tmp;
1624
1625         /* if this is the first time a port requires extra_fifo_pool_size,
1626          * the total extra_fifo_pool_size must be initialized to 1 buffer per
1627          * port
1628          */
1629         if (extra_fifo && !fman->state->extra_fifo_pool_size)
1630                 fman->state->extra_fifo_pool_size =
1631                         fman->state->num_of_rx_ports * FMAN_BMI_FIFO_UNITS;
1632
1633         fman->state->extra_fifo_pool_size =
1634                 max(fman->state->extra_fifo_pool_size, extra_fifo);
1635
1636         /* check that there are enough uncommitted fifo size */
1637         if ((fman->state->accumulated_fifo_size + fifo) >
1638             (fman->state->total_fifo_size -
1639             fman->state->extra_fifo_pool_size)) {
1640                 dev_err(fman->dev, "%s: Requested fifo size and extra size exceed total FIFO size.\n",
1641                         __func__);
1642                 return -EAGAIN;
1643         }
1644
1645         /* Read, modify and write to HW */
1646         tmp = (fifo / FMAN_BMI_FIFO_UNITS - 1) |
1647                ((extra_fifo / FMAN_BMI_FIFO_UNITS) <<
1648                BMI_EXTRA_FIFO_SIZE_SHIFT);
1649         iowrite32be(tmp, &bmi_rg->fmbm_pfs[port_id - 1]);
1650
1651         /* update accumulated */
1652         fman->state->accumulated_fifo_size += fifo;
1653
1654         return 0;
1655 }
1656
1657 static int set_num_of_tasks(struct fman *fman, u8 port_id, u8 *num_of_tasks,
1658                             u8 *num_of_extra_tasks)
1659 {
1660         struct fman_bmi_regs __iomem *bmi_rg = fman->bmi_regs;
1661         u8 tasks = *num_of_tasks;
1662         u8 extra_tasks = *num_of_extra_tasks;
1663         u32 tmp;
1664
1665         if (extra_tasks)
1666                 fman->state->extra_tasks_pool_size =
1667                 max(fman->state->extra_tasks_pool_size, extra_tasks);
1668
1669         /* check that there are enough uncommitted tasks */
1670         if ((fman->state->accumulated_num_of_tasks + tasks) >
1671             (fman->state->total_num_of_tasks -
1672              fman->state->extra_tasks_pool_size)) {
1673                 dev_err(fman->dev, "%s: Requested num_of_tasks and extra tasks pool for fm%d exceed total num_of_tasks.\n",
1674                         __func__, fman->state->fm_id);
1675                 return -EAGAIN;
1676         }
1677         /* update accumulated */
1678         fman->state->accumulated_num_of_tasks += tasks;
1679
1680         /* Write to HW */
1681         tmp = ioread32be(&bmi_rg->fmbm_pp[port_id - 1]) &
1682             ~(BMI_NUM_OF_TASKS_MASK | BMI_NUM_OF_EXTRA_TASKS_MASK);
1683         tmp |= ((u32)((tasks - 1) << BMI_NUM_OF_TASKS_SHIFT) |
1684                 (u32)(extra_tasks << BMI_EXTRA_NUM_OF_TASKS_SHIFT));
1685         iowrite32be(tmp, &bmi_rg->fmbm_pp[port_id - 1]);
1686
1687         return 0;
1688 }
1689
1690 static int set_num_of_open_dmas(struct fman *fman, u8 port_id,
1691                                 u8 *num_of_open_dmas,
1692                                 u8 *num_of_extra_open_dmas)
1693 {
1694         struct fman_bmi_regs __iomem *bmi_rg = fman->bmi_regs;
1695         u8 open_dmas = *num_of_open_dmas;
1696         u8 extra_open_dmas = *num_of_extra_open_dmas;
1697         u8 total_num_dmas = 0, current_val = 0, current_extra_val = 0;
1698         u32 tmp;
1699
1700         if (!open_dmas) {
1701                 /* Configuration according to values in the HW.
1702                  * read the current number of open Dma's
1703                  */
1704                 tmp = ioread32be(&bmi_rg->fmbm_pp[port_id - 1]);
1705                 current_extra_val = (u8)((tmp & BMI_NUM_OF_EXTRA_DMAS_MASK) >>
1706                                          BMI_EXTRA_NUM_OF_DMAS_SHIFT);
1707
1708                 tmp = ioread32be(&bmi_rg->fmbm_pp[port_id - 1]);
1709                 current_val = (u8)(((tmp & BMI_NUM_OF_DMAS_MASK) >>
1710                                    BMI_NUM_OF_DMAS_SHIFT) + 1);
1711
1712                 /* This is the first configuration and user did not
1713                  * specify value (!open_dmas), reset values will be used
1714                  * and we just save these values for resource management
1715                  */
1716                 fman->state->extra_open_dmas_pool_size =
1717                         (u8)max(fman->state->extra_open_dmas_pool_size,
1718                                 current_extra_val);
1719                 fman->state->accumulated_num_of_open_dmas += current_val;
1720                 *num_of_open_dmas = current_val;
1721                 *num_of_extra_open_dmas = current_extra_val;
1722                 return 0;
1723         }
1724
1725         if (extra_open_dmas > current_extra_val)
1726                 fman->state->extra_open_dmas_pool_size =
1727                     (u8)max(fman->state->extra_open_dmas_pool_size,
1728                             extra_open_dmas);
1729
1730         if ((fman->state->rev_info.major < 6) &&
1731             (fman->state->accumulated_num_of_open_dmas - current_val +
1732              open_dmas > fman->state->max_num_of_open_dmas)) {
1733                 dev_err(fman->dev, "%s: Requested num_of_open_dmas for fm%d exceeds total num_of_open_dmas.\n",
1734                         __func__, fman->state->fm_id);
1735                 return -EAGAIN;
1736         } else if ((fman->state->rev_info.major >= 6) &&
1737                    !((fman->state->rev_info.major == 6) &&
1738                    (fman->state->rev_info.minor == 0)) &&
1739                    (fman->state->accumulated_num_of_open_dmas -
1740                    current_val + open_dmas >
1741                    fman->state->dma_thresh_max_commq + 1)) {
1742                 dev_err(fman->dev, "%s: Requested num_of_open_dmas for fm%d exceeds DMA Command queue (%d)\n",
1743                         __func__, fman->state->fm_id,
1744                        fman->state->dma_thresh_max_commq + 1);
1745                 return -EAGAIN;
1746         }
1747
1748         WARN_ON(fman->state->accumulated_num_of_open_dmas < current_val);
1749         /* update acummulated */
1750         fman->state->accumulated_num_of_open_dmas -= current_val;
1751         fman->state->accumulated_num_of_open_dmas += open_dmas;
1752
1753         if (fman->state->rev_info.major < 6)
1754                 total_num_dmas =
1755                     (u8)(fman->state->accumulated_num_of_open_dmas +
1756                     fman->state->extra_open_dmas_pool_size);
1757
1758         /* calculate reg */
1759         tmp = ioread32be(&bmi_rg->fmbm_pp[port_id - 1]) &
1760             ~(BMI_NUM_OF_DMAS_MASK | BMI_NUM_OF_EXTRA_DMAS_MASK);
1761         tmp |= (u32)(((open_dmas - 1) << BMI_NUM_OF_DMAS_SHIFT) |
1762                            (extra_open_dmas << BMI_EXTRA_NUM_OF_DMAS_SHIFT));
1763         iowrite32be(tmp, &bmi_rg->fmbm_pp[port_id - 1]);
1764
1765         /* update total num of DMA's with committed number of open DMAS,
1766          * and max uncommitted pool.
1767          */
1768         if (total_num_dmas) {
1769                 tmp = ioread32be(&bmi_rg->fmbm_cfg2) & ~BMI_CFG2_DMAS_MASK;
1770                 tmp |= (u32)(total_num_dmas - 1) << BMI_CFG2_DMAS_SHIFT;
1771                 iowrite32be(tmp, &bmi_rg->fmbm_cfg2);
1772         }
1773
1774         return 0;
1775 }
1776
1777 static int fman_config(struct fman *fman)
1778 {
1779         void __iomem *base_addr;
1780         int err;
1781
1782         base_addr = fman->dts_params.base_addr;
1783
1784         fman->state = kzalloc(sizeof(*fman->state), GFP_KERNEL);
1785         if (!fman->state)
1786                 goto err_fm_state;
1787
1788         /* Allocate the FM driver's parameters structure */
1789         fman->cfg = kzalloc(sizeof(*fman->cfg), GFP_KERNEL);
1790         if (!fman->cfg)
1791                 goto err_fm_drv;
1792
1793         /* Initialize MURAM block */
1794         fman->muram =
1795                 fman_muram_init(fman->dts_params.muram_res.start,
1796                                 resource_size(&fman->dts_params.muram_res));
1797         if (!fman->muram)
1798                 goto err_fm_soc_specific;
1799
1800         /* Initialize FM parameters which will be kept by the driver */
1801         fman->state->fm_id = fman->dts_params.id;
1802         fman->state->fm_clk_freq = fman->dts_params.clk_freq;
1803         fman->state->qman_channel_base = fman->dts_params.qman_channel_base;
1804         fman->state->num_of_qman_channels =
1805                 fman->dts_params.num_of_qman_channels;
1806         fman->state->res = fman->dts_params.res;
1807         fman->exception_cb = fman_exceptions;
1808         fman->bus_error_cb = fman_bus_error;
1809         fman->fpm_regs = base_addr + FPM_OFFSET;
1810         fman->bmi_regs = base_addr + BMI_OFFSET;
1811         fman->qmi_regs = base_addr + QMI_OFFSET;
1812         fman->dma_regs = base_addr + DMA_OFFSET;
1813         fman->hwp_regs = base_addr + HWP_OFFSET;
1814         fman->base_addr = base_addr;
1815
1816         spin_lock_init(&fman->spinlock);
1817         fman_defconfig(fman->cfg);
1818
1819         fman->state->extra_fifo_pool_size = 0;
1820         fman->state->exceptions = (EX_DMA_BUS_ERROR                 |
1821                                         EX_DMA_READ_ECC              |
1822                                         EX_DMA_SYSTEM_WRITE_ECC      |
1823                                         EX_DMA_FM_WRITE_ECC          |
1824                                         EX_FPM_STALL_ON_TASKS        |
1825                                         EX_FPM_SINGLE_ECC            |
1826                                         EX_FPM_DOUBLE_ECC            |
1827                                         EX_QMI_DEQ_FROM_UNKNOWN_PORTID |
1828                                         EX_BMI_LIST_RAM_ECC          |
1829                                         EX_BMI_STORAGE_PROFILE_ECC   |
1830                                         EX_BMI_STATISTICS_RAM_ECC    |
1831                                         EX_MURAM_ECC                 |
1832                                         EX_BMI_DISPATCH_RAM_ECC      |
1833                                         EX_QMI_DOUBLE_ECC            |
1834                                         EX_QMI_SINGLE_ECC);
1835
1836         /* Read FMan revision for future use*/
1837         fman_get_revision(fman, &fman->state->rev_info);
1838
1839         err = fill_soc_specific_params(fman->state);
1840         if (err)
1841                 goto err_fm_soc_specific;
1842
1843         /* FM_AID_MODE_NO_TNUM_SW005 Errata workaround */
1844         if (fman->state->rev_info.major >= 6)
1845                 fman->cfg->dma_aid_mode = FMAN_DMA_AID_OUT_PORT_ID;
1846
1847         fman->cfg->qmi_def_tnums_thresh = fman->state->qmi_def_tnums_thresh;
1848
1849         fman->state->total_num_of_tasks =
1850         (u8)DFLT_TOTAL_NUM_OF_TASKS(fman->state->rev_info.major,
1851                                     fman->state->rev_info.minor,
1852                                     fman->state->bmi_max_num_of_tasks);
1853
1854         if (fman->state->rev_info.major < 6) {
1855                 fman->cfg->dma_comm_qtsh_clr_emer =
1856                 (u8)DFLT_DMA_COMM_Q_LOW(fman->state->rev_info.major,
1857                                         fman->state->dma_thresh_max_commq);
1858
1859                 fman->cfg->dma_comm_qtsh_asrt_emer =
1860                 (u8)DFLT_DMA_COMM_Q_HIGH(fman->state->rev_info.major,
1861                                          fman->state->dma_thresh_max_commq);
1862
1863                 fman->cfg->dma_cam_num_of_entries =
1864                 DFLT_DMA_CAM_NUM_OF_ENTRIES(fman->state->rev_info.major);
1865
1866                 fman->cfg->dma_read_buf_tsh_clr_emer =
1867                 DFLT_DMA_READ_INT_BUF_LOW(fman->state->dma_thresh_max_buf);
1868
1869                 fman->cfg->dma_read_buf_tsh_asrt_emer =
1870                 DFLT_DMA_READ_INT_BUF_HIGH(fman->state->dma_thresh_max_buf);
1871
1872                 fman->cfg->dma_write_buf_tsh_clr_emer =
1873                 DFLT_DMA_WRITE_INT_BUF_LOW(fman->state->dma_thresh_max_buf);
1874
1875                 fman->cfg->dma_write_buf_tsh_asrt_emer =
1876                 DFLT_DMA_WRITE_INT_BUF_HIGH(fman->state->dma_thresh_max_buf);
1877
1878                 fman->cfg->dma_axi_dbg_num_of_beats =
1879                 DFLT_AXI_DBG_NUM_OF_BEATS;
1880         }
1881
1882         return 0;
1883
1884 err_fm_soc_specific:
1885         kfree(fman->cfg);
1886 err_fm_drv:
1887         kfree(fman->state);
1888 err_fm_state:
1889         kfree(fman);
1890         return -EINVAL;
1891 }
1892
1893 static int fman_reset(struct fman *fman)
1894 {
1895         u32 count;
1896         int err = 0;
1897
1898         if (fman->state->rev_info.major < 6) {
1899                 iowrite32be(FPM_RSTC_FM_RESET, &fman->fpm_regs->fm_rstc);
1900                 /* Wait for reset completion */
1901                 count = 100;
1902                 do {
1903                         udelay(1);
1904                 } while (((ioread32be(&fman->fpm_regs->fm_rstc)) &
1905                          FPM_RSTC_FM_RESET) && --count);
1906                 if (count == 0)
1907                         err = -EBUSY;
1908
1909                 goto _return;
1910         } else {
1911 #ifdef CONFIG_PPC
1912                 struct device_node *guts_node;
1913                 struct ccsr_guts __iomem *guts_regs;
1914                 u32 devdisr2, reg;
1915
1916                 /* Errata A007273 */
1917                 guts_node =
1918                         of_find_compatible_node(NULL, NULL,
1919                                                 "fsl,qoriq-device-config-2.0");
1920                 if (!guts_node) {
1921                         dev_err(fman->dev, "%s: Couldn't find guts node\n",
1922                                 __func__);
1923                         goto guts_node;
1924                 }
1925
1926                 guts_regs = of_iomap(guts_node, 0);
1927                 if (!guts_regs) {
1928                         dev_err(fman->dev, "%s: Couldn't map %s regs\n",
1929                                 __func__, guts_node->full_name);
1930                         goto guts_regs;
1931                 }
1932 #define FMAN1_ALL_MACS_MASK     0xFCC00000
1933 #define FMAN2_ALL_MACS_MASK     0x000FCC00
1934                 /* Read current state */
1935                 devdisr2 = ioread32be(&guts_regs->devdisr2);
1936                 if (fman->dts_params.id == 0)
1937                         reg = devdisr2 & ~FMAN1_ALL_MACS_MASK;
1938                 else
1939                         reg = devdisr2 & ~FMAN2_ALL_MACS_MASK;
1940
1941                 /* Enable all MACs */
1942                 iowrite32be(reg, &guts_regs->devdisr2);
1943 #endif
1944
1945                 /* Perform FMan reset */
1946                 iowrite32be(FPM_RSTC_FM_RESET, &fman->fpm_regs->fm_rstc);
1947
1948                 /* Wait for reset completion */
1949                 count = 100;
1950                 do {
1951                         udelay(1);
1952                 } while (((ioread32be(&fman->fpm_regs->fm_rstc)) &
1953                          FPM_RSTC_FM_RESET) && --count);
1954                 if (count == 0) {
1955 #ifdef CONFIG_PPC
1956                         iounmap(guts_regs);
1957                         of_node_put(guts_node);
1958 #endif
1959                         err = -EBUSY;
1960                         goto _return;
1961                 }
1962 #ifdef CONFIG_PPC
1963
1964                 /* Restore devdisr2 value */
1965                 iowrite32be(devdisr2, &guts_regs->devdisr2);
1966
1967                 iounmap(guts_regs);
1968                 of_node_put(guts_node);
1969 #endif
1970
1971                 goto _return;
1972
1973 #ifdef CONFIG_PPC
1974 guts_regs:
1975                 of_node_put(guts_node);
1976 guts_node:
1977                 dev_dbg(fman->dev, "%s: Didn't perform FManV3 reset due to Errata A007273!\n",
1978                         __func__);
1979 #endif
1980         }
1981 _return:
1982         return err;
1983 }
1984
1985 static int fman_init(struct fman *fman)
1986 {
1987         struct fman_cfg *cfg = NULL;
1988         int err = 0, i, count;
1989
1990         if (is_init_done(fman->cfg))
1991                 return -EINVAL;
1992
1993         fman->state->count1_micro_bit = FM_TIMESTAMP_1_USEC_BIT;
1994
1995         cfg = fman->cfg;
1996
1997         /* clear revision-dependent non existing exception */
1998         if (fman->state->rev_info.major < 6)
1999                 fman->state->exceptions &= ~FMAN_EX_BMI_DISPATCH_RAM_ECC;
2000
2001         if (fman->state->rev_info.major >= 6)
2002                 fman->state->exceptions &= ~FMAN_EX_QMI_SINGLE_ECC;
2003
2004         /* clear CPG */
2005         memset_io((void __iomem *)(fman->base_addr + CGP_OFFSET), 0,
2006                   fman->state->fm_port_num_of_cg);
2007
2008         /* Save LIODN info before FMan reset
2009          * Skipping non-existent port 0 (i = 1)
2010          */
2011         for (i = 1; i < FMAN_LIODN_TBL; i++) {
2012                 u32 liodn_base;
2013
2014                 fman->liodn_offset[i] =
2015                         ioread32be(&fman->bmi_regs->fmbm_spliodn[i - 1]);
2016                 liodn_base = ioread32be(&fman->dma_regs->fmdmplr[i / 2]);
2017                 if (i % 2) {
2018                         /* FMDM_PLR LSB holds LIODN base for odd ports */
2019                         liodn_base &= DMA_LIODN_BASE_MASK;
2020                 } else {
2021                         /* FMDM_PLR MSB holds LIODN base for even ports */
2022                         liodn_base >>= DMA_LIODN_SHIFT;
2023                         liodn_base &= DMA_LIODN_BASE_MASK;
2024                 }
2025                 fman->liodn_base[i] = liodn_base;
2026         }
2027
2028         err = fman_reset(fman);
2029         if (err)
2030                 return err;
2031
2032         if (ioread32be(&fman->qmi_regs->fmqm_gs) & QMI_GS_HALT_NOT_BUSY) {
2033                 resume(fman->fpm_regs);
2034                 /* Wait until QMI is not in halt not busy state */
2035                 count = 100;
2036                 do {
2037                         udelay(1);
2038                 } while (((ioread32be(&fman->qmi_regs->fmqm_gs)) &
2039                          QMI_GS_HALT_NOT_BUSY) && --count);
2040                 if (count == 0)
2041                         dev_warn(fman->dev, "%s: QMI is in halt not busy state\n",
2042                                  __func__);
2043         }
2044
2045         if (clear_iram(fman) != 0)
2046                 return -EINVAL;
2047
2048         cfg->exceptions = fman->state->exceptions;
2049
2050         /* Init DMA Registers */
2051
2052         err = dma_init(fman);
2053         if (err != 0) {
2054                 free_init_resources(fman);
2055                 return err;
2056         }
2057
2058         /* Init FPM Registers */
2059         fpm_init(fman->fpm_regs, fman->cfg);
2060
2061         /* define common resources */
2062         /* allocate MURAM for FIFO according to total size */
2063         fman->fifo_offset = fman_muram_alloc(fman->muram,
2064                                              fman->state->total_fifo_size);
2065         if (IS_ERR_VALUE(fman->fifo_offset)) {
2066                 free_init_resources(fman);
2067                 dev_err(fman->dev, "%s: MURAM alloc for BMI FIFO failed\n",
2068                         __func__);
2069                 return -ENOMEM;
2070         }
2071
2072         cfg->fifo_base_addr = fman->fifo_offset;
2073         cfg->total_fifo_size = fman->state->total_fifo_size;
2074         cfg->total_num_of_tasks = fman->state->total_num_of_tasks;
2075         cfg->clk_freq = fman->state->fm_clk_freq;
2076
2077         /* Init BMI Registers */
2078         bmi_init(fman->bmi_regs, fman->cfg);
2079
2080         /* Init QMI Registers */
2081         qmi_init(fman->qmi_regs, fman->cfg);
2082
2083         /* Init HW Parser */
2084         hwp_init(fman->hwp_regs);
2085
2086         err = enable(fman, cfg);
2087         if (err != 0)
2088                 return err;
2089
2090         enable_time_stamp(fman);
2091
2092         kfree(fman->cfg);
2093         fman->cfg = NULL;
2094
2095         return 0;
2096 }
2097
2098 static int fman_set_exception(struct fman *fman,
2099                               enum fman_exceptions exception, bool enable)
2100 {
2101         u32 bit_mask = 0;
2102
2103         if (!is_init_done(fman->cfg))
2104                 return -EINVAL;
2105
2106         bit_mask = get_exception_flag(exception);
2107         if (bit_mask) {
2108                 if (enable)
2109                         fman->state->exceptions |= bit_mask;
2110                 else
2111                         fman->state->exceptions &= ~bit_mask;
2112         } else {
2113                 dev_err(fman->dev, "%s: Undefined exception (%d)\n",
2114                         __func__, exception);
2115                 return -EINVAL;
2116         }
2117
2118         return set_exception(fman, exception, enable);
2119 }
2120
2121 /**
2122  * fman_register_intr
2123  * @fman:       A Pointer to FMan device
2124  * @mod:        Calling module
2125  * @mod_id:     Module id (if more than 1 exists, '0' if not)
2126  * @intr_type:  Interrupt type (error/normal) selection.
2127  * @f_isr:      The interrupt service routine.
2128  * @h_src_arg:  Argument to be passed to f_isr.
2129  *
2130  * Used to register an event handler to be processed by FMan
2131  *
2132  * Return: 0 on success; Error code otherwise.
2133  */
2134 void fman_register_intr(struct fman *fman, enum fman_event_modules module,
2135                         u8 mod_id, enum fman_intr_type intr_type,
2136                         void (*isr_cb)(void *src_arg), void *src_arg)
2137 {
2138         int event = 0;
2139
2140         event = get_module_event(module, mod_id, intr_type);
2141         WARN_ON(event >= FMAN_EV_CNT);
2142
2143         /* register in local FM structure */
2144         fman->intr_mng[event].isr_cb = isr_cb;
2145         fman->intr_mng[event].src_handle = src_arg;
2146 }
2147 EXPORT_SYMBOL(fman_register_intr);
2148
2149 /**
2150  * fman_unregister_intr
2151  * @fman:       A Pointer to FMan device
2152  * @mod:        Calling module
2153  * @mod_id:     Module id (if more than 1 exists, '0' if not)
2154  * @intr_type:  Interrupt type (error/normal) selection.
2155  *
2156  * Used to unregister an event handler to be processed by FMan
2157  *
2158  * Return: 0 on success; Error code otherwise.
2159  */
2160 void fman_unregister_intr(struct fman *fman, enum fman_event_modules module,
2161                           u8 mod_id, enum fman_intr_type intr_type)
2162 {
2163         int event = 0;
2164
2165         event = get_module_event(module, mod_id, intr_type);
2166         WARN_ON(event >= FMAN_EV_CNT);
2167
2168         fman->intr_mng[event].isr_cb = NULL;
2169         fman->intr_mng[event].src_handle = NULL;
2170 }
2171 EXPORT_SYMBOL(fman_unregister_intr);
2172
2173 /**
2174  * fman_set_port_params
2175  * @fman:               A Pointer to FMan device
2176  * @port_params:        Port parameters
2177  *
2178  * Used by FMan Port to pass parameters to the FMan
2179  *
2180  * Return: 0 on success; Error code otherwise.
2181  */
2182 int fman_set_port_params(struct fman *fman,
2183                          struct fman_port_init_params *port_params)
2184 {
2185         int err;
2186         unsigned long flags;
2187         u8 port_id = port_params->port_id, mac_id;
2188
2189         spin_lock_irqsave(&fman->spinlock, flags);
2190
2191         err = set_num_of_tasks(fman, port_params->port_id,
2192                                &port_params->num_of_tasks,
2193                                &port_params->num_of_extra_tasks);
2194         if (err)
2195                 goto return_err;
2196
2197         /* TX Ports */
2198         if (port_params->port_type != FMAN_PORT_TYPE_RX) {
2199                 u32 enq_th, deq_th, reg;
2200
2201                 /* update qmi ENQ/DEQ threshold */
2202                 fman->state->accumulated_num_of_deq_tnums +=
2203                         port_params->deq_pipeline_depth;
2204                 enq_th = (ioread32be(&fman->qmi_regs->fmqm_gc) &
2205                           QMI_CFG_ENQ_MASK) >> QMI_CFG_ENQ_SHIFT;
2206                 /* if enq_th is too big, we reduce it to the max value
2207                  * that is still 0
2208                  */
2209                 if (enq_th >= (fman->state->qmi_max_num_of_tnums -
2210                     fman->state->accumulated_num_of_deq_tnums)) {
2211                         enq_th =
2212                         fman->state->qmi_max_num_of_tnums -
2213                         fman->state->accumulated_num_of_deq_tnums - 1;
2214
2215                         reg = ioread32be(&fman->qmi_regs->fmqm_gc);
2216                         reg &= ~QMI_CFG_ENQ_MASK;
2217                         reg |= (enq_th << QMI_CFG_ENQ_SHIFT);
2218                         iowrite32be(reg, &fman->qmi_regs->fmqm_gc);
2219                 }
2220
2221                 deq_th = ioread32be(&fman->qmi_regs->fmqm_gc) &
2222                                     QMI_CFG_DEQ_MASK;
2223                 /* if deq_th is too small, we enlarge it to the min
2224                  * value that is still 0.
2225                  * depTh may not be larger than 63
2226                  * (fman->state->qmi_max_num_of_tnums-1).
2227                  */
2228                 if ((deq_th <= fman->state->accumulated_num_of_deq_tnums) &&
2229                     (deq_th < fman->state->qmi_max_num_of_tnums - 1)) {
2230                         deq_th = fman->state->accumulated_num_of_deq_tnums + 1;
2231                         reg = ioread32be(&fman->qmi_regs->fmqm_gc);
2232                         reg &= ~QMI_CFG_DEQ_MASK;
2233                         reg |= deq_th;
2234                         iowrite32be(reg, &fman->qmi_regs->fmqm_gc);
2235                 }
2236         }
2237
2238         err = set_size_of_fifo(fman, port_params->port_id,
2239                                &port_params->size_of_fifo,
2240                                &port_params->extra_size_of_fifo);
2241         if (err)
2242                 goto return_err;
2243
2244         err = set_num_of_open_dmas(fman, port_params->port_id,
2245                                    &port_params->num_of_open_dmas,
2246                                    &port_params->num_of_extra_open_dmas);
2247         if (err)
2248                 goto return_err;
2249
2250         set_port_liodn(fman, port_id, fman->liodn_base[port_id],
2251                        fman->liodn_offset[port_id]);
2252
2253         if (fman->state->rev_info.major < 6)
2254                 set_port_order_restoration(fman->fpm_regs, port_id);
2255
2256         mac_id = hw_port_id_to_sw_port_id(fman->state->rev_info.major, port_id);
2257
2258         if (port_params->max_frame_length >= fman->state->mac_mfl[mac_id]) {
2259                 fman->state->port_mfl[mac_id] = port_params->max_frame_length;
2260         } else {
2261                 dev_warn(fman->dev, "%s: Port (%d) max_frame_length is smaller than MAC (%d) current MTU\n",
2262                          __func__, port_id, mac_id);
2263                 err = -EINVAL;
2264                 goto return_err;
2265         }
2266
2267         spin_unlock_irqrestore(&fman->spinlock, flags);
2268
2269         return 0;
2270
2271 return_err:
2272         spin_unlock_irqrestore(&fman->spinlock, flags);
2273         return err;
2274 }
2275 EXPORT_SYMBOL(fman_set_port_params);
2276
2277 /**
2278  * fman_reset_mac
2279  * @fman:       A Pointer to FMan device
2280  * @mac_id:     MAC id to be reset
2281  *
2282  * Reset a specific MAC
2283  *
2284  * Return: 0 on success; Error code otherwise.
2285  */
2286 int fman_reset_mac(struct fman *fman, u8 mac_id)
2287 {
2288         struct fman_fpm_regs __iomem *fpm_rg = fman->fpm_regs;
2289         u32 msk, timeout = 100;
2290
2291         if (fman->state->rev_info.major >= 6) {
2292                 dev_err(fman->dev, "%s: FMan MAC reset no available for FMan V3!\n",
2293                         __func__);
2294                 return -EINVAL;
2295         }
2296
2297         /* Get the relevant bit mask */
2298         switch (mac_id) {
2299         case 0:
2300                 msk = FPM_RSTC_MAC0_RESET;
2301                 break;
2302         case 1:
2303                 msk = FPM_RSTC_MAC1_RESET;
2304                 break;
2305         case 2:
2306                 msk = FPM_RSTC_MAC2_RESET;
2307                 break;
2308         case 3:
2309                 msk = FPM_RSTC_MAC3_RESET;
2310                 break;
2311         case 4:
2312                 msk = FPM_RSTC_MAC4_RESET;
2313                 break;
2314         case 5:
2315                 msk = FPM_RSTC_MAC5_RESET;
2316                 break;
2317         case 6:
2318                 msk = FPM_RSTC_MAC6_RESET;
2319                 break;
2320         case 7:
2321                 msk = FPM_RSTC_MAC7_RESET;
2322                 break;
2323         case 8:
2324                 msk = FPM_RSTC_MAC8_RESET;
2325                 break;
2326         case 9:
2327                 msk = FPM_RSTC_MAC9_RESET;
2328                 break;
2329         default:
2330                 dev_warn(fman->dev, "%s: Illegal MAC Id [%d]\n",
2331                          __func__, mac_id);
2332                 return -EINVAL;
2333         }
2334
2335         /* reset */
2336         iowrite32be(msk, &fpm_rg->fm_rstc);
2337         while ((ioread32be(&fpm_rg->fm_rstc) & msk) && --timeout)
2338                 udelay(10);
2339
2340         if (!timeout)
2341                 return -EIO;
2342
2343         return 0;
2344 }
2345 EXPORT_SYMBOL(fman_reset_mac);
2346
2347 /**
2348  * fman_set_mac_max_frame
2349  * @fman:       A Pointer to FMan device
2350  * @mac_id:     MAC id
2351  * @mfl:        Maximum frame length
2352  *
2353  * Set maximum frame length of specific MAC in FMan driver
2354  *
2355  * Return: 0 on success; Error code otherwise.
2356  */
2357 int fman_set_mac_max_frame(struct fman *fman, u8 mac_id, u16 mfl)
2358 {
2359         /* if port is already initialized, check that MaxFrameLength is smaller
2360          * or equal to the port's max
2361          */
2362         if ((!fman->state->port_mfl[mac_id]) ||
2363             (mfl <= fman->state->port_mfl[mac_id])) {
2364                 fman->state->mac_mfl[mac_id] = mfl;
2365         } else {
2366                 dev_warn(fman->dev, "%s: MAC max_frame_length is larger than Port max_frame_length\n",
2367                          __func__);
2368                 return -EINVAL;
2369         }
2370         return 0;
2371 }
2372 EXPORT_SYMBOL(fman_set_mac_max_frame);
2373
2374 /**
2375  * fman_get_clock_freq
2376  * @fman:       A Pointer to FMan device
2377  *
2378  * Get FMan clock frequency
2379  *
2380  * Return: FMan clock frequency
2381  */
2382 u16 fman_get_clock_freq(struct fman *fman)
2383 {
2384         return fman->state->fm_clk_freq;
2385 }
2386
2387 /**
2388  * fman_get_bmi_max_fifo_size
2389  * @fman:       A Pointer to FMan device
2390  *
2391  * Get FMan maximum FIFO size
2392  *
2393  * Return: FMan Maximum FIFO size
2394  */
2395 u32 fman_get_bmi_max_fifo_size(struct fman *fman)
2396 {
2397         return fman->state->bmi_max_fifo_size;
2398 }
2399 EXPORT_SYMBOL(fman_get_bmi_max_fifo_size);
2400
2401 /**
2402  * fman_get_revision
2403  * @fman                - Pointer to the FMan module
2404  * @rev_info            - A structure of revision information parameters.
2405  *
2406  * Returns the FM revision
2407  *
2408  * Allowed only following fman_init().
2409  *
2410  * Return: 0 on success; Error code otherwise.
2411  */
2412 void fman_get_revision(struct fman *fman, struct fman_rev_info *rev_info)
2413 {
2414         u32 tmp;
2415
2416         tmp = ioread32be(&fman->fpm_regs->fm_ip_rev_1);
2417         rev_info->major = (u8)((tmp & FPM_REV1_MAJOR_MASK) >>
2418                                 FPM_REV1_MAJOR_SHIFT);
2419         rev_info->minor = tmp & FPM_REV1_MINOR_MASK;
2420 }
2421 EXPORT_SYMBOL(fman_get_revision);
2422
2423 /**
2424  * fman_get_qman_channel_id
2425  * @fman:       A Pointer to FMan device
2426  * @port_id:    Port id
2427  *
2428  * Get QMan channel ID associated to the Port id
2429  *
2430  * Return: QMan channel ID
2431  */
2432 u32 fman_get_qman_channel_id(struct fman *fman, u32 port_id)
2433 {
2434         int i;
2435
2436         if (fman->state->rev_info.major >= 6) {
2437                 u32 port_ids[] = {0x30, 0x31, 0x28, 0x29, 0x2a, 0x2b,
2438                                   0x2c, 0x2d, 0x2, 0x3, 0x4, 0x5, 0x7, 0x7};
2439                 for (i = 0; i < fman->state->num_of_qman_channels; i++) {
2440                         if (port_ids[i] == port_id)
2441                                 break;
2442                 }
2443         } else {
2444                 u32 port_ids[] = {0x30, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x1,
2445                                   0x2, 0x3, 0x4, 0x5, 0x7, 0x7};
2446                 for (i = 0; i < fman->state->num_of_qman_channels; i++) {
2447                         if (port_ids[i] == port_id)
2448                                 break;
2449                 }
2450         }
2451
2452         if (i == fman->state->num_of_qman_channels)
2453                 return 0;
2454
2455         return fman->state->qman_channel_base + i;
2456 }
2457 EXPORT_SYMBOL(fman_get_qman_channel_id);
2458
2459 /**
2460  * fman_get_mem_region
2461  * @fman:       A Pointer to FMan device
2462  *
2463  * Get FMan memory region
2464  *
2465  * Return: A structure with FMan memory region information
2466  */
2467 struct resource *fman_get_mem_region(struct fman *fman)
2468 {
2469         return fman->state->res;
2470 }
2471 EXPORT_SYMBOL(fman_get_mem_region);
2472
2473 /* Bootargs defines */
2474 /* Extra headroom for RX buffers - Default, min and max */
2475 #define FSL_FM_RX_EXTRA_HEADROOM        64
2476 #define FSL_FM_RX_EXTRA_HEADROOM_MIN    16
2477 #define FSL_FM_RX_EXTRA_HEADROOM_MAX    384
2478
2479 /* Maximum frame length */
2480 #define FSL_FM_MAX_FRAME_SIZE                   1522
2481 #define FSL_FM_MAX_POSSIBLE_FRAME_SIZE          9600
2482 #define FSL_FM_MIN_POSSIBLE_FRAME_SIZE          64
2483
2484 /* Extra headroom for Rx buffers.
2485  * FMan is instructed to allocate, on the Rx path, this amount of
2486  * space at the beginning of a data buffer, beside the DPA private
2487  * data area and the IC fields.
2488  * Does not impact Tx buffer layout.
2489  * Configurable from bootargs. 64 by default, it's needed on
2490  * particular forwarding scenarios that add extra headers to the
2491  * forwarded frame.
2492  */
2493 static int fsl_fm_rx_extra_headroom = FSL_FM_RX_EXTRA_HEADROOM;
2494 module_param(fsl_fm_rx_extra_headroom, int, 0);
2495 MODULE_PARM_DESC(fsl_fm_rx_extra_headroom, "Extra headroom for Rx buffers");
2496
2497 /* Max frame size, across all interfaces.
2498  * Configurable from bootargs, to avoid allocating oversized (socket)
2499  * buffers when not using jumbo frames.
2500  * Must be large enough to accommodate the network MTU, but small enough
2501  * to avoid wasting skb memory.
2502  *
2503  * Could be overridden once, at boot-time, via the
2504  * fm_set_max_frm() callback.
2505  */
2506 static int fsl_fm_max_frm = FSL_FM_MAX_FRAME_SIZE;
2507 module_param(fsl_fm_max_frm, int, 0);
2508 MODULE_PARM_DESC(fsl_fm_max_frm, "Maximum frame size, across all interfaces");
2509
2510 /**
2511  * fman_get_max_frm
2512  *
2513  * Return: Max frame length configured in the FM driver
2514  */
2515 u16 fman_get_max_frm(void)
2516 {
2517         static bool fm_check_mfl;
2518
2519         if (!fm_check_mfl) {
2520                 if (fsl_fm_max_frm > FSL_FM_MAX_POSSIBLE_FRAME_SIZE ||
2521                     fsl_fm_max_frm < FSL_FM_MIN_POSSIBLE_FRAME_SIZE) {
2522                         pr_warn("Invalid fsl_fm_max_frm value (%d) in bootargs, valid range is %d-%d. Falling back to the default (%d)\n",
2523                                 fsl_fm_max_frm,
2524                                 FSL_FM_MIN_POSSIBLE_FRAME_SIZE,
2525                                 FSL_FM_MAX_POSSIBLE_FRAME_SIZE,
2526                                 FSL_FM_MAX_FRAME_SIZE);
2527                         fsl_fm_max_frm = FSL_FM_MAX_FRAME_SIZE;
2528                 }
2529                 fm_check_mfl = true;
2530         }
2531
2532         return fsl_fm_max_frm;
2533 }
2534 EXPORT_SYMBOL(fman_get_max_frm);
2535
2536 /**
2537  * fman_get_rx_extra_headroom
2538  *
2539  * Return: Extra headroom size configured in the FM driver
2540  */
2541 int fman_get_rx_extra_headroom(void)
2542 {
2543         static bool fm_check_rx_extra_headroom;
2544
2545         if (!fm_check_rx_extra_headroom) {
2546                 if (fsl_fm_rx_extra_headroom > FSL_FM_RX_EXTRA_HEADROOM_MAX ||
2547                     fsl_fm_rx_extra_headroom < FSL_FM_RX_EXTRA_HEADROOM_MIN) {
2548                         pr_warn("Invalid fsl_fm_rx_extra_headroom value (%d) in bootargs, valid range is %d-%d. Falling back to the default (%d)\n",
2549                                 fsl_fm_rx_extra_headroom,
2550                                 FSL_FM_RX_EXTRA_HEADROOM_MIN,
2551                                 FSL_FM_RX_EXTRA_HEADROOM_MAX,
2552                                 FSL_FM_RX_EXTRA_HEADROOM);
2553                         fsl_fm_rx_extra_headroom = FSL_FM_RX_EXTRA_HEADROOM;
2554                 }
2555
2556                 fm_check_rx_extra_headroom = true;
2557                 fsl_fm_rx_extra_headroom = ALIGN(fsl_fm_rx_extra_headroom, 16);
2558         }
2559
2560         return fsl_fm_rx_extra_headroom;
2561 }
2562 EXPORT_SYMBOL(fman_get_rx_extra_headroom);
2563
2564 /**
2565  * fman_bind
2566  * @dev:        FMan OF device pointer
2567  *
2568  * Bind to a specific FMan device.
2569  *
2570  * Allowed only after the port was created.
2571  *
2572  * Return: A pointer to the FMan device
2573  */
2574 struct fman *fman_bind(struct device *fm_dev)
2575 {
2576         return (struct fman *)(dev_get_drvdata(get_device(fm_dev)));
2577 }
2578 EXPORT_SYMBOL(fman_bind);
2579
2580 static irqreturn_t fman_err_irq(int irq, void *handle)
2581 {
2582         struct fman *fman = (struct fman *)handle;
2583         u32 pending;
2584         struct fman_fpm_regs __iomem *fpm_rg;
2585         irqreturn_t single_ret, ret = IRQ_NONE;
2586
2587         if (!is_init_done(fman->cfg))
2588                 return IRQ_NONE;
2589
2590         fpm_rg = fman->fpm_regs;
2591
2592         /* error interrupts */
2593         pending = ioread32be(&fpm_rg->fm_epi);
2594         if (!pending)
2595                 return IRQ_NONE;
2596
2597         if (pending & ERR_INTR_EN_BMI) {
2598                 single_ret = bmi_err_event(fman);
2599                 if (single_ret == IRQ_HANDLED)
2600                         ret = IRQ_HANDLED;
2601         }
2602         if (pending & ERR_INTR_EN_QMI) {
2603                 single_ret = qmi_err_event(fman);
2604                 if (single_ret == IRQ_HANDLED)
2605                         ret = IRQ_HANDLED;
2606         }
2607         if (pending & ERR_INTR_EN_FPM) {
2608                 single_ret = fpm_err_event(fman);
2609                 if (single_ret == IRQ_HANDLED)
2610                         ret = IRQ_HANDLED;
2611         }
2612         if (pending & ERR_INTR_EN_DMA) {
2613                 single_ret = dma_err_event(fman);
2614                 if (single_ret == IRQ_HANDLED)
2615                         ret = IRQ_HANDLED;
2616         }
2617         if (pending & ERR_INTR_EN_MURAM) {
2618                 single_ret = muram_err_intr(fman);
2619                 if (single_ret == IRQ_HANDLED)
2620                         ret = IRQ_HANDLED;
2621         }
2622
2623         /* MAC error interrupts */
2624         if (pending & ERR_INTR_EN_MAC0) {
2625                 single_ret = call_mac_isr(fman, FMAN_EV_ERR_MAC0 + 0);
2626                 if (single_ret == IRQ_HANDLED)
2627                         ret = IRQ_HANDLED;
2628         }
2629         if (pending & ERR_INTR_EN_MAC1) {
2630                 single_ret = call_mac_isr(fman, FMAN_EV_ERR_MAC0 + 1);
2631                 if (single_ret == IRQ_HANDLED)
2632                         ret = IRQ_HANDLED;
2633         }
2634         if (pending & ERR_INTR_EN_MAC2) {
2635                 single_ret = call_mac_isr(fman, FMAN_EV_ERR_MAC0 + 2);
2636                 if (single_ret == IRQ_HANDLED)
2637                         ret = IRQ_HANDLED;
2638         }
2639         if (pending & ERR_INTR_EN_MAC3) {
2640                 single_ret = call_mac_isr(fman, FMAN_EV_ERR_MAC0 + 3);
2641                 if (single_ret == IRQ_HANDLED)
2642                         ret = IRQ_HANDLED;
2643         }
2644         if (pending & ERR_INTR_EN_MAC4) {
2645                 single_ret = call_mac_isr(fman, FMAN_EV_ERR_MAC0 + 4);
2646                 if (single_ret == IRQ_HANDLED)
2647                         ret = IRQ_HANDLED;
2648         }
2649         if (pending & ERR_INTR_EN_MAC5) {
2650                 single_ret = call_mac_isr(fman, FMAN_EV_ERR_MAC0 + 5);
2651                 if (single_ret == IRQ_HANDLED)
2652                         ret = IRQ_HANDLED;
2653         }
2654         if (pending & ERR_INTR_EN_MAC6) {
2655                 single_ret = call_mac_isr(fman, FMAN_EV_ERR_MAC0 + 6);
2656                 if (single_ret == IRQ_HANDLED)
2657                         ret = IRQ_HANDLED;
2658         }
2659         if (pending & ERR_INTR_EN_MAC7) {
2660                 single_ret = call_mac_isr(fman, FMAN_EV_ERR_MAC0 + 7);
2661                 if (single_ret == IRQ_HANDLED)
2662                         ret = IRQ_HANDLED;
2663         }
2664         if (pending & ERR_INTR_EN_MAC8) {
2665                 single_ret = call_mac_isr(fman, FMAN_EV_ERR_MAC0 + 8);
2666                 if (single_ret == IRQ_HANDLED)
2667                         ret = IRQ_HANDLED;
2668         }
2669         if (pending & ERR_INTR_EN_MAC9) {
2670                 single_ret = call_mac_isr(fman, FMAN_EV_ERR_MAC0 + 9);
2671                 if (single_ret == IRQ_HANDLED)
2672                         ret = IRQ_HANDLED;
2673         }
2674
2675         return ret;
2676 }
2677
2678 static irqreturn_t fman_irq(int irq, void *handle)
2679 {
2680         struct fman *fman = (struct fman *)handle;
2681         u32 pending;
2682         struct fman_fpm_regs __iomem *fpm_rg;
2683         irqreturn_t single_ret, ret = IRQ_NONE;
2684
2685         if (!is_init_done(fman->cfg))
2686                 return IRQ_NONE;
2687
2688         fpm_rg = fman->fpm_regs;
2689
2690         /* normal interrupts */
2691         pending = ioread32be(&fpm_rg->fm_npi);
2692         if (!pending)
2693                 return IRQ_NONE;
2694
2695         if (pending & INTR_EN_QMI) {
2696                 single_ret = qmi_event(fman);
2697                 if (single_ret == IRQ_HANDLED)
2698                         ret = IRQ_HANDLED;
2699         }
2700
2701         /* MAC interrupts */
2702         if (pending & INTR_EN_MAC0) {
2703                 single_ret = call_mac_isr(fman, FMAN_EV_MAC0 + 0);
2704                 if (single_ret == IRQ_HANDLED)
2705                         ret = IRQ_HANDLED;
2706         }
2707         if (pending & INTR_EN_MAC1) {
2708                 single_ret = call_mac_isr(fman, FMAN_EV_MAC0 + 1);
2709                 if (single_ret == IRQ_HANDLED)
2710                         ret = IRQ_HANDLED;
2711         }
2712         if (pending & INTR_EN_MAC2) {
2713                 single_ret = call_mac_isr(fman, FMAN_EV_MAC0 + 2);
2714                 if (single_ret == IRQ_HANDLED)
2715                         ret = IRQ_HANDLED;
2716         }
2717         if (pending & INTR_EN_MAC3) {
2718                 single_ret = call_mac_isr(fman, FMAN_EV_MAC0 + 3);
2719                 if (single_ret == IRQ_HANDLED)
2720                         ret = IRQ_HANDLED;
2721         }
2722         if (pending & INTR_EN_MAC4) {
2723                 single_ret = call_mac_isr(fman, FMAN_EV_MAC0 + 4);
2724                 if (single_ret == IRQ_HANDLED)
2725                         ret = IRQ_HANDLED;
2726         }
2727         if (pending & INTR_EN_MAC5) {
2728                 single_ret = call_mac_isr(fman, FMAN_EV_MAC0 + 5);
2729                 if (single_ret == IRQ_HANDLED)
2730                         ret = IRQ_HANDLED;
2731         }
2732         if (pending & INTR_EN_MAC6) {
2733                 single_ret = call_mac_isr(fman, FMAN_EV_MAC0 + 6);
2734                 if (single_ret == IRQ_HANDLED)
2735                         ret = IRQ_HANDLED;
2736         }
2737         if (pending & INTR_EN_MAC7) {
2738                 single_ret = call_mac_isr(fman, FMAN_EV_MAC0 + 7);
2739                 if (single_ret == IRQ_HANDLED)
2740                         ret = IRQ_HANDLED;
2741         }
2742         if (pending & INTR_EN_MAC8) {
2743                 single_ret = call_mac_isr(fman, FMAN_EV_MAC0 + 8);
2744                 if (single_ret == IRQ_HANDLED)
2745                         ret = IRQ_HANDLED;
2746         }
2747         if (pending & INTR_EN_MAC9) {
2748                 single_ret = call_mac_isr(fman, FMAN_EV_MAC0 + 9);
2749                 if (single_ret == IRQ_HANDLED)
2750                         ret = IRQ_HANDLED;
2751         }
2752
2753         return ret;
2754 }
2755
2756 static const struct of_device_id fman_muram_match[] = {
2757         {
2758                 .compatible = "fsl,fman-muram"},
2759         {}
2760 };
2761 MODULE_DEVICE_TABLE(of, fman_muram_match);
2762
2763 static struct fman *read_dts_node(struct platform_device *of_dev)
2764 {
2765         struct fman *fman;
2766         struct device_node *fm_node, *muram_node;
2767         struct resource *res;
2768         u32 val, range[2];
2769         int err, irq;
2770         struct clk *clk;
2771         u32 clk_rate;
2772         phys_addr_t phys_base_addr;
2773         resource_size_t mem_size;
2774
2775         fman = kzalloc(sizeof(*fman), GFP_KERNEL);
2776         if (!fman)
2777                 return NULL;
2778
2779         fm_node = of_node_get(of_dev->dev.of_node);
2780
2781         err = of_property_read_u32(fm_node, "cell-index", &val);
2782         if (err) {
2783                 dev_err(&of_dev->dev, "%s: failed to read cell-index for %s\n",
2784                         __func__, fm_node->full_name);
2785                 goto fman_node_put;
2786         }
2787         fman->dts_params.id = (u8)val;
2788
2789         /* Get the FM interrupt */
2790         res = platform_get_resource(of_dev, IORESOURCE_IRQ, 0);
2791         if (!res) {
2792                 dev_err(&of_dev->dev, "%s: Can't get FMan IRQ resource\n",
2793                         __func__);
2794                 goto fman_node_put;
2795         }
2796         irq = res->start;
2797
2798         /* Get the FM error interrupt */
2799         res = platform_get_resource(of_dev, IORESOURCE_IRQ, 1);
2800         if (!res) {
2801                 dev_err(&of_dev->dev, "%s: Can't get FMan Error IRQ resource\n",
2802                         __func__);
2803                 goto fman_node_put;
2804         }
2805         fman->dts_params.err_irq = res->start;
2806
2807         /* Get the FM address */
2808         res = platform_get_resource(of_dev, IORESOURCE_MEM, 0);
2809         if (!res) {
2810                 dev_err(&of_dev->dev, "%s: Can't get FMan memory resource\n",
2811                         __func__);
2812                 goto fman_node_put;
2813         }
2814
2815         phys_base_addr = res->start;
2816         mem_size = resource_size(res);
2817
2818         clk = of_clk_get(fm_node, 0);
2819         if (IS_ERR(clk)) {
2820                 dev_err(&of_dev->dev, "%s: Failed to get FM%d clock structure\n",
2821                         __func__, fman->dts_params.id);
2822                 goto fman_node_put;
2823         }
2824
2825         clk_rate = clk_get_rate(clk);
2826         if (!clk_rate) {
2827                 dev_err(&of_dev->dev, "%s: Failed to determine FM%d clock rate\n",
2828                         __func__, fman->dts_params.id);
2829                 goto fman_node_put;
2830         }
2831         /* Rounding to MHz */
2832         fman->dts_params.clk_freq = DIV_ROUND_UP(clk_rate, 1000000);
2833
2834         err = of_property_read_u32_array(fm_node, "fsl,qman-channel-range",
2835                                          &range[0], 2);
2836         if (err) {
2837                 dev_err(&of_dev->dev, "%s: failed to read fsl,qman-channel-range for %s\n",
2838                         __func__, fm_node->full_name);
2839                 goto fman_node_put;
2840         }
2841         fman->dts_params.qman_channel_base = range[0];
2842         fman->dts_params.num_of_qman_channels = range[1];
2843
2844         /* Get the MURAM base address and size */
2845         muram_node = of_find_matching_node(fm_node, fman_muram_match);
2846         if (!muram_node) {
2847                 dev_err(&of_dev->dev, "%s: could not find MURAM node\n",
2848                         __func__);
2849                 goto fman_node_put;
2850         }
2851
2852         err = of_address_to_resource(muram_node, 0,
2853                                      &fman->dts_params.muram_res);
2854         if (err) {
2855                 of_node_put(muram_node);
2856                 dev_err(&of_dev->dev, "%s: of_address_to_resource() = %d\n",
2857                         __func__, err);
2858                 goto fman_node_put;
2859         }
2860
2861         of_node_put(muram_node);
2862         of_node_put(fm_node);
2863
2864         err = devm_request_irq(&of_dev->dev, irq, fman_irq, 0, "fman", fman);
2865         if (err < 0) {
2866                 dev_err(&of_dev->dev, "%s: irq %d allocation failed (error = %d)\n",
2867                         __func__, irq, err);
2868                 goto fman_free;
2869         }
2870
2871         if (fman->dts_params.err_irq != 0) {
2872                 err = devm_request_irq(&of_dev->dev, fman->dts_params.err_irq,
2873                                        fman_err_irq, IRQF_SHARED,
2874                                        "fman-err", fman);
2875                 if (err < 0) {
2876                         dev_err(&of_dev->dev, "%s: irq %d allocation failed (error = %d)\n",
2877                                 __func__, fman->dts_params.err_irq, err);
2878                         goto fman_free;
2879                 }
2880         }
2881
2882         fman->dts_params.res =
2883                 devm_request_mem_region(&of_dev->dev, phys_base_addr,
2884                                         mem_size, "fman");
2885         if (!fman->dts_params.res) {
2886                 dev_err(&of_dev->dev, "%s: request_mem_region() failed\n",
2887                         __func__);
2888                 goto fman_free;
2889         }
2890
2891         fman->dts_params.base_addr =
2892                 devm_ioremap(&of_dev->dev, phys_base_addr, mem_size);
2893         if (!fman->dts_params.base_addr) {
2894                 dev_err(&of_dev->dev, "%s: devm_ioremap() failed\n", __func__);
2895                 goto fman_free;
2896         }
2897
2898         fman->dev = &of_dev->dev;
2899
2900         err = of_platform_populate(fm_node, NULL, NULL, &of_dev->dev);
2901         if (err) {
2902                 dev_err(&of_dev->dev, "%s: of_platform_populate() failed\n",
2903                         __func__);
2904                 goto fman_free;
2905         }
2906
2907         return fman;
2908
2909 fman_node_put:
2910         of_node_put(fm_node);
2911 fman_free:
2912         kfree(fman);
2913         return NULL;
2914 }
2915
2916 static int fman_probe(struct platform_device *of_dev)
2917 {
2918         struct fman *fman;
2919         struct device *dev;
2920         int err;
2921
2922         dev = &of_dev->dev;
2923
2924         fman = read_dts_node(of_dev);
2925         if (!fman)
2926                 return -EIO;
2927
2928         err = fman_config(fman);
2929         if (err) {
2930                 dev_err(dev, "%s: FMan config failed\n", __func__);
2931                 return -EINVAL;
2932         }
2933
2934         if (fman_init(fman) != 0) {
2935                 dev_err(dev, "%s: FMan init failed\n", __func__);
2936                 return -EINVAL;
2937         }
2938
2939         if (fman->dts_params.err_irq == 0) {
2940                 fman_set_exception(fman, FMAN_EX_DMA_BUS_ERROR, false);
2941                 fman_set_exception(fman, FMAN_EX_DMA_READ_ECC, false);
2942                 fman_set_exception(fman, FMAN_EX_DMA_SYSTEM_WRITE_ECC, false);
2943                 fman_set_exception(fman, FMAN_EX_DMA_FM_WRITE_ECC, false);
2944                 fman_set_exception(fman, FMAN_EX_DMA_SINGLE_PORT_ECC, false);
2945                 fman_set_exception(fman, FMAN_EX_FPM_STALL_ON_TASKS, false);
2946                 fman_set_exception(fman, FMAN_EX_FPM_SINGLE_ECC, false);
2947                 fman_set_exception(fman, FMAN_EX_FPM_DOUBLE_ECC, false);
2948                 fman_set_exception(fman, FMAN_EX_QMI_SINGLE_ECC, false);
2949                 fman_set_exception(fman, FMAN_EX_QMI_DOUBLE_ECC, false);
2950                 fman_set_exception(fman,
2951                                    FMAN_EX_QMI_DEQ_FROM_UNKNOWN_PORTID, false);
2952                 fman_set_exception(fman, FMAN_EX_BMI_LIST_RAM_ECC, false);
2953                 fman_set_exception(fman, FMAN_EX_BMI_STORAGE_PROFILE_ECC,
2954                                    false);
2955                 fman_set_exception(fman, FMAN_EX_BMI_STATISTICS_RAM_ECC, false);
2956                 fman_set_exception(fman, FMAN_EX_BMI_DISPATCH_RAM_ECC, false);
2957         }
2958
2959         dev_set_drvdata(dev, fman);
2960
2961         dev_dbg(dev, "FMan%d probed\n", fman->dts_params.id);
2962
2963         return 0;
2964 }
2965
2966 static const struct of_device_id fman_match[] = {
2967         {
2968                 .compatible = "fsl,fman"},
2969         {}
2970 };
2971
2972 MODULE_DEVICE_TABLE(of, fman_match);
2973
2974 static struct platform_driver fman_driver = {
2975         .driver = {
2976                 .name = "fsl-fman",
2977                 .of_match_table = fman_match,
2978         },
2979         .probe = fman_probe,
2980 };
2981
2982 static int __init fman_load(void)
2983 {
2984         int err;
2985
2986         pr_debug("FSL DPAA FMan driver\n");
2987
2988         err = platform_driver_register(&fman_driver);
2989         if (err < 0)
2990                 pr_err("Error, platform_driver_register() = %d\n", err);
2991
2992         return err;
2993 }
2994 module_init(fman_load);
2995
2996 static void __exit fman_unload(void)
2997 {
2998         platform_driver_unregister(&fman_driver);
2999 }
3000 module_exit(fman_unload);
3001
3002 MODULE_LICENSE("Dual BSD/GPL");
3003 MODULE_DESCRIPTION("Freescale DPAA Frame Manager driver");