drivers/mtd/nand/raw/rockchip-nand-controller.c

   1 // SPDX-License-Identifier: GPL-2.0 OR MIT
   2 /*
   3  * Rockchip NAND Flash controller driver.
   4  * Copyright (C) 2020 Rockchip Inc.
   5  * Author: Yifeng Zhao <yifeng.zhao@rock-chips.com>
   6  */
   7
   8 #include <linux/clk.h>
   9 #include <linux/delay.h>
  10 #include <linux/dma-mapping.h>
  11 #include <linux/dmaengine.h>
  12 #include <linux/interrupt.h>
  13 #include <linux/iopoll.h>
  14 #include <linux/module.h>
  15 #include <linux/mtd/mtd.h>
  16 #include <linux/mtd/rawnand.h>
  17 #include <linux/of.h>
  18 #include <linux/of_device.h>
  19 #include <linux/platform_device.h>
  20 #include <linux/slab.h>
  21
  22 /*
  23  * NFC Page Data Layout:
  24  *      1024 bytes data + 4Bytes sys data + 28Bytes~124Bytes ECC data +
  25  *      1024 bytes data + 4Bytes sys data + 28Bytes~124Bytes ECC data +
  26  *      ......
  27  * NAND Page Data Layout:
  28  *      1024 * n data + m Bytes oob
  29  * Original Bad Block Mask Location:
  30  *      First byte of oob(spare).
  31  * nand_chip->oob_poi data layout:
  32  *      4Bytes sys data + .... + 4Bytes sys data + ECC data.
  33  */
  34
  35 /* NAND controller register definition */
  36 #define NFC_READ                        (0)
  37 #define NFC_WRITE                       (1)
  38
  39 #define NFC_FMCTL                       (0x00)
  40 #define   FMCTL_CE_SEL_M                0xFF
  41 #define   FMCTL_CE_SEL(x)               (1 << (x))
  42 #define   FMCTL_WP                      BIT(8)
  43 #define   FMCTL_RDY                     BIT(9)
  44
  45 #define NFC_FMWAIT                      (0x04)
  46 #define   FLCTL_RST                     BIT(0)
  47 #define   FLCTL_WR                      (1)     /* 0: read, 1: write */
  48 #define   FLCTL_XFER_ST                 BIT(2)
  49 #define   FLCTL_XFER_EN                 BIT(3)
  50 #define   FLCTL_ACORRECT                BIT(10) /* Auto correct error bits. */
  51 #define   FLCTL_XFER_READY              BIT(20)
  52 #define   FLCTL_XFER_SECTOR             (22)
  53 #define   FLCTL_TOG_FIX                 BIT(29)
  54
  55 #define   BCHCTL_BANK_M                 (7 << 5)
  56 #define   BCHCTL_BANK                   (5)
  57
  58 #define   DMA_ST                        BIT(0)
  59 #define   DMA_WR                        (1)     /* 0: write, 1: read */
  60 #define   DMA_EN                        BIT(2)
  61 #define   DMA_AHB_SIZE                  (3)     /* 0: 1, 1: 2, 2: 4 */
  62 #define   DMA_BURST_SIZE                (6)     /* 0: 1, 3: 4, 5: 8, 7: 16 */
  63 #define   DMA_INC_NUM                   (9)     /* 1 - 16 */
  64
  65 #define ECC_ERR_CNT(x, e) ((((x) >> (e).low) & (e).low_mask) |\
  66           (((x) >> (e).high) & (e).high_mask) << (e).low_bn)
  67 #define   INT_DMA                       BIT(0)
  68 #define NFC_BANK                        (0x800)
  69 #define NFC_BANK_STEP                   (0x100)
  70 #define   BANK_DATA                     (0x00)
  71 #define   BANK_ADDR                     (0x04)
  72 #define   BANK_CMD                      (0x08)
  73 #define NFC_SRAM0                       (0x1000)
  74 #define NFC_SRAM1                       (0x1400)
  75 #define NFC_SRAM_SIZE                   (0x400)
  76 #define NFC_TIMEOUT                     (500000)
  77 #define NFC_MAX_OOB_PER_STEP            128
  78 #define NFC_MIN_OOB_PER_STEP            64
  79 #define MAX_DATA_SIZE                   0xFFFC
  80 #define MAX_ADDRESS_CYC                 6
  81 #define NFC_ECC_MAX_MODES               4
  82 #define NFC_MAX_NSELS                   (8) /* Some Socs only have 1 or 2 CSs. */
  83 #define NFC_SYS_DATA_SIZE               (4) /* 4 bytes sys data in oob pre 1024 data.*/
  84 #define RK_DEFAULT_CLOCK_RATE           (150 * 1000 * 1000) /* 150 Mhz */
  85 #define ACCTIMING(csrw, rwpw, rwcs)     ((csrw) << 12 | (rwpw) << 5 | (rwcs))
  86
  87 enum nfc_type {
  88         NFC_V6,
  89         NFC_V8,
  90         NFC_V9,
  91 };
  92
  93 /**
  94  * struct rk_ecc_cnt_status: represent a ecc status data.
  95  * @err_flag_bit: error flag bit index at register.
  96  * @low: ECC count low bit index at register.
  97  * @low_mask: mask bit.
  98  * @low_bn: ECC count low bit number.
  99  * @high: ECC count high bit index at register.
 100  * @high_mask: mask bit
 101  */
 102 struct ecc_cnt_status {
 103         u8 err_flag_bit;
 104         u8 low;
 105         u8 low_mask;
 106         u8 low_bn;
 107         u8 high;
 108         u8 high_mask;
 109 };
 110
 111 /**
 112  * @type: NFC version
 113  * @ecc_strengths: ECC strengths
 114  * @ecc_cfgs: ECC config values
 115  * @flctl_off: FLCTL register offset
 116  * @bchctl_off: BCHCTL register offset
 117  * @dma_data_buf_off: DMA_DATA_BUF register offset
 118  * @dma_oob_buf_off: DMA_OOB_BUF register offset
 119  * @dma_cfg_off: DMA_CFG register offset
 120  * @dma_st_off: DMA_ST register offset
 121  * @bch_st_off: BCG_ST register offset
 122  * @randmz_off: RANDMZ register offset
 123  * @int_en_off: interrupt enable register offset
 124  * @int_clr_off: interrupt clean register offset
 125  * @int_st_off: interrupt status register offset
 126  * @oob0_off: oob0 register offset
 127  * @oob1_off: oob1 register offset
 128  * @ecc0: represent ECC0 status data
 129  * @ecc1: represent ECC1 status data
 130  */
 131 struct nfc_cfg {
 132         enum nfc_type type;
 133         u8 ecc_strengths[NFC_ECC_MAX_MODES];
 134         u32 ecc_cfgs[NFC_ECC_MAX_MODES];
 135         u32 flctl_off;
 136         u32 bchctl_off;
 137         u32 dma_cfg_off;
 138         u32 dma_data_buf_off;
 139         u32 dma_oob_buf_off;
 140         u32 dma_st_off;
 141         u32 bch_st_off;
 142         u32 randmz_off;
 143         u32 int_en_off;
 144         u32 int_clr_off;
 145         u32 int_st_off;
 146         u32 oob0_off;
 147         u32 oob1_off;
 148         struct ecc_cnt_status ecc0;
 149         struct ecc_cnt_status ecc1;
 150 };
 151
 152 struct rk_nfc_nand_chip {
 153         struct list_head node;
 154         struct nand_chip chip;
 155
 156         u16 boot_blks;
 157         u16 metadata_size;
 158         u32 boot_ecc;
 159         u32 timing;
 160
 161         u8 nsels;
 162         u8 sels[];
 163         /* Nothing after this field. */
 164 };
 165
 166 struct rk_nfc {
 167         struct nand_controller controller;
 168         const struct nfc_cfg *cfg;
 169         struct device *dev;
 170
 171         struct clk *nfc_clk;
 172         struct clk *ahb_clk;
 173         void __iomem *regs;
 174
 175         u32 selected_bank;
 176         u32 band_offset;
 177         u32 cur_ecc;
 178         u32 cur_timing;
 179
 180         struct completion done;
 181         struct list_head chips;
 182
 183         u8 *page_buf;
 184         u32 *oob_buf;
 185         u32 page_buf_size;
 186         u32 oob_buf_size;
 187
 188         unsigned long assigned_cs;
 189 };
 190
 191 static inline struct rk_nfc_nand_chip *rk_nfc_to_rknand(struct nand_chip *chip)
 192 {
 193         return container_of(chip, struct rk_nfc_nand_chip, chip);
 194 }
 195
 196 static inline u8 *rk_nfc_buf_to_data_ptr(struct nand_chip *chip, const u8 *p, int i)
 197 {
 198         return (u8 *)p + i * chip->ecc.size;
 199 }
 200
 201 static inline u8 *rk_nfc_buf_to_oob_ptr(struct nand_chip *chip, int i)
 202 {
 203         u8 *poi;
 204
 205         poi = chip->oob_poi + i * NFC_SYS_DATA_SIZE;
 206
 207         return poi;
 208 }
 209
 210 static inline u8 *rk_nfc_buf_to_oob_ecc_ptr(struct nand_chip *chip, int i)
 211 {
 212         struct rk_nfc_nand_chip *rknand = rk_nfc_to_rknand(chip);
 213         u8 *poi;
 214
 215         poi = chip->oob_poi + rknand->metadata_size + chip->ecc.bytes * i;
 216
 217         return poi;
 218 }
 219
 220 static inline int rk_nfc_data_len(struct nand_chip *chip)
 221 {
 222         return chip->ecc.size + chip->ecc.bytes + NFC_SYS_DATA_SIZE;
 223 }
 224
 225 static inline u8 *rk_nfc_data_ptr(struct nand_chip *chip, int i)
 226 {
 227         struct rk_nfc *nfc = nand_get_controller_data(chip);
 228
 229         return nfc->page_buf + i * rk_nfc_data_len(chip);
 230 }
 231
 232 static inline u8 *rk_nfc_oob_ptr(struct nand_chip *chip, int i)
 233 {
 234         struct rk_nfc *nfc = nand_get_controller_data(chip);
 235
 236         return nfc->page_buf + i * rk_nfc_data_len(chip) + chip->ecc.size;
 237 }
 238
 239 static int rk_nfc_hw_ecc_setup(struct nand_chip *chip, u32 strength)
 240 {
 241         struct rk_nfc *nfc = nand_get_controller_data(chip);
 242         u32 reg, i;
 243
 244         for (i = 0; i < NFC_ECC_MAX_MODES; i++) {
 245                 if (strength == nfc->cfg->ecc_strengths[i]) {
 246                         reg = nfc->cfg->ecc_cfgs[i];
 247                         break;
 248                 }
 249         }
 250
 251         if (i >= NFC_ECC_MAX_MODES)
 252                 return -EINVAL;
 253
 254         writel(reg, nfc->regs + nfc->cfg->bchctl_off);
 255
 256         /* Save chip ECC setting */
 257         nfc->cur_ecc = strength;
 258
 259         return 0;
 260 }
 261
 262 static void rk_nfc_select_chip(struct nand_chip *chip, int cs)
 263 {
 264         struct rk_nfc *nfc = nand_get_controller_data(chip);
 265         struct rk_nfc_nand_chip *rknand = rk_nfc_to_rknand(chip);
 266         struct nand_ecc_ctrl *ecc = &chip->ecc;
 267         u32 val;
 268
 269         if (cs < 0) {
 270                 nfc->selected_bank = -1;
 271                 /* Deselect the currently selected target. */
 272                 val = readl_relaxed(nfc->regs + NFC_FMCTL);
 273                 val &= ~FMCTL_CE_SEL_M;
 274                 writel(val, nfc->regs + NFC_FMCTL);
 275                 return;
 276         }
 277
 278         nfc->selected_bank = rknand->sels[cs];
 279         nfc->band_offset = NFC_BANK + nfc->selected_bank * NFC_BANK_STEP;
 280
 281         val = readl_relaxed(nfc->regs + NFC_FMCTL);
 282         val &= ~FMCTL_CE_SEL_M;
 283         val |= FMCTL_CE_SEL(nfc->selected_bank);
 284
 285         writel(val, nfc->regs + NFC_FMCTL);
 286
 287         /*
 288          * Compare current chip timing with selected chip timing and
 289          * change if needed.
 290          */
 291         if (nfc->cur_timing != rknand->timing) {
 292                 writel(rknand->timing, nfc->regs + NFC_FMWAIT);
 293                 nfc->cur_timing = rknand->timing;
 294         }
 295
 296         /*
 297          * Compare current chip ECC setting with selected chip ECC setting and
 298          * change if needed.
 299          */
 300         if (nfc->cur_ecc != ecc->strength)
 301                 rk_nfc_hw_ecc_setup(chip, ecc->strength);
 302 }
 303
 304 static inline int rk_nfc_wait_ioready(struct rk_nfc *nfc)
 305 {
 306         int rc;
 307         u32 val;
 308
 309         rc = readl_relaxed_poll_timeout(nfc->regs + NFC_FMCTL, val,
 310                                         val & FMCTL_RDY, 10, NFC_TIMEOUT);
 311
 312         return rc;
 313 }
 314
 315 static void rk_nfc_read_buf(struct rk_nfc *nfc, u8 *buf, int len)
 316 {
 317         int i;
 318
 319         for (i = 0; i < len; i++)
 320                 buf[i] = readb_relaxed(nfc->regs + nfc->band_offset +
 321                                        BANK_DATA);
 322 }
 323
 324 static void rk_nfc_write_buf(struct rk_nfc *nfc, const u8 *buf, int len)
 325 {
 326         int i;
 327
 328         for (i = 0; i < len; i++)
 329                 writeb(buf[i], nfc->regs + nfc->band_offset + BANK_DATA);
 330 }
 331
 332 static int rk_nfc_cmd(struct nand_chip *chip,
 333                       const struct nand_subop *subop)
 334 {
 335         struct rk_nfc *nfc = nand_get_controller_data(chip);
 336         unsigned int i, j, remaining, start;
 337         int reg_offset = nfc->band_offset;
 338         u8 *inbuf = NULL;
 339         const u8 *outbuf;
 340         u32 cnt = 0;
 341         int ret = 0;
 342
 343         for (i = 0; i < subop->ninstrs; i++) {
 344                 const struct nand_op_instr *instr = &subop->instrs[i];
 345
 346                 switch (instr->type) {
 347                 case NAND_OP_CMD_INSTR:
 348                         writeb(instr->ctx.cmd.opcode,
 349                                nfc->regs + reg_offset + BANK_CMD);
 350                         break;
 351
 352                 case NAND_OP_ADDR_INSTR:
 353                         remaining = nand_subop_get_num_addr_cyc(subop, i);
 354                         start = nand_subop_get_addr_start_off(subop, i);
 355
 356                         for (j = 0; j < 8 && j + start < remaining; j++)
 357                                 writeb(instr->ctx.addr.addrs[j + start],
 358                                        nfc->regs + reg_offset + BANK_ADDR);
 359                         break;
 360
 361                 case NAND_OP_DATA_IN_INSTR:
 362                 case NAND_OP_DATA_OUT_INSTR:
 363                         start = nand_subop_get_data_start_off(subop, i);
 364                         cnt = nand_subop_get_data_len(subop, i);
 365
 366                         if (instr->type == NAND_OP_DATA_OUT_INSTR) {
 367                                 outbuf = instr->ctx.data.buf.out + start;
 368                                 rk_nfc_write_buf(nfc, outbuf, cnt);
 369                         } else {
 370                                 inbuf = instr->ctx.data.buf.in + start;
 371                                 rk_nfc_read_buf(nfc, inbuf, cnt);
 372                         }
 373                         break;
 374
 375                 case NAND_OP_WAITRDY_INSTR:
 376                         if (rk_nfc_wait_ioready(nfc) < 0) {
 377                                 ret = -ETIMEDOUT;
 378                                 dev_err(nfc->dev, "IO not ready\n");
 379                         }
 380                         break;
 381                 }
 382         }
 383
 384         return ret;
 385 }
 386
 387 static const struct nand_op_parser rk_nfc_op_parser = NAND_OP_PARSER(
 388         NAND_OP_PARSER_PATTERN(
 389                 rk_nfc_cmd,
 390                 NAND_OP_PARSER_PAT_CMD_ELEM(true),
 391                 NAND_OP_PARSER_PAT_ADDR_ELEM(true, MAX_ADDRESS_CYC),
 392                 NAND_OP_PARSER_PAT_CMD_ELEM(true),
 393                 NAND_OP_PARSER_PAT_WAITRDY_ELEM(true),
 394                 NAND_OP_PARSER_PAT_DATA_IN_ELEM(true, MAX_DATA_SIZE)),
 395         NAND_OP_PARSER_PATTERN(
 396                 rk_nfc_cmd,
 397                 NAND_OP_PARSER_PAT_CMD_ELEM(true),
 398                 NAND_OP_PARSER_PAT_ADDR_ELEM(true, MAX_ADDRESS_CYC),
 399                 NAND_OP_PARSER_PAT_DATA_OUT_ELEM(true, MAX_DATA_SIZE),
 400                 NAND_OP_PARSER_PAT_CMD_ELEM(true),
 401                 NAND_OP_PARSER_PAT_WAITRDY_ELEM(true)),
 402 );
 403
 404 static int rk_nfc_exec_op(struct nand_chip *chip,
 405                           const struct nand_operation *op,
 406                           bool check_only)
 407 {
 408         if (!check_only)
 409                 rk_nfc_select_chip(chip, op->cs);
 410
 411         return nand_op_parser_exec_op(chip, &rk_nfc_op_parser, op,
 412                                       check_only);
 413 }
 414
 415 static int rk_nfc_setup_interface(struct nand_chip *chip, int target,
 416                                   const struct nand_interface_config *conf)
 417 {
 418         struct rk_nfc_nand_chip *rknand = rk_nfc_to_rknand(chip);
 419         struct rk_nfc *nfc = nand_get_controller_data(chip);
 420         const struct nand_sdr_timings *timings;
 421         u32 rate, tc2rw, trwpw, trw2c;
 422         u32 temp;
 423
 424         if (target < 0)
 425                 return 0;
 426
 427         timings = nand_get_sdr_timings(conf);
 428         if (IS_ERR(timings))
 429                 return -EOPNOTSUPP;
 430
 431         if (IS_ERR(nfc->nfc_clk))
 432                 rate = clk_get_rate(nfc->ahb_clk);
 433         else
 434                 rate = clk_get_rate(nfc->nfc_clk);
 435
 436         /* Turn clock rate into kHz. */
 437         rate /= 1000;
 438
 439         tc2rw = 1;
 440         trw2c = 1;
 441
 442         trwpw = max(timings->tWC_min, timings->tRC_min) / 1000;
 443         trwpw = DIV_ROUND_UP(trwpw * rate, 1000000);
 444
 445         temp = timings->tREA_max / 1000;
 446         temp = DIV_ROUND_UP(temp * rate, 1000000);
 447
 448         if (trwpw < temp)
 449                 trwpw = temp;
 450
 451         /*
 452          * ACCON: access timing control register
 453          * -------------------------------------
 454          * 31:18: reserved
 455          * 17:12: csrw, clock cycles from the falling edge of CSn to the
 456          *   falling edge of RDn or WRn
 457          * 11:11: reserved
 458          * 10:05: rwpw, the width of RDn or WRn in processor clock cycles
 459          * 04:00: rwcs, clock cycles from the rising edge of RDn or WRn to the
 460          *   rising edge of CSn
 461          */
 462
 463         /* Save chip timing */
 464         rknand->timing = ACCTIMING(tc2rw, trwpw, trw2c);
 465
 466         return 0;
 467 }
 468
 469 static void rk_nfc_xfer_start(struct rk_nfc *nfc, u8 rw, u8 n_KB,
 470                               dma_addr_t dma_data, dma_addr_t dma_oob)
 471 {
 472         u32 dma_reg, fl_reg, bch_reg;
 473
 474         dma_reg = DMA_ST | ((!rw) << DMA_WR) | DMA_EN | (2 << DMA_AHB_SIZE) |
 475               (7 << DMA_BURST_SIZE) | (16 << DMA_INC_NUM);
 476
 477         fl_reg = (rw << FLCTL_WR) | FLCTL_XFER_EN | FLCTL_ACORRECT |
 478                  (n_KB << FLCTL_XFER_SECTOR) | FLCTL_TOG_FIX;
 479
 480         if (nfc->cfg->type == NFC_V6 || nfc->cfg->type == NFC_V8) {
 481                 bch_reg = readl_relaxed(nfc->regs + nfc->cfg->bchctl_off);
 482                 bch_reg = (bch_reg & (~BCHCTL_BANK_M)) |
 483                           (nfc->selected_bank << BCHCTL_BANK);
 484                 writel(bch_reg, nfc->regs + nfc->cfg->bchctl_off);
 485         }
 486
 487         writel(dma_reg, nfc->regs + nfc->cfg->dma_cfg_off);
 488         writel((u32)dma_data, nfc->regs + nfc->cfg->dma_data_buf_off);
 489         writel((u32)dma_oob, nfc->regs + nfc->cfg->dma_oob_buf_off);
 490         writel(fl_reg, nfc->regs + nfc->cfg->flctl_off);
 491         fl_reg |= FLCTL_XFER_ST;
 492         writel(fl_reg, nfc->regs + nfc->cfg->flctl_off);
 493 }
 494
 495 static int rk_nfc_wait_for_xfer_done(struct rk_nfc *nfc)
 496 {
 497         void __iomem *ptr;
 498         u32 reg;
 499
 500         ptr = nfc->regs + nfc->cfg->flctl_off;
 501
 502         return readl_relaxed_poll_timeout(ptr, reg,
 503                                          reg & FLCTL_XFER_READY,
 504                                          10, NFC_TIMEOUT);
 505 }
 506
 507 static int rk_nfc_write_page_raw(struct nand_chip *chip, const u8 *buf,
 508                                  int oob_on, int page)
 509 {
 510         struct rk_nfc_nand_chip *rknand = rk_nfc_to_rknand(chip);
 511         struct rk_nfc *nfc = nand_get_controller_data(chip);
 512         struct mtd_info *mtd = nand_to_mtd(chip);
 513         struct nand_ecc_ctrl *ecc = &chip->ecc;
 514         int i, pages_per_blk;
 515
 516         pages_per_blk = mtd->erasesize / mtd->writesize;
 517         if ((chip->options & NAND_IS_BOOT_MEDIUM) &&
 518             (page < (pages_per_blk * rknand->boot_blks)) &&
 519             rknand->boot_ecc != ecc->strength) {
 520                 /*
 521                  * There's currently no method to notify the MTD framework that
 522                  * a different ECC strength is in use for the boot blocks.
 523                  */
 524                 return -EIO;
 525         }
 526
 527         if (!buf)
 528                 memset(nfc->page_buf, 0xff, mtd->writesize + mtd->oobsize);
 529
 530         for (i = 0; i < ecc->steps; i++) {
 531                 /* Copy data to the NFC buffer. */
 532                 if (buf)
 533                         memcpy(rk_nfc_data_ptr(chip, i),
 534                                rk_nfc_buf_to_data_ptr(chip, buf, i),
 535                                ecc->size);
 536                 /*
 537                  * The first four bytes of OOB are reserved for the
 538                  * boot ROM. In some debugging cases, such as with a
 539                  * read, erase and write back test these 4 bytes stored
 540                  * in OOB also need to be written back.
 541                  *
 542                  * The function nand_block_bad detects bad blocks like:
 543                  *
 544                  * bad = chip->oob_poi[chip->badblockpos];
 545                  *
 546                  * chip->badblockpos == 0 for a large page NAND Flash,
 547                  * so chip->oob_poi[0] is the bad block mask (BBM).
 548                  *
 549                  * The OOB data layout on the NFC is:
 550                  *
 551                  *    PA0  PA1  PA2  PA3  | BBM OOB1 OOB2 OOB3 | ...
 552                  *
 553                  * or
 554                  *
 555                  *    0xFF 0xFF 0xFF 0xFF | BBM OOB1 OOB2 OOB3 | ...
 556                  *
 557                  * The code here just swaps the first 4 bytes with the last
 558                  * 4 bytes without losing any data.
 559                  *
 560                  * The chip->oob_poi data layout:
 561                  *
 562                  *    BBM  OOB1 OOB2 OOB3 |......|  PA0  PA1  PA2  PA3
 563                  *
 564                  * The rk_nfc_ooblayout_free() function already has reserved
 565                  * these 4 bytes together with 2 bytes for BBM
 566                  * by reducing it's length:
 567                  *
 568                  * oob_region->length = rknand->metadata_size - NFC_SYS_DATA_SIZE - 2;
 569                  */
 570                 if (!i)
 571                         memcpy(rk_nfc_oob_ptr(chip, i),
 572                                rk_nfc_buf_to_oob_ptr(chip, ecc->steps - 1),
 573                                NFC_SYS_DATA_SIZE);
 574                 else
 575                         memcpy(rk_nfc_oob_ptr(chip, i),
 576                                rk_nfc_buf_to_oob_ptr(chip, i - 1),
 577                                NFC_SYS_DATA_SIZE);
 578                 /* Copy ECC data to the NFC buffer. */
 579                 memcpy(rk_nfc_oob_ptr(chip, i) + NFC_SYS_DATA_SIZE,
 580                        rk_nfc_buf_to_oob_ecc_ptr(chip, i),
 581                        ecc->bytes);
 582         }
 583
 584         nand_prog_page_begin_op(chip, page, 0, NULL, 0);
 585         rk_nfc_write_buf(nfc, buf, mtd->writesize + mtd->oobsize);
 586         return nand_prog_page_end_op(chip);
 587 }
 588
 589 static int rk_nfc_write_page_hwecc(struct nand_chip *chip, const u8 *buf,
 590                                    int oob_on, int page)
 591 {
 592         struct mtd_info *mtd = nand_to_mtd(chip);
 593         struct rk_nfc *nfc = nand_get_controller_data(chip);
 594         struct rk_nfc_nand_chip *rknand = rk_nfc_to_rknand(chip);
 595         struct nand_ecc_ctrl *ecc = &chip->ecc;
 596         int oob_step = (ecc->bytes > 60) ? NFC_MAX_OOB_PER_STEP :
 597                         NFC_MIN_OOB_PER_STEP;
 598         int pages_per_blk = mtd->erasesize / mtd->writesize;
 599         int ret = 0, i, boot_rom_mode = 0;
 600         dma_addr_t dma_data, dma_oob;
 601         u32 tmp;
 602         u8 *oob;
 603
 604         nand_prog_page_begin_op(chip, page, 0, NULL, 0);
 605
 606         if (buf)
 607                 memcpy(nfc->page_buf, buf, mtd->writesize);
 608         else
 609                 memset(nfc->page_buf, 0xFF, mtd->writesize);
 610
 611         /*
 612          * The first blocks (4, 8 or 16 depending on the device) are used
 613          * by the boot ROM and the first 32 bits of OOB need to link to
 614          * the next page address in the same block. We can't directly copy
 615          * OOB data from the MTD framework, because this page address
 616          * conflicts for example with the bad block marker (BBM),
 617          * so we shift all OOB data including the BBM with 4 byte positions.
 618          * As a consequence the OOB size available to the MTD framework is
 619          * also reduced with 4 bytes.
 620          *
 621          *    PA0  PA1  PA2  PA3 | BBM OOB1 OOB2 OOB3 | ...
 622          *
 623          * If a NAND is not a boot medium or the page is not a boot block,
 624          * the first 4 bytes are left untouched by writing 0xFF to them.
 625          *
 626          *   0xFF 0xFF 0xFF 0xFF | BBM OOB1 OOB2 OOB3 | ...
 627          *
 628          * The code here just swaps the first 4 bytes with the last
 629          * 4 bytes without losing any data.
 630          *
 631          * The chip->oob_poi data layout:
 632          *
 633          *    BBM  OOB1 OOB2 OOB3 |......|  PA0  PA1  PA2  PA3
 634          *
 635          * Configure the ECC algorithm supported by the boot ROM.
 636          */
 637         if ((page < (pages_per_blk * rknand->boot_blks)) &&
 638             (chip->options & NAND_IS_BOOT_MEDIUM)) {
 639                 boot_rom_mode = 1;
 640                 if (rknand->boot_ecc != ecc->strength)
 641                         rk_nfc_hw_ecc_setup(chip, rknand->boot_ecc);
 642         }
 643
 644         for (i = 0; i < ecc->steps; i++) {
 645                 if (!i)
 646                         oob = chip->oob_poi + (ecc->steps - 1) * NFC_SYS_DATA_SIZE;
 647                 else
 648                         oob = chip->oob_poi + (i - 1) * NFC_SYS_DATA_SIZE;
 649
 650                 tmp = oob[0] | oob[1] << 8 | oob[2] << 16 | oob[3] << 24;
 651
 652                 if (nfc->cfg->type == NFC_V9)
 653                         nfc->oob_buf[i] = tmp;
 654                 else
 655                         nfc->oob_buf[i * (oob_step / 4)] = tmp;
 656         }
 657
 658         dma_data = dma_map_single(nfc->dev, (void *)nfc->page_buf,
 659                                   mtd->writesize, DMA_TO_DEVICE);
 660         dma_oob = dma_map_single(nfc->dev, nfc->oob_buf,
 661                                  ecc->steps * oob_step,
 662                                  DMA_TO_DEVICE);
 663
 664         reinit_completion(&nfc->done);
 665         writel(INT_DMA, nfc->regs + nfc->cfg->int_en_off);
 666
 667         rk_nfc_xfer_start(nfc, NFC_WRITE, ecc->steps, dma_data,
 668                           dma_oob);
 669         ret = wait_for_completion_timeout(&nfc->done,
 670                                           msecs_to_jiffies(100));
 671         if (!ret)
 672                 dev_warn(nfc->dev, "write: wait dma done timeout.\n");
 673         /*
 674          * Whether the DMA transfer is completed or not. The driver
 675          * needs to check the NFC`s status register to see if the data
 676          * transfer was completed.
 677          */
 678         ret = rk_nfc_wait_for_xfer_done(nfc);
 679
 680         dma_unmap_single(nfc->dev, dma_data, mtd->writesize,
 681                          DMA_TO_DEVICE);
 682         dma_unmap_single(nfc->dev, dma_oob, ecc->steps * oob_step,
 683                          DMA_TO_DEVICE);
 684
 685         if (boot_rom_mode && rknand->boot_ecc != ecc->strength)
 686                 rk_nfc_hw_ecc_setup(chip, ecc->strength);
 687
 688         if (ret) {
 689                 dev_err(nfc->dev, "write: wait transfer done timeout.\n");
 690                 return -ETIMEDOUT;
 691         }
 692
 693         return nand_prog_page_end_op(chip);
 694 }
 695
 696 static int rk_nfc_write_oob(struct nand_chip *chip, int page)
 697 {
 698         return rk_nfc_write_page_hwecc(chip, NULL, 1, page);
 699 }
 700
 701 static int rk_nfc_read_page_raw(struct nand_chip *chip, u8 *buf, int oob_on,
 702                                 int page)
 703 {
 704         struct rk_nfc_nand_chip *rknand = rk_nfc_to_rknand(chip);
 705         struct rk_nfc *nfc = nand_get_controller_data(chip);
 706         struct mtd_info *mtd = nand_to_mtd(chip);
 707         struct nand_ecc_ctrl *ecc = &chip->ecc;
 708         int i, pages_per_blk;
 709
 710         pages_per_blk = mtd->erasesize / mtd->writesize;
 711         if ((chip->options & NAND_IS_BOOT_MEDIUM) &&
 712             (page < (pages_per_blk * rknand->boot_blks)) &&
 713             rknand->boot_ecc != ecc->strength) {
 714                 /*
 715                  * There's currently no method to notify the MTD framework that
 716                  * a different ECC strength is in use for the boot blocks.
 717                  */
 718                 return -EIO;
 719         }
 720
 721         nand_read_page_op(chip, page, 0, NULL, 0);
 722         rk_nfc_read_buf(nfc, nfc->page_buf, mtd->writesize + mtd->oobsize);
 723         for (i = 0; i < ecc->steps; i++) {
 724                 /*
 725                  * The first four bytes of OOB are reserved for the
 726                  * boot ROM. In some debugging cases, such as with a read,
 727                  * erase and write back test, these 4 bytes also must be
 728                  * saved somewhere, otherwise this information will be
 729                  * lost during a write back.
 730                  */
 731                 if (!i)
 732                         memcpy(rk_nfc_buf_to_oob_ptr(chip, ecc->steps - 1),
 733                                rk_nfc_oob_ptr(chip, i),
 734                                NFC_SYS_DATA_SIZE);
 735                 else
 736                         memcpy(rk_nfc_buf_to_oob_ptr(chip, i - 1),
 737                                rk_nfc_oob_ptr(chip, i),
 738                                NFC_SYS_DATA_SIZE);
 739
 740                 /* Copy ECC data from the NFC buffer. */
 741                 memcpy(rk_nfc_buf_to_oob_ecc_ptr(chip, i),
 742                        rk_nfc_oob_ptr(chip, i) + NFC_SYS_DATA_SIZE,
 743                        ecc->bytes);
 744
 745                 /* Copy data from the NFC buffer. */
 746                 if (buf)
 747                         memcpy(rk_nfc_buf_to_data_ptr(chip, buf, i),
 748                                rk_nfc_data_ptr(chip, i),
 749                                ecc->size);
 750         }
 751
 752         return 0;
 753 }
 754
 755 static int rk_nfc_read_page_hwecc(struct nand_chip *chip, u8 *buf, int oob_on,
 756                                   int page)
 757 {
 758         struct mtd_info *mtd = nand_to_mtd(chip);
 759         struct rk_nfc *nfc = nand_get_controller_data(chip);
 760         struct rk_nfc_nand_chip *rknand = rk_nfc_to_rknand(chip);
 761         struct nand_ecc_ctrl *ecc = &chip->ecc;
 762         int oob_step = (ecc->bytes > 60) ? NFC_MAX_OOB_PER_STEP :
 763                         NFC_MIN_OOB_PER_STEP;
 764         int pages_per_blk = mtd->erasesize / mtd->writesize;
 765         dma_addr_t dma_data, dma_oob;
 766         int ret = 0, i, cnt, boot_rom_mode = 0;
 767         int max_bitflips = 0, bch_st, ecc_fail = 0;
 768         u8 *oob;
 769         u32 tmp;
 770
 771         nand_read_page_op(chip, page, 0, NULL, 0);
 772
 773         dma_data = dma_map_single(nfc->dev, nfc->page_buf,
 774                                   mtd->writesize,
 775                                   DMA_FROM_DEVICE);
 776         dma_oob = dma_map_single(nfc->dev, nfc->oob_buf,
 777                                  ecc->steps * oob_step,
 778                                  DMA_FROM_DEVICE);
 779
 780         /*
 781          * The first blocks (4, 8 or 16 depending on the device)
 782          * are used by the boot ROM.
 783          * Configure the ECC algorithm supported by the boot ROM.
 784          */
 785         if ((page < (pages_per_blk * rknand->boot_blks)) &&
 786             (chip->options & NAND_IS_BOOT_MEDIUM)) {
 787                 boot_rom_mode = 1;
 788                 if (rknand->boot_ecc != ecc->strength)
 789                         rk_nfc_hw_ecc_setup(chip, rknand->boot_ecc);
 790         }
 791
 792         reinit_completion(&nfc->done);
 793         writel(INT_DMA, nfc->regs + nfc->cfg->int_en_off);
 794         rk_nfc_xfer_start(nfc, NFC_READ, ecc->steps, dma_data,
 795                           dma_oob);
 796         ret = wait_for_completion_timeout(&nfc->done,
 797                                           msecs_to_jiffies(100));
 798         if (!ret)
 799                 dev_warn(nfc->dev, "read: wait dma done timeout.\n");
 800         /*
 801          * Whether the DMA transfer is completed or not. The driver
 802          * needs to check the NFC`s status register to see if the data
 803          * transfer was completed.
 804          */
 805         ret = rk_nfc_wait_for_xfer_done(nfc);
 806
 807         dma_unmap_single(nfc->dev, dma_data, mtd->writesize,
 808                          DMA_FROM_DEVICE);
 809         dma_unmap_single(nfc->dev, dma_oob, ecc->steps * oob_step,
 810                          DMA_FROM_DEVICE);
 811
 812         if (ret) {
 813                 ret = -ETIMEDOUT;
 814                 dev_err(nfc->dev, "read: wait transfer done timeout.\n");
 815                 goto timeout_err;
 816         }
 817
 818         for (i = 0; i < ecc->steps; i++) {
 819                 if (!i)
 820                         oob = chip->oob_poi + (ecc->steps - 1) * NFC_SYS_DATA_SIZE;
 821                 else
 822                         oob = chip->oob_poi + (i - 1) * NFC_SYS_DATA_SIZE;
 823
 824                 if (nfc->cfg->type == NFC_V9)
 825                         tmp = nfc->oob_buf[i];
 826                 else
 827                         tmp = nfc->oob_buf[i * (oob_step / 4)];
 828
 829                 *oob++ = (u8)tmp;
 830                 *oob++ = (u8)(tmp >> 8);
 831                 *oob++ = (u8)(tmp >> 16);
 832                 *oob++ = (u8)(tmp >> 24);
 833         }
 834
 835         for (i = 0; i < (ecc->steps / 2); i++) {
 836                 bch_st = readl_relaxed(nfc->regs +
 837                                        nfc->cfg->bch_st_off + i * 4);
 838                 if (bch_st & BIT(nfc->cfg->ecc0.err_flag_bit) ||
 839                     bch_st & BIT(nfc->cfg->ecc1.err_flag_bit)) {
 840                         mtd->ecc_stats.failed++;
 841                         ecc_fail = 1;
 842                 } else {
 843                         cnt = ECC_ERR_CNT(bch_st, nfc->cfg->ecc0);
 844                         mtd->ecc_stats.corrected += cnt;
 845                         max_bitflips = max_t(u32, max_bitflips, cnt);
 846
 847                         cnt = ECC_ERR_CNT(bch_st, nfc->cfg->ecc1);
 848                         mtd->ecc_stats.corrected += cnt;
 849                         max_bitflips = max_t(u32, max_bitflips, cnt);
 850                 }
 851         }
 852
 853         if (buf)
 854                 memcpy(buf, nfc->page_buf, mtd->writesize);
 855
 856 timeout_err:
 857         if (boot_rom_mode && rknand->boot_ecc != ecc->strength)
 858                 rk_nfc_hw_ecc_setup(chip, ecc->strength);
 859
 860         if (ret)
 861                 return ret;
 862
 863         if (ecc_fail) {
 864                 dev_err(nfc->dev, "read page: %x ecc error!\n", page);
 865                 return 0;
 866         }
 867
 868         return max_bitflips;
 869 }
 870
 871 static int rk_nfc_read_oob(struct nand_chip *chip, int page)
 872 {
 873         return rk_nfc_read_page_hwecc(chip, NULL, 1, page);
 874 }
 875
 876 static inline void rk_nfc_hw_init(struct rk_nfc *nfc)
 877 {
 878         /* Disable flash wp. */
 879         writel(FMCTL_WP, nfc->regs + NFC_FMCTL);
 880         /* Config default timing 40ns at 150 Mhz NFC clock. */
 881         writel(0x1081, nfc->regs + NFC_FMWAIT);
 882         nfc->cur_timing = 0x1081;
 883         /* Disable randomizer and DMA. */
 884         writel(0, nfc->regs + nfc->cfg->randmz_off);
 885         writel(0, nfc->regs + nfc->cfg->dma_cfg_off);
 886         writel(FLCTL_RST, nfc->regs + nfc->cfg->flctl_off);
 887 }
 888
 889 static irqreturn_t rk_nfc_irq(int irq, void *id)
 890 {
 891         struct rk_nfc *nfc = id;
 892         u32 sta, ien;
 893
 894         sta = readl_relaxed(nfc->regs + nfc->cfg->int_st_off);
 895         ien = readl_relaxed(nfc->regs + nfc->cfg->int_en_off);
 896
 897         if (!(sta & ien))
 898                 return IRQ_NONE;
 899
 900         writel(sta, nfc->regs + nfc->cfg->int_clr_off);
 901         writel(~sta & ien, nfc->regs + nfc->cfg->int_en_off);
 902
 903         complete(&nfc->done);
 904
 905         return IRQ_HANDLED;
 906 }
 907
 908 static int rk_nfc_enable_clks(struct device *dev, struct rk_nfc *nfc)
 909 {
 910         int ret;
 911
 912         if (!IS_ERR(nfc->nfc_clk)) {
 913                 ret = clk_prepare_enable(nfc->nfc_clk);
 914                 if (ret) {
 915                         dev_err(dev, "failed to enable NFC clk\n");
 916                         return ret;
 917                 }
 918         }
 919
 920         ret = clk_prepare_enable(nfc->ahb_clk);
 921         if (ret) {
 922                 dev_err(dev, "failed to enable ahb clk\n");
 923                 clk_disable_unprepare(nfc->nfc_clk);
 924                 return ret;
 925         }
 926
 927         return 0;
 928 }
 929
 930 static void rk_nfc_disable_clks(struct rk_nfc *nfc)
 931 {
 932         clk_disable_unprepare(nfc->nfc_clk);
 933         clk_disable_unprepare(nfc->ahb_clk);
 934 }
 935
 936 static int rk_nfc_ooblayout_free(struct mtd_info *mtd, int section,
 937                                  struct mtd_oob_region *oob_region)
 938 {
 939         struct nand_chip *chip = mtd_to_nand(mtd);
 940         struct rk_nfc_nand_chip *rknand = rk_nfc_to_rknand(chip);
 941
 942         if (section)
 943                 return -ERANGE;
 944
 945         oob_region->length = rknand->metadata_size - NFC_SYS_DATA_SIZE - 2;
 946         oob_region->offset = 2;
 947
 948         return 0;
 949 }
 950
 951 static int rk_nfc_ooblayout_ecc(struct mtd_info *mtd, int section,
 952                                 struct mtd_oob_region *oob_region)
 953 {
 954         struct nand_chip *chip = mtd_to_nand(mtd);
 955         struct rk_nfc_nand_chip *rknand = rk_nfc_to_rknand(chip);
 956
 957         if (section)
 958                 return -ERANGE;
 959
 960         oob_region->length = mtd->oobsize - rknand->metadata_size;
 961         oob_region->offset = rknand->metadata_size;
 962
 963         return 0;
 964 }
 965
 966 static const struct mtd_ooblayout_ops rk_nfc_ooblayout_ops = {
 967         .free = rk_nfc_ooblayout_free,
 968         .ecc = rk_nfc_ooblayout_ecc,
 969 };
 970
 971 static int rk_nfc_ecc_init(struct device *dev, struct mtd_info *mtd)
 972 {
 973         struct nand_chip *chip = mtd_to_nand(mtd);
 974         struct rk_nfc *nfc = nand_get_controller_data(chip);
 975         struct nand_ecc_ctrl *ecc = &chip->ecc;
 976         const u8 *strengths = nfc->cfg->ecc_strengths;
 977         u8 max_strength, nfc_max_strength;
 978         int i;
 979
 980         nfc_max_strength = nfc->cfg->ecc_strengths[0];
 981         /* If optional dt settings not present. */
 982         if (!ecc->size || !ecc->strength ||
 983             ecc->strength > nfc_max_strength) {
 984                 chip->ecc.size = 1024;
 985                 ecc->steps = mtd->writesize / ecc->size;
 986
 987                 /*
 988                  * HW ECC always requests the number of ECC bytes per 1024 byte
 989                  * blocks. The first 4 OOB bytes are reserved for sys data.
 990                  */
 991                 max_strength = ((mtd->oobsize / ecc->steps) - 4) * 8 /
 992                                  fls(8 * 1024);
 993                 if (max_strength > nfc_max_strength)
 994                         max_strength = nfc_max_strength;
 995
 996                 for (i = 0; i < 4; i++) {
 997                         if (max_strength >= strengths[i])
 998                                 break;
 999                 }
1000
1001                 if (i >= 4) {
1002                         dev_err(nfc->dev, "unsupported ECC strength\n");
1003                         return -EOPNOTSUPP;
1004                 }
1005
1006                 ecc->strength = strengths[i];
1007         }
1008         ecc->steps = mtd->writesize / ecc->size;
1009         ecc->bytes = DIV_ROUND_UP(ecc->strength * fls(8 * chip->ecc.size), 8);
1010
1011         return 0;
1012 }
1013
1014 static int rk_nfc_attach_chip(struct nand_chip *chip)
1015 {
1016         struct mtd_info *mtd = nand_to_mtd(chip);
1017         struct device *dev = mtd->dev.parent;
1018         struct rk_nfc *nfc = nand_get_controller_data(chip);
1019         struct rk_nfc_nand_chip *rknand = rk_nfc_to_rknand(chip);
1020         struct nand_ecc_ctrl *ecc = &chip->ecc;
1021         int new_page_len, new_oob_len;
1022         void *buf;
1023         int ret;
1024
1025         if (chip->options & NAND_BUSWIDTH_16) {
1026                 dev_err(dev, "16 bits bus width not supported");
1027                 return -EINVAL;
1028         }
1029
1030         if (ecc->engine_type != NAND_ECC_ENGINE_TYPE_ON_HOST)
1031                 return 0;
1032
1033         ret = rk_nfc_ecc_init(dev, mtd);
1034         if (ret)
1035                 return ret;
1036
1037         rknand->metadata_size = NFC_SYS_DATA_SIZE * ecc->steps;
1038
1039         if (rknand->metadata_size < NFC_SYS_DATA_SIZE + 2) {
1040                 dev_err(dev,
1041                         "driver needs at least %d bytes of meta data\n",
1042                         NFC_SYS_DATA_SIZE + 2);
1043                 return -EIO;
1044         }
1045
1046         /* Check buffer first, avoid duplicate alloc buffer. */
1047         new_page_len = mtd->writesize + mtd->oobsize;
1048         if (nfc->page_buf && new_page_len > nfc->page_buf_size) {
1049                 buf = krealloc(nfc->page_buf, new_page_len,
1050                                GFP_KERNEL | GFP_DMA);
1051                 if (!buf)
1052                         return -ENOMEM;
1053                 nfc->page_buf = buf;
1054                 nfc->page_buf_size = new_page_len;
1055         }
1056
1057         new_oob_len = ecc->steps * NFC_MAX_OOB_PER_STEP;
1058         if (nfc->oob_buf && new_oob_len > nfc->oob_buf_size) {
1059                 buf = krealloc(nfc->oob_buf, new_oob_len,
1060                                GFP_KERNEL | GFP_DMA);
1061                 if (!buf) {
1062                         kfree(nfc->page_buf);
1063                         nfc->page_buf = NULL;
1064                         return -ENOMEM;
1065                 }
1066                 nfc->oob_buf = buf;
1067                 nfc->oob_buf_size = new_oob_len;
1068         }
1069
1070         if (!nfc->page_buf) {
1071                 nfc->page_buf = kzalloc(new_page_len, GFP_KERNEL | GFP_DMA);
1072                 if (!nfc->page_buf)
1073                         return -ENOMEM;
1074                 nfc->page_buf_size = new_page_len;
1075         }
1076
1077         if (!nfc->oob_buf) {
1078                 nfc->oob_buf = kzalloc(new_oob_len, GFP_KERNEL | GFP_DMA);
1079                 if (!nfc->oob_buf) {
1080                         kfree(nfc->page_buf);
1081                         nfc->page_buf = NULL;
1082                         return -ENOMEM;
1083                 }
1084                 nfc->oob_buf_size = new_oob_len;
1085         }
1086
1087         chip->ecc.write_page_raw = rk_nfc_write_page_raw;
1088         chip->ecc.write_page = rk_nfc_write_page_hwecc;
1089         chip->ecc.write_oob = rk_nfc_write_oob;
1090
1091         chip->ecc.read_page_raw = rk_nfc_read_page_raw;
1092         chip->ecc.read_page = rk_nfc_read_page_hwecc;
1093         chip->ecc.read_oob = rk_nfc_read_oob;
1094
1095         return 0;
1096 }
1097
1098 static const struct nand_controller_ops rk_nfc_controller_ops = {
1099         .attach_chip = rk_nfc_attach_chip,
1100         .exec_op = rk_nfc_exec_op,
1101         .setup_interface = rk_nfc_setup_interface,
1102 };
1103
1104 static int rk_nfc_nand_chip_init(struct device *dev, struct rk_nfc *nfc,
1105                                  struct device_node *np)
1106 {
1107         struct rk_nfc_nand_chip *rknand;
1108         struct nand_chip *chip;
1109         struct mtd_info *mtd;
1110         int nsels;
1111         u32 tmp;
1112         int ret;
1113         int i;
1114
1115         if (!of_get_property(np, "reg", &nsels))
1116                 return -ENODEV;
1117         nsels /= sizeof(u32);
1118         if (!nsels || nsels > NFC_MAX_NSELS) {
1119                 dev_err(dev, "invalid reg property size %d\n", nsels);
1120                 return -EINVAL;
1121         }
1122
1123         rknand = devm_kzalloc(dev, sizeof(*rknand) + nsels * sizeof(u8),
1124                               GFP_KERNEL);
1125         if (!rknand)
1126                 return -ENOMEM;
1127
1128         rknand->nsels = nsels;
1129         for (i = 0; i < nsels; i++) {
1130                 ret = of_property_read_u32_index(np, "reg", i, &tmp);
1131                 if (ret) {
1132                         dev_err(dev, "reg property failure : %d\n", ret);
1133                         return ret;
1134                 }
1135
1136                 if (tmp >= NFC_MAX_NSELS) {
1137                         dev_err(dev, "invalid CS: %u\n", tmp);
1138                         return -EINVAL;
1139                 }
1140
1141                 if (test_and_set_bit(tmp, &nfc->assigned_cs)) {
1142                         dev_err(dev, "CS %u already assigned\n", tmp);
1143                         return -EINVAL;
1144                 }
1145
1146                 rknand->sels[i] = tmp;
1147         }
1148
1149         chip = &rknand->chip;
1150         chip->controller = &nfc->controller;
1151
1152         nand_set_flash_node(chip, np);
1153
1154         nand_set_controller_data(chip, nfc);
1155
1156         chip->options |= NAND_USES_DMA | NAND_NO_SUBPAGE_WRITE;
1157         chip->bbt_options = NAND_BBT_USE_FLASH | NAND_BBT_NO_OOB;
1158
1159         /* Set default mode in case dt entry is missing. */
1160         chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST;
1161
1162         mtd = nand_to_mtd(chip);
1163         mtd->owner = THIS_MODULE;
1164         mtd->dev.parent = dev;
1165
1166         if (!mtd->name) {
1167                 dev_err(nfc->dev, "NAND label property is mandatory\n");
1168                 return -EINVAL;
1169         }
1170
1171         mtd_set_ooblayout(mtd, &rk_nfc_ooblayout_ops);
1172         rk_nfc_hw_init(nfc);
1173         ret = nand_scan(chip, nsels);
1174         if (ret)
1175                 return ret;
1176
1177         if (chip->options & NAND_IS_BOOT_MEDIUM) {
1178                 ret = of_property_read_u32(np, "rockchip,boot-blks", &tmp);
1179                 rknand->boot_blks = ret ? 0 : tmp;
1180
1181                 ret = of_property_read_u32(np, "rockchip,boot-ecc-strength",
1182                                            &tmp);
1183                 rknand->boot_ecc = ret ? chip->ecc.strength : tmp;
1184         }
1185
1186         ret = mtd_device_register(mtd, NULL, 0);
1187         if (ret) {
1188                 dev_err(dev, "MTD parse partition error\n");
1189                 nand_cleanup(chip);
1190                 return ret;
1191         }
1192
1193         list_add_tail(&rknand->node, &nfc->chips);
1194
1195         return 0;
1196 }
1197
1198 static void rk_nfc_chips_cleanup(struct rk_nfc *nfc)
1199 {
1200         struct rk_nfc_nand_chip *rknand, *tmp;
1201         struct nand_chip *chip;
1202         int ret;
1203
1204         list_for_each_entry_safe(rknand, tmp, &nfc->chips, node) {
1205                 chip = &rknand->chip;
1206                 ret = mtd_device_unregister(nand_to_mtd(chip));
1207                 WARN_ON(ret);
1208                 nand_cleanup(chip);
1209                 list_del(&rknand->node);
1210         }
1211 }
1212
1213 static int rk_nfc_nand_chips_init(struct device *dev, struct rk_nfc *nfc)
1214 {
1215         struct device_node *np = dev->of_node, *nand_np;
1216         int nchips = of_get_child_count(np);
1217         int ret;
1218
1219         if (!nchips || nchips > NFC_MAX_NSELS) {
1220                 dev_err(nfc->dev, "incorrect number of NAND chips (%d)\n",
1221                         nchips);
1222                 return -EINVAL;
1223         }
1224
1225         for_each_child_of_node(np, nand_np) {
1226                 ret = rk_nfc_nand_chip_init(dev, nfc, nand_np);
1227                 if (ret) {
1228                         of_node_put(nand_np);
1229                         rk_nfc_chips_cleanup(nfc);
1230                         return ret;
1231                 }
1232         }
1233
1234         return 0;
1235 }
1236
1237 static struct nfc_cfg nfc_v6_cfg = {
1238                 .type                   = NFC_V6,
1239                 .ecc_strengths          = {60, 40, 24, 16},
1240                 .ecc_cfgs               = {
1241                         0x00040011, 0x00040001, 0x00000011, 0x00000001,
1242                 },
1243                 .flctl_off              = 0x08,
1244                 .bchctl_off             = 0x0C,
1245                 .dma_cfg_off            = 0x10,
1246                 .dma_data_buf_off       = 0x14,
1247                 .dma_oob_buf_off        = 0x18,
1248                 .dma_st_off             = 0x1C,
1249                 .bch_st_off             = 0x20,
1250                 .randmz_off             = 0x150,
1251                 .int_en_off             = 0x16C,
1252                 .int_clr_off            = 0x170,
1253                 .int_st_off             = 0x174,
1254                 .oob0_off               = 0x200,
1255                 .oob1_off               = 0x230,
1256                 .ecc0                   = {
1257                         .err_flag_bit   = 2,
1258                         .low            = 3,
1259                         .low_mask       = 0x1F,
1260                         .low_bn         = 5,
1261                         .high           = 27,
1262                         .high_mask      = 0x1,
1263                 },
1264                 .ecc1                   = {
1265                         .err_flag_bit   = 15,
1266                         .low            = 16,
1267                         .low_mask       = 0x1F,
1268                         .low_bn         = 5,
1269                         .high           = 29,
1270                         .high_mask      = 0x1,
1271                 },
1272 };
1273
1274 static struct nfc_cfg nfc_v8_cfg = {
1275                 .type                   = NFC_V8,
1276                 .ecc_strengths          = {16, 16, 16, 16},
1277                 .ecc_cfgs               = {
1278                         0x00000001, 0x00000001, 0x00000001, 0x00000001,
1279                 },
1280                 .flctl_off              = 0x08,
1281                 .bchctl_off             = 0x0C,
1282                 .dma_cfg_off            = 0x10,
1283                 .dma_data_buf_off       = 0x14,
1284                 .dma_oob_buf_off        = 0x18,
1285                 .dma_st_off             = 0x1C,
1286                 .bch_st_off             = 0x20,
1287                 .randmz_off             = 0x150,
1288                 .int_en_off             = 0x16C,
1289                 .int_clr_off            = 0x170,
1290                 .int_st_off             = 0x174,
1291                 .oob0_off               = 0x200,
1292                 .oob1_off               = 0x230,
1293                 .ecc0                   = {
1294                         .err_flag_bit   = 2,
1295                         .low            = 3,
1296                         .low_mask       = 0x1F,
1297                         .low_bn         = 5,
1298                         .high           = 27,
1299                         .high_mask      = 0x1,
1300                 },
1301                 .ecc1                   = {
1302                         .err_flag_bit   = 15,
1303                         .low            = 16,
1304                         .low_mask       = 0x1F,
1305                         .low_bn         = 5,
1306                         .high           = 29,
1307                         .high_mask      = 0x1,
1308                 },
1309 };
1310
1311 static struct nfc_cfg nfc_v9_cfg = {
1312                 .type                   = NFC_V9,
1313                 .ecc_strengths          = {70, 60, 40, 16},
1314                 .ecc_cfgs               = {
1315                         0x00000001, 0x06000001, 0x04000001, 0x02000001,
1316                 },
1317                 .flctl_off              = 0x10,
1318                 .bchctl_off             = 0x20,
1319                 .dma_cfg_off            = 0x30,
1320                 .dma_data_buf_off       = 0x34,
1321                 .dma_oob_buf_off        = 0x38,
1322                 .dma_st_off             = 0x3C,
1323                 .bch_st_off             = 0x150,
1324                 .randmz_off             = 0x208,
1325                 .int_en_off             = 0x120,
1326                 .int_clr_off            = 0x124,
1327                 .int_st_off             = 0x128,
1328                 .oob0_off               = 0x200,
1329                 .oob1_off               = 0x204,
1330                 .ecc0                   = {
1331                         .err_flag_bit   = 2,
1332                         .low            = 3,
1333                         .low_mask       = 0x7F,
1334                         .low_bn         = 7,
1335                         .high           = 0,
1336                         .high_mask      = 0x0,
1337                 },
1338                 .ecc1                   = {
1339                         .err_flag_bit   = 18,
1340                         .low            = 19,
1341                         .low_mask       = 0x7F,
1342                         .low_bn         = 7,
1343                         .high           = 0,
1344                         .high_mask      = 0x0,
1345                 },
1346 };
1347
1348 static const struct of_device_id rk_nfc_id_table[] = {
1349         {
1350                 .compatible = "rockchip,px30-nfc",
1351                 .data = &nfc_v9_cfg
1352         },
1353         {
1354                 .compatible = "rockchip,rk2928-nfc",
1355                 .data = &nfc_v6_cfg
1356         },
1357         {
1358                 .compatible = "rockchip,rv1108-nfc",
1359                 .data = &nfc_v8_cfg
1360         },
1361         { /* sentinel */ }
1362 };
1363 MODULE_DEVICE_TABLE(of, rk_nfc_id_table);
1364
1365 static int rk_nfc_probe(struct platform_device *pdev)
1366 {
1367         struct device *dev = &pdev->dev;
1368         struct rk_nfc *nfc;
1369         int ret, irq;
1370
1371         nfc = devm_kzalloc(dev, sizeof(*nfc), GFP_KERNEL);
1372         if (!nfc)
1373                 return -ENOMEM;
1374
1375         nand_controller_init(&nfc->controller);
1376         INIT_LIST_HEAD(&nfc->chips);
1377         nfc->controller.ops = &rk_nfc_controller_ops;
1378
1379         nfc->cfg = of_device_get_match_data(dev);
1380         nfc->dev = dev;
1381
1382         init_completion(&nfc->done);
1383
1384         nfc->regs = devm_platform_ioremap_resource(pdev, 0);
1385         if (IS_ERR(nfc->regs)) {
1386                 ret = PTR_ERR(nfc->regs);
1387                 goto release_nfc;
1388         }
1389
1390         nfc->nfc_clk = devm_clk_get(dev, "nfc");
1391         if (IS_ERR(nfc->nfc_clk)) {
1392                 dev_dbg(dev, "no NFC clk\n");
1393                 /* Some earlier models, such as rk3066, have no NFC clk. */
1394         }
1395
1396         nfc->ahb_clk = devm_clk_get(dev, "ahb");
1397         if (IS_ERR(nfc->ahb_clk)) {
1398                 dev_err(dev, "no ahb clk\n");
1399                 ret = PTR_ERR(nfc->ahb_clk);
1400                 goto release_nfc;
1401         }
1402
1403         ret = rk_nfc_enable_clks(dev, nfc);
1404         if (ret)
1405                 goto release_nfc;
1406
1407         irq = platform_get_irq(pdev, 0);
1408         if (irq < 0) {
1409                 ret = -EINVAL;
1410                 goto clk_disable;
1411         }
1412
1413         writel(0, nfc->regs + nfc->cfg->int_en_off);
1414         ret = devm_request_irq(dev, irq, rk_nfc_irq, 0x0, "rk-nand", nfc);
1415         if (ret) {
1416                 dev_err(dev, "failed to request NFC irq\n");
1417                 goto clk_disable;
1418         }
1419
1420         platform_set_drvdata(pdev, nfc);
1421
1422         ret = rk_nfc_nand_chips_init(dev, nfc);
1423         if (ret) {
1424                 dev_err(dev, "failed to init NAND chips\n");
1425                 goto clk_disable;
1426         }
1427         return 0;
1428
1429 clk_disable:
1430         rk_nfc_disable_clks(nfc);
1431 release_nfc:
1432         return ret;
1433 }
1434
1435 static void rk_nfc_remove(struct platform_device *pdev)
1436 {
1437         struct rk_nfc *nfc = platform_get_drvdata(pdev);
1438
1439         kfree(nfc->page_buf);
1440         kfree(nfc->oob_buf);
1441         rk_nfc_chips_cleanup(nfc);
1442         rk_nfc_disable_clks(nfc);
1443 }
1444
1445 static int __maybe_unused rk_nfc_suspend(struct device *dev)
1446 {
1447         struct rk_nfc *nfc = dev_get_drvdata(dev);
1448
1449         rk_nfc_disable_clks(nfc);
1450
1451         return 0;
1452 }
1453
1454 static int __maybe_unused rk_nfc_resume(struct device *dev)
1455 {
1456         struct rk_nfc *nfc = dev_get_drvdata(dev);
1457         struct rk_nfc_nand_chip *rknand;
1458         struct nand_chip *chip;
1459         int ret;
1460         u32 i;
1461
1462         ret = rk_nfc_enable_clks(dev, nfc);
1463         if (ret)
1464                 return ret;
1465
1466         /* Reset NAND chip if VCC was powered off. */
1467         list_for_each_entry(rknand, &nfc->chips, node) {
1468                 chip = &rknand->chip;
1469                 for (i = 0; i < rknand->nsels; i++)
1470                         nand_reset(chip, i);
1471         }
1472
1473         return 0;
1474 }
1475
1476 static const struct dev_pm_ops rk_nfc_pm_ops = {
1477         SET_SYSTEM_SLEEP_PM_OPS(rk_nfc_suspend, rk_nfc_resume)
1478 };
1479
1480 static struct platform_driver rk_nfc_driver = {
1481         .probe = rk_nfc_probe,
1482         .remove_new = rk_nfc_remove,
1483         .driver = {
1484                 .name = "rockchip-nfc",
1485                 .of_match_table = rk_nfc_id_table,
1486                 .pm = &rk_nfc_pm_ops,
1487         },
1488 };
1489
1490 module_platform_driver(rk_nfc_driver);
1491
1492 MODULE_LICENSE("Dual MIT/GPL");
1493 MODULE_AUTHOR("Yifeng Zhao <yifeng.zhao@rock-chips.com>");
1494 MODULE_DESCRIPTION("Rockchip Nand Flash Controller Driver");
1495 MODULE_ALIAS("platform:rockchip-nand-controller");