1 // SPDX-License-Identifier: GPL-2.0 OR MIT
3 * Rockchip NAND Flash controller driver.
4 * Copyright (C) 2020 Rockchip Inc.
5 * Author: Yifeng Zhao <yifeng.zhao@rock-chips.com>
9 #include <linux/delay.h>
10 #include <linux/dma-mapping.h>
11 #include <linux/dmaengine.h>
12 #include <linux/interrupt.h>
13 #include <linux/iopoll.h>
14 #include <linux/module.h>
15 #include <linux/mtd/mtd.h>
16 #include <linux/mtd/rawnand.h>
18 #include <linux/of_device.h>
19 #include <linux/platform_device.h>
20 #include <linux/slab.h>
23 * NFC Page Data Layout:
24 * 1024 bytes data + 4Bytes sys data + 28Bytes~124Bytes ECC data +
25 * 1024 bytes data + 4Bytes sys data + 28Bytes~124Bytes ECC data +
27 * NAND Page Data Layout:
28 * 1024 * n data + m Bytes oob
29 * Original Bad Block Mask Location:
30 * First byte of oob(spare).
31 * nand_chip->oob_poi data layout:
32 * 4Bytes sys data + .... + 4Bytes sys data + ECC data.
35 /* NAND controller register definition */
39 #define NFC_FMCTL (0x00)
40 #define FMCTL_CE_SEL_M 0xFF
41 #define FMCTL_CE_SEL(x) (1 << (x))
42 #define FMCTL_WP BIT(8)
43 #define FMCTL_RDY BIT(9)
45 #define NFC_FMWAIT (0x04)
46 #define FLCTL_RST BIT(0)
47 #define FLCTL_WR (1) /* 0: read, 1: write */
48 #define FLCTL_XFER_ST BIT(2)
49 #define FLCTL_XFER_EN BIT(3)
50 #define FLCTL_ACORRECT BIT(10) /* Auto correct error bits. */
51 #define FLCTL_XFER_READY BIT(20)
52 #define FLCTL_XFER_SECTOR (22)
53 #define FLCTL_TOG_FIX BIT(29)
55 #define BCHCTL_BANK_M (7 << 5)
56 #define BCHCTL_BANK (5)
59 #define DMA_WR (1) /* 0: write, 1: read */
61 #define DMA_AHB_SIZE (3) /* 0: 1, 1: 2, 2: 4 */
62 #define DMA_BURST_SIZE (6) /* 0: 1, 3: 4, 5: 8, 7: 16 */
63 #define DMA_INC_NUM (9) /* 1 - 16 */
65 #define ECC_ERR_CNT(x, e) ((((x) >> (e).low) & (e).low_mask) |\
66 (((x) >> (e).high) & (e).high_mask) << (e).low_bn)
67 #define INT_DMA BIT(0)
68 #define NFC_BANK (0x800)
69 #define NFC_BANK_STEP (0x100)
70 #define BANK_DATA (0x00)
71 #define BANK_ADDR (0x04)
72 #define BANK_CMD (0x08)
73 #define NFC_SRAM0 (0x1000)
74 #define NFC_SRAM1 (0x1400)
75 #define NFC_SRAM_SIZE (0x400)
76 #define NFC_TIMEOUT (500000)
77 #define NFC_MAX_OOB_PER_STEP 128
78 #define NFC_MIN_OOB_PER_STEP 64
79 #define MAX_DATA_SIZE 0xFFFC
80 #define MAX_ADDRESS_CYC 6
81 #define NFC_ECC_MAX_MODES 4
82 #define NFC_MAX_NSELS (8) /* Some Socs only have 1 or 2 CSs. */
83 #define NFC_SYS_DATA_SIZE (4) /* 4 bytes sys data in oob pre 1024 data.*/
84 #define RK_DEFAULT_CLOCK_RATE (150 * 1000 * 1000) /* 150 Mhz */
85 #define ACCTIMING(csrw, rwpw, rwcs) ((csrw) << 12 | (rwpw) << 5 | (rwcs))
94 * struct rk_ecc_cnt_status: represent a ecc status data.
95 * @err_flag_bit: error flag bit index at register.
96 * @low: ECC count low bit index at register.
97 * @low_mask: mask bit.
98 * @low_bn: ECC count low bit number.
99 * @high: ECC count high bit index at register.
100 * @high_mask: mask bit
102 struct ecc_cnt_status {
113 * @ecc_strengths: ECC strengths
114 * @ecc_cfgs: ECC config values
115 * @flctl_off: FLCTL register offset
116 * @bchctl_off: BCHCTL register offset
117 * @dma_data_buf_off: DMA_DATA_BUF register offset
118 * @dma_oob_buf_off: DMA_OOB_BUF register offset
119 * @dma_cfg_off: DMA_CFG register offset
120 * @dma_st_off: DMA_ST register offset
121 * @bch_st_off: BCG_ST register offset
122 * @randmz_off: RANDMZ register offset
123 * @int_en_off: interrupt enable register offset
124 * @int_clr_off: interrupt clean register offset
125 * @int_st_off: interrupt status register offset
126 * @oob0_off: oob0 register offset
127 * @oob1_off: oob1 register offset
128 * @ecc0: represent ECC0 status data
129 * @ecc1: represent ECC1 status data
133 u8 ecc_strengths[NFC_ECC_MAX_MODES];
134 u32 ecc_cfgs[NFC_ECC_MAX_MODES];
138 u32 dma_data_buf_off;
148 struct ecc_cnt_status ecc0;
149 struct ecc_cnt_status ecc1;
152 struct rk_nfc_nand_chip {
153 struct list_head node;
154 struct nand_chip chip;
163 /* Nothing after this field. */
167 struct nand_controller controller;
168 const struct nfc_cfg *cfg;
180 struct completion done;
181 struct list_head chips;
188 unsigned long assigned_cs;
191 static inline struct rk_nfc_nand_chip *rk_nfc_to_rknand(struct nand_chip *chip)
193 return container_of(chip, struct rk_nfc_nand_chip, chip);
196 static inline u8 *rk_nfc_buf_to_data_ptr(struct nand_chip *chip, const u8 *p, int i)
198 return (u8 *)p + i * chip->ecc.size;
201 static inline u8 *rk_nfc_buf_to_oob_ptr(struct nand_chip *chip, int i)
205 poi = chip->oob_poi + i * NFC_SYS_DATA_SIZE;
210 static inline u8 *rk_nfc_buf_to_oob_ecc_ptr(struct nand_chip *chip, int i)
212 struct rk_nfc_nand_chip *rknand = rk_nfc_to_rknand(chip);
215 poi = chip->oob_poi + rknand->metadata_size + chip->ecc.bytes * i;
220 static inline int rk_nfc_data_len(struct nand_chip *chip)
222 return chip->ecc.size + chip->ecc.bytes + NFC_SYS_DATA_SIZE;
225 static inline u8 *rk_nfc_data_ptr(struct nand_chip *chip, int i)
227 struct rk_nfc *nfc = nand_get_controller_data(chip);
229 return nfc->page_buf + i * rk_nfc_data_len(chip);
232 static inline u8 *rk_nfc_oob_ptr(struct nand_chip *chip, int i)
234 struct rk_nfc *nfc = nand_get_controller_data(chip);
236 return nfc->page_buf + i * rk_nfc_data_len(chip) + chip->ecc.size;
239 static int rk_nfc_hw_ecc_setup(struct nand_chip *chip, u32 strength)
241 struct rk_nfc *nfc = nand_get_controller_data(chip);
244 for (i = 0; i < NFC_ECC_MAX_MODES; i++) {
245 if (strength == nfc->cfg->ecc_strengths[i]) {
246 reg = nfc->cfg->ecc_cfgs[i];
251 if (i >= NFC_ECC_MAX_MODES)
254 writel(reg, nfc->regs + nfc->cfg->bchctl_off);
256 /* Save chip ECC setting */
257 nfc->cur_ecc = strength;
262 static void rk_nfc_select_chip(struct nand_chip *chip, int cs)
264 struct rk_nfc *nfc = nand_get_controller_data(chip);
265 struct rk_nfc_nand_chip *rknand = rk_nfc_to_rknand(chip);
266 struct nand_ecc_ctrl *ecc = &chip->ecc;
270 nfc->selected_bank = -1;
271 /* Deselect the currently selected target. */
272 val = readl_relaxed(nfc->regs + NFC_FMCTL);
273 val &= ~FMCTL_CE_SEL_M;
274 writel(val, nfc->regs + NFC_FMCTL);
278 nfc->selected_bank = rknand->sels[cs];
279 nfc->band_offset = NFC_BANK + nfc->selected_bank * NFC_BANK_STEP;
281 val = readl_relaxed(nfc->regs + NFC_FMCTL);
282 val &= ~FMCTL_CE_SEL_M;
283 val |= FMCTL_CE_SEL(nfc->selected_bank);
285 writel(val, nfc->regs + NFC_FMCTL);
288 * Compare current chip timing with selected chip timing and
291 if (nfc->cur_timing != rknand->timing) {
292 writel(rknand->timing, nfc->regs + NFC_FMWAIT);
293 nfc->cur_timing = rknand->timing;
297 * Compare current chip ECC setting with selected chip ECC setting and
300 if (nfc->cur_ecc != ecc->strength)
301 rk_nfc_hw_ecc_setup(chip, ecc->strength);
304 static inline int rk_nfc_wait_ioready(struct rk_nfc *nfc)
309 rc = readl_relaxed_poll_timeout(nfc->regs + NFC_FMCTL, val,
310 val & FMCTL_RDY, 10, NFC_TIMEOUT);
315 static void rk_nfc_read_buf(struct rk_nfc *nfc, u8 *buf, int len)
319 for (i = 0; i < len; i++)
320 buf[i] = readb_relaxed(nfc->regs + nfc->band_offset +
324 static void rk_nfc_write_buf(struct rk_nfc *nfc, const u8 *buf, int len)
328 for (i = 0; i < len; i++)
329 writeb(buf[i], nfc->regs + nfc->band_offset + BANK_DATA);
332 static int rk_nfc_cmd(struct nand_chip *chip,
333 const struct nand_subop *subop)
335 struct rk_nfc *nfc = nand_get_controller_data(chip);
336 unsigned int i, j, remaining, start;
337 int reg_offset = nfc->band_offset;
343 for (i = 0; i < subop->ninstrs; i++) {
344 const struct nand_op_instr *instr = &subop->instrs[i];
346 switch (instr->type) {
347 case NAND_OP_CMD_INSTR:
348 writeb(instr->ctx.cmd.opcode,
349 nfc->regs + reg_offset + BANK_CMD);
352 case NAND_OP_ADDR_INSTR:
353 remaining = nand_subop_get_num_addr_cyc(subop, i);
354 start = nand_subop_get_addr_start_off(subop, i);
356 for (j = 0; j < 8 && j + start < remaining; j++)
357 writeb(instr->ctx.addr.addrs[j + start],
358 nfc->regs + reg_offset + BANK_ADDR);
361 case NAND_OP_DATA_IN_INSTR:
362 case NAND_OP_DATA_OUT_INSTR:
363 start = nand_subop_get_data_start_off(subop, i);
364 cnt = nand_subop_get_data_len(subop, i);
366 if (instr->type == NAND_OP_DATA_OUT_INSTR) {
367 outbuf = instr->ctx.data.buf.out + start;
368 rk_nfc_write_buf(nfc, outbuf, cnt);
370 inbuf = instr->ctx.data.buf.in + start;
371 rk_nfc_read_buf(nfc, inbuf, cnt);
375 case NAND_OP_WAITRDY_INSTR:
376 if (rk_nfc_wait_ioready(nfc) < 0) {
378 dev_err(nfc->dev, "IO not ready\n");
387 static const struct nand_op_parser rk_nfc_op_parser = NAND_OP_PARSER(
388 NAND_OP_PARSER_PATTERN(
390 NAND_OP_PARSER_PAT_CMD_ELEM(true),
391 NAND_OP_PARSER_PAT_ADDR_ELEM(true, MAX_ADDRESS_CYC),
392 NAND_OP_PARSER_PAT_CMD_ELEM(true),
393 NAND_OP_PARSER_PAT_WAITRDY_ELEM(true),
394 NAND_OP_PARSER_PAT_DATA_IN_ELEM(true, MAX_DATA_SIZE)),
395 NAND_OP_PARSER_PATTERN(
397 NAND_OP_PARSER_PAT_CMD_ELEM(true),
398 NAND_OP_PARSER_PAT_ADDR_ELEM(true, MAX_ADDRESS_CYC),
399 NAND_OP_PARSER_PAT_DATA_OUT_ELEM(true, MAX_DATA_SIZE),
400 NAND_OP_PARSER_PAT_CMD_ELEM(true),
401 NAND_OP_PARSER_PAT_WAITRDY_ELEM(true)),
404 static int rk_nfc_exec_op(struct nand_chip *chip,
405 const struct nand_operation *op,
409 rk_nfc_select_chip(chip, op->cs);
411 return nand_op_parser_exec_op(chip, &rk_nfc_op_parser, op,
415 static int rk_nfc_setup_interface(struct nand_chip *chip, int target,
416 const struct nand_interface_config *conf)
418 struct rk_nfc_nand_chip *rknand = rk_nfc_to_rknand(chip);
419 struct rk_nfc *nfc = nand_get_controller_data(chip);
420 const struct nand_sdr_timings *timings;
421 u32 rate, tc2rw, trwpw, trw2c;
427 timings = nand_get_sdr_timings(conf);
431 if (IS_ERR(nfc->nfc_clk))
432 rate = clk_get_rate(nfc->ahb_clk);
434 rate = clk_get_rate(nfc->nfc_clk);
436 /* Turn clock rate into kHz. */
442 trwpw = max(timings->tWC_min, timings->tRC_min) / 1000;
443 trwpw = DIV_ROUND_UP(trwpw * rate, 1000000);
445 temp = timings->tREA_max / 1000;
446 temp = DIV_ROUND_UP(temp * rate, 1000000);
452 * ACCON: access timing control register
453 * -------------------------------------
455 * 17:12: csrw, clock cycles from the falling edge of CSn to the
456 * falling edge of RDn or WRn
458 * 10:05: rwpw, the width of RDn or WRn in processor clock cycles
459 * 04:00: rwcs, clock cycles from the rising edge of RDn or WRn to the
463 /* Save chip timing */
464 rknand->timing = ACCTIMING(tc2rw, trwpw, trw2c);
469 static void rk_nfc_xfer_start(struct rk_nfc *nfc, u8 rw, u8 n_KB,
470 dma_addr_t dma_data, dma_addr_t dma_oob)
472 u32 dma_reg, fl_reg, bch_reg;
474 dma_reg = DMA_ST | ((!rw) << DMA_WR) | DMA_EN | (2 << DMA_AHB_SIZE) |
475 (7 << DMA_BURST_SIZE) | (16 << DMA_INC_NUM);
477 fl_reg = (rw << FLCTL_WR) | FLCTL_XFER_EN | FLCTL_ACORRECT |
478 (n_KB << FLCTL_XFER_SECTOR) | FLCTL_TOG_FIX;
480 if (nfc->cfg->type == NFC_V6 || nfc->cfg->type == NFC_V8) {
481 bch_reg = readl_relaxed(nfc->regs + nfc->cfg->bchctl_off);
482 bch_reg = (bch_reg & (~BCHCTL_BANK_M)) |
483 (nfc->selected_bank << BCHCTL_BANK);
484 writel(bch_reg, nfc->regs + nfc->cfg->bchctl_off);
487 writel(dma_reg, nfc->regs + nfc->cfg->dma_cfg_off);
488 writel((u32)dma_data, nfc->regs + nfc->cfg->dma_data_buf_off);
489 writel((u32)dma_oob, nfc->regs + nfc->cfg->dma_oob_buf_off);
490 writel(fl_reg, nfc->regs + nfc->cfg->flctl_off);
491 fl_reg |= FLCTL_XFER_ST;
492 writel(fl_reg, nfc->regs + nfc->cfg->flctl_off);
495 static int rk_nfc_wait_for_xfer_done(struct rk_nfc *nfc)
500 ptr = nfc->regs + nfc->cfg->flctl_off;
502 return readl_relaxed_poll_timeout(ptr, reg,
503 reg & FLCTL_XFER_READY,
507 static int rk_nfc_write_page_raw(struct nand_chip *chip, const u8 *buf,
508 int oob_on, int page)
510 struct rk_nfc_nand_chip *rknand = rk_nfc_to_rknand(chip);
511 struct rk_nfc *nfc = nand_get_controller_data(chip);
512 struct mtd_info *mtd = nand_to_mtd(chip);
513 struct nand_ecc_ctrl *ecc = &chip->ecc;
514 int i, pages_per_blk;
516 pages_per_blk = mtd->erasesize / mtd->writesize;
517 if ((chip->options & NAND_IS_BOOT_MEDIUM) &&
518 (page < (pages_per_blk * rknand->boot_blks)) &&
519 rknand->boot_ecc != ecc->strength) {
521 * There's currently no method to notify the MTD framework that
522 * a different ECC strength is in use for the boot blocks.
528 memset(nfc->page_buf, 0xff, mtd->writesize + mtd->oobsize);
530 for (i = 0; i < ecc->steps; i++) {
531 /* Copy data to the NFC buffer. */
533 memcpy(rk_nfc_data_ptr(chip, i),
534 rk_nfc_buf_to_data_ptr(chip, buf, i),
537 * The first four bytes of OOB are reserved for the
538 * boot ROM. In some debugging cases, such as with a
539 * read, erase and write back test these 4 bytes stored
540 * in OOB also need to be written back.
542 * The function nand_block_bad detects bad blocks like:
544 * bad = chip->oob_poi[chip->badblockpos];
546 * chip->badblockpos == 0 for a large page NAND Flash,
547 * so chip->oob_poi[0] is the bad block mask (BBM).
549 * The OOB data layout on the NFC is:
551 * PA0 PA1 PA2 PA3 | BBM OOB1 OOB2 OOB3 | ...
555 * 0xFF 0xFF 0xFF 0xFF | BBM OOB1 OOB2 OOB3 | ...
557 * The code here just swaps the first 4 bytes with the last
558 * 4 bytes without losing any data.
560 * The chip->oob_poi data layout:
562 * BBM OOB1 OOB2 OOB3 |......| PA0 PA1 PA2 PA3
564 * The rk_nfc_ooblayout_free() function already has reserved
565 * these 4 bytes with:
567 * oob_region->offset = NFC_SYS_DATA_SIZE + 2;
570 memcpy(rk_nfc_oob_ptr(chip, i),
571 rk_nfc_buf_to_oob_ptr(chip, ecc->steps - 1),
574 memcpy(rk_nfc_oob_ptr(chip, i),
575 rk_nfc_buf_to_oob_ptr(chip, i - 1),
577 /* Copy ECC data to the NFC buffer. */
578 memcpy(rk_nfc_oob_ptr(chip, i) + NFC_SYS_DATA_SIZE,
579 rk_nfc_buf_to_oob_ecc_ptr(chip, i),
583 nand_prog_page_begin_op(chip, page, 0, NULL, 0);
584 rk_nfc_write_buf(nfc, buf, mtd->writesize + mtd->oobsize);
585 return nand_prog_page_end_op(chip);
588 static int rk_nfc_write_page_hwecc(struct nand_chip *chip, const u8 *buf,
589 int oob_on, int page)
591 struct mtd_info *mtd = nand_to_mtd(chip);
592 struct rk_nfc *nfc = nand_get_controller_data(chip);
593 struct rk_nfc_nand_chip *rknand = rk_nfc_to_rknand(chip);
594 struct nand_ecc_ctrl *ecc = &chip->ecc;
595 int oob_step = (ecc->bytes > 60) ? NFC_MAX_OOB_PER_STEP :
596 NFC_MIN_OOB_PER_STEP;
597 int pages_per_blk = mtd->erasesize / mtd->writesize;
598 int ret = 0, i, boot_rom_mode = 0;
599 dma_addr_t dma_data, dma_oob;
603 nand_prog_page_begin_op(chip, page, 0, NULL, 0);
606 memcpy(nfc->page_buf, buf, mtd->writesize);
608 memset(nfc->page_buf, 0xFF, mtd->writesize);
611 * The first blocks (4, 8 or 16 depending on the device) are used
612 * by the boot ROM and the first 32 bits of OOB need to link to
613 * the next page address in the same block. We can't directly copy
614 * OOB data from the MTD framework, because this page address
615 * conflicts for example with the bad block marker (BBM),
616 * so we shift all OOB data including the BBM with 4 byte positions.
617 * As a consequence the OOB size available to the MTD framework is
618 * also reduced with 4 bytes.
620 * PA0 PA1 PA2 PA3 | BBM OOB1 OOB2 OOB3 | ...
622 * If a NAND is not a boot medium or the page is not a boot block,
623 * the first 4 bytes are left untouched by writing 0xFF to them.
625 * 0xFF 0xFF 0xFF 0xFF | BBM OOB1 OOB2 OOB3 | ...
627 * Configure the ECC algorithm supported by the boot ROM.
629 if ((page < (pages_per_blk * rknand->boot_blks)) &&
630 (chip->options & NAND_IS_BOOT_MEDIUM)) {
632 if (rknand->boot_ecc != ecc->strength)
633 rk_nfc_hw_ecc_setup(chip, rknand->boot_ecc);
636 for (i = 0; i < ecc->steps; i++) {
640 oob = chip->oob_poi + (i - 1) * NFC_SYS_DATA_SIZE;
641 reg = oob[0] | oob[1] << 8 | oob[2] << 16 |
645 if (!i && boot_rom_mode)
646 reg = (page & (pages_per_blk - 1)) * 4;
648 if (nfc->cfg->type == NFC_V9)
649 nfc->oob_buf[i] = reg;
651 nfc->oob_buf[i * (oob_step / 4)] = reg;
654 dma_data = dma_map_single(nfc->dev, (void *)nfc->page_buf,
655 mtd->writesize, DMA_TO_DEVICE);
656 dma_oob = dma_map_single(nfc->dev, nfc->oob_buf,
657 ecc->steps * oob_step,
660 reinit_completion(&nfc->done);
661 writel(INT_DMA, nfc->regs + nfc->cfg->int_en_off);
663 rk_nfc_xfer_start(nfc, NFC_WRITE, ecc->steps, dma_data,
665 ret = wait_for_completion_timeout(&nfc->done,
666 msecs_to_jiffies(100));
668 dev_warn(nfc->dev, "write: wait dma done timeout.\n");
670 * Whether the DMA transfer is completed or not. The driver
671 * needs to check the NFC`s status register to see if the data
672 * transfer was completed.
674 ret = rk_nfc_wait_for_xfer_done(nfc);
676 dma_unmap_single(nfc->dev, dma_data, mtd->writesize,
678 dma_unmap_single(nfc->dev, dma_oob, ecc->steps * oob_step,
681 if (boot_rom_mode && rknand->boot_ecc != ecc->strength)
682 rk_nfc_hw_ecc_setup(chip, ecc->strength);
685 dev_err(nfc->dev, "write: wait transfer done timeout.\n");
689 return nand_prog_page_end_op(chip);
692 static int rk_nfc_write_oob(struct nand_chip *chip, int page)
694 return rk_nfc_write_page_hwecc(chip, NULL, 1, page);
697 static int rk_nfc_read_page_raw(struct nand_chip *chip, u8 *buf, int oob_on,
700 struct rk_nfc_nand_chip *rknand = rk_nfc_to_rknand(chip);
701 struct rk_nfc *nfc = nand_get_controller_data(chip);
702 struct mtd_info *mtd = nand_to_mtd(chip);
703 struct nand_ecc_ctrl *ecc = &chip->ecc;
704 int i, pages_per_blk;
706 pages_per_blk = mtd->erasesize / mtd->writesize;
707 if ((chip->options & NAND_IS_BOOT_MEDIUM) &&
708 (page < (pages_per_blk * rknand->boot_blks)) &&
709 rknand->boot_ecc != ecc->strength) {
711 * There's currently no method to notify the MTD framework that
712 * a different ECC strength is in use for the boot blocks.
717 nand_read_page_op(chip, page, 0, NULL, 0);
718 rk_nfc_read_buf(nfc, nfc->page_buf, mtd->writesize + mtd->oobsize);
719 for (i = 0; i < ecc->steps; i++) {
721 * The first four bytes of OOB are reserved for the
722 * boot ROM. In some debugging cases, such as with a read,
723 * erase and write back test, these 4 bytes also must be
724 * saved somewhere, otherwise this information will be
725 * lost during a write back.
728 memcpy(rk_nfc_buf_to_oob_ptr(chip, ecc->steps - 1),
729 rk_nfc_oob_ptr(chip, i),
732 memcpy(rk_nfc_buf_to_oob_ptr(chip, i - 1),
733 rk_nfc_oob_ptr(chip, i),
736 /* Copy ECC data from the NFC buffer. */
737 memcpy(rk_nfc_buf_to_oob_ecc_ptr(chip, i),
738 rk_nfc_oob_ptr(chip, i) + NFC_SYS_DATA_SIZE,
741 /* Copy data from the NFC buffer. */
743 memcpy(rk_nfc_buf_to_data_ptr(chip, buf, i),
744 rk_nfc_data_ptr(chip, i),
751 static int rk_nfc_read_page_hwecc(struct nand_chip *chip, u8 *buf, int oob_on,
754 struct mtd_info *mtd = nand_to_mtd(chip);
755 struct rk_nfc *nfc = nand_get_controller_data(chip);
756 struct rk_nfc_nand_chip *rknand = rk_nfc_to_rknand(chip);
757 struct nand_ecc_ctrl *ecc = &chip->ecc;
758 int oob_step = (ecc->bytes > 60) ? NFC_MAX_OOB_PER_STEP :
759 NFC_MIN_OOB_PER_STEP;
760 int pages_per_blk = mtd->erasesize / mtd->writesize;
761 dma_addr_t dma_data, dma_oob;
762 int ret = 0, i, cnt, boot_rom_mode = 0;
763 int max_bitflips = 0, bch_st, ecc_fail = 0;
767 nand_read_page_op(chip, page, 0, NULL, 0);
769 dma_data = dma_map_single(nfc->dev, nfc->page_buf,
772 dma_oob = dma_map_single(nfc->dev, nfc->oob_buf,
773 ecc->steps * oob_step,
777 * The first blocks (4, 8 or 16 depending on the device)
778 * are used by the boot ROM.
779 * Configure the ECC algorithm supported by the boot ROM.
781 if ((page < (pages_per_blk * rknand->boot_blks)) &&
782 (chip->options & NAND_IS_BOOT_MEDIUM)) {
784 if (rknand->boot_ecc != ecc->strength)
785 rk_nfc_hw_ecc_setup(chip, rknand->boot_ecc);
788 reinit_completion(&nfc->done);
789 writel(INT_DMA, nfc->regs + nfc->cfg->int_en_off);
790 rk_nfc_xfer_start(nfc, NFC_READ, ecc->steps, dma_data,
792 ret = wait_for_completion_timeout(&nfc->done,
793 msecs_to_jiffies(100));
795 dev_warn(nfc->dev, "read: wait dma done timeout.\n");
797 * Whether the DMA transfer is completed or not. The driver
798 * needs to check the NFC`s status register to see if the data
799 * transfer was completed.
801 ret = rk_nfc_wait_for_xfer_done(nfc);
803 dma_unmap_single(nfc->dev, dma_data, mtd->writesize,
805 dma_unmap_single(nfc->dev, dma_oob, ecc->steps * oob_step,
810 dev_err(nfc->dev, "read: wait transfer done timeout.\n");
814 for (i = 1; i < ecc->steps; i++) {
815 oob = chip->oob_poi + (i - 1) * NFC_SYS_DATA_SIZE;
816 if (nfc->cfg->type == NFC_V9)
817 tmp = nfc->oob_buf[i];
819 tmp = nfc->oob_buf[i * (oob_step / 4)];
821 *oob++ = (u8)(tmp >> 8);
822 *oob++ = (u8)(tmp >> 16);
823 *oob++ = (u8)(tmp >> 24);
826 for (i = 0; i < (ecc->steps / 2); i++) {
827 bch_st = readl_relaxed(nfc->regs +
828 nfc->cfg->bch_st_off + i * 4);
829 if (bch_st & BIT(nfc->cfg->ecc0.err_flag_bit) ||
830 bch_st & BIT(nfc->cfg->ecc1.err_flag_bit)) {
831 mtd->ecc_stats.failed++;
834 cnt = ECC_ERR_CNT(bch_st, nfc->cfg->ecc0);
835 mtd->ecc_stats.corrected += cnt;
836 max_bitflips = max_t(u32, max_bitflips, cnt);
838 cnt = ECC_ERR_CNT(bch_st, nfc->cfg->ecc1);
839 mtd->ecc_stats.corrected += cnt;
840 max_bitflips = max_t(u32, max_bitflips, cnt);
845 memcpy(buf, nfc->page_buf, mtd->writesize);
848 if (boot_rom_mode && rknand->boot_ecc != ecc->strength)
849 rk_nfc_hw_ecc_setup(chip, ecc->strength);
855 dev_err(nfc->dev, "read page: %x ecc error!\n", page);
862 static int rk_nfc_read_oob(struct nand_chip *chip, int page)
864 return rk_nfc_read_page_hwecc(chip, NULL, 1, page);
867 static inline void rk_nfc_hw_init(struct rk_nfc *nfc)
869 /* Disable flash wp. */
870 writel(FMCTL_WP, nfc->regs + NFC_FMCTL);
871 /* Config default timing 40ns at 150 Mhz NFC clock. */
872 writel(0x1081, nfc->regs + NFC_FMWAIT);
873 nfc->cur_timing = 0x1081;
874 /* Disable randomizer and DMA. */
875 writel(0, nfc->regs + nfc->cfg->randmz_off);
876 writel(0, nfc->regs + nfc->cfg->dma_cfg_off);
877 writel(FLCTL_RST, nfc->regs + nfc->cfg->flctl_off);
880 static irqreturn_t rk_nfc_irq(int irq, void *id)
882 struct rk_nfc *nfc = id;
885 sta = readl_relaxed(nfc->regs + nfc->cfg->int_st_off);
886 ien = readl_relaxed(nfc->regs + nfc->cfg->int_en_off);
891 writel(sta, nfc->regs + nfc->cfg->int_clr_off);
892 writel(~sta & ien, nfc->regs + nfc->cfg->int_en_off);
894 complete(&nfc->done);
899 static int rk_nfc_enable_clks(struct device *dev, struct rk_nfc *nfc)
903 if (!IS_ERR(nfc->nfc_clk)) {
904 ret = clk_prepare_enable(nfc->nfc_clk);
906 dev_err(dev, "failed to enable NFC clk\n");
911 ret = clk_prepare_enable(nfc->ahb_clk);
913 dev_err(dev, "failed to enable ahb clk\n");
914 clk_disable_unprepare(nfc->nfc_clk);
921 static void rk_nfc_disable_clks(struct rk_nfc *nfc)
923 clk_disable_unprepare(nfc->nfc_clk);
924 clk_disable_unprepare(nfc->ahb_clk);
927 static int rk_nfc_ooblayout_free(struct mtd_info *mtd, int section,
928 struct mtd_oob_region *oob_region)
930 struct nand_chip *chip = mtd_to_nand(mtd);
931 struct rk_nfc_nand_chip *rknand = rk_nfc_to_rknand(chip);
937 * The beginning of the OOB area stores the reserved data for the NFC,
938 * the size of the reserved data is NFC_SYS_DATA_SIZE bytes.
940 oob_region->length = rknand->metadata_size - NFC_SYS_DATA_SIZE - 2;
941 oob_region->offset = NFC_SYS_DATA_SIZE + 2;
946 static int rk_nfc_ooblayout_ecc(struct mtd_info *mtd, int section,
947 struct mtd_oob_region *oob_region)
949 struct nand_chip *chip = mtd_to_nand(mtd);
950 struct rk_nfc_nand_chip *rknand = rk_nfc_to_rknand(chip);
955 oob_region->length = mtd->oobsize - rknand->metadata_size;
956 oob_region->offset = rknand->metadata_size;
961 static const struct mtd_ooblayout_ops rk_nfc_ooblayout_ops = {
962 .free = rk_nfc_ooblayout_free,
963 .ecc = rk_nfc_ooblayout_ecc,
966 static int rk_nfc_ecc_init(struct device *dev, struct mtd_info *mtd)
968 struct nand_chip *chip = mtd_to_nand(mtd);
969 struct rk_nfc *nfc = nand_get_controller_data(chip);
970 struct nand_ecc_ctrl *ecc = &chip->ecc;
971 const u8 *strengths = nfc->cfg->ecc_strengths;
972 u8 max_strength, nfc_max_strength;
975 nfc_max_strength = nfc->cfg->ecc_strengths[0];
976 /* If optional dt settings not present. */
977 if (!ecc->size || !ecc->strength ||
978 ecc->strength > nfc_max_strength) {
979 chip->ecc.size = 1024;
980 ecc->steps = mtd->writesize / ecc->size;
983 * HW ECC always requests the number of ECC bytes per 1024 byte
984 * blocks. The first 4 OOB bytes are reserved for sys data.
986 max_strength = ((mtd->oobsize / ecc->steps) - 4) * 8 /
988 if (max_strength > nfc_max_strength)
989 max_strength = nfc_max_strength;
991 for (i = 0; i < 4; i++) {
992 if (max_strength >= strengths[i])
997 dev_err(nfc->dev, "unsupported ECC strength\n");
1001 ecc->strength = strengths[i];
1003 ecc->steps = mtd->writesize / ecc->size;
1004 ecc->bytes = DIV_ROUND_UP(ecc->strength * fls(8 * chip->ecc.size), 8);
1009 static int rk_nfc_attach_chip(struct nand_chip *chip)
1011 struct mtd_info *mtd = nand_to_mtd(chip);
1012 struct device *dev = mtd->dev.parent;
1013 struct rk_nfc *nfc = nand_get_controller_data(chip);
1014 struct rk_nfc_nand_chip *rknand = rk_nfc_to_rknand(chip);
1015 struct nand_ecc_ctrl *ecc = &chip->ecc;
1016 int new_page_len, new_oob_len;
1020 if (chip->options & NAND_BUSWIDTH_16) {
1021 dev_err(dev, "16 bits bus width not supported");
1025 if (ecc->engine_type != NAND_ECC_ENGINE_TYPE_ON_HOST)
1028 ret = rk_nfc_ecc_init(dev, mtd);
1032 rknand->metadata_size = NFC_SYS_DATA_SIZE * ecc->steps;
1034 if (rknand->metadata_size < NFC_SYS_DATA_SIZE + 2) {
1036 "driver needs at least %d bytes of meta data\n",
1037 NFC_SYS_DATA_SIZE + 2);
1041 /* Check buffer first, avoid duplicate alloc buffer. */
1042 new_page_len = mtd->writesize + mtd->oobsize;
1043 if (nfc->page_buf && new_page_len > nfc->page_buf_size) {
1044 buf = krealloc(nfc->page_buf, new_page_len,
1045 GFP_KERNEL | GFP_DMA);
1048 nfc->page_buf = buf;
1049 nfc->page_buf_size = new_page_len;
1052 new_oob_len = ecc->steps * NFC_MAX_OOB_PER_STEP;
1053 if (nfc->oob_buf && new_oob_len > nfc->oob_buf_size) {
1054 buf = krealloc(nfc->oob_buf, new_oob_len,
1055 GFP_KERNEL | GFP_DMA);
1057 kfree(nfc->page_buf);
1058 nfc->page_buf = NULL;
1062 nfc->oob_buf_size = new_oob_len;
1065 if (!nfc->page_buf) {
1066 nfc->page_buf = kzalloc(new_page_len, GFP_KERNEL | GFP_DMA);
1069 nfc->page_buf_size = new_page_len;
1072 if (!nfc->oob_buf) {
1073 nfc->oob_buf = kzalloc(new_oob_len, GFP_KERNEL | GFP_DMA);
1074 if (!nfc->oob_buf) {
1075 kfree(nfc->page_buf);
1076 nfc->page_buf = NULL;
1079 nfc->oob_buf_size = new_oob_len;
1082 chip->ecc.write_page_raw = rk_nfc_write_page_raw;
1083 chip->ecc.write_page = rk_nfc_write_page_hwecc;
1084 chip->ecc.write_oob = rk_nfc_write_oob;
1086 chip->ecc.read_page_raw = rk_nfc_read_page_raw;
1087 chip->ecc.read_page = rk_nfc_read_page_hwecc;
1088 chip->ecc.read_oob = rk_nfc_read_oob;
1093 static const struct nand_controller_ops rk_nfc_controller_ops = {
1094 .attach_chip = rk_nfc_attach_chip,
1095 .exec_op = rk_nfc_exec_op,
1096 .setup_interface = rk_nfc_setup_interface,
1099 static int rk_nfc_nand_chip_init(struct device *dev, struct rk_nfc *nfc,
1100 struct device_node *np)
1102 struct rk_nfc_nand_chip *rknand;
1103 struct nand_chip *chip;
1104 struct mtd_info *mtd;
1110 if (!of_get_property(np, "reg", &nsels))
1112 nsels /= sizeof(u32);
1113 if (!nsels || nsels > NFC_MAX_NSELS) {
1114 dev_err(dev, "invalid reg property size %d\n", nsels);
1118 rknand = devm_kzalloc(dev, sizeof(*rknand) + nsels * sizeof(u8),
1123 rknand->nsels = nsels;
1124 for (i = 0; i < nsels; i++) {
1125 ret = of_property_read_u32_index(np, "reg", i, &tmp);
1127 dev_err(dev, "reg property failure : %d\n", ret);
1131 if (tmp >= NFC_MAX_NSELS) {
1132 dev_err(dev, "invalid CS: %u\n", tmp);
1136 if (test_and_set_bit(tmp, &nfc->assigned_cs)) {
1137 dev_err(dev, "CS %u already assigned\n", tmp);
1141 rknand->sels[i] = tmp;
1144 chip = &rknand->chip;
1145 chip->controller = &nfc->controller;
1147 nand_set_flash_node(chip, np);
1149 nand_set_controller_data(chip, nfc);
1151 chip->options |= NAND_USES_DMA | NAND_NO_SUBPAGE_WRITE;
1152 chip->bbt_options = NAND_BBT_USE_FLASH | NAND_BBT_NO_OOB;
1154 /* Set default mode in case dt entry is missing. */
1155 chip->ecc.engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST;
1157 mtd = nand_to_mtd(chip);
1158 mtd->owner = THIS_MODULE;
1159 mtd->dev.parent = dev;
1162 dev_err(nfc->dev, "NAND label property is mandatory\n");
1166 mtd_set_ooblayout(mtd, &rk_nfc_ooblayout_ops);
1167 rk_nfc_hw_init(nfc);
1168 ret = nand_scan(chip, nsels);
1172 if (chip->options & NAND_IS_BOOT_MEDIUM) {
1173 ret = of_property_read_u32(np, "rockchip,boot-blks", &tmp);
1174 rknand->boot_blks = ret ? 0 : tmp;
1176 ret = of_property_read_u32(np, "rockchip,boot-ecc-strength",
1178 rknand->boot_ecc = ret ? chip->ecc.strength : tmp;
1181 ret = mtd_device_register(mtd, NULL, 0);
1183 dev_err(dev, "MTD parse partition error\n");
1188 list_add_tail(&rknand->node, &nfc->chips);
1193 static void rk_nfc_chips_cleanup(struct rk_nfc *nfc)
1195 struct rk_nfc_nand_chip *rknand, *tmp;
1196 struct nand_chip *chip;
1199 list_for_each_entry_safe(rknand, tmp, &nfc->chips, node) {
1200 chip = &rknand->chip;
1201 ret = mtd_device_unregister(nand_to_mtd(chip));
1204 list_del(&rknand->node);
1208 static int rk_nfc_nand_chips_init(struct device *dev, struct rk_nfc *nfc)
1210 struct device_node *np = dev->of_node, *nand_np;
1211 int nchips = of_get_child_count(np);
1214 if (!nchips || nchips > NFC_MAX_NSELS) {
1215 dev_err(nfc->dev, "incorrect number of NAND chips (%d)\n",
1220 for_each_child_of_node(np, nand_np) {
1221 ret = rk_nfc_nand_chip_init(dev, nfc, nand_np);
1223 of_node_put(nand_np);
1224 rk_nfc_chips_cleanup(nfc);
1232 static struct nfc_cfg nfc_v6_cfg = {
1234 .ecc_strengths = {60, 40, 24, 16},
1236 0x00040011, 0x00040001, 0x00000011, 0x00000001,
1240 .dma_cfg_off = 0x10,
1241 .dma_data_buf_off = 0x14,
1242 .dma_oob_buf_off = 0x18,
1245 .randmz_off = 0x150,
1246 .int_en_off = 0x16C,
1247 .int_clr_off = 0x170,
1248 .int_st_off = 0x174,
1269 static struct nfc_cfg nfc_v8_cfg = {
1271 .ecc_strengths = {16, 16, 16, 16},
1273 0x00000001, 0x00000001, 0x00000001, 0x00000001,
1277 .dma_cfg_off = 0x10,
1278 .dma_data_buf_off = 0x14,
1279 .dma_oob_buf_off = 0x18,
1282 .randmz_off = 0x150,
1283 .int_en_off = 0x16C,
1284 .int_clr_off = 0x170,
1285 .int_st_off = 0x174,
1306 static struct nfc_cfg nfc_v9_cfg = {
1308 .ecc_strengths = {70, 60, 40, 16},
1310 0x00000001, 0x06000001, 0x04000001, 0x02000001,
1314 .dma_cfg_off = 0x30,
1315 .dma_data_buf_off = 0x34,
1316 .dma_oob_buf_off = 0x38,
1318 .bch_st_off = 0x150,
1319 .randmz_off = 0x208,
1320 .int_en_off = 0x120,
1321 .int_clr_off = 0x124,
1322 .int_st_off = 0x128,
1343 static const struct of_device_id rk_nfc_id_table[] = {
1345 .compatible = "rockchip,px30-nfc",
1349 .compatible = "rockchip,rk2928-nfc",
1353 .compatible = "rockchip,rv1108-nfc",
1358 MODULE_DEVICE_TABLE(of, rk_nfc_id_table);
1360 static int rk_nfc_probe(struct platform_device *pdev)
1362 struct device *dev = &pdev->dev;
1366 nfc = devm_kzalloc(dev, sizeof(*nfc), GFP_KERNEL);
1370 nand_controller_init(&nfc->controller);
1371 INIT_LIST_HEAD(&nfc->chips);
1372 nfc->controller.ops = &rk_nfc_controller_ops;
1374 nfc->cfg = of_device_get_match_data(dev);
1377 init_completion(&nfc->done);
1379 nfc->regs = devm_platform_ioremap_resource(pdev, 0);
1380 if (IS_ERR(nfc->regs)) {
1381 ret = PTR_ERR(nfc->regs);
1385 nfc->nfc_clk = devm_clk_get(dev, "nfc");
1386 if (IS_ERR(nfc->nfc_clk)) {
1387 dev_dbg(dev, "no NFC clk\n");
1388 /* Some earlier models, such as rk3066, have no NFC clk. */
1391 nfc->ahb_clk = devm_clk_get(dev, "ahb");
1392 if (IS_ERR(nfc->ahb_clk)) {
1393 dev_err(dev, "no ahb clk\n");
1394 ret = PTR_ERR(nfc->ahb_clk);
1398 ret = rk_nfc_enable_clks(dev, nfc);
1402 irq = platform_get_irq(pdev, 0);
1408 writel(0, nfc->regs + nfc->cfg->int_en_off);
1409 ret = devm_request_irq(dev, irq, rk_nfc_irq, 0x0, "rk-nand", nfc);
1411 dev_err(dev, "failed to request NFC irq\n");
1415 platform_set_drvdata(pdev, nfc);
1417 ret = rk_nfc_nand_chips_init(dev, nfc);
1419 dev_err(dev, "failed to init NAND chips\n");
1425 rk_nfc_disable_clks(nfc);
1430 static void rk_nfc_remove(struct platform_device *pdev)
1432 struct rk_nfc *nfc = platform_get_drvdata(pdev);
1434 kfree(nfc->page_buf);
1435 kfree(nfc->oob_buf);
1436 rk_nfc_chips_cleanup(nfc);
1437 rk_nfc_disable_clks(nfc);
1440 static int __maybe_unused rk_nfc_suspend(struct device *dev)
1442 struct rk_nfc *nfc = dev_get_drvdata(dev);
1444 rk_nfc_disable_clks(nfc);
1449 static int __maybe_unused rk_nfc_resume(struct device *dev)
1451 struct rk_nfc *nfc = dev_get_drvdata(dev);
1452 struct rk_nfc_nand_chip *rknand;
1453 struct nand_chip *chip;
1457 ret = rk_nfc_enable_clks(dev, nfc);
1461 /* Reset NAND chip if VCC was powered off. */
1462 list_for_each_entry(rknand, &nfc->chips, node) {
1463 chip = &rknand->chip;
1464 for (i = 0; i < rknand->nsels; i++)
1465 nand_reset(chip, i);
1471 static const struct dev_pm_ops rk_nfc_pm_ops = {
1472 SET_SYSTEM_SLEEP_PM_OPS(rk_nfc_suspend, rk_nfc_resume)
1475 static struct platform_driver rk_nfc_driver = {
1476 .probe = rk_nfc_probe,
1477 .remove_new = rk_nfc_remove,
1479 .name = "rockchip-nfc",
1480 .of_match_table = rk_nfc_id_table,
1481 .pm = &rk_nfc_pm_ops,
1485 module_platform_driver(rk_nfc_driver);
1487 MODULE_LICENSE("Dual MIT/GPL");
1488 MODULE_AUTHOR("Yifeng Zhao <yifeng.zhao@rock-chips.com>");
1489 MODULE_DESCRIPTION("Rockchip Nand Flash Controller Driver");
1490 MODULE_ALIAS("platform:rockchip-nand-controller");