1 // SPDX-License-Identifier: GPL-2.0+
3 * Freescale GPMI NAND Flash Driver
5 * Copyright (C) 2010-2015 Freescale Semiconductor, Inc.
6 * Copyright (C) 2008 Embedded Alley Solutions, Inc.
9 #include <linux/delay.h>
10 #include <linux/slab.h>
11 #include <linux/sched/task_stack.h>
12 #include <linux/interrupt.h>
13 #include <linux/module.h>
14 #include <linux/mtd/partitions.h>
16 #include <linux/of_device.h>
17 #include <linux/pm_runtime.h>
18 #include <linux/dma/mxs-dma.h>
19 #include "gpmi-nand.h"
20 #include "gpmi-regs.h"
23 /* Resource names for the GPMI NAND driver. */
24 #define GPMI_NAND_GPMI_REGS_ADDR_RES_NAME "gpmi-nand"
25 #define GPMI_NAND_BCH_REGS_ADDR_RES_NAME "bch"
26 #define GPMI_NAND_BCH_INTERRUPT_RES_NAME "bch"
28 /* Converts time to clock cycles */
29 #define TO_CYCLES(duration, period) DIV_ROUND_UP_ULL(duration, period)
31 #define MXS_SET_ADDR 0x4
32 #define MXS_CLR_ADDR 0x8
34 * Clear the bit and poll it cleared. This is usually called with
35 * a reset address and mask being either SFTRST(bit 31) or CLKGATE
38 static int clear_poll_bit(void __iomem *addr, u32 mask)
43 writel(mask, addr + MXS_CLR_ADDR);
46 * SFTRST needs 3 GPMI clocks to settle, the reference manual
47 * recommends to wait 1us.
51 /* poll the bit becoming clear */
52 while ((readl(addr) & mask) && --timeout)
58 #define MODULE_CLKGATE (1 << 30)
59 #define MODULE_SFTRST (1 << 31)
61 * The current mxs_reset_block() will do two things:
62 * [1] enable the module.
63 * [2] reset the module.
65 * In most of the cases, it's ok.
66 * But in MX23, there is a hardware bug in the BCH block (see erratum #2847).
67 * If you try to soft reset the BCH block, it becomes unusable until
68 * the next hard reset. This case occurs in the NAND boot mode. When the board
69 * boots by NAND, the ROM of the chip will initialize the BCH blocks itself.
70 * So If the driver tries to reset the BCH again, the BCH will not work anymore.
71 * You will see a DMA timeout in this case. The bug has been fixed
72 * in the following chips, such as MX28.
74 * To avoid this bug, just add a new parameter `just_enable` for
75 * the mxs_reset_block(), and rewrite it here.
77 static int gpmi_reset_block(void __iomem *reset_addr, bool just_enable)
82 /* clear and poll SFTRST */
83 ret = clear_poll_bit(reset_addr, MODULE_SFTRST);
88 writel(MODULE_CLKGATE, reset_addr + MXS_CLR_ADDR);
91 /* set SFTRST to reset the block */
92 writel(MODULE_SFTRST, reset_addr + MXS_SET_ADDR);
95 /* poll CLKGATE becoming set */
96 while ((!(readl(reset_addr) & MODULE_CLKGATE)) && --timeout)
98 if (unlikely(!timeout))
102 /* clear and poll SFTRST */
103 ret = clear_poll_bit(reset_addr, MODULE_SFTRST);
107 /* clear and poll CLKGATE */
108 ret = clear_poll_bit(reset_addr, MODULE_CLKGATE);
115 pr_err("%s(%p): module reset timeout\n", __func__, reset_addr);
119 static int __gpmi_enable_clk(struct gpmi_nand_data *this, bool v)
125 for (i = 0; i < GPMI_CLK_MAX; i++) {
126 clk = this->resources.clock[i];
131 ret = clk_prepare_enable(clk);
135 clk_disable_unprepare(clk);
142 clk_disable_unprepare(this->resources.clock[i - 1]);
146 static int gpmi_init(struct gpmi_nand_data *this)
148 struct resources *r = &this->resources;
151 ret = pm_runtime_get_sync(this->dev);
153 pm_runtime_put_noidle(this->dev);
157 ret = gpmi_reset_block(r->gpmi_regs, false);
162 * Reset BCH here, too. We got failures otherwise :(
163 * See later BCH reset for explanation of MX23 and MX28 handling
165 ret = gpmi_reset_block(r->bch_regs, GPMI_IS_MXS(this));
169 /* Choose NAND mode. */
170 writel(BM_GPMI_CTRL1_GPMI_MODE, r->gpmi_regs + HW_GPMI_CTRL1_CLR);
172 /* Set the IRQ polarity. */
173 writel(BM_GPMI_CTRL1_ATA_IRQRDY_POLARITY,
174 r->gpmi_regs + HW_GPMI_CTRL1_SET);
176 /* Disable Write-Protection. */
177 writel(BM_GPMI_CTRL1_DEV_RESET, r->gpmi_regs + HW_GPMI_CTRL1_SET);
179 /* Select BCH ECC. */
180 writel(BM_GPMI_CTRL1_BCH_MODE, r->gpmi_regs + HW_GPMI_CTRL1_SET);
183 * Decouple the chip select from dma channel. We use dma0 for all
184 * the chips, force all NAND RDY_BUSY inputs to be sourced from
187 writel(BM_GPMI_CTRL1_DECOUPLE_CS | BM_GPMI_CTRL1_GANGED_RDYBUSY,
188 r->gpmi_regs + HW_GPMI_CTRL1_SET);
191 pm_runtime_mark_last_busy(this->dev);
192 pm_runtime_put_autosuspend(this->dev);
196 /* This function is very useful. It is called only when the bug occur. */
197 static void gpmi_dump_info(struct gpmi_nand_data *this)
199 struct resources *r = &this->resources;
200 struct bch_geometry *geo = &this->bch_geometry;
204 dev_err(this->dev, "Show GPMI registers :\n");
205 for (i = 0; i <= HW_GPMI_DEBUG / 0x10 + 1; i++) {
206 reg = readl(r->gpmi_regs + i * 0x10);
207 dev_err(this->dev, "offset 0x%.3x : 0x%.8x\n", i * 0x10, reg);
210 /* start to print out the BCH info */
211 dev_err(this->dev, "Show BCH registers :\n");
212 for (i = 0; i <= HW_BCH_VERSION / 0x10 + 1; i++) {
213 reg = readl(r->bch_regs + i * 0x10);
214 dev_err(this->dev, "offset 0x%.3x : 0x%.8x\n", i * 0x10, reg);
216 dev_err(this->dev, "BCH Geometry :\n"
218 "ECC Strength : %u\n"
219 "Page Size in Bytes : %u\n"
220 "Metadata Size in Bytes : %u\n"
221 "ECC0 Chunk Size in Bytes: %u\n"
222 "ECCn Chunk Size in Bytes: %u\n"
223 "ECC Chunk Count : %u\n"
224 "Payload Size in Bytes : %u\n"
225 "Auxiliary Size in Bytes: %u\n"
226 "Auxiliary Status Offset: %u\n"
227 "Block Mark Byte Offset : %u\n"
228 "Block Mark Bit Offset : %u\n",
233 geo->ecc0_chunk_size,
234 geo->eccn_chunk_size,
235 geo->ecc_chunk_count,
238 geo->auxiliary_status_offset,
239 geo->block_mark_byte_offset,
240 geo->block_mark_bit_offset);
243 static bool gpmi_check_ecc(struct gpmi_nand_data *this)
245 struct nand_chip *chip = &this->nand;
246 struct bch_geometry *geo = &this->bch_geometry;
247 struct nand_device *nand = &chip->base;
248 struct nand_ecc_props *conf = &nand->ecc.ctx.conf;
250 conf->step_size = geo->eccn_chunk_size;
251 conf->strength = geo->ecc_strength;
253 /* Do the sanity check. */
254 if (GPMI_IS_MXS(this)) {
255 /* The mx23/mx28 only support the GF13. */
256 if (geo->gf_len == 14)
260 if (geo->ecc_strength > this->devdata->bch_max_ecc_strength)
263 if (!nand_ecc_is_strong_enough(nand))
269 /* check if bbm locates in data chunk rather than ecc chunk */
270 static bool bbm_in_data_chunk(struct gpmi_nand_data *this,
271 unsigned int *chunk_num)
273 struct bch_geometry *geo = &this->bch_geometry;
274 struct nand_chip *chip = &this->nand;
275 struct mtd_info *mtd = nand_to_mtd(chip);
278 if (geo->ecc0_chunk_size != geo->eccn_chunk_size) {
280 "The size of ecc0_chunk must equal to eccn_chunk\n");
284 i = (mtd->writesize * 8 - geo->metadata_size * 8) /
285 (geo->gf_len * geo->ecc_strength +
286 geo->eccn_chunk_size * 8);
288 j = (mtd->writesize * 8 - geo->metadata_size * 8) -
289 (geo->gf_len * geo->ecc_strength +
290 geo->eccn_chunk_size * 8) * i;
292 if (j < geo->eccn_chunk_size * 8) {
294 dev_dbg(this->dev, "Set ecc to %d and bbm in chunk %d\n",
295 geo->ecc_strength, *chunk_num);
303 * If we can get the ECC information from the nand chip, we do not
304 * need to calculate them ourselves.
306 * We may have available oob space in this case.
308 static int set_geometry_by_ecc_info(struct gpmi_nand_data *this,
309 unsigned int ecc_strength,
310 unsigned int ecc_step)
312 struct bch_geometry *geo = &this->bch_geometry;
313 struct nand_chip *chip = &this->nand;
314 struct mtd_info *mtd = nand_to_mtd(chip);
315 unsigned int block_mark_bit_offset;
326 "unsupported nand chip. ecc bits : %d, ecc size : %d\n",
327 nanddev_get_ecc_requirements(&chip->base)->strength,
328 nanddev_get_ecc_requirements(&chip->base)->step_size);
331 geo->ecc0_chunk_size = ecc_step;
332 geo->eccn_chunk_size = ecc_step;
333 geo->ecc_strength = round_up(ecc_strength, 2);
334 if (!gpmi_check_ecc(this))
337 /* Keep the C >= O */
338 if (geo->eccn_chunk_size < mtd->oobsize) {
340 "unsupported nand chip. ecc size: %d, oob size : %d\n",
341 ecc_step, mtd->oobsize);
345 /* The default value, see comment in the legacy_set_geometry(). */
346 geo->metadata_size = 10;
348 geo->ecc_chunk_count = mtd->writesize / geo->eccn_chunk_size;
351 * Now, the NAND chip with 2K page(data chunk is 512byte) shows below:
354 * |<----------------------------------------------------->|
358 * |<-------------------------------------------->| D | | O' |
361 * +---+----------+-+----------+-+----------+-+----------+-+-----+
362 * | M | data |E| data |E| data |E| data |E| |
363 * +---+----------+-+----------+-+----------+-+----------+-+-----+
369 * P : the page size for BCH module.
370 * E : The ECC strength.
371 * G : the length of Galois Field.
372 * N : The chunk count of per page.
373 * M : the metasize of per page.
374 * C : the ecc chunk size, aka the "data" above.
375 * P': the nand chip's page size.
376 * O : the nand chip's oob size.
379 * The formula for P is :
382 * P = ------------ + P' + M
385 * The position of block mark moves forward in the ECC-based view
386 * of page, and the delta is:
389 * D = (---------------- + M)
392 * Please see the comment in legacy_set_geometry().
393 * With the condition C >= O , we still can get same result.
394 * So the bit position of the physical block mark within the ECC-based
395 * view of the page is :
398 geo->page_size = mtd->writesize + geo->metadata_size +
399 (geo->gf_len * geo->ecc_strength * geo->ecc_chunk_count) / 8;
401 geo->payload_size = mtd->writesize;
403 geo->auxiliary_status_offset = ALIGN(geo->metadata_size, 4);
404 geo->auxiliary_size = ALIGN(geo->metadata_size, 4)
405 + ALIGN(geo->ecc_chunk_count, 4);
407 if (!this->swap_block_mark)
411 block_mark_bit_offset = mtd->writesize * 8 -
412 (geo->ecc_strength * geo->gf_len * (geo->ecc_chunk_count - 1)
413 + geo->metadata_size * 8);
415 geo->block_mark_byte_offset = block_mark_bit_offset / 8;
416 geo->block_mark_bit_offset = block_mark_bit_offset % 8;
421 * Calculate the ECC strength by hand:
422 * E : The ECC strength.
423 * G : the length of Galois Field.
424 * N : The chunk count of per page.
425 * O : the oobsize of the NAND chip.
426 * M : the metasize of per page.
430 * ------------ <= (O - M)
438 static inline int get_ecc_strength(struct gpmi_nand_data *this)
440 struct bch_geometry *geo = &this->bch_geometry;
441 struct mtd_info *mtd = nand_to_mtd(&this->nand);
444 ecc_strength = ((mtd->oobsize - geo->metadata_size) * 8)
445 / (geo->gf_len * geo->ecc_chunk_count);
447 /* We need the minor even number. */
448 return round_down(ecc_strength, 2);
451 static int set_geometry_for_large_oob(struct gpmi_nand_data *this)
453 struct bch_geometry *geo = &this->bch_geometry;
454 struct nand_chip *chip = &this->nand;
455 struct mtd_info *mtd = nand_to_mtd(chip);
456 const struct nand_ecc_props *requirements =
457 nanddev_get_ecc_requirements(&chip->base);
458 unsigned int block_mark_bit_offset;
459 unsigned int max_ecc;
460 unsigned int bbm_chunk;
463 /* sanity check for the minimum ecc nand required */
464 if (!(requirements->strength > 0 &&
465 requirements->step_size > 0))
467 geo->ecc_strength = requirements->strength;
469 /* check if platform can support this nand */
470 if (!gpmi_check_ecc(this)) {
472 "unsupported NAND chip, minimum ecc required %d\n",
477 /* calculate the maximum ecc platform can support*/
478 geo->metadata_size = 10;
480 geo->ecc0_chunk_size = 1024;
481 geo->eccn_chunk_size = 1024;
482 geo->ecc_chunk_count = mtd->writesize / geo->eccn_chunk_size;
483 max_ecc = min(get_ecc_strength(this),
484 this->devdata->bch_max_ecc_strength);
487 * search a supported ecc strength that makes bbm
488 * located in data chunk
490 geo->ecc_strength = max_ecc;
491 while (!(geo->ecc_strength < requirements->strength)) {
492 if (bbm_in_data_chunk(this, &bbm_chunk))
494 geo->ecc_strength -= 2;
497 /* if none of them works, keep using the minimum ecc */
498 /* nand required but changing ecc page layout */
499 geo->ecc_strength = requirements->strength;
500 /* add extra ecc for meta data */
501 geo->ecc0_chunk_size = 0;
502 geo->ecc_chunk_count = (mtd->writesize / geo->eccn_chunk_size) + 1;
503 geo->ecc_for_meta = 1;
504 /* check if oob can afford this extra ecc chunk */
505 if (mtd->oobsize * 8 < geo->metadata_size * 8 +
506 geo->gf_len * geo->ecc_strength * geo->ecc_chunk_count) {
507 dev_err(this->dev, "unsupported NAND chip with new layout\n");
511 /* calculate in which chunk bbm located */
512 bbm_chunk = (mtd->writesize * 8 - geo->metadata_size * 8 -
513 geo->gf_len * geo->ecc_strength) /
514 (geo->gf_len * geo->ecc_strength +
515 geo->eccn_chunk_size * 8) + 1;
519 geo->page_size = mtd->writesize + geo->metadata_size +
520 (geo->gf_len * geo->ecc_strength * geo->ecc_chunk_count) / 8;
521 geo->payload_size = mtd->writesize;
524 * The auxiliary buffer contains the metadata and the ECC status. The
525 * metadata is padded to the nearest 32-bit boundary. The ECC status
526 * contains one byte for every ECC chunk, and is also padded to the
527 * nearest 32-bit boundary.
529 geo->auxiliary_status_offset = ALIGN(geo->metadata_size, 4);
530 geo->auxiliary_size = ALIGN(geo->metadata_size, 4)
531 + ALIGN(geo->ecc_chunk_count, 4);
533 if (!this->swap_block_mark)
536 /* calculate the number of ecc chunk behind the bbm */
537 i = (mtd->writesize / geo->eccn_chunk_size) - bbm_chunk + 1;
539 block_mark_bit_offset = mtd->writesize * 8 -
540 (geo->ecc_strength * geo->gf_len * (geo->ecc_chunk_count - i)
541 + geo->metadata_size * 8);
543 geo->block_mark_byte_offset = block_mark_bit_offset / 8;
544 geo->block_mark_bit_offset = block_mark_bit_offset % 8;
546 dev_dbg(this->dev, "BCH Geometry :\n"
548 "ECC Strength : %u\n"
549 "Page Size in Bytes : %u\n"
550 "Metadata Size in Bytes : %u\n"
551 "ECC0 Chunk Size in Bytes: %u\n"
552 "ECCn Chunk Size in Bytes: %u\n"
553 "ECC Chunk Count : %u\n"
554 "Payload Size in Bytes : %u\n"
555 "Auxiliary Size in Bytes: %u\n"
556 "Auxiliary Status Offset: %u\n"
557 "Block Mark Byte Offset : %u\n"
558 "Block Mark Bit Offset : %u\n"
559 "Block Mark in chunk : %u\n"
560 "Ecc for Meta data : %u\n",
565 geo->ecc0_chunk_size,
566 geo->eccn_chunk_size,
567 geo->ecc_chunk_count,
570 geo->auxiliary_status_offset,
571 geo->block_mark_byte_offset,
572 geo->block_mark_bit_offset,
579 static int legacy_set_geometry(struct gpmi_nand_data *this)
581 struct bch_geometry *geo = &this->bch_geometry;
582 struct mtd_info *mtd = nand_to_mtd(&this->nand);
583 unsigned int metadata_size;
584 unsigned int status_size;
585 unsigned int block_mark_bit_offset;
588 * The size of the metadata can be changed, though we set it to 10
589 * bytes now. But it can't be too large, because we have to save
590 * enough space for BCH.
592 geo->metadata_size = 10;
594 /* The default for the length of Galois Field. */
597 /* The default for chunk size. */
598 geo->ecc0_chunk_size = 512;
599 geo->eccn_chunk_size = 512;
600 while (geo->eccn_chunk_size < mtd->oobsize) {
601 geo->ecc0_chunk_size *= 2; /* keep C >= O */
602 geo->eccn_chunk_size *= 2; /* keep C >= O */
606 geo->ecc_chunk_count = mtd->writesize / geo->eccn_chunk_size;
608 /* We use the same ECC strength for all chunks. */
609 geo->ecc_strength = get_ecc_strength(this);
610 if (!gpmi_check_ecc(this)) {
612 "ecc strength: %d cannot be supported by the controller (%d)\n"
613 "try to use minimum ecc strength that NAND chip required\n",
615 this->devdata->bch_max_ecc_strength);
619 geo->page_size = mtd->writesize + geo->metadata_size +
620 (geo->gf_len * geo->ecc_strength * geo->ecc_chunk_count) / 8;
621 geo->payload_size = mtd->writesize;
624 * The auxiliary buffer contains the metadata and the ECC status. The
625 * metadata is padded to the nearest 32-bit boundary. The ECC status
626 * contains one byte for every ECC chunk, and is also padded to the
627 * nearest 32-bit boundary.
629 metadata_size = ALIGN(geo->metadata_size, 4);
630 status_size = ALIGN(geo->ecc_chunk_count, 4);
632 geo->auxiliary_size = metadata_size + status_size;
633 geo->auxiliary_status_offset = metadata_size;
635 if (!this->swap_block_mark)
639 * We need to compute the byte and bit offsets of
640 * the physical block mark within the ECC-based view of the page.
642 * NAND chip with 2K page shows below:
648 * +---+----------+-+----------+-+----------+-+----------+-+
649 * | M | data |E| data |E| data |E| data |E|
650 * +---+----------+-+----------+-+----------+-+----------+-+
652 * The position of block mark moves forward in the ECC-based view
653 * of page, and the delta is:
656 * D = (---------------- + M)
659 * With the formula to compute the ECC strength, and the condition
660 * : C >= O (C is the ecc chunk size)
662 * It's easy to deduce to the following result:
664 * E * G (O - M) C - M C - M
665 * ----------- <= ------- <= -------- < ---------
671 * D = (---------------- + M) < C
674 * The above inequality means the position of block mark
675 * within the ECC-based view of the page is still in the data chunk,
676 * and it's NOT in the ECC bits of the chunk.
678 * Use the following to compute the bit position of the
679 * physical block mark within the ECC-based view of the page:
680 * (page_size - D) * 8
684 block_mark_bit_offset = mtd->writesize * 8 -
685 (geo->ecc_strength * geo->gf_len * (geo->ecc_chunk_count - 1)
686 + geo->metadata_size * 8);
688 geo->block_mark_byte_offset = block_mark_bit_offset / 8;
689 geo->block_mark_bit_offset = block_mark_bit_offset % 8;
693 static int common_nfc_set_geometry(struct gpmi_nand_data *this)
695 struct nand_chip *chip = &this->nand;
696 struct mtd_info *mtd = nand_to_mtd(&this->nand);
697 const struct nand_ecc_props *requirements =
698 nanddev_get_ecc_requirements(&chip->base);
699 bool use_minimun_ecc;
702 use_minimun_ecc = of_property_read_bool(this->dev->of_node,
703 "fsl,use-minimum-ecc");
705 /* use legacy bch geometry settings by default*/
706 if ((!use_minimun_ecc && mtd->oobsize < 1024) ||
707 !(requirements->strength > 0 && requirements->step_size > 0)) {
708 dev_dbg(this->dev, "use legacy bch geometry\n");
709 err = legacy_set_geometry(this);
714 /* for large oob nand */
715 if (mtd->oobsize > 1024) {
716 dev_dbg(this->dev, "use large oob bch geometry\n");
717 err = set_geometry_for_large_oob(this);
722 /* otherwise use the minimum ecc nand chip required */
723 dev_dbg(this->dev, "use minimum ecc bch geometry\n");
724 err = set_geometry_by_ecc_info(this, requirements->strength,
725 requirements->step_size);
727 dev_err(this->dev, "none of the bch geometry setting works\n");
732 /* Configures the geometry for BCH. */
733 static int bch_set_geometry(struct gpmi_nand_data *this)
735 struct resources *r = &this->resources;
738 ret = common_nfc_set_geometry(this);
742 ret = pm_runtime_get_sync(this->dev);
744 pm_runtime_put_autosuspend(this->dev);
749 * Due to erratum #2847 of the MX23, the BCH cannot be soft reset on this
750 * chip, otherwise it will lock up. So we skip resetting BCH on the MX23.
753 ret = gpmi_reset_block(r->bch_regs, GPMI_IS_MXS(this));
757 /* Set *all* chip selects to use layout 0. */
758 writel(0, r->bch_regs + HW_BCH_LAYOUTSELECT);
762 pm_runtime_mark_last_busy(this->dev);
763 pm_runtime_put_autosuspend(this->dev);
769 * <1> Firstly, we should know what's the GPMI-clock means.
770 * The GPMI-clock is the internal clock in the gpmi nand controller.
771 * If you set 100MHz to gpmi nand controller, the GPMI-clock's period
772 * is 10ns. Mark the GPMI-clock's period as GPMI-clock-period.
774 * <2> Secondly, we should know what's the frequency on the nand chip pins.
775 * The frequency on the nand chip pins is derived from the GPMI-clock.
776 * We can get it from the following equation:
780 * F : the frequency on the nand chip pins.
781 * G : the GPMI clock, such as 100MHz.
782 * DS : GPMI_HW_GPMI_TIMING0:DATA_SETUP
783 * DH : GPMI_HW_GPMI_TIMING0:DATA_HOLD
785 * <3> Thirdly, when the frequency on the nand chip pins is above 33MHz,
786 * the nand EDO(extended Data Out) timing could be applied.
787 * The GPMI implements a feedback read strobe to sample the read data.
788 * The feedback read strobe can be delayed to support the nand EDO timing
789 * where the read strobe may deasserts before the read data is valid, and
790 * read data is valid for some time after read strobe.
792 * The following figure illustrates some aspects of a NAND Flash read:
799 * __ ___|__________________________________
803 * Read Data --------------< >---------
807 * FeedbackRDN ________ ____________
810 * D stands for delay, set in the HW_GPMI_CTRL1:RDN_DELAY.
813 * <4> Now, we begin to describe how to compute the right RDN_DELAY.
815 * 4.1) From the aspect of the nand chip pins:
816 * Delay = (tREA + C - tRP) {1}
818 * tREA : the maximum read access time.
819 * C : a constant to adjust the delay. default is 4000ps.
820 * tRP : the read pulse width, which is exactly:
821 * tRP = (GPMI-clock-period) * DATA_SETUP
823 * 4.2) From the aspect of the GPMI nand controller:
824 * Delay = RDN_DELAY * 0.125 * RP {2}
826 * RP : the DLL reference period.
827 * if (GPMI-clock-period > DLL_THRETHOLD)
828 * RP = GPMI-clock-period / 2;
830 * RP = GPMI-clock-period;
832 * Set the HW_GPMI_CTRL1:HALF_PERIOD if GPMI-clock-period
833 * is greater DLL_THRETHOLD. In other SOCs, the DLL_THRETHOLD
834 * is 16000ps, but in mx6q, we use 12000ps.
836 * 4.3) since {1} equals {2}, we get:
838 * (tREA + 4000 - tRP) * 8
839 * RDN_DELAY = ----------------------- {3}
842 static int gpmi_nfc_compute_timings(struct gpmi_nand_data *this,
843 const struct nand_sdr_timings *sdr)
845 struct gpmi_nfc_hardware_timing *hw = &this->hw;
846 struct resources *r = &this->resources;
847 unsigned int dll_threshold_ps = this->devdata->max_chain_delay;
848 unsigned int period_ps, reference_period_ps;
849 unsigned int data_setup_cycles, data_hold_cycles, addr_setup_cycles;
851 bool use_half_period;
852 int sample_delay_ps, sample_delay_factor;
853 u16 busy_timeout_cycles;
855 unsigned long clk_rate, min_rate;
857 if (sdr->tRC_min >= 30000) {
858 /* ONFI non-EDO modes [0-3] */
859 hw->clk_rate = 22000000;
861 wrn_dly_sel = BV_GPMI_CTRL1_WRN_DLY_SEL_4_TO_8NS;
862 } else if (sdr->tRC_min >= 25000) {
863 /* ONFI EDO mode 4 */
864 hw->clk_rate = 80000000;
866 wrn_dly_sel = BV_GPMI_CTRL1_WRN_DLY_SEL_NO_DELAY;
868 /* ONFI EDO mode 5 */
869 hw->clk_rate = 100000000;
871 wrn_dly_sel = BV_GPMI_CTRL1_WRN_DLY_SEL_NO_DELAY;
874 clk_rate = clk_round_rate(r->clock[0], hw->clk_rate);
875 if (clk_rate <= min_rate) {
876 dev_err(this->dev, "clock setting: expected %ld, got %ld\n",
877 hw->clk_rate, clk_rate);
881 hw->clk_rate = clk_rate;
882 /* SDR core timings are given in picoseconds */
883 period_ps = div_u64((u64)NSEC_PER_SEC * 1000, hw->clk_rate);
885 addr_setup_cycles = TO_CYCLES(sdr->tALS_min, period_ps);
886 data_setup_cycles = TO_CYCLES(sdr->tDS_min, period_ps);
887 data_hold_cycles = TO_CYCLES(sdr->tDH_min, period_ps);
888 busy_timeout_cycles = TO_CYCLES(sdr->tWB_max + sdr->tR_max, period_ps);
890 hw->timing0 = BF_GPMI_TIMING0_ADDRESS_SETUP(addr_setup_cycles) |
891 BF_GPMI_TIMING0_DATA_HOLD(data_hold_cycles) |
892 BF_GPMI_TIMING0_DATA_SETUP(data_setup_cycles);
893 hw->timing1 = BF_GPMI_TIMING1_BUSY_TIMEOUT(busy_timeout_cycles * 4096);
896 * Derive NFC ideal delay from {3}:
898 * (tREA + 4000 - tRP) * 8
899 * RDN_DELAY = -----------------------
902 if (period_ps > dll_threshold_ps) {
903 use_half_period = true;
904 reference_period_ps = period_ps / 2;
906 use_half_period = false;
907 reference_period_ps = period_ps;
910 tRP_ps = data_setup_cycles * period_ps;
911 sample_delay_ps = (sdr->tREA_max + 4000 - tRP_ps) * 8;
912 if (sample_delay_ps > 0)
913 sample_delay_factor = sample_delay_ps / reference_period_ps;
915 sample_delay_factor = 0;
917 hw->ctrl1n = BF_GPMI_CTRL1_WRN_DLY_SEL(wrn_dly_sel);
918 if (sample_delay_factor)
919 hw->ctrl1n |= BF_GPMI_CTRL1_RDN_DELAY(sample_delay_factor) |
920 BM_GPMI_CTRL1_DLL_ENABLE |
921 (use_half_period ? BM_GPMI_CTRL1_HALF_PERIOD : 0);
925 static int gpmi_nfc_apply_timings(struct gpmi_nand_data *this)
927 struct gpmi_nfc_hardware_timing *hw = &this->hw;
928 struct resources *r = &this->resources;
929 void __iomem *gpmi_regs = r->gpmi_regs;
930 unsigned int dll_wait_time_us;
933 /* Clock dividers do NOT guarantee a clean clock signal on its output
934 * during the change of the divide factor on i.MX6Q/UL/SX. On i.MX7/8,
935 * all clock dividers provide these guarantee.
937 if (GPMI_IS_MX6Q(this) || GPMI_IS_MX6SX(this))
938 clk_disable_unprepare(r->clock[0]);
940 ret = clk_set_rate(r->clock[0], hw->clk_rate);
942 dev_err(this->dev, "cannot set clock rate to %lu Hz: %d\n", hw->clk_rate, ret);
946 if (GPMI_IS_MX6Q(this) || GPMI_IS_MX6SX(this)) {
947 ret = clk_prepare_enable(r->clock[0]);
952 writel(hw->timing0, gpmi_regs + HW_GPMI_TIMING0);
953 writel(hw->timing1, gpmi_regs + HW_GPMI_TIMING1);
956 * Clear several CTRL1 fields, DLL must be disabled when setting
957 * RDN_DELAY or HALF_PERIOD.
959 writel(BM_GPMI_CTRL1_CLEAR_MASK, gpmi_regs + HW_GPMI_CTRL1_CLR);
960 writel(hw->ctrl1n, gpmi_regs + HW_GPMI_CTRL1_SET);
962 /* Wait 64 clock cycles before using the GPMI after enabling the DLL */
963 dll_wait_time_us = USEC_PER_SEC / hw->clk_rate * 64;
964 if (!dll_wait_time_us)
965 dll_wait_time_us = 1;
967 /* Wait for the DLL to settle. */
968 udelay(dll_wait_time_us);
973 static int gpmi_setup_interface(struct nand_chip *chip, int chipnr,
974 const struct nand_interface_config *conf)
976 struct gpmi_nand_data *this = nand_get_controller_data(chip);
977 const struct nand_sdr_timings *sdr;
980 /* Retrieve required NAND timings */
981 sdr = nand_get_sdr_timings(conf);
985 /* Only MX28/MX6 GPMI controller can reach EDO timings */
986 if (sdr->tRC_min <= 25000 && !GPMI_IS_MX28(this) && !GPMI_IS_MX6(this))
989 /* Stop here if this call was just a check */
993 /* Do the actual derivation of the controller timings */
994 ret = gpmi_nfc_compute_timings(this, sdr);
998 this->hw.must_apply_timings = true;
1003 /* Clears a BCH interrupt. */
1004 static void gpmi_clear_bch(struct gpmi_nand_data *this)
1006 struct resources *r = &this->resources;
1007 writel(BM_BCH_CTRL_COMPLETE_IRQ, r->bch_regs + HW_BCH_CTRL_CLR);
1010 static struct dma_chan *get_dma_chan(struct gpmi_nand_data *this)
1012 /* We use the DMA channel 0 to access all the nand chips. */
1013 return this->dma_chans[0];
1016 /* This will be called after the DMA operation is finished. */
1017 static void dma_irq_callback(void *param)
1019 struct gpmi_nand_data *this = param;
1020 struct completion *dma_c = &this->dma_done;
1025 static irqreturn_t bch_irq(int irq, void *cookie)
1027 struct gpmi_nand_data *this = cookie;
1029 gpmi_clear_bch(this);
1030 complete(&this->bch_done);
1034 static int gpmi_raw_len_to_len(struct gpmi_nand_data *this, int raw_len)
1037 * raw_len is the length to read/write including bch data which
1038 * we are passed in exec_op. Calculate the data length from it.
1041 return ALIGN_DOWN(raw_len, this->bch_geometry.eccn_chunk_size);
1046 /* Can we use the upper's buffer directly for DMA? */
1047 static bool prepare_data_dma(struct gpmi_nand_data *this, const void *buf,
1048 int raw_len, struct scatterlist *sgl,
1049 enum dma_data_direction dr)
1052 int len = gpmi_raw_len_to_len(this, raw_len);
1054 /* first try to map the upper buffer directly */
1055 if (virt_addr_valid(buf) && !object_is_on_stack(buf)) {
1056 sg_init_one(sgl, buf, len);
1057 ret = dma_map_sg(this->dev, sgl, 1, dr);
1065 /* We have to use our own DMA buffer. */
1066 sg_init_one(sgl, this->data_buffer_dma, len);
1068 if (dr == DMA_TO_DEVICE && buf != this->data_buffer_dma)
1069 memcpy(this->data_buffer_dma, buf, len);
1071 dma_map_sg(this->dev, sgl, 1, dr);
1076 /* add our owner bbt descriptor */
1077 static uint8_t scan_ff_pattern[] = { 0xff };
1078 static struct nand_bbt_descr gpmi_bbt_descr = {
1082 .pattern = scan_ff_pattern
1086 * We may change the layout if we can get the ECC info from the datasheet,
1087 * else we will use all the (page + OOB).
1089 static int gpmi_ooblayout_ecc(struct mtd_info *mtd, int section,
1090 struct mtd_oob_region *oobregion)
1092 struct nand_chip *chip = mtd_to_nand(mtd);
1093 struct gpmi_nand_data *this = nand_get_controller_data(chip);
1094 struct bch_geometry *geo = &this->bch_geometry;
1099 oobregion->offset = 0;
1100 oobregion->length = geo->page_size - mtd->writesize;
1105 static int gpmi_ooblayout_free(struct mtd_info *mtd, int section,
1106 struct mtd_oob_region *oobregion)
1108 struct nand_chip *chip = mtd_to_nand(mtd);
1109 struct gpmi_nand_data *this = nand_get_controller_data(chip);
1110 struct bch_geometry *geo = &this->bch_geometry;
1115 /* The available oob size we have. */
1116 if (geo->page_size < mtd->writesize + mtd->oobsize) {
1117 oobregion->offset = geo->page_size - mtd->writesize;
1118 oobregion->length = mtd->oobsize - oobregion->offset;
1124 static const char * const gpmi_clks_for_mx2x[] = {
1128 static const struct mtd_ooblayout_ops gpmi_ooblayout_ops = {
1129 .ecc = gpmi_ooblayout_ecc,
1130 .free = gpmi_ooblayout_free,
1133 static const struct gpmi_devdata gpmi_devdata_imx23 = {
1135 .bch_max_ecc_strength = 20,
1136 .max_chain_delay = 16000,
1137 .clks = gpmi_clks_for_mx2x,
1138 .clks_count = ARRAY_SIZE(gpmi_clks_for_mx2x),
1141 static const struct gpmi_devdata gpmi_devdata_imx28 = {
1143 .bch_max_ecc_strength = 20,
1144 .max_chain_delay = 16000,
1145 .clks = gpmi_clks_for_mx2x,
1146 .clks_count = ARRAY_SIZE(gpmi_clks_for_mx2x),
1149 static const char * const gpmi_clks_for_mx6[] = {
1150 "gpmi_io", "gpmi_apb", "gpmi_bch", "gpmi_bch_apb", "per1_bch",
1153 static const struct gpmi_devdata gpmi_devdata_imx6q = {
1155 .bch_max_ecc_strength = 40,
1156 .max_chain_delay = 12000,
1157 .clks = gpmi_clks_for_mx6,
1158 .clks_count = ARRAY_SIZE(gpmi_clks_for_mx6),
1161 static const struct gpmi_devdata gpmi_devdata_imx6sx = {
1163 .bch_max_ecc_strength = 62,
1164 .max_chain_delay = 12000,
1165 .clks = gpmi_clks_for_mx6,
1166 .clks_count = ARRAY_SIZE(gpmi_clks_for_mx6),
1169 static const char * const gpmi_clks_for_mx7d[] = {
1170 "gpmi_io", "gpmi_bch_apb",
1173 static const struct gpmi_devdata gpmi_devdata_imx7d = {
1175 .bch_max_ecc_strength = 62,
1176 .max_chain_delay = 12000,
1177 .clks = gpmi_clks_for_mx7d,
1178 .clks_count = ARRAY_SIZE(gpmi_clks_for_mx7d),
1181 static int acquire_register_block(struct gpmi_nand_data *this,
1182 const char *res_name)
1184 struct platform_device *pdev = this->pdev;
1185 struct resources *res = &this->resources;
1188 p = devm_platform_ioremap_resource_byname(pdev, res_name);
1192 if (!strcmp(res_name, GPMI_NAND_GPMI_REGS_ADDR_RES_NAME))
1194 else if (!strcmp(res_name, GPMI_NAND_BCH_REGS_ADDR_RES_NAME))
1197 dev_err(this->dev, "unknown resource name : %s\n", res_name);
1202 static int acquire_bch_irq(struct gpmi_nand_data *this, irq_handler_t irq_h)
1204 struct platform_device *pdev = this->pdev;
1205 const char *res_name = GPMI_NAND_BCH_INTERRUPT_RES_NAME;
1208 err = platform_get_irq_byname(pdev, res_name);
1212 err = devm_request_irq(this->dev, err, irq_h, 0, res_name, this);
1214 dev_err(this->dev, "error requesting BCH IRQ\n");
1219 static void release_dma_channels(struct gpmi_nand_data *this)
1222 for (i = 0; i < DMA_CHANS; i++)
1223 if (this->dma_chans[i]) {
1224 dma_release_channel(this->dma_chans[i]);
1225 this->dma_chans[i] = NULL;
1229 static int acquire_dma_channels(struct gpmi_nand_data *this)
1231 struct platform_device *pdev = this->pdev;
1232 struct dma_chan *dma_chan;
1235 /* request dma channel */
1236 dma_chan = dma_request_chan(&pdev->dev, "rx-tx");
1237 if (IS_ERR(dma_chan)) {
1238 ret = dev_err_probe(this->dev, PTR_ERR(dma_chan),
1239 "DMA channel request failed\n");
1240 release_dma_channels(this);
1242 this->dma_chans[0] = dma_chan;
1248 static int gpmi_get_clks(struct gpmi_nand_data *this)
1250 struct resources *r = &this->resources;
1254 for (i = 0; i < this->devdata->clks_count; i++) {
1255 clk = devm_clk_get(this->dev, this->devdata->clks[i]);
1267 dev_dbg(this->dev, "failed in finding the clocks.\n");
1271 static int acquire_resources(struct gpmi_nand_data *this)
1275 ret = acquire_register_block(this, GPMI_NAND_GPMI_REGS_ADDR_RES_NAME);
1279 ret = acquire_register_block(this, GPMI_NAND_BCH_REGS_ADDR_RES_NAME);
1283 ret = acquire_bch_irq(this, bch_irq);
1287 ret = acquire_dma_channels(this);
1291 ret = gpmi_get_clks(this);
1297 release_dma_channels(this);
1302 static void release_resources(struct gpmi_nand_data *this)
1304 release_dma_channels(this);
1307 static void gpmi_free_dma_buffer(struct gpmi_nand_data *this)
1309 struct device *dev = this->dev;
1310 struct bch_geometry *geo = &this->bch_geometry;
1312 if (this->auxiliary_virt && virt_addr_valid(this->auxiliary_virt))
1313 dma_free_coherent(dev, geo->auxiliary_size,
1314 this->auxiliary_virt,
1315 this->auxiliary_phys);
1316 kfree(this->data_buffer_dma);
1317 kfree(this->raw_buffer);
1319 this->data_buffer_dma = NULL;
1320 this->raw_buffer = NULL;
1323 /* Allocate the DMA buffers */
1324 static int gpmi_alloc_dma_buffer(struct gpmi_nand_data *this)
1326 struct bch_geometry *geo = &this->bch_geometry;
1327 struct device *dev = this->dev;
1328 struct mtd_info *mtd = nand_to_mtd(&this->nand);
1331 * [2] Allocate a read/write data buffer.
1332 * The gpmi_alloc_dma_buffer can be called twice.
1333 * We allocate a PAGE_SIZE length buffer if gpmi_alloc_dma_buffer
1334 * is called before the NAND identification; and we allocate a
1335 * buffer of the real NAND page size when the gpmi_alloc_dma_buffer
1338 this->data_buffer_dma = kzalloc(mtd->writesize ?: PAGE_SIZE,
1339 GFP_DMA | GFP_KERNEL);
1340 if (this->data_buffer_dma == NULL)
1343 this->auxiliary_virt = dma_alloc_coherent(dev, geo->auxiliary_size,
1344 &this->auxiliary_phys, GFP_DMA);
1345 if (!this->auxiliary_virt)
1348 this->raw_buffer = kzalloc((mtd->writesize ?: PAGE_SIZE) + mtd->oobsize, GFP_KERNEL);
1349 if (!this->raw_buffer)
1355 gpmi_free_dma_buffer(this);
1360 * Handles block mark swapping.
1361 * It can be called in swapping the block mark, or swapping it back,
1362 * because the the operations are the same.
1364 static void block_mark_swapping(struct gpmi_nand_data *this,
1365 void *payload, void *auxiliary)
1367 struct bch_geometry *nfc_geo = &this->bch_geometry;
1372 unsigned char from_data;
1373 unsigned char from_oob;
1375 if (!this->swap_block_mark)
1379 * If control arrives here, we're swapping. Make some convenience
1382 bit = nfc_geo->block_mark_bit_offset;
1383 p = payload + nfc_geo->block_mark_byte_offset;
1387 * Get the byte from the data area that overlays the block mark. Since
1388 * the ECC engine applies its own view to the bits in the page, the
1389 * physical block mark won't (in general) appear on a byte boundary in
1392 from_data = (p[0] >> bit) | (p[1] << (8 - bit));
1394 /* Get the byte from the OOB. */
1400 mask = (0x1 << bit) - 1;
1401 p[0] = (p[0] & mask) | (from_oob << bit);
1404 p[1] = (p[1] & mask) | (from_oob >> (8 - bit));
1407 static int gpmi_count_bitflips(struct nand_chip *chip, void *buf, int first,
1410 struct gpmi_nand_data *this = nand_get_controller_data(chip);
1411 struct bch_geometry *nfc_geo = &this->bch_geometry;
1412 struct mtd_info *mtd = nand_to_mtd(chip);
1414 unsigned char *status;
1415 unsigned int max_bitflips = 0;
1417 /* Loop over status bytes, accumulating ECC status. */
1418 status = this->auxiliary_virt + ALIGN(meta, 4);
1420 for (i = first; i < last; i++, status++) {
1421 if ((*status == STATUS_GOOD) || (*status == STATUS_ERASED))
1424 if (*status == STATUS_UNCORRECTABLE) {
1425 int eccbits = nfc_geo->ecc_strength * nfc_geo->gf_len;
1426 u8 *eccbuf = this->raw_buffer;
1427 int offset, bitoffset;
1431 /* Read ECC bytes into our internal raw_buffer */
1432 offset = nfc_geo->metadata_size * 8;
1433 offset += ((8 * nfc_geo->eccn_chunk_size) + eccbits) * (i + 1);
1435 bitoffset = offset % 8;
1436 eccbytes = DIV_ROUND_UP(offset + eccbits, 8);
1439 nand_change_read_column_op(chip, offset, eccbuf,
1443 * ECC data are not byte aligned and we may have
1444 * in-band data in the first and last byte of
1445 * eccbuf. Set non-eccbits to one so that
1446 * nand_check_erased_ecc_chunk() does not count them
1450 eccbuf[0] |= GENMASK(bitoffset - 1, 0);
1452 bitoffset = (bitoffset + eccbits) % 8;
1454 eccbuf[eccbytes - 1] |= GENMASK(7, bitoffset);
1457 * The ECC hardware has an uncorrectable ECC status
1458 * code in case we have bitflips in an erased page. As
1459 * nothing was written into this subpage the ECC is
1460 * obviously wrong and we can not trust it. We assume
1461 * at this point that we are reading an erased page and
1462 * try to correct the bitflips in buffer up to
1463 * ecc_strength bitflips. If this is a page with random
1464 * data, we exceed this number of bitflips and have a
1465 * ECC failure. Otherwise we use the corrected buffer.
1468 /* The first block includes metadata */
1469 flips = nand_check_erased_ecc_chunk(
1470 buf + i * nfc_geo->eccn_chunk_size,
1471 nfc_geo->eccn_chunk_size,
1473 this->auxiliary_virt,
1474 nfc_geo->metadata_size,
1475 nfc_geo->ecc_strength);
1477 flips = nand_check_erased_ecc_chunk(
1478 buf + i * nfc_geo->eccn_chunk_size,
1479 nfc_geo->eccn_chunk_size,
1482 nfc_geo->ecc_strength);
1486 max_bitflips = max_t(unsigned int, max_bitflips,
1488 mtd->ecc_stats.corrected += flips;
1492 mtd->ecc_stats.failed++;
1496 mtd->ecc_stats.corrected += *status;
1497 max_bitflips = max_t(unsigned int, max_bitflips, *status);
1500 return max_bitflips;
1503 static void gpmi_bch_layout_std(struct gpmi_nand_data *this)
1505 struct bch_geometry *geo = &this->bch_geometry;
1506 unsigned int ecc_strength = geo->ecc_strength >> 1;
1507 unsigned int gf_len = geo->gf_len;
1508 unsigned int block0_size = geo->ecc0_chunk_size;
1509 unsigned int blockn_size = geo->eccn_chunk_size;
1511 this->bch_flashlayout0 =
1512 BF_BCH_FLASH0LAYOUT0_NBLOCKS(geo->ecc_chunk_count - 1) |
1513 BF_BCH_FLASH0LAYOUT0_META_SIZE(geo->metadata_size) |
1514 BF_BCH_FLASH0LAYOUT0_ECC0(ecc_strength, this) |
1515 BF_BCH_FLASH0LAYOUT0_GF(gf_len, this) |
1516 BF_BCH_FLASH0LAYOUT0_DATA0_SIZE(block0_size, this);
1518 this->bch_flashlayout1 =
1519 BF_BCH_FLASH0LAYOUT1_PAGE_SIZE(geo->page_size) |
1520 BF_BCH_FLASH0LAYOUT1_ECCN(ecc_strength, this) |
1521 BF_BCH_FLASH0LAYOUT1_GF(gf_len, this) |
1522 BF_BCH_FLASH0LAYOUT1_DATAN_SIZE(blockn_size, this);
1525 static int gpmi_ecc_read_page(struct nand_chip *chip, uint8_t *buf,
1526 int oob_required, int page)
1528 struct gpmi_nand_data *this = nand_get_controller_data(chip);
1529 struct mtd_info *mtd = nand_to_mtd(chip);
1530 struct bch_geometry *geo = &this->bch_geometry;
1531 unsigned int max_bitflips;
1534 gpmi_bch_layout_std(this);
1537 ret = nand_read_page_op(chip, page, 0, buf, geo->page_size);
1541 max_bitflips = gpmi_count_bitflips(chip, buf, 0,
1542 geo->ecc_chunk_count,
1543 geo->auxiliary_status_offset);
1545 /* handle the block mark swapping */
1546 block_mark_swapping(this, buf, this->auxiliary_virt);
1550 * It's time to deliver the OOB bytes. See gpmi_ecc_read_oob()
1551 * for details about our policy for delivering the OOB.
1553 * We fill the caller's buffer with set bits, and then copy the
1554 * block mark to th caller's buffer. Note that, if block mark
1555 * swapping was necessary, it has already been done, so we can
1556 * rely on the first byte of the auxiliary buffer to contain
1559 memset(chip->oob_poi, ~0, mtd->oobsize);
1560 chip->oob_poi[0] = ((uint8_t *)this->auxiliary_virt)[0];
1563 return max_bitflips;
1566 /* Fake a virtual small page for the subpage read */
1567 static int gpmi_ecc_read_subpage(struct nand_chip *chip, uint32_t offs,
1568 uint32_t len, uint8_t *buf, int page)
1570 struct gpmi_nand_data *this = nand_get_controller_data(chip);
1571 struct bch_geometry *geo = &this->bch_geometry;
1572 int size = chip->ecc.size; /* ECC chunk size */
1573 int meta, n, page_size;
1574 unsigned int max_bitflips;
1575 unsigned int ecc_strength;
1576 int first, last, marker_pos;
1577 int ecc_parity_size;
1581 /* The size of ECC parity */
1582 ecc_parity_size = geo->gf_len * geo->ecc_strength / 8;
1584 /* Align it with the chunk size */
1585 first = offs / size;
1586 last = (offs + len - 1) / size;
1588 if (this->swap_block_mark) {
1590 * Find the chunk which contains the Block Marker.
1591 * If this chunk is in the range of [first, last],
1592 * we have to read out the whole page.
1593 * Why? since we had swapped the data at the position of Block
1594 * Marker to the metadata which is bound with the chunk 0.
1596 marker_pos = geo->block_mark_byte_offset / size;
1597 if (last >= marker_pos && first <= marker_pos) {
1599 "page:%d, first:%d, last:%d, marker at:%d\n",
1600 page, first, last, marker_pos);
1601 return gpmi_ecc_read_page(chip, buf, 0, page);
1606 * if there is an ECC dedicate for meta:
1607 * - need to add an extra ECC size when calculating col and page_size,
1608 * if the meta size is NOT zero.
1609 * - ecc0_chunk size need to set to the same size as other chunks,
1610 * if the meta size is zero.
1613 meta = geo->metadata_size;
1615 if (geo->ecc_for_meta)
1616 col = meta + ecc_parity_size
1617 + (size + ecc_parity_size) * first;
1619 col = meta + (size + ecc_parity_size) * first;
1622 buf = buf + first * size;
1625 ecc_parity_size = geo->gf_len * geo->ecc_strength / 8;
1626 n = last - first + 1;
1628 if (geo->ecc_for_meta && meta)
1629 page_size = meta + ecc_parity_size
1630 + (size + ecc_parity_size) * n;
1632 page_size = meta + (size + ecc_parity_size) * n;
1634 ecc_strength = geo->ecc_strength >> 1;
1636 this->bch_flashlayout0 = BF_BCH_FLASH0LAYOUT0_NBLOCKS(
1637 (geo->ecc_for_meta ? n : n - 1)) |
1638 BF_BCH_FLASH0LAYOUT0_META_SIZE(meta) |
1639 BF_BCH_FLASH0LAYOUT0_ECC0(ecc_strength, this) |
1640 BF_BCH_FLASH0LAYOUT0_GF(geo->gf_len, this) |
1641 BF_BCH_FLASH0LAYOUT0_DATA0_SIZE((geo->ecc_for_meta ?
1642 0 : geo->ecc0_chunk_size), this);
1644 this->bch_flashlayout1 = BF_BCH_FLASH0LAYOUT1_PAGE_SIZE(page_size) |
1645 BF_BCH_FLASH0LAYOUT1_ECCN(ecc_strength, this) |
1646 BF_BCH_FLASH0LAYOUT1_GF(geo->gf_len, this) |
1647 BF_BCH_FLASH0LAYOUT1_DATAN_SIZE(geo->eccn_chunk_size, this);
1651 ret = nand_read_page_op(chip, page, col, buf, page_size);
1655 dev_dbg(this->dev, "page:%d(%d:%d)%d, chunk:(%d:%d), BCH PG size:%d\n",
1656 page, offs, len, col, first, n, page_size);
1658 max_bitflips = gpmi_count_bitflips(chip, buf, first, last, meta);
1660 return max_bitflips;
1663 static int gpmi_ecc_write_page(struct nand_chip *chip, const uint8_t *buf,
1664 int oob_required, int page)
1666 struct mtd_info *mtd = nand_to_mtd(chip);
1667 struct gpmi_nand_data *this = nand_get_controller_data(chip);
1668 struct bch_geometry *nfc_geo = &this->bch_geometry;
1670 dev_dbg(this->dev, "ecc write page.\n");
1672 gpmi_bch_layout_std(this);
1675 memcpy(this->auxiliary_virt, chip->oob_poi, nfc_geo->auxiliary_size);
1677 if (this->swap_block_mark) {
1679 * When doing bad block marker swapping we must always copy the
1680 * input buffer as we can't modify the const buffer.
1682 memcpy(this->data_buffer_dma, buf, mtd->writesize);
1683 buf = this->data_buffer_dma;
1684 block_mark_swapping(this, this->data_buffer_dma,
1685 this->auxiliary_virt);
1688 return nand_prog_page_op(chip, page, 0, buf, nfc_geo->page_size);
1692 * There are several places in this driver where we have to handle the OOB and
1693 * block marks. This is the function where things are the most complicated, so
1694 * this is where we try to explain it all. All the other places refer back to
1697 * These are the rules, in order of decreasing importance:
1699 * 1) Nothing the caller does can be allowed to imperil the block mark.
1701 * 2) In read operations, the first byte of the OOB we return must reflect the
1702 * true state of the block mark, no matter where that block mark appears in
1703 * the physical page.
1705 * 3) ECC-based read operations return an OOB full of set bits (since we never
1706 * allow ECC-based writes to the OOB, it doesn't matter what ECC-based reads
1709 * 4) "Raw" read operations return a direct view of the physical bytes in the
1710 * page, using the conventional definition of which bytes are data and which
1711 * are OOB. This gives the caller a way to see the actual, physical bytes
1712 * in the page, without the distortions applied by our ECC engine.
1715 * What we do for this specific read operation depends on two questions:
1717 * 1) Are we doing a "raw" read, or an ECC-based read?
1719 * 2) Are we using block mark swapping or transcription?
1721 * There are four cases, illustrated by the following Karnaugh map:
1723 * | Raw | ECC-based |
1724 * -------------+-------------------------+-------------------------+
1725 * | Read the conventional | |
1726 * | OOB at the end of the | |
1727 * Swapping | page and return it. It | |
1728 * | contains exactly what | |
1729 * | we want. | Read the block mark and |
1730 * -------------+-------------------------+ return it in a buffer |
1731 * | Read the conventional | full of set bits. |
1732 * | OOB at the end of the | |
1733 * | page and also the block | |
1734 * Transcribing | mark in the metadata. | |
1735 * | Copy the block mark | |
1736 * | into the first byte of | |
1738 * -------------+-------------------------+-------------------------+
1740 * Note that we break rule #4 in the Transcribing/Raw case because we're not
1741 * giving an accurate view of the actual, physical bytes in the page (we're
1742 * overwriting the block mark). That's OK because it's more important to follow
1745 * It turns out that knowing whether we want an "ECC-based" or "raw" read is not
1746 * easy. When reading a page, for example, the NAND Flash MTD code calls our
1747 * ecc.read_page or ecc.read_page_raw function. Thus, the fact that MTD wants an
1748 * ECC-based or raw view of the page is implicit in which function it calls
1749 * (there is a similar pair of ECC-based/raw functions for writing).
1751 static int gpmi_ecc_read_oob(struct nand_chip *chip, int page)
1753 struct mtd_info *mtd = nand_to_mtd(chip);
1754 struct gpmi_nand_data *this = nand_get_controller_data(chip);
1757 /* clear the OOB buffer */
1758 memset(chip->oob_poi, ~0, mtd->oobsize);
1760 /* Read out the conventional OOB. */
1761 ret = nand_read_page_op(chip, page, mtd->writesize, chip->oob_poi,
1767 * Now, we want to make sure the block mark is correct. In the
1768 * non-transcribing case (!GPMI_IS_MX23()), we already have it.
1769 * Otherwise, we need to explicitly read it.
1771 if (GPMI_IS_MX23(this)) {
1772 /* Read the block mark into the first byte of the OOB buffer. */
1773 ret = nand_read_page_op(chip, page, 0, chip->oob_poi, 1);
1781 static int gpmi_ecc_write_oob(struct nand_chip *chip, int page)
1783 struct mtd_info *mtd = nand_to_mtd(chip);
1784 struct mtd_oob_region of = { };
1786 /* Do we have available oob area? */
1787 mtd_ooblayout_free(mtd, 0, &of);
1791 if (!nand_is_slc(chip))
1794 return nand_prog_page_op(chip, page, mtd->writesize + of.offset,
1795 chip->oob_poi + of.offset, of.length);
1799 * This function reads a NAND page without involving the ECC engine (no HW
1801 * The tricky part in the GPMI/BCH controller is that it stores ECC bits
1802 * inline (interleaved with payload DATA), and do not align data chunk on
1804 * We thus need to take care moving the payload data and ECC bits stored in the
1805 * page into the provided buffers, which is why we're using nand_extract_bits().
1807 * See set_geometry_by_ecc_info inline comments to have a full description
1808 * of the layout used by the GPMI controller.
1810 static int gpmi_ecc_read_page_raw(struct nand_chip *chip, uint8_t *buf,
1811 int oob_required, int page)
1813 struct mtd_info *mtd = nand_to_mtd(chip);
1814 struct gpmi_nand_data *this = nand_get_controller_data(chip);
1815 struct bch_geometry *nfc_geo = &this->bch_geometry;
1816 int eccsize = nfc_geo->eccn_chunk_size;
1817 int eccbits = nfc_geo->ecc_strength * nfc_geo->gf_len;
1818 u8 *tmp_buf = this->raw_buffer;
1821 size_t oob_byte_off;
1822 uint8_t *oob = chip->oob_poi;
1826 ret = nand_read_page_op(chip, page, 0, tmp_buf,
1827 mtd->writesize + mtd->oobsize);
1832 * If required, swap the bad block marker and the data stored in the
1833 * metadata section, so that we don't wrongly consider a block as bad.
1835 * See the layout description for a detailed explanation on why this
1838 if (this->swap_block_mark)
1839 swap(tmp_buf[0], tmp_buf[mtd->writesize]);
1842 * Copy the metadata section into the oob buffer (this section is
1843 * guaranteed to be aligned on a byte boundary).
1846 memcpy(oob, tmp_buf, nfc_geo->metadata_size);
1848 oob_bit_off = nfc_geo->metadata_size * 8;
1849 src_bit_off = oob_bit_off;
1851 /* Extract interleaved payload data and ECC bits */
1852 for (step = 0; step < nfc_geo->ecc_chunk_count; step++) {
1854 nand_extract_bits(buf, step * eccsize * 8, tmp_buf,
1855 src_bit_off, eccsize * 8);
1856 src_bit_off += eccsize * 8;
1858 /* Align last ECC block to align a byte boundary */
1859 if (step == nfc_geo->ecc_chunk_count - 1 &&
1860 (oob_bit_off + eccbits) % 8)
1861 eccbits += 8 - ((oob_bit_off + eccbits) % 8);
1864 nand_extract_bits(oob, oob_bit_off, tmp_buf,
1865 src_bit_off, eccbits);
1867 src_bit_off += eccbits;
1868 oob_bit_off += eccbits;
1872 oob_byte_off = oob_bit_off / 8;
1874 if (oob_byte_off < mtd->oobsize)
1875 memcpy(oob + oob_byte_off,
1876 tmp_buf + mtd->writesize + oob_byte_off,
1877 mtd->oobsize - oob_byte_off);
1884 * This function writes a NAND page without involving the ECC engine (no HW
1886 * The tricky part in the GPMI/BCH controller is that it stores ECC bits
1887 * inline (interleaved with payload DATA), and do not align data chunk on
1889 * We thus need to take care moving the OOB area at the right place in the
1890 * final page, which is why we're using nand_extract_bits().
1892 * See set_geometry_by_ecc_info inline comments to have a full description
1893 * of the layout used by the GPMI controller.
1895 static int gpmi_ecc_write_page_raw(struct nand_chip *chip, const uint8_t *buf,
1896 int oob_required, int page)
1898 struct mtd_info *mtd = nand_to_mtd(chip);
1899 struct gpmi_nand_data *this = nand_get_controller_data(chip);
1900 struct bch_geometry *nfc_geo = &this->bch_geometry;
1901 int eccsize = nfc_geo->eccn_chunk_size;
1902 int eccbits = nfc_geo->ecc_strength * nfc_geo->gf_len;
1903 u8 *tmp_buf = this->raw_buffer;
1904 uint8_t *oob = chip->oob_poi;
1907 size_t oob_byte_off;
1911 * Initialize all bits to 1 in case we don't have a buffer for the
1912 * payload or oob data in order to leave unspecified bits of data
1913 * to their initial state.
1915 if (!buf || !oob_required)
1916 memset(tmp_buf, 0xff, mtd->writesize + mtd->oobsize);
1919 * First copy the metadata section (stored in oob buffer) at the
1920 * beginning of the page, as imposed by the GPMI layout.
1922 memcpy(tmp_buf, oob, nfc_geo->metadata_size);
1923 oob_bit_off = nfc_geo->metadata_size * 8;
1924 dst_bit_off = oob_bit_off;
1926 /* Interleave payload data and ECC bits */
1927 for (step = 0; step < nfc_geo->ecc_chunk_count; step++) {
1929 nand_extract_bits(tmp_buf, dst_bit_off, buf,
1930 step * eccsize * 8, eccsize * 8);
1931 dst_bit_off += eccsize * 8;
1933 /* Align last ECC block to align a byte boundary */
1934 if (step == nfc_geo->ecc_chunk_count - 1 &&
1935 (oob_bit_off + eccbits) % 8)
1936 eccbits += 8 - ((oob_bit_off + eccbits) % 8);
1939 nand_extract_bits(tmp_buf, dst_bit_off, oob,
1940 oob_bit_off, eccbits);
1942 dst_bit_off += eccbits;
1943 oob_bit_off += eccbits;
1946 oob_byte_off = oob_bit_off / 8;
1948 if (oob_required && oob_byte_off < mtd->oobsize)
1949 memcpy(tmp_buf + mtd->writesize + oob_byte_off,
1950 oob + oob_byte_off, mtd->oobsize - oob_byte_off);
1953 * If required, swap the bad block marker and the first byte of the
1954 * metadata section, so that we don't modify the bad block marker.
1956 * See the layout description for a detailed explanation on why this
1959 if (this->swap_block_mark)
1960 swap(tmp_buf[0], tmp_buf[mtd->writesize]);
1962 return nand_prog_page_op(chip, page, 0, tmp_buf,
1963 mtd->writesize + mtd->oobsize);
1966 static int gpmi_ecc_read_oob_raw(struct nand_chip *chip, int page)
1968 return gpmi_ecc_read_page_raw(chip, NULL, 1, page);
1971 static int gpmi_ecc_write_oob_raw(struct nand_chip *chip, int page)
1973 return gpmi_ecc_write_page_raw(chip, NULL, 1, page);
1976 static int gpmi_block_markbad(struct nand_chip *chip, loff_t ofs)
1978 struct mtd_info *mtd = nand_to_mtd(chip);
1979 struct gpmi_nand_data *this = nand_get_controller_data(chip);
1981 uint8_t *block_mark;
1982 int column, page, chipnr;
1984 chipnr = (int)(ofs >> chip->chip_shift);
1985 nand_select_target(chip, chipnr);
1987 column = !GPMI_IS_MX23(this) ? mtd->writesize : 0;
1989 /* Write the block mark. */
1990 block_mark = this->data_buffer_dma;
1991 block_mark[0] = 0; /* bad block marker */
1993 /* Shift to get page */
1994 page = (int)(ofs >> chip->page_shift);
1996 ret = nand_prog_page_op(chip, page, column, block_mark, 1);
1998 nand_deselect_target(chip);
2003 static int nand_boot_set_geometry(struct gpmi_nand_data *this)
2005 struct boot_rom_geometry *geometry = &this->rom_geometry;
2008 * Set the boot block stride size.
2010 * In principle, we should be reading this from the OTP bits, since
2011 * that's where the ROM is going to get it. In fact, we don't have any
2012 * way to read the OTP bits, so we go with the default and hope for the
2015 geometry->stride_size_in_pages = 64;
2018 * Set the search area stride exponent.
2020 * In principle, we should be reading this from the OTP bits, since
2021 * that's where the ROM is going to get it. In fact, we don't have any
2022 * way to read the OTP bits, so we go with the default and hope for the
2025 geometry->search_area_stride_exponent = 2;
2029 static const char *fingerprint = "STMP";
2030 static int mx23_check_transcription_stamp(struct gpmi_nand_data *this)
2032 struct boot_rom_geometry *rom_geo = &this->rom_geometry;
2033 struct device *dev = this->dev;
2034 struct nand_chip *chip = &this->nand;
2035 unsigned int search_area_size_in_strides;
2036 unsigned int stride;
2038 u8 *buffer = nand_get_data_buf(chip);
2039 int found_an_ncb_fingerprint = false;
2042 /* Compute the number of strides in a search area. */
2043 search_area_size_in_strides = 1 << rom_geo->search_area_stride_exponent;
2045 nand_select_target(chip, 0);
2048 * Loop through the first search area, looking for the NCB fingerprint.
2050 dev_dbg(dev, "Scanning for an NCB fingerprint...\n");
2052 for (stride = 0; stride < search_area_size_in_strides; stride++) {
2053 /* Compute the page addresses. */
2054 page = stride * rom_geo->stride_size_in_pages;
2056 dev_dbg(dev, "Looking for a fingerprint in page 0x%x\n", page);
2059 * Read the NCB fingerprint. The fingerprint is four bytes long
2060 * and starts in the 12th byte of the page.
2062 ret = nand_read_page_op(chip, page, 12, buffer,
2063 strlen(fingerprint));
2067 /* Look for the fingerprint. */
2068 if (!memcmp(buffer, fingerprint, strlen(fingerprint))) {
2069 found_an_ncb_fingerprint = true;
2075 nand_deselect_target(chip);
2077 if (found_an_ncb_fingerprint)
2078 dev_dbg(dev, "\tFound a fingerprint\n");
2080 dev_dbg(dev, "\tNo fingerprint found\n");
2081 return found_an_ncb_fingerprint;
2084 /* Writes a transcription stamp. */
2085 static int mx23_write_transcription_stamp(struct gpmi_nand_data *this)
2087 struct device *dev = this->dev;
2088 struct boot_rom_geometry *rom_geo = &this->rom_geometry;
2089 struct nand_chip *chip = &this->nand;
2090 struct mtd_info *mtd = nand_to_mtd(chip);
2091 unsigned int block_size_in_pages;
2092 unsigned int search_area_size_in_strides;
2093 unsigned int search_area_size_in_pages;
2094 unsigned int search_area_size_in_blocks;
2096 unsigned int stride;
2098 u8 *buffer = nand_get_data_buf(chip);
2101 /* Compute the search area geometry. */
2102 block_size_in_pages = mtd->erasesize / mtd->writesize;
2103 search_area_size_in_strides = 1 << rom_geo->search_area_stride_exponent;
2104 search_area_size_in_pages = search_area_size_in_strides *
2105 rom_geo->stride_size_in_pages;
2106 search_area_size_in_blocks =
2107 (search_area_size_in_pages + (block_size_in_pages - 1)) /
2108 block_size_in_pages;
2110 dev_dbg(dev, "Search Area Geometry :\n");
2111 dev_dbg(dev, "\tin Blocks : %u\n", search_area_size_in_blocks);
2112 dev_dbg(dev, "\tin Strides: %u\n", search_area_size_in_strides);
2113 dev_dbg(dev, "\tin Pages : %u\n", search_area_size_in_pages);
2115 nand_select_target(chip, 0);
2117 /* Loop over blocks in the first search area, erasing them. */
2118 dev_dbg(dev, "Erasing the search area...\n");
2120 for (block = 0; block < search_area_size_in_blocks; block++) {
2121 /* Erase this block. */
2122 dev_dbg(dev, "\tErasing block 0x%x\n", block);
2123 status = nand_erase_op(chip, block);
2125 dev_err(dev, "[%s] Erase failed.\n", __func__);
2128 /* Write the NCB fingerprint into the page buffer. */
2129 memset(buffer, ~0, mtd->writesize);
2130 memcpy(buffer + 12, fingerprint, strlen(fingerprint));
2132 /* Loop through the first search area, writing NCB fingerprints. */
2133 dev_dbg(dev, "Writing NCB fingerprints...\n");
2134 for (stride = 0; stride < search_area_size_in_strides; stride++) {
2135 /* Compute the page addresses. */
2136 page = stride * rom_geo->stride_size_in_pages;
2138 /* Write the first page of the current stride. */
2139 dev_dbg(dev, "Writing an NCB fingerprint in page 0x%x\n", page);
2141 status = chip->ecc.write_page_raw(chip, buffer, 0, page);
2143 dev_err(dev, "[%s] Write failed.\n", __func__);
2146 nand_deselect_target(chip);
2151 static int mx23_boot_init(struct gpmi_nand_data *this)
2153 struct device *dev = this->dev;
2154 struct nand_chip *chip = &this->nand;
2155 struct mtd_info *mtd = nand_to_mtd(chip);
2156 unsigned int block_count;
2165 * If control arrives here, we can't use block mark swapping, which
2166 * means we're forced to use transcription. First, scan for the
2167 * transcription stamp. If we find it, then we don't have to do
2168 * anything -- the block marks are already transcribed.
2170 if (mx23_check_transcription_stamp(this))
2174 * If control arrives here, we couldn't find a transcription stamp, so
2175 * so we presume the block marks are in the conventional location.
2177 dev_dbg(dev, "Transcribing bad block marks...\n");
2179 /* Compute the number of blocks in the entire medium. */
2180 block_count = nanddev_eraseblocks_per_target(&chip->base);
2183 * Loop over all the blocks in the medium, transcribing block marks as
2186 for (block = 0; block < block_count; block++) {
2188 * Compute the chip, page and byte addresses for this block's
2189 * conventional mark.
2191 chipnr = block >> (chip->chip_shift - chip->phys_erase_shift);
2192 page = block << (chip->phys_erase_shift - chip->page_shift);
2193 byte = block << chip->phys_erase_shift;
2195 /* Send the command to read the conventional block mark. */
2196 nand_select_target(chip, chipnr);
2197 ret = nand_read_page_op(chip, page, mtd->writesize, &block_mark,
2199 nand_deselect_target(chip);
2205 * Check if the block is marked bad. If so, we need to mark it
2206 * again, but this time the result will be a mark in the
2207 * location where we transcribe block marks.
2209 if (block_mark != 0xff) {
2210 dev_dbg(dev, "Transcribing mark in block %u\n", block);
2211 ret = chip->legacy.block_markbad(chip, byte);
2214 "Failed to mark block bad with ret %d\n",
2219 /* Write the stamp that indicates we've transcribed the block marks. */
2220 mx23_write_transcription_stamp(this);
2224 static int nand_boot_init(struct gpmi_nand_data *this)
2226 nand_boot_set_geometry(this);
2228 /* This is ROM arch-specific initilization before the BBT scanning. */
2229 if (GPMI_IS_MX23(this))
2230 return mx23_boot_init(this);
2234 static int gpmi_set_geometry(struct gpmi_nand_data *this)
2238 /* Free the temporary DMA memory for reading ID. */
2239 gpmi_free_dma_buffer(this);
2241 /* Set up the NFC geometry which is used by BCH. */
2242 ret = bch_set_geometry(this);
2244 dev_err(this->dev, "Error setting BCH geometry : %d\n", ret);
2248 /* Alloc the new DMA buffers according to the pagesize and oobsize */
2249 return gpmi_alloc_dma_buffer(this);
2252 static int gpmi_init_last(struct gpmi_nand_data *this)
2254 struct nand_chip *chip = &this->nand;
2255 struct mtd_info *mtd = nand_to_mtd(chip);
2256 struct nand_ecc_ctrl *ecc = &chip->ecc;
2257 struct bch_geometry *bch_geo = &this->bch_geometry;
2260 /* Set up the medium geometry */
2261 ret = gpmi_set_geometry(this);
2265 /* Init the nand_ecc_ctrl{} */
2266 ecc->read_page = gpmi_ecc_read_page;
2267 ecc->write_page = gpmi_ecc_write_page;
2268 ecc->read_oob = gpmi_ecc_read_oob;
2269 ecc->write_oob = gpmi_ecc_write_oob;
2270 ecc->read_page_raw = gpmi_ecc_read_page_raw;
2271 ecc->write_page_raw = gpmi_ecc_write_page_raw;
2272 ecc->read_oob_raw = gpmi_ecc_read_oob_raw;
2273 ecc->write_oob_raw = gpmi_ecc_write_oob_raw;
2274 ecc->engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST;
2275 ecc->size = bch_geo->eccn_chunk_size;
2276 ecc->strength = bch_geo->ecc_strength;
2277 mtd_set_ooblayout(mtd, &gpmi_ooblayout_ops);
2280 * We only enable the subpage read when:
2281 * (1) the chip is imx6, and
2282 * (2) the size of the ECC parity is byte aligned.
2284 if (GPMI_IS_MX6(this) &&
2285 ((bch_geo->gf_len * bch_geo->ecc_strength) % 8) == 0) {
2286 ecc->read_subpage = gpmi_ecc_read_subpage;
2287 chip->options |= NAND_SUBPAGE_READ;
2293 static int gpmi_nand_attach_chip(struct nand_chip *chip)
2295 struct gpmi_nand_data *this = nand_get_controller_data(chip);
2298 if (chip->bbt_options & NAND_BBT_USE_FLASH) {
2299 chip->bbt_options |= NAND_BBT_NO_OOB;
2301 if (of_property_read_bool(this->dev->of_node,
2302 "fsl,no-blockmark-swap"))
2303 this->swap_block_mark = false;
2305 dev_dbg(this->dev, "Blockmark swapping %sabled\n",
2306 this->swap_block_mark ? "en" : "dis");
2308 ret = gpmi_init_last(this);
2312 chip->options |= NAND_SKIP_BBTSCAN;
2317 static struct gpmi_transfer *get_next_transfer(struct gpmi_nand_data *this)
2319 struct gpmi_transfer *transfer = &this->transfers[this->ntransfers];
2323 if (this->ntransfers == GPMI_MAX_TRANSFERS)
2329 static struct dma_async_tx_descriptor *gpmi_chain_command(
2330 struct gpmi_nand_data *this, u8 cmd, const u8 *addr, int naddr)
2332 struct dma_chan *channel = get_dma_chan(this);
2333 struct dma_async_tx_descriptor *desc;
2334 struct gpmi_transfer *transfer;
2335 int chip = this->nand.cur_cs;
2338 /* [1] send out the PIO words */
2339 pio[0] = BF_GPMI_CTRL0_COMMAND_MODE(BV_GPMI_CTRL0_COMMAND_MODE__WRITE)
2340 | BM_GPMI_CTRL0_WORD_LENGTH
2341 | BF_GPMI_CTRL0_CS(chip, this)
2342 | BF_GPMI_CTRL0_LOCK_CS(LOCK_CS_ENABLE, this)
2343 | BF_GPMI_CTRL0_ADDRESS(BV_GPMI_CTRL0_ADDRESS__NAND_CLE)
2344 | BM_GPMI_CTRL0_ADDRESS_INCREMENT
2345 | BF_GPMI_CTRL0_XFER_COUNT(naddr + 1);
2348 desc = mxs_dmaengine_prep_pio(channel, pio, ARRAY_SIZE(pio),
2353 transfer = get_next_transfer(this);
2357 transfer->cmdbuf[0] = cmd;
2359 memcpy(&transfer->cmdbuf[1], addr, naddr);
2361 sg_init_one(&transfer->sgl, transfer->cmdbuf, naddr + 1);
2362 dma_map_sg(this->dev, &transfer->sgl, 1, DMA_TO_DEVICE);
2364 transfer->direction = DMA_TO_DEVICE;
2366 desc = dmaengine_prep_slave_sg(channel, &transfer->sgl, 1, DMA_MEM_TO_DEV,
2367 MXS_DMA_CTRL_WAIT4END);
2371 static struct dma_async_tx_descriptor *gpmi_chain_wait_ready(
2372 struct gpmi_nand_data *this)
2374 struct dma_chan *channel = get_dma_chan(this);
2377 pio[0] = BF_GPMI_CTRL0_COMMAND_MODE(BV_GPMI_CTRL0_COMMAND_MODE__WAIT_FOR_READY)
2378 | BM_GPMI_CTRL0_WORD_LENGTH
2379 | BF_GPMI_CTRL0_CS(this->nand.cur_cs, this)
2380 | BF_GPMI_CTRL0_LOCK_CS(LOCK_CS_ENABLE, this)
2381 | BF_GPMI_CTRL0_ADDRESS(BV_GPMI_CTRL0_ADDRESS__NAND_DATA)
2382 | BF_GPMI_CTRL0_XFER_COUNT(0);
2385 return mxs_dmaengine_prep_pio(channel, pio, 2, DMA_TRANS_NONE,
2386 MXS_DMA_CTRL_WAIT4END | MXS_DMA_CTRL_WAIT4RDY);
2389 static struct dma_async_tx_descriptor *gpmi_chain_data_read(
2390 struct gpmi_nand_data *this, void *buf, int raw_len, bool *direct)
2392 struct dma_async_tx_descriptor *desc;
2393 struct dma_chan *channel = get_dma_chan(this);
2394 struct gpmi_transfer *transfer;
2397 transfer = get_next_transfer(this);
2401 transfer->direction = DMA_FROM_DEVICE;
2403 *direct = prepare_data_dma(this, buf, raw_len, &transfer->sgl,
2406 pio[0] = BF_GPMI_CTRL0_COMMAND_MODE(BV_GPMI_CTRL0_COMMAND_MODE__READ)
2407 | BM_GPMI_CTRL0_WORD_LENGTH
2408 | BF_GPMI_CTRL0_CS(this->nand.cur_cs, this)
2409 | BF_GPMI_CTRL0_LOCK_CS(LOCK_CS_ENABLE, this)
2410 | BF_GPMI_CTRL0_ADDRESS(BV_GPMI_CTRL0_ADDRESS__NAND_DATA)
2411 | BF_GPMI_CTRL0_XFER_COUNT(raw_len);
2414 pio[2] = BM_GPMI_ECCCTRL_ENABLE_ECC
2415 | BF_GPMI_ECCCTRL_ECC_CMD(BV_GPMI_ECCCTRL_ECC_CMD__BCH_DECODE)
2416 | BF_GPMI_ECCCTRL_BUFFER_MASK(BV_GPMI_ECCCTRL_BUFFER_MASK__BCH_PAGE
2417 | BV_GPMI_ECCCTRL_BUFFER_MASK__BCH_AUXONLY);
2419 pio[4] = transfer->sgl.dma_address;
2420 pio[5] = this->auxiliary_phys;
2423 desc = mxs_dmaengine_prep_pio(channel, pio, ARRAY_SIZE(pio),
2429 desc = dmaengine_prep_slave_sg(channel, &transfer->sgl, 1,
2431 MXS_DMA_CTRL_WAIT4END);
2436 static struct dma_async_tx_descriptor *gpmi_chain_data_write(
2437 struct gpmi_nand_data *this, const void *buf, int raw_len)
2439 struct dma_chan *channel = get_dma_chan(this);
2440 struct dma_async_tx_descriptor *desc;
2441 struct gpmi_transfer *transfer;
2444 transfer = get_next_transfer(this);
2448 transfer->direction = DMA_TO_DEVICE;
2450 prepare_data_dma(this, buf, raw_len, &transfer->sgl, DMA_TO_DEVICE);
2452 pio[0] = BF_GPMI_CTRL0_COMMAND_MODE(BV_GPMI_CTRL0_COMMAND_MODE__WRITE)
2453 | BM_GPMI_CTRL0_WORD_LENGTH
2454 | BF_GPMI_CTRL0_CS(this->nand.cur_cs, this)
2455 | BF_GPMI_CTRL0_LOCK_CS(LOCK_CS_ENABLE, this)
2456 | BF_GPMI_CTRL0_ADDRESS(BV_GPMI_CTRL0_ADDRESS__NAND_DATA)
2457 | BF_GPMI_CTRL0_XFER_COUNT(raw_len);
2460 pio[2] = BM_GPMI_ECCCTRL_ENABLE_ECC
2461 | BF_GPMI_ECCCTRL_ECC_CMD(BV_GPMI_ECCCTRL_ECC_CMD__BCH_ENCODE)
2462 | BF_GPMI_ECCCTRL_BUFFER_MASK(BV_GPMI_ECCCTRL_BUFFER_MASK__BCH_PAGE |
2463 BV_GPMI_ECCCTRL_BUFFER_MASK__BCH_AUXONLY);
2465 pio[4] = transfer->sgl.dma_address;
2466 pio[5] = this->auxiliary_phys;
2469 desc = mxs_dmaengine_prep_pio(channel, pio, ARRAY_SIZE(pio),
2471 (this->bch ? MXS_DMA_CTRL_WAIT4END : 0));
2476 desc = dmaengine_prep_slave_sg(channel, &transfer->sgl, 1,
2478 MXS_DMA_CTRL_WAIT4END);
2483 static int gpmi_nfc_exec_op(struct nand_chip *chip,
2484 const struct nand_operation *op,
2487 const struct nand_op_instr *instr;
2488 struct gpmi_nand_data *this = nand_get_controller_data(chip);
2489 struct dma_async_tx_descriptor *desc = NULL;
2490 int i, ret, buf_len = 0, nbufs = 0;
2492 void *buf_read = NULL;
2493 const void *buf_write = NULL;
2494 bool direct = false;
2495 struct completion *dma_completion, *bch_completion;
2501 this->ntransfers = 0;
2502 for (i = 0; i < GPMI_MAX_TRANSFERS; i++)
2503 this->transfers[i].direction = DMA_NONE;
2505 ret = pm_runtime_get_sync(this->dev);
2507 pm_runtime_put_noidle(this->dev);
2512 * This driver currently supports only one NAND chip. Plus, dies share
2513 * the same configuration. So once timings have been applied on the
2514 * controller side, they will not change anymore. When the time will
2515 * come, the check on must_apply_timings will have to be dropped.
2517 if (this->hw.must_apply_timings) {
2518 this->hw.must_apply_timings = false;
2519 ret = gpmi_nfc_apply_timings(this);
2524 dev_dbg(this->dev, "%s: %d instructions\n", __func__, op->ninstrs);
2526 for (i = 0; i < op->ninstrs; i++) {
2527 instr = &op->instrs[i];
2529 nand_op_trace(" ", instr);
2531 switch (instr->type) {
2532 case NAND_OP_WAITRDY_INSTR:
2533 desc = gpmi_chain_wait_ready(this);
2535 case NAND_OP_CMD_INSTR:
2536 cmd = instr->ctx.cmd.opcode;
2539 * When this command has an address cycle chain it
2540 * together with the address cycle
2542 if (i + 1 != op->ninstrs &&
2543 op->instrs[i + 1].type == NAND_OP_ADDR_INSTR)
2546 desc = gpmi_chain_command(this, cmd, NULL, 0);
2549 case NAND_OP_ADDR_INSTR:
2550 desc = gpmi_chain_command(this, cmd, instr->ctx.addr.addrs,
2551 instr->ctx.addr.naddrs);
2553 case NAND_OP_DATA_OUT_INSTR:
2554 buf_write = instr->ctx.data.buf.out;
2555 buf_len = instr->ctx.data.len;
2558 desc = gpmi_chain_data_write(this, buf_write, buf_len);
2561 case NAND_OP_DATA_IN_INSTR:
2562 if (!instr->ctx.data.len)
2564 buf_read = instr->ctx.data.buf.in;
2565 buf_len = instr->ctx.data.len;
2568 desc = gpmi_chain_data_read(this, buf_read, buf_len,
2579 dev_dbg(this->dev, "%s setup done\n", __func__);
2582 dev_err(this->dev, "Multiple data instructions not supported\n");
2588 writel(this->bch_flashlayout0,
2589 this->resources.bch_regs + HW_BCH_FLASH0LAYOUT0);
2590 writel(this->bch_flashlayout1,
2591 this->resources.bch_regs + HW_BCH_FLASH0LAYOUT1);
2594 desc->callback = dma_irq_callback;
2595 desc->callback_param = this;
2596 dma_completion = &this->dma_done;
2597 bch_completion = NULL;
2599 init_completion(dma_completion);
2601 if (this->bch && buf_read) {
2602 writel(BM_BCH_CTRL_COMPLETE_IRQ_EN,
2603 this->resources.bch_regs + HW_BCH_CTRL_SET);
2604 bch_completion = &this->bch_done;
2605 init_completion(bch_completion);
2608 dmaengine_submit(desc);
2609 dma_async_issue_pending(get_dma_chan(this));
2611 to = wait_for_completion_timeout(dma_completion, msecs_to_jiffies(1000));
2613 dev_err(this->dev, "DMA timeout, last DMA\n");
2614 gpmi_dump_info(this);
2619 if (this->bch && buf_read) {
2620 to = wait_for_completion_timeout(bch_completion, msecs_to_jiffies(1000));
2622 dev_err(this->dev, "BCH timeout, last DMA\n");
2623 gpmi_dump_info(this);
2629 writel(BM_BCH_CTRL_COMPLETE_IRQ_EN,
2630 this->resources.bch_regs + HW_BCH_CTRL_CLR);
2631 gpmi_clear_bch(this);
2636 for (i = 0; i < this->ntransfers; i++) {
2637 struct gpmi_transfer *transfer = &this->transfers[i];
2639 if (transfer->direction != DMA_NONE)
2640 dma_unmap_sg(this->dev, &transfer->sgl, 1,
2641 transfer->direction);
2644 if (!ret && buf_read && !direct)
2645 memcpy(buf_read, this->data_buffer_dma,
2646 gpmi_raw_len_to_len(this, buf_len));
2651 pm_runtime_mark_last_busy(this->dev);
2652 pm_runtime_put_autosuspend(this->dev);
2657 static const struct nand_controller_ops gpmi_nand_controller_ops = {
2658 .attach_chip = gpmi_nand_attach_chip,
2659 .setup_interface = gpmi_setup_interface,
2660 .exec_op = gpmi_nfc_exec_op,
2663 static int gpmi_nand_init(struct gpmi_nand_data *this)
2665 struct nand_chip *chip = &this->nand;
2666 struct mtd_info *mtd = nand_to_mtd(chip);
2669 /* init the MTD data structures */
2670 mtd->name = "gpmi-nand";
2671 mtd->dev.parent = this->dev;
2673 /* init the nand_chip{}, we don't support a 16-bit NAND Flash bus. */
2674 nand_set_controller_data(chip, this);
2675 nand_set_flash_node(chip, this->pdev->dev.of_node);
2676 chip->legacy.block_markbad = gpmi_block_markbad;
2677 chip->badblock_pattern = &gpmi_bbt_descr;
2678 chip->options |= NAND_NO_SUBPAGE_WRITE;
2680 /* Set up swap_block_mark, must be set before the gpmi_set_geometry() */
2681 this->swap_block_mark = !GPMI_IS_MX23(this);
2684 * Allocate a temporary DMA buffer for reading ID in the
2685 * nand_scan_ident().
2687 this->bch_geometry.payload_size = 1024;
2688 this->bch_geometry.auxiliary_size = 128;
2689 ret = gpmi_alloc_dma_buffer(this);
2693 nand_controller_init(&this->base);
2694 this->base.ops = &gpmi_nand_controller_ops;
2695 chip->controller = &this->base;
2697 ret = nand_scan(chip, GPMI_IS_MX6(this) ? 2 : 1);
2701 ret = nand_boot_init(this);
2703 goto err_nand_cleanup;
2704 ret = nand_create_bbt(chip);
2706 goto err_nand_cleanup;
2708 ret = mtd_device_register(mtd, NULL, 0);
2710 goto err_nand_cleanup;
2716 gpmi_free_dma_buffer(this);
2720 static const struct of_device_id gpmi_nand_id_table[] = {
2721 { .compatible = "fsl,imx23-gpmi-nand", .data = &gpmi_devdata_imx23, },
2722 { .compatible = "fsl,imx28-gpmi-nand", .data = &gpmi_devdata_imx28, },
2723 { .compatible = "fsl,imx6q-gpmi-nand", .data = &gpmi_devdata_imx6q, },
2724 { .compatible = "fsl,imx6sx-gpmi-nand", .data = &gpmi_devdata_imx6sx, },
2725 { .compatible = "fsl,imx7d-gpmi-nand", .data = &gpmi_devdata_imx7d,},
2728 MODULE_DEVICE_TABLE(of, gpmi_nand_id_table);
2730 static int gpmi_nand_probe(struct platform_device *pdev)
2732 struct gpmi_nand_data *this;
2735 this = devm_kzalloc(&pdev->dev, sizeof(*this), GFP_KERNEL);
2739 this->devdata = of_device_get_match_data(&pdev->dev);
2740 platform_set_drvdata(pdev, this);
2742 this->dev = &pdev->dev;
2744 ret = acquire_resources(this);
2746 goto exit_acquire_resources;
2748 ret = __gpmi_enable_clk(this, true);
2750 goto exit_acquire_resources;
2752 pm_runtime_set_autosuspend_delay(&pdev->dev, 500);
2753 pm_runtime_use_autosuspend(&pdev->dev);
2754 pm_runtime_set_active(&pdev->dev);
2755 pm_runtime_enable(&pdev->dev);
2756 pm_runtime_get_sync(&pdev->dev);
2758 ret = gpmi_init(this);
2762 ret = gpmi_nand_init(this);
2766 pm_runtime_mark_last_busy(&pdev->dev);
2767 pm_runtime_put_autosuspend(&pdev->dev);
2769 dev_info(this->dev, "driver registered.\n");
2774 pm_runtime_put(&pdev->dev);
2775 pm_runtime_disable(&pdev->dev);
2776 release_resources(this);
2777 exit_acquire_resources:
2782 static int gpmi_nand_remove(struct platform_device *pdev)
2784 struct gpmi_nand_data *this = platform_get_drvdata(pdev);
2785 struct nand_chip *chip = &this->nand;
2788 pm_runtime_put_sync(&pdev->dev);
2789 pm_runtime_disable(&pdev->dev);
2791 ret = mtd_device_unregister(nand_to_mtd(chip));
2794 gpmi_free_dma_buffer(this);
2795 release_resources(this);
2799 #ifdef CONFIG_PM_SLEEP
2800 static int gpmi_pm_suspend(struct device *dev)
2802 struct gpmi_nand_data *this = dev_get_drvdata(dev);
2804 release_dma_channels(this);
2808 static int gpmi_pm_resume(struct device *dev)
2810 struct gpmi_nand_data *this = dev_get_drvdata(dev);
2813 ret = acquire_dma_channels(this);
2817 /* re-init the GPMI registers */
2818 ret = gpmi_init(this);
2820 dev_err(this->dev, "Error setting GPMI : %d\n", ret);
2824 /* Set flag to get timing setup restored for next exec_op */
2825 if (this->hw.clk_rate)
2826 this->hw.must_apply_timings = true;
2828 /* re-init the BCH registers */
2829 ret = bch_set_geometry(this);
2831 dev_err(this->dev, "Error setting BCH : %d\n", ret);
2837 #endif /* CONFIG_PM_SLEEP */
2839 static int __maybe_unused gpmi_runtime_suspend(struct device *dev)
2841 struct gpmi_nand_data *this = dev_get_drvdata(dev);
2843 return __gpmi_enable_clk(this, false);
2846 static int __maybe_unused gpmi_runtime_resume(struct device *dev)
2848 struct gpmi_nand_data *this = dev_get_drvdata(dev);
2850 return __gpmi_enable_clk(this, true);
2853 static const struct dev_pm_ops gpmi_pm_ops = {
2854 SET_SYSTEM_SLEEP_PM_OPS(gpmi_pm_suspend, gpmi_pm_resume)
2855 SET_RUNTIME_PM_OPS(gpmi_runtime_suspend, gpmi_runtime_resume, NULL)
2858 static struct platform_driver gpmi_nand_driver = {
2860 .name = "gpmi-nand",
2862 .of_match_table = gpmi_nand_id_table,
2864 .probe = gpmi_nand_probe,
2865 .remove = gpmi_nand_remove,
2867 module_platform_driver(gpmi_nand_driver);
2869 MODULE_AUTHOR("Freescale Semiconductor, Inc.");
2870 MODULE_DESCRIPTION("i.MX GPMI NAND Flash Controller Driver");
2871 MODULE_LICENSE("GPL");