1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (c) 2016, The Linux Foundation. All rights reserved.
6 #include <linux/slab.h>
7 #include <linux/bitops.h>
8 #include <linux/dma/qcom_adm.h>
9 #include <linux/dma-mapping.h>
10 #include <linux/dmaengine.h>
11 #include <linux/module.h>
12 #include <linux/mtd/rawnand.h>
13 #include <linux/mtd/partitions.h>
15 #include <linux/of_device.h>
16 #include <linux/delay.h>
17 #include <linux/dma/qcom_bam_dma.h>
19 /* NANDc reg offsets */
20 #define NAND_FLASH_CMD 0x00
21 #define NAND_ADDR0 0x04
22 #define NAND_ADDR1 0x08
23 #define NAND_FLASH_CHIP_SELECT 0x0c
24 #define NAND_EXEC_CMD 0x10
25 #define NAND_FLASH_STATUS 0x14
26 #define NAND_BUFFER_STATUS 0x18
27 #define NAND_DEV0_CFG0 0x20
28 #define NAND_DEV0_CFG1 0x24
29 #define NAND_DEV0_ECC_CFG 0x28
30 #define NAND_AUTO_STATUS_EN 0x2c
31 #define NAND_DEV1_CFG0 0x30
32 #define NAND_DEV1_CFG1 0x34
33 #define NAND_READ_ID 0x40
34 #define NAND_READ_STATUS 0x44
35 #define NAND_DEV_CMD0 0xa0
36 #define NAND_DEV_CMD1 0xa4
37 #define NAND_DEV_CMD2 0xa8
38 #define NAND_DEV_CMD_VLD 0xac
39 #define SFLASHC_BURST_CFG 0xe0
40 #define NAND_ERASED_CW_DETECT_CFG 0xe8
41 #define NAND_ERASED_CW_DETECT_STATUS 0xec
42 #define NAND_EBI2_ECC_BUF_CFG 0xf0
43 #define FLASH_BUF_ACC 0x100
45 #define NAND_CTRL 0xf00
46 #define NAND_VERSION 0xf08
47 #define NAND_READ_LOCATION_0 0xf20
48 #define NAND_READ_LOCATION_1 0xf24
49 #define NAND_READ_LOCATION_2 0xf28
50 #define NAND_READ_LOCATION_3 0xf2c
51 #define NAND_READ_LOCATION_LAST_CW_0 0xf40
52 #define NAND_READ_LOCATION_LAST_CW_1 0xf44
53 #define NAND_READ_LOCATION_LAST_CW_2 0xf48
54 #define NAND_READ_LOCATION_LAST_CW_3 0xf4c
56 /* dummy register offsets, used by write_reg_dma */
57 #define NAND_DEV_CMD1_RESTORE 0xdead
58 #define NAND_DEV_CMD_VLD_RESTORE 0xbeef
60 /* NAND_FLASH_CMD bits */
61 #define PAGE_ACC BIT(4)
62 #define LAST_PAGE BIT(5)
64 /* NAND_FLASH_CHIP_SELECT bits */
65 #define NAND_DEV_SEL 0
68 /* NAND_FLASH_STATUS bits */
69 #define FS_OP_ERR BIT(4)
70 #define FS_READY_BSY_N BIT(5)
71 #define FS_MPU_ERR BIT(8)
72 #define FS_DEVICE_STS_ERR BIT(16)
73 #define FS_DEVICE_WP BIT(23)
75 /* NAND_BUFFER_STATUS bits */
76 #define BS_UNCORRECTABLE_BIT BIT(8)
77 #define BS_CORRECTABLE_ERR_MSK 0x1f
79 /* NAND_DEVn_CFG0 bits */
80 #define DISABLE_STATUS_AFTER_WRITE 4
82 #define UD_SIZE_BYTES 9
83 #define ECC_PARITY_SIZE_BYTES_RS 19
84 #define SPARE_SIZE_BYTES 23
85 #define NUM_ADDR_CYCLES 27
86 #define STATUS_BFR_READ 30
87 #define SET_RD_MODE_AFTER_STATUS 31
89 /* NAND_DEVn_CFG0 bits */
90 #define DEV0_CFG1_ECC_DISABLE 0
92 #define NAND_RECOVERY_CYCLES 2
93 #define CS_ACTIVE_BSY 5
94 #define BAD_BLOCK_BYTE_NUM 6
95 #define BAD_BLOCK_IN_SPARE_AREA 16
96 #define WR_RD_BSY_GAP 17
97 #define ENABLE_BCH_ECC 27
99 /* NAND_DEV0_ECC_CFG bits */
100 #define ECC_CFG_ECC_DISABLE 0
101 #define ECC_SW_RESET 1
103 #define ECC_PARITY_SIZE_BYTES_BCH 8
104 #define ECC_NUM_DATA_BYTES 16
105 #define ECC_FORCE_CLK_OPEN 30
107 /* NAND_DEV_CMD1 bits */
110 /* NAND_DEV_CMD_VLD bits */
111 #define READ_START_VLD BIT(0)
112 #define READ_STOP_VLD BIT(1)
113 #define WRITE_START_VLD BIT(2)
114 #define ERASE_START_VLD BIT(3)
115 #define SEQ_READ_START_VLD BIT(4)
117 /* NAND_EBI2_ECC_BUF_CFG bits */
120 /* NAND_ERASED_CW_DETECT_CFG bits */
121 #define ERASED_CW_ECC_MASK 1
122 #define AUTO_DETECT_RES 0
123 #define MASK_ECC (1 << ERASED_CW_ECC_MASK)
124 #define RESET_ERASED_DET (1 << AUTO_DETECT_RES)
125 #define ACTIVE_ERASED_DET (0 << AUTO_DETECT_RES)
126 #define CLR_ERASED_PAGE_DET (RESET_ERASED_DET | MASK_ECC)
127 #define SET_ERASED_PAGE_DET (ACTIVE_ERASED_DET | MASK_ECC)
129 /* NAND_ERASED_CW_DETECT_STATUS bits */
130 #define PAGE_ALL_ERASED BIT(7)
131 #define CODEWORD_ALL_ERASED BIT(6)
132 #define PAGE_ERASED BIT(5)
133 #define CODEWORD_ERASED BIT(4)
134 #define ERASED_PAGE (PAGE_ALL_ERASED | PAGE_ERASED)
135 #define ERASED_CW (CODEWORD_ALL_ERASED | CODEWORD_ERASED)
137 /* NAND_READ_LOCATION_n bits */
138 #define READ_LOCATION_OFFSET 0
139 #define READ_LOCATION_SIZE 16
140 #define READ_LOCATION_LAST 31
143 #define NAND_VERSION_MAJOR_MASK 0xf0000000
144 #define NAND_VERSION_MAJOR_SHIFT 28
145 #define NAND_VERSION_MINOR_MASK 0x0fff0000
146 #define NAND_VERSION_MINOR_SHIFT 16
149 #define OP_PAGE_READ 0x2
150 #define OP_PAGE_READ_WITH_ECC 0x3
151 #define OP_PAGE_READ_WITH_ECC_SPARE 0x4
152 #define OP_PAGE_READ_ONFI_READ 0x5
153 #define OP_PROGRAM_PAGE 0x6
154 #define OP_PAGE_PROGRAM_WITH_ECC 0x7
155 #define OP_PROGRAM_PAGE_SPARE 0x9
156 #define OP_BLOCK_ERASE 0xa
157 #define OP_FETCH_ID 0xb
158 #define OP_RESET_DEVICE 0xd
160 /* Default Value for NAND_DEV_CMD_VLD */
161 #define NAND_DEV_CMD_VLD_VAL (READ_START_VLD | WRITE_START_VLD | \
162 ERASE_START_VLD | SEQ_READ_START_VLD)
165 #define BAM_MODE_EN BIT(0)
168 * the NAND controller performs reads/writes with ECC in 516 byte chunks.
169 * the driver calls the chunks 'step' or 'codeword' interchangeably
171 #define NANDC_STEP_SIZE 512
174 * the largest page size we support is 8K, this will have 16 steps/codewords
177 #define MAX_NUM_STEPS (SZ_8K / NANDC_STEP_SIZE)
179 /* we read at most 3 registers per codeword scan */
180 #define MAX_REG_RD (3 * MAX_NUM_STEPS)
182 /* ECC modes supported by the controller */
183 #define ECC_NONE BIT(0)
184 #define ECC_RS_4BIT BIT(1)
185 #define ECC_BCH_4BIT BIT(2)
186 #define ECC_BCH_8BIT BIT(3)
188 #define nandc_set_read_loc_first(chip, reg, cw_offset, read_size, is_last_read_loc) \
189 nandc_set_reg(chip, reg, \
190 ((cw_offset) << READ_LOCATION_OFFSET) | \
191 ((read_size) << READ_LOCATION_SIZE) | \
192 ((is_last_read_loc) << READ_LOCATION_LAST))
194 #define nandc_set_read_loc_last(chip, reg, cw_offset, read_size, is_last_read_loc) \
195 nandc_set_reg(chip, reg, \
196 ((cw_offset) << READ_LOCATION_OFFSET) | \
197 ((read_size) << READ_LOCATION_SIZE) | \
198 ((is_last_read_loc) << READ_LOCATION_LAST))
200 * Returns the actual register address for all NAND_DEV_ registers
201 * (i.e. NAND_DEV_CMD0, NAND_DEV_CMD1, NAND_DEV_CMD2 and NAND_DEV_CMD_VLD)
203 #define dev_cmd_reg_addr(nandc, reg) ((nandc)->props->dev_cmd_reg_start + (reg))
205 /* Returns the NAND register physical address */
206 #define nandc_reg_phys(chip, offset) ((chip)->base_phys + (offset))
208 /* Returns the dma address for reg read buffer */
209 #define reg_buf_dma_addr(chip, vaddr) \
210 ((chip)->reg_read_dma + \
211 ((uint8_t *)(vaddr) - (uint8_t *)(chip)->reg_read_buf))
213 #define QPIC_PER_CW_CMD_ELEMENTS 32
214 #define QPIC_PER_CW_CMD_SGL 32
215 #define QPIC_PER_CW_DATA_SGL 8
217 #define QPIC_NAND_COMPLETION_TIMEOUT msecs_to_jiffies(2000)
220 * Flags used in DMA descriptor preparation helper functions
221 * (i.e. read_reg_dma/write_reg_dma/read_data_dma/write_data_dma)
223 /* Don't set the EOT in current tx BAM sgl */
224 #define NAND_BAM_NO_EOT BIT(0)
225 /* Set the NWD flag in current BAM sgl */
226 #define NAND_BAM_NWD BIT(1)
227 /* Finish writing in the current BAM sgl and start writing in another BAM sgl */
228 #define NAND_BAM_NEXT_SGL BIT(2)
230 * Erased codeword status is being used two times in single transfer so this
231 * flag will determine the current value of erased codeword status register
233 #define NAND_ERASED_CW_SET BIT(4)
236 * This data type corresponds to the BAM transaction which will be used for all
238 * @bam_ce - the array of BAM command elements
239 * @cmd_sgl - sgl for NAND BAM command pipe
240 * @data_sgl - sgl for NAND BAM consumer/producer pipe
241 * @bam_ce_pos - the index in bam_ce which is available for next sgl
242 * @bam_ce_start - the index in bam_ce which marks the start position ce
243 * for current sgl. It will be used for size calculation
245 * @cmd_sgl_pos - current index in command sgl.
246 * @cmd_sgl_start - start index in command sgl.
247 * @tx_sgl_pos - current index in data sgl for tx.
248 * @tx_sgl_start - start index in data sgl for tx.
249 * @rx_sgl_pos - current index in data sgl for rx.
250 * @rx_sgl_start - start index in data sgl for rx.
251 * @wait_second_completion - wait for second DMA desc completion before making
252 * the NAND transfer completion.
253 * @txn_done - completion for NAND transfer.
254 * @last_data_desc - last DMA desc in data channel (tx/rx).
255 * @last_cmd_desc - last DMA desc in command channel.
257 struct bam_transaction {
258 struct bam_cmd_element *bam_ce;
259 struct scatterlist *cmd_sgl;
260 struct scatterlist *data_sgl;
269 bool wait_second_completion;
270 struct completion txn_done;
271 struct dma_async_tx_descriptor *last_data_desc;
272 struct dma_async_tx_descriptor *last_cmd_desc;
276 * This data type corresponds to the nand dma descriptor
277 * @list - list for desc_info
278 * @dir - DMA transfer direction
279 * @adm_sgl - sgl which will be used for single sgl dma descriptor. Only used by
281 * @bam_sgl - sgl which will be used for dma descriptor. Only used by BAM
282 * @sgl_cnt - number of SGL in bam_sgl. Only used by BAM
283 * @dma_desc - low level DMA engine descriptor
286 struct list_head node;
288 enum dma_data_direction dir;
290 struct scatterlist adm_sgl;
292 struct scatterlist *bam_sgl;
296 struct dma_async_tx_descriptor *dma_desc;
300 * holds the current register values that we want to write. acts as a contiguous
301 * chunk of memory which we use to write the controller registers through DMA.
314 __le32 clrflashstatus;
315 __le32 clrreadstatus;
324 __le32 read_location0;
325 __le32 read_location1;
326 __le32 read_location2;
327 __le32 read_location3;
328 __le32 read_location_last0;
329 __le32 read_location_last1;
330 __le32 read_location_last2;
331 __le32 read_location_last3;
333 __le32 erased_cw_detect_cfg_clr;
334 __le32 erased_cw_detect_cfg_set;
338 * NAND controller data struct
340 * @controller: base controller structure
341 * @host_list: list containing all the chips attached to the
343 * @dev: parent device
345 * @base_phys: physical base address of controller registers
346 * @base_dma: dma base address of controller registers
347 * @core_clk: controller clock
348 * @aon_clk: another controller clock
351 * @cmd_crci: ADM DMA CRCI for command flow control
352 * @data_crci: ADM DMA CRCI for data flow control
353 * @desc_list: DMA descriptor list (list of desc_infos)
355 * @data_buffer: our local DMA buffer for page read/writes,
356 * used when we can't use the buffer provided
357 * by upper layers directly
358 * @buf_size/count/start: markers for chip->legacy.read_buf/write_buf
360 * @reg_read_buf: local buffer for reading back registers via DMA
361 * @reg_read_dma: contains dma address for register read buffer
362 * @reg_read_pos: marker for data read in reg_read_buf
364 * @regs: a contiguous chunk of memory for DMA register
365 * writes. contains the register values to be
366 * written to controller
367 * @cmd1/vld: some fixed controller register values
368 * @props: properties of current NAND controller,
369 * initialized via DT match data
370 * @max_cwperpage: maximum QPIC codewords required. calculated
371 * from all connected NAND devices pagesize
373 struct qcom_nand_controller {
374 struct nand_controller controller;
375 struct list_head host_list;
380 phys_addr_t base_phys;
383 struct clk *core_clk;
387 /* will be used only by QPIC for BAM DMA */
389 struct dma_chan *tx_chan;
390 struct dma_chan *rx_chan;
391 struct dma_chan *cmd_chan;
394 /* will be used only by EBI2 for ADM DMA */
396 struct dma_chan *chan;
397 unsigned int cmd_crci;
398 unsigned int data_crci;
402 struct list_head desc_list;
403 struct bam_transaction *bam_txn;
409 unsigned int max_cwperpage;
411 __le32 *reg_read_buf;
412 dma_addr_t reg_read_dma;
415 struct nandc_regs *regs;
418 const struct qcom_nandc_props *props;
422 * NAND chip structure
424 * @chip: base NAND chip structure
425 * @node: list node to add itself to host_list in
426 * qcom_nand_controller
428 * @cs: chip select value for this chip
429 * @cw_size: the number of bytes in a single step/codeword
430 * of a page, consisting of all data, ecc, spare
432 * @cw_data: the number of bytes within a codeword protected
434 * @use_ecc: request the controller to use ECC for the
435 * upcoming read/write
436 * @bch_enabled: flag to tell whether BCH ECC mode is used
437 * @ecc_bytes_hw: ECC bytes used by controller hardware for this
439 * @status: value to be returned if NAND_CMD_STATUS command
441 * @last_command: keeps track of last command on this chip. used
442 * for reading correct status
444 * @cfg0, cfg1, cfg0_raw..: NANDc register configurations needed for
445 * ecc/non-ecc mode for the current nand flash
448 struct qcom_nand_host {
449 struct nand_chip chip;
450 struct list_head node;
464 u32 cfg0_raw, cfg1_raw;
472 * This data type corresponds to the NAND controller properties which varies
473 * among different NAND controllers.
474 * @ecc_modes - ecc mode for NAND
475 * @is_bam - whether NAND controller is using BAM
476 * @is_qpic - whether NAND CTRL is part of qpic IP
477 * @qpic_v2 - flag to indicate QPIC IP version 2
478 * @dev_cmd_reg_start - NAND_DEV_CMD_* registers starting offset
480 struct qcom_nandc_props {
485 u32 dev_cmd_reg_start;
488 /* Frees the BAM transaction memory */
489 static void free_bam_transaction(struct qcom_nand_controller *nandc)
491 struct bam_transaction *bam_txn = nandc->bam_txn;
493 devm_kfree(nandc->dev, bam_txn);
496 /* Allocates and Initializes the BAM transaction */
497 static struct bam_transaction *
498 alloc_bam_transaction(struct qcom_nand_controller *nandc)
500 struct bam_transaction *bam_txn;
502 unsigned int num_cw = nandc->max_cwperpage;
506 sizeof(*bam_txn) + num_cw *
507 ((sizeof(*bam_txn->bam_ce) * QPIC_PER_CW_CMD_ELEMENTS) +
508 (sizeof(*bam_txn->cmd_sgl) * QPIC_PER_CW_CMD_SGL) +
509 (sizeof(*bam_txn->data_sgl) * QPIC_PER_CW_DATA_SGL));
511 bam_txn_buf = devm_kzalloc(nandc->dev, bam_txn_size, GFP_KERNEL);
515 bam_txn = bam_txn_buf;
516 bam_txn_buf += sizeof(*bam_txn);
518 bam_txn->bam_ce = bam_txn_buf;
520 sizeof(*bam_txn->bam_ce) * QPIC_PER_CW_CMD_ELEMENTS * num_cw;
522 bam_txn->cmd_sgl = bam_txn_buf;
524 sizeof(*bam_txn->cmd_sgl) * QPIC_PER_CW_CMD_SGL * num_cw;
526 bam_txn->data_sgl = bam_txn_buf;
528 init_completion(&bam_txn->txn_done);
533 /* Clears the BAM transaction indexes */
534 static void clear_bam_transaction(struct qcom_nand_controller *nandc)
536 struct bam_transaction *bam_txn = nandc->bam_txn;
538 if (!nandc->props->is_bam)
541 bam_txn->bam_ce_pos = 0;
542 bam_txn->bam_ce_start = 0;
543 bam_txn->cmd_sgl_pos = 0;
544 bam_txn->cmd_sgl_start = 0;
545 bam_txn->tx_sgl_pos = 0;
546 bam_txn->tx_sgl_start = 0;
547 bam_txn->rx_sgl_pos = 0;
548 bam_txn->rx_sgl_start = 0;
549 bam_txn->last_data_desc = NULL;
550 bam_txn->wait_second_completion = false;
552 sg_init_table(bam_txn->cmd_sgl, nandc->max_cwperpage *
553 QPIC_PER_CW_CMD_SGL);
554 sg_init_table(bam_txn->data_sgl, nandc->max_cwperpage *
555 QPIC_PER_CW_DATA_SGL);
557 reinit_completion(&bam_txn->txn_done);
560 /* Callback for DMA descriptor completion */
561 static void qpic_bam_dma_done(void *data)
563 struct bam_transaction *bam_txn = data;
566 * In case of data transfer with NAND, 2 callbacks will be generated.
567 * One for command channel and another one for data channel.
568 * If current transaction has data descriptors
569 * (i.e. wait_second_completion is true), then set this to false
570 * and wait for second DMA descriptor completion.
572 if (bam_txn->wait_second_completion)
573 bam_txn->wait_second_completion = false;
575 complete(&bam_txn->txn_done);
578 static inline struct qcom_nand_host *to_qcom_nand_host(struct nand_chip *chip)
580 return container_of(chip, struct qcom_nand_host, chip);
583 static inline struct qcom_nand_controller *
584 get_qcom_nand_controller(struct nand_chip *chip)
586 return container_of(chip->controller, struct qcom_nand_controller,
590 static inline u32 nandc_read(struct qcom_nand_controller *nandc, int offset)
592 return ioread32(nandc->base + offset);
595 static inline void nandc_write(struct qcom_nand_controller *nandc, int offset,
598 iowrite32(val, nandc->base + offset);
601 static inline void nandc_read_buffer_sync(struct qcom_nand_controller *nandc,
604 if (!nandc->props->is_bam)
608 dma_sync_single_for_cpu(nandc->dev, nandc->reg_read_dma,
610 sizeof(*nandc->reg_read_buf),
613 dma_sync_single_for_device(nandc->dev, nandc->reg_read_dma,
615 sizeof(*nandc->reg_read_buf),
619 static __le32 *offset_to_nandc_reg(struct nandc_regs *regs, int offset)
628 case NAND_FLASH_CHIP_SELECT:
629 return ®s->chip_sel;
632 case NAND_FLASH_STATUS:
633 return ®s->clrflashstatus;
638 case NAND_DEV0_ECC_CFG:
639 return ®s->ecc_bch_cfg;
640 case NAND_READ_STATUS:
641 return ®s->clrreadstatus;
644 case NAND_DEV_CMD1_RESTORE:
645 return ®s->orig_cmd1;
646 case NAND_DEV_CMD_VLD:
648 case NAND_DEV_CMD_VLD_RESTORE:
649 return ®s->orig_vld;
650 case NAND_EBI2_ECC_BUF_CFG:
651 return ®s->ecc_buf_cfg;
652 case NAND_READ_LOCATION_0:
653 return ®s->read_location0;
654 case NAND_READ_LOCATION_1:
655 return ®s->read_location1;
656 case NAND_READ_LOCATION_2:
657 return ®s->read_location2;
658 case NAND_READ_LOCATION_3:
659 return ®s->read_location3;
660 case NAND_READ_LOCATION_LAST_CW_0:
661 return ®s->read_location_last0;
662 case NAND_READ_LOCATION_LAST_CW_1:
663 return ®s->read_location_last1;
664 case NAND_READ_LOCATION_LAST_CW_2:
665 return ®s->read_location_last2;
666 case NAND_READ_LOCATION_LAST_CW_3:
667 return ®s->read_location_last3;
673 static void nandc_set_reg(struct nand_chip *chip, int offset,
676 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
677 struct nandc_regs *regs = nandc->regs;
680 reg = offset_to_nandc_reg(regs, offset);
683 *reg = cpu_to_le32(val);
686 /* Helper to check the code word, whether it is last cw or not */
687 static bool qcom_nandc_is_last_cw(struct nand_ecc_ctrl *ecc, int cw)
689 return cw == (ecc->steps - 1);
692 /* helper to configure location register values */
693 static void nandc_set_read_loc(struct nand_chip *chip, int cw, int reg,
694 int cw_offset, int read_size, int is_last_read_loc)
696 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
697 struct nand_ecc_ctrl *ecc = &chip->ecc;
698 int reg_base = NAND_READ_LOCATION_0;
700 if (nandc->props->qpic_v2 && qcom_nandc_is_last_cw(ecc, cw))
701 reg_base = NAND_READ_LOCATION_LAST_CW_0;
705 if (nandc->props->qpic_v2 && qcom_nandc_is_last_cw(ecc, cw))
706 return nandc_set_read_loc_last(chip, reg_base, cw_offset,
707 read_size, is_last_read_loc);
709 return nandc_set_read_loc_first(chip, reg_base, cw_offset,
710 read_size, is_last_read_loc);
713 /* helper to configure address register values */
714 static void set_address(struct qcom_nand_host *host, u16 column, int page)
716 struct nand_chip *chip = &host->chip;
718 if (chip->options & NAND_BUSWIDTH_16)
721 nandc_set_reg(chip, NAND_ADDR0, page << 16 | column);
722 nandc_set_reg(chip, NAND_ADDR1, page >> 16 & 0xff);
726 * update_rw_regs: set up read/write register values, these will be
727 * written to the NAND controller registers via DMA
729 * @num_cw: number of steps for the read/write operation
730 * @read: read or write operation
731 * @cw : which code word
733 static void update_rw_regs(struct qcom_nand_host *host, int num_cw, bool read, int cw)
735 struct nand_chip *chip = &host->chip;
736 u32 cmd, cfg0, cfg1, ecc_bch_cfg;
737 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
741 cmd = OP_PAGE_READ_WITH_ECC | PAGE_ACC | LAST_PAGE;
743 cmd = OP_PAGE_READ | PAGE_ACC | LAST_PAGE;
745 cmd = OP_PROGRAM_PAGE | PAGE_ACC | LAST_PAGE;
749 cfg0 = (host->cfg0 & ~(7U << CW_PER_PAGE)) |
750 (num_cw - 1) << CW_PER_PAGE;
753 ecc_bch_cfg = host->ecc_bch_cfg;
755 cfg0 = (host->cfg0_raw & ~(7U << CW_PER_PAGE)) |
756 (num_cw - 1) << CW_PER_PAGE;
758 cfg1 = host->cfg1_raw;
759 ecc_bch_cfg = 1 << ECC_CFG_ECC_DISABLE;
762 nandc_set_reg(chip, NAND_FLASH_CMD, cmd);
763 nandc_set_reg(chip, NAND_DEV0_CFG0, cfg0);
764 nandc_set_reg(chip, NAND_DEV0_CFG1, cfg1);
765 nandc_set_reg(chip, NAND_DEV0_ECC_CFG, ecc_bch_cfg);
766 if (!nandc->props->qpic_v2)
767 nandc_set_reg(chip, NAND_EBI2_ECC_BUF_CFG, host->ecc_buf_cfg);
768 nandc_set_reg(chip, NAND_FLASH_STATUS, host->clrflashstatus);
769 nandc_set_reg(chip, NAND_READ_STATUS, host->clrreadstatus);
770 nandc_set_reg(chip, NAND_EXEC_CMD, 1);
773 nandc_set_read_loc(chip, cw, 0, 0, host->use_ecc ?
774 host->cw_data : host->cw_size, 1);
778 * Maps the scatter gather list for DMA transfer and forms the DMA descriptor
779 * for BAM. This descriptor will be added in the NAND DMA descriptor queue
780 * which will be submitted to DMA engine.
782 static int prepare_bam_async_desc(struct qcom_nand_controller *nandc,
783 struct dma_chan *chan,
786 struct desc_info *desc;
787 struct scatterlist *sgl;
788 unsigned int sgl_cnt;
790 struct bam_transaction *bam_txn = nandc->bam_txn;
791 enum dma_transfer_direction dir_eng;
792 struct dma_async_tx_descriptor *dma_desc;
794 desc = kzalloc(sizeof(*desc), GFP_KERNEL);
798 if (chan == nandc->cmd_chan) {
799 sgl = &bam_txn->cmd_sgl[bam_txn->cmd_sgl_start];
800 sgl_cnt = bam_txn->cmd_sgl_pos - bam_txn->cmd_sgl_start;
801 bam_txn->cmd_sgl_start = bam_txn->cmd_sgl_pos;
802 dir_eng = DMA_MEM_TO_DEV;
803 desc->dir = DMA_TO_DEVICE;
804 } else if (chan == nandc->tx_chan) {
805 sgl = &bam_txn->data_sgl[bam_txn->tx_sgl_start];
806 sgl_cnt = bam_txn->tx_sgl_pos - bam_txn->tx_sgl_start;
807 bam_txn->tx_sgl_start = bam_txn->tx_sgl_pos;
808 dir_eng = DMA_MEM_TO_DEV;
809 desc->dir = DMA_TO_DEVICE;
811 sgl = &bam_txn->data_sgl[bam_txn->rx_sgl_start];
812 sgl_cnt = bam_txn->rx_sgl_pos - bam_txn->rx_sgl_start;
813 bam_txn->rx_sgl_start = bam_txn->rx_sgl_pos;
814 dir_eng = DMA_DEV_TO_MEM;
815 desc->dir = DMA_FROM_DEVICE;
818 sg_mark_end(sgl + sgl_cnt - 1);
819 ret = dma_map_sg(nandc->dev, sgl, sgl_cnt, desc->dir);
821 dev_err(nandc->dev, "failure in mapping desc\n");
826 desc->sgl_cnt = sgl_cnt;
829 dma_desc = dmaengine_prep_slave_sg(chan, sgl, sgl_cnt, dir_eng,
833 dev_err(nandc->dev, "failure in prep desc\n");
834 dma_unmap_sg(nandc->dev, sgl, sgl_cnt, desc->dir);
839 desc->dma_desc = dma_desc;
841 /* update last data/command descriptor */
842 if (chan == nandc->cmd_chan)
843 bam_txn->last_cmd_desc = dma_desc;
845 bam_txn->last_data_desc = dma_desc;
847 list_add_tail(&desc->node, &nandc->desc_list);
853 * Prepares the command descriptor for BAM DMA which will be used for NAND
854 * register reads and writes. The command descriptor requires the command
855 * to be formed in command element type so this function uses the command
856 * element from bam transaction ce array and fills the same with required
857 * data. A single SGL can contain multiple command elements so
858 * NAND_BAM_NEXT_SGL will be used for starting the separate SGL
859 * after the current command element.
861 static int prep_bam_dma_desc_cmd(struct qcom_nand_controller *nandc, bool read,
862 int reg_off, const void *vaddr,
863 int size, unsigned int flags)
867 struct bam_cmd_element *bam_ce_buffer;
868 struct bam_transaction *bam_txn = nandc->bam_txn;
870 bam_ce_buffer = &bam_txn->bam_ce[bam_txn->bam_ce_pos];
872 /* fill the command desc */
873 for (i = 0; i < size; i++) {
875 bam_prep_ce(&bam_ce_buffer[i],
876 nandc_reg_phys(nandc, reg_off + 4 * i),
878 reg_buf_dma_addr(nandc,
879 (__le32 *)vaddr + i));
881 bam_prep_ce_le32(&bam_ce_buffer[i],
882 nandc_reg_phys(nandc, reg_off + 4 * i),
884 *((__le32 *)vaddr + i));
887 bam_txn->bam_ce_pos += size;
889 /* use the separate sgl after this command */
890 if (flags & NAND_BAM_NEXT_SGL) {
891 bam_ce_buffer = &bam_txn->bam_ce[bam_txn->bam_ce_start];
892 bam_ce_size = (bam_txn->bam_ce_pos -
893 bam_txn->bam_ce_start) *
894 sizeof(struct bam_cmd_element);
895 sg_set_buf(&bam_txn->cmd_sgl[bam_txn->cmd_sgl_pos],
896 bam_ce_buffer, bam_ce_size);
897 bam_txn->cmd_sgl_pos++;
898 bam_txn->bam_ce_start = bam_txn->bam_ce_pos;
900 if (flags & NAND_BAM_NWD) {
901 ret = prepare_bam_async_desc(nandc, nandc->cmd_chan,
913 * Prepares the data descriptor for BAM DMA which will be used for NAND
914 * data reads and writes.
916 static int prep_bam_dma_desc_data(struct qcom_nand_controller *nandc, bool read,
918 int size, unsigned int flags)
921 struct bam_transaction *bam_txn = nandc->bam_txn;
924 sg_set_buf(&bam_txn->data_sgl[bam_txn->rx_sgl_pos],
926 bam_txn->rx_sgl_pos++;
928 sg_set_buf(&bam_txn->data_sgl[bam_txn->tx_sgl_pos],
930 bam_txn->tx_sgl_pos++;
933 * BAM will only set EOT for DMA_PREP_INTERRUPT so if this flag
934 * is not set, form the DMA descriptor
936 if (!(flags & NAND_BAM_NO_EOT)) {
937 ret = prepare_bam_async_desc(nandc, nandc->tx_chan,
947 static int prep_adm_dma_desc(struct qcom_nand_controller *nandc, bool read,
948 int reg_off, const void *vaddr, int size,
951 struct desc_info *desc;
952 struct dma_async_tx_descriptor *dma_desc;
953 struct scatterlist *sgl;
954 struct dma_slave_config slave_conf;
955 struct qcom_adm_peripheral_config periph_conf = {};
956 enum dma_transfer_direction dir_eng;
959 desc = kzalloc(sizeof(*desc), GFP_KERNEL);
963 sgl = &desc->adm_sgl;
965 sg_init_one(sgl, vaddr, size);
968 dir_eng = DMA_DEV_TO_MEM;
969 desc->dir = DMA_FROM_DEVICE;
971 dir_eng = DMA_MEM_TO_DEV;
972 desc->dir = DMA_TO_DEVICE;
975 ret = dma_map_sg(nandc->dev, sgl, 1, desc->dir);
981 memset(&slave_conf, 0x00, sizeof(slave_conf));
983 slave_conf.device_fc = flow_control;
985 slave_conf.src_maxburst = 16;
986 slave_conf.src_addr = nandc->base_dma + reg_off;
987 if (nandc->data_crci) {
988 periph_conf.crci = nandc->data_crci;
989 slave_conf.peripheral_config = &periph_conf;
990 slave_conf.peripheral_size = sizeof(periph_conf);
993 slave_conf.dst_maxburst = 16;
994 slave_conf.dst_addr = nandc->base_dma + reg_off;
995 if (nandc->cmd_crci) {
996 periph_conf.crci = nandc->cmd_crci;
997 slave_conf.peripheral_config = &periph_conf;
998 slave_conf.peripheral_size = sizeof(periph_conf);
1002 ret = dmaengine_slave_config(nandc->chan, &slave_conf);
1004 dev_err(nandc->dev, "failed to configure dma channel\n");
1008 dma_desc = dmaengine_prep_slave_sg(nandc->chan, sgl, 1, dir_eng, 0);
1010 dev_err(nandc->dev, "failed to prepare desc\n");
1015 desc->dma_desc = dma_desc;
1017 list_add_tail(&desc->node, &nandc->desc_list);
1027 * read_reg_dma: prepares a descriptor to read a given number of
1028 * contiguous registers to the reg_read_buf pointer
1030 * @first: offset of the first register in the contiguous block
1031 * @num_regs: number of registers to read
1032 * @flags: flags to control DMA descriptor preparation
1034 static int read_reg_dma(struct qcom_nand_controller *nandc, int first,
1035 int num_regs, unsigned int flags)
1037 bool flow_control = false;
1040 vaddr = nandc->reg_read_buf + nandc->reg_read_pos;
1041 nandc->reg_read_pos += num_regs;
1043 if (first == NAND_DEV_CMD_VLD || first == NAND_DEV_CMD1)
1044 first = dev_cmd_reg_addr(nandc, first);
1046 if (nandc->props->is_bam)
1047 return prep_bam_dma_desc_cmd(nandc, true, first, vaddr,
1050 if (first == NAND_READ_ID || first == NAND_FLASH_STATUS)
1051 flow_control = true;
1053 return prep_adm_dma_desc(nandc, true, first, vaddr,
1054 num_regs * sizeof(u32), flow_control);
1058 * write_reg_dma: prepares a descriptor to write a given number of
1059 * contiguous registers
1061 * @first: offset of the first register in the contiguous block
1062 * @num_regs: number of registers to write
1063 * @flags: flags to control DMA descriptor preparation
1065 static int write_reg_dma(struct qcom_nand_controller *nandc, int first,
1066 int num_regs, unsigned int flags)
1068 bool flow_control = false;
1069 struct nandc_regs *regs = nandc->regs;
1072 vaddr = offset_to_nandc_reg(regs, first);
1074 if (first == NAND_ERASED_CW_DETECT_CFG) {
1075 if (flags & NAND_ERASED_CW_SET)
1076 vaddr = ®s->erased_cw_detect_cfg_set;
1078 vaddr = ®s->erased_cw_detect_cfg_clr;
1081 if (first == NAND_EXEC_CMD)
1082 flags |= NAND_BAM_NWD;
1084 if (first == NAND_DEV_CMD1_RESTORE || first == NAND_DEV_CMD1)
1085 first = dev_cmd_reg_addr(nandc, NAND_DEV_CMD1);
1087 if (first == NAND_DEV_CMD_VLD_RESTORE || first == NAND_DEV_CMD_VLD)
1088 first = dev_cmd_reg_addr(nandc, NAND_DEV_CMD_VLD);
1090 if (nandc->props->is_bam)
1091 return prep_bam_dma_desc_cmd(nandc, false, first, vaddr,
1094 if (first == NAND_FLASH_CMD)
1095 flow_control = true;
1097 return prep_adm_dma_desc(nandc, false, first, vaddr,
1098 num_regs * sizeof(u32), flow_control);
1102 * read_data_dma: prepares a DMA descriptor to transfer data from the
1103 * controller's internal buffer to the buffer 'vaddr'
1105 * @reg_off: offset within the controller's data buffer
1106 * @vaddr: virtual address of the buffer we want to write to
1107 * @size: DMA transaction size in bytes
1108 * @flags: flags to control DMA descriptor preparation
1110 static int read_data_dma(struct qcom_nand_controller *nandc, int reg_off,
1111 const u8 *vaddr, int size, unsigned int flags)
1113 if (nandc->props->is_bam)
1114 return prep_bam_dma_desc_data(nandc, true, vaddr, size, flags);
1116 return prep_adm_dma_desc(nandc, true, reg_off, vaddr, size, false);
1120 * write_data_dma: prepares a DMA descriptor to transfer data from
1121 * 'vaddr' to the controller's internal buffer
1123 * @reg_off: offset within the controller's data buffer
1124 * @vaddr: virtual address of the buffer we want to read from
1125 * @size: DMA transaction size in bytes
1126 * @flags: flags to control DMA descriptor preparation
1128 static int write_data_dma(struct qcom_nand_controller *nandc, int reg_off,
1129 const u8 *vaddr, int size, unsigned int flags)
1131 if (nandc->props->is_bam)
1132 return prep_bam_dma_desc_data(nandc, false, vaddr, size, flags);
1134 return prep_adm_dma_desc(nandc, false, reg_off, vaddr, size, false);
1138 * Helper to prepare DMA descriptors for configuring registers
1139 * before reading a NAND page.
1141 static void config_nand_page_read(struct nand_chip *chip)
1143 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1145 write_reg_dma(nandc, NAND_ADDR0, 2, 0);
1146 write_reg_dma(nandc, NAND_DEV0_CFG0, 3, 0);
1147 if (!nandc->props->qpic_v2)
1148 write_reg_dma(nandc, NAND_EBI2_ECC_BUF_CFG, 1, 0);
1149 write_reg_dma(nandc, NAND_ERASED_CW_DETECT_CFG, 1, 0);
1150 write_reg_dma(nandc, NAND_ERASED_CW_DETECT_CFG, 1,
1151 NAND_ERASED_CW_SET | NAND_BAM_NEXT_SGL);
1155 * Helper to prepare DMA descriptors for configuring registers
1156 * before reading each codeword in NAND page.
1159 config_nand_cw_read(struct nand_chip *chip, bool use_ecc, int cw)
1161 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1162 struct nand_ecc_ctrl *ecc = &chip->ecc;
1164 int reg = NAND_READ_LOCATION_0;
1166 if (nandc->props->qpic_v2 && qcom_nandc_is_last_cw(ecc, cw))
1167 reg = NAND_READ_LOCATION_LAST_CW_0;
1169 if (nandc->props->is_bam)
1170 write_reg_dma(nandc, reg, 4, NAND_BAM_NEXT_SGL);
1172 write_reg_dma(nandc, NAND_FLASH_CMD, 1, NAND_BAM_NEXT_SGL);
1173 write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL);
1176 read_reg_dma(nandc, NAND_FLASH_STATUS, 2, 0);
1177 read_reg_dma(nandc, NAND_ERASED_CW_DETECT_STATUS, 1,
1180 read_reg_dma(nandc, NAND_FLASH_STATUS, 1, NAND_BAM_NEXT_SGL);
1185 * Helper to prepare dma descriptors to configure registers needed for reading a
1186 * single codeword in page
1189 config_nand_single_cw_page_read(struct nand_chip *chip,
1190 bool use_ecc, int cw)
1192 config_nand_page_read(chip);
1193 config_nand_cw_read(chip, use_ecc, cw);
1197 * Helper to prepare DMA descriptors used to configure registers needed for
1198 * before writing a NAND page.
1200 static void config_nand_page_write(struct nand_chip *chip)
1202 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1204 write_reg_dma(nandc, NAND_ADDR0, 2, 0);
1205 write_reg_dma(nandc, NAND_DEV0_CFG0, 3, 0);
1206 if (!nandc->props->qpic_v2)
1207 write_reg_dma(nandc, NAND_EBI2_ECC_BUF_CFG, 1,
1212 * Helper to prepare DMA descriptors for configuring registers
1213 * before writing each codeword in NAND page.
1215 static void config_nand_cw_write(struct nand_chip *chip)
1217 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1219 write_reg_dma(nandc, NAND_FLASH_CMD, 1, NAND_BAM_NEXT_SGL);
1220 write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL);
1222 read_reg_dma(nandc, NAND_FLASH_STATUS, 1, NAND_BAM_NEXT_SGL);
1224 write_reg_dma(nandc, NAND_FLASH_STATUS, 1, 0);
1225 write_reg_dma(nandc, NAND_READ_STATUS, 1, NAND_BAM_NEXT_SGL);
1229 * the following functions are used within chip->legacy.cmdfunc() to
1230 * perform different NAND_CMD_* commands
1233 /* sets up descriptors for NAND_CMD_PARAM */
1234 static int nandc_param(struct qcom_nand_host *host)
1236 struct nand_chip *chip = &host->chip;
1237 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1240 * NAND_CMD_PARAM is called before we know much about the FLASH chip
1241 * in use. we configure the controller to perform a raw read of 512
1242 * bytes to read onfi params
1244 if (nandc->props->qpic_v2)
1245 nandc_set_reg(chip, NAND_FLASH_CMD, OP_PAGE_READ_ONFI_READ |
1246 PAGE_ACC | LAST_PAGE);
1248 nandc_set_reg(chip, NAND_FLASH_CMD, OP_PAGE_READ |
1249 PAGE_ACC | LAST_PAGE);
1251 nandc_set_reg(chip, NAND_ADDR0, 0);
1252 nandc_set_reg(chip, NAND_ADDR1, 0);
1253 nandc_set_reg(chip, NAND_DEV0_CFG0, 0 << CW_PER_PAGE
1254 | 512 << UD_SIZE_BYTES
1255 | 5 << NUM_ADDR_CYCLES
1256 | 0 << SPARE_SIZE_BYTES);
1257 nandc_set_reg(chip, NAND_DEV0_CFG1, 7 << NAND_RECOVERY_CYCLES
1258 | 0 << CS_ACTIVE_BSY
1259 | 17 << BAD_BLOCK_BYTE_NUM
1260 | 1 << BAD_BLOCK_IN_SPARE_AREA
1261 | 2 << WR_RD_BSY_GAP
1263 | 1 << DEV0_CFG1_ECC_DISABLE);
1264 if (!nandc->props->qpic_v2)
1265 nandc_set_reg(chip, NAND_EBI2_ECC_BUF_CFG, 1 << ECC_CFG_ECC_DISABLE);
1267 /* configure CMD1 and VLD for ONFI param probing in QPIC v1 */
1268 if (!nandc->props->qpic_v2) {
1269 nandc_set_reg(chip, NAND_DEV_CMD_VLD,
1270 (nandc->vld & ~READ_START_VLD));
1271 nandc_set_reg(chip, NAND_DEV_CMD1,
1272 (nandc->cmd1 & ~(0xFF << READ_ADDR))
1273 | NAND_CMD_PARAM << READ_ADDR);
1276 nandc_set_reg(chip, NAND_EXEC_CMD, 1);
1278 if (!nandc->props->qpic_v2) {
1279 nandc_set_reg(chip, NAND_DEV_CMD1_RESTORE, nandc->cmd1);
1280 nandc_set_reg(chip, NAND_DEV_CMD_VLD_RESTORE, nandc->vld);
1283 nandc_set_read_loc(chip, 0, 0, 0, 512, 1);
1285 if (!nandc->props->qpic_v2) {
1286 write_reg_dma(nandc, NAND_DEV_CMD_VLD, 1, 0);
1287 write_reg_dma(nandc, NAND_DEV_CMD1, 1, NAND_BAM_NEXT_SGL);
1290 nandc->buf_count = 512;
1291 memset(nandc->data_buffer, 0xff, nandc->buf_count);
1293 config_nand_single_cw_page_read(chip, false, 0);
1295 read_data_dma(nandc, FLASH_BUF_ACC, nandc->data_buffer,
1296 nandc->buf_count, 0);
1298 /* restore CMD1 and VLD regs */
1299 if (!nandc->props->qpic_v2) {
1300 write_reg_dma(nandc, NAND_DEV_CMD1_RESTORE, 1, 0);
1301 write_reg_dma(nandc, NAND_DEV_CMD_VLD_RESTORE, 1, NAND_BAM_NEXT_SGL);
1307 /* sets up descriptors for NAND_CMD_ERASE1 */
1308 static int erase_block(struct qcom_nand_host *host, int page_addr)
1310 struct nand_chip *chip = &host->chip;
1311 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1313 nandc_set_reg(chip, NAND_FLASH_CMD,
1314 OP_BLOCK_ERASE | PAGE_ACC | LAST_PAGE);
1315 nandc_set_reg(chip, NAND_ADDR0, page_addr);
1316 nandc_set_reg(chip, NAND_ADDR1, 0);
1317 nandc_set_reg(chip, NAND_DEV0_CFG0,
1318 host->cfg0_raw & ~(7 << CW_PER_PAGE));
1319 nandc_set_reg(chip, NAND_DEV0_CFG1, host->cfg1_raw);
1320 nandc_set_reg(chip, NAND_EXEC_CMD, 1);
1321 nandc_set_reg(chip, NAND_FLASH_STATUS, host->clrflashstatus);
1322 nandc_set_reg(chip, NAND_READ_STATUS, host->clrreadstatus);
1324 write_reg_dma(nandc, NAND_FLASH_CMD, 3, NAND_BAM_NEXT_SGL);
1325 write_reg_dma(nandc, NAND_DEV0_CFG0, 2, NAND_BAM_NEXT_SGL);
1326 write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL);
1328 read_reg_dma(nandc, NAND_FLASH_STATUS, 1, NAND_BAM_NEXT_SGL);
1330 write_reg_dma(nandc, NAND_FLASH_STATUS, 1, 0);
1331 write_reg_dma(nandc, NAND_READ_STATUS, 1, NAND_BAM_NEXT_SGL);
1336 /* sets up descriptors for NAND_CMD_READID */
1337 static int read_id(struct qcom_nand_host *host, int column)
1339 struct nand_chip *chip = &host->chip;
1340 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1345 nandc_set_reg(chip, NAND_FLASH_CMD, OP_FETCH_ID);
1346 nandc_set_reg(chip, NAND_ADDR0, column);
1347 nandc_set_reg(chip, NAND_ADDR1, 0);
1348 nandc_set_reg(chip, NAND_FLASH_CHIP_SELECT,
1349 nandc->props->is_bam ? 0 : DM_EN);
1350 nandc_set_reg(chip, NAND_EXEC_CMD, 1);
1352 write_reg_dma(nandc, NAND_FLASH_CMD, 4, NAND_BAM_NEXT_SGL);
1353 write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL);
1355 read_reg_dma(nandc, NAND_READ_ID, 1, NAND_BAM_NEXT_SGL);
1360 /* sets up descriptors for NAND_CMD_RESET */
1361 static int reset(struct qcom_nand_host *host)
1363 struct nand_chip *chip = &host->chip;
1364 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1366 nandc_set_reg(chip, NAND_FLASH_CMD, OP_RESET_DEVICE);
1367 nandc_set_reg(chip, NAND_EXEC_CMD, 1);
1369 write_reg_dma(nandc, NAND_FLASH_CMD, 1, NAND_BAM_NEXT_SGL);
1370 write_reg_dma(nandc, NAND_EXEC_CMD, 1, NAND_BAM_NEXT_SGL);
1372 read_reg_dma(nandc, NAND_FLASH_STATUS, 1, NAND_BAM_NEXT_SGL);
1377 /* helpers to submit/free our list of dma descriptors */
1378 static int submit_descs(struct qcom_nand_controller *nandc)
1380 struct desc_info *desc;
1381 dma_cookie_t cookie = 0;
1382 struct bam_transaction *bam_txn = nandc->bam_txn;
1385 if (nandc->props->is_bam) {
1386 if (bam_txn->rx_sgl_pos > bam_txn->rx_sgl_start) {
1387 r = prepare_bam_async_desc(nandc, nandc->rx_chan, 0);
1392 if (bam_txn->tx_sgl_pos > bam_txn->tx_sgl_start) {
1393 r = prepare_bam_async_desc(nandc, nandc->tx_chan,
1394 DMA_PREP_INTERRUPT);
1399 if (bam_txn->cmd_sgl_pos > bam_txn->cmd_sgl_start) {
1400 r = prepare_bam_async_desc(nandc, nandc->cmd_chan,
1407 list_for_each_entry(desc, &nandc->desc_list, node)
1408 cookie = dmaengine_submit(desc->dma_desc);
1410 if (nandc->props->is_bam) {
1411 bam_txn->last_cmd_desc->callback = qpic_bam_dma_done;
1412 bam_txn->last_cmd_desc->callback_param = bam_txn;
1413 if (bam_txn->last_data_desc) {
1414 bam_txn->last_data_desc->callback = qpic_bam_dma_done;
1415 bam_txn->last_data_desc->callback_param = bam_txn;
1416 bam_txn->wait_second_completion = true;
1419 dma_async_issue_pending(nandc->tx_chan);
1420 dma_async_issue_pending(nandc->rx_chan);
1421 dma_async_issue_pending(nandc->cmd_chan);
1423 if (!wait_for_completion_timeout(&bam_txn->txn_done,
1424 QPIC_NAND_COMPLETION_TIMEOUT))
1427 if (dma_sync_wait(nandc->chan, cookie) != DMA_COMPLETE)
1434 static void free_descs(struct qcom_nand_controller *nandc)
1436 struct desc_info *desc, *n;
1438 list_for_each_entry_safe(desc, n, &nandc->desc_list, node) {
1439 list_del(&desc->node);
1441 if (nandc->props->is_bam)
1442 dma_unmap_sg(nandc->dev, desc->bam_sgl,
1443 desc->sgl_cnt, desc->dir);
1445 dma_unmap_sg(nandc->dev, &desc->adm_sgl, 1,
1452 /* reset the register read buffer for next NAND operation */
1453 static void clear_read_regs(struct qcom_nand_controller *nandc)
1455 nandc->reg_read_pos = 0;
1456 nandc_read_buffer_sync(nandc, false);
1459 static void pre_command(struct qcom_nand_host *host, int command)
1461 struct nand_chip *chip = &host->chip;
1462 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1464 nandc->buf_count = 0;
1465 nandc->buf_start = 0;
1466 host->use_ecc = false;
1467 host->last_command = command;
1469 clear_read_regs(nandc);
1471 if (command == NAND_CMD_RESET || command == NAND_CMD_READID ||
1472 command == NAND_CMD_PARAM || command == NAND_CMD_ERASE1)
1473 clear_bam_transaction(nandc);
1477 * this is called after NAND_CMD_PAGEPROG and NAND_CMD_ERASE1 to set our
1478 * privately maintained status byte, this status byte can be read after
1479 * NAND_CMD_STATUS is called
1481 static void parse_erase_write_errors(struct qcom_nand_host *host, int command)
1483 struct nand_chip *chip = &host->chip;
1484 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1485 struct nand_ecc_ctrl *ecc = &chip->ecc;
1489 num_cw = command == NAND_CMD_PAGEPROG ? ecc->steps : 1;
1490 nandc_read_buffer_sync(nandc, true);
1492 for (i = 0; i < num_cw; i++) {
1493 u32 flash_status = le32_to_cpu(nandc->reg_read_buf[i]);
1495 if (flash_status & FS_MPU_ERR)
1496 host->status &= ~NAND_STATUS_WP;
1498 if (flash_status & FS_OP_ERR || (i == (num_cw - 1) &&
1500 FS_DEVICE_STS_ERR)))
1501 host->status |= NAND_STATUS_FAIL;
1505 static void post_command(struct qcom_nand_host *host, int command)
1507 struct nand_chip *chip = &host->chip;
1508 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1511 case NAND_CMD_READID:
1512 nandc_read_buffer_sync(nandc, true);
1513 memcpy(nandc->data_buffer, nandc->reg_read_buf,
1516 case NAND_CMD_PAGEPROG:
1517 case NAND_CMD_ERASE1:
1518 parse_erase_write_errors(host, command);
1526 * Implements chip->legacy.cmdfunc. It's only used for a limited set of
1527 * commands. The rest of the commands wouldn't be called by upper layers.
1528 * For example, NAND_CMD_READOOB would never be called because we have our own
1529 * versions of read_oob ops for nand_ecc_ctrl.
1531 static void qcom_nandc_command(struct nand_chip *chip, unsigned int command,
1532 int column, int page_addr)
1534 struct qcom_nand_host *host = to_qcom_nand_host(chip);
1535 struct nand_ecc_ctrl *ecc = &chip->ecc;
1536 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1540 pre_command(host, command);
1543 case NAND_CMD_RESET:
1548 case NAND_CMD_READID:
1549 nandc->buf_count = 4;
1550 ret = read_id(host, column);
1554 case NAND_CMD_PARAM:
1555 ret = nandc_param(host);
1559 case NAND_CMD_ERASE1:
1560 ret = erase_block(host, page_addr);
1564 case NAND_CMD_READ0:
1565 /* we read the entire page for now */
1566 WARN_ON(column != 0);
1568 host->use_ecc = true;
1569 set_address(host, 0, page_addr);
1570 update_rw_regs(host, ecc->steps, true, 0);
1573 case NAND_CMD_SEQIN:
1574 WARN_ON(column != 0);
1575 set_address(host, 0, page_addr);
1578 case NAND_CMD_PAGEPROG:
1579 case NAND_CMD_STATUS:
1586 dev_err(nandc->dev, "failure executing command %d\n",
1593 ret = submit_descs(nandc);
1596 "failure submitting descs for command %d\n",
1602 post_command(host, command);
1606 * when using BCH ECC, the HW flags an error in NAND_FLASH_STATUS if it read
1607 * an erased CW, and reports an erased CW in NAND_ERASED_CW_DETECT_STATUS.
1609 * when using RS ECC, the HW reports the same erros when reading an erased CW,
1610 * but it notifies that it is an erased CW by placing special characters at
1611 * certain offsets in the buffer.
1613 * verify if the page is erased or not, and fix up the page for RS ECC by
1614 * replacing the special characters with 0xff.
1616 static bool erased_chunk_check_and_fixup(u8 *data_buf, int data_len)
1621 * an erased page flags an error in NAND_FLASH_STATUS, check if the page
1622 * is erased by looking for 0x54s at offsets 3 and 175 from the
1623 * beginning of each codeword
1626 empty1 = data_buf[3];
1627 empty2 = data_buf[175];
1630 * if the erased codework markers, if they exist override them with
1633 if ((empty1 == 0x54 && empty2 == 0xff) ||
1634 (empty1 == 0xff && empty2 == 0x54)) {
1636 data_buf[175] = 0xff;
1640 * check if the entire chunk contains 0xffs or not. if it doesn't, then
1641 * restore the original values at the special offsets
1643 if (memchr_inv(data_buf, 0xff, data_len)) {
1644 data_buf[3] = empty1;
1645 data_buf[175] = empty2;
1659 /* reads back FLASH_STATUS register set by the controller */
1660 static int check_flash_errors(struct qcom_nand_host *host, int cw_cnt)
1662 struct nand_chip *chip = &host->chip;
1663 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1666 nandc_read_buffer_sync(nandc, true);
1668 for (i = 0; i < cw_cnt; i++) {
1669 u32 flash = le32_to_cpu(nandc->reg_read_buf[i]);
1671 if (flash & (FS_OP_ERR | FS_MPU_ERR))
1678 /* performs raw read for one codeword */
1680 qcom_nandc_read_cw_raw(struct mtd_info *mtd, struct nand_chip *chip,
1681 u8 *data_buf, u8 *oob_buf, int page, int cw)
1683 struct qcom_nand_host *host = to_qcom_nand_host(chip);
1684 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1685 struct nand_ecc_ctrl *ecc = &chip->ecc;
1686 int data_size1, data_size2, oob_size1, oob_size2;
1687 int ret, reg_off = FLASH_BUF_ACC, read_loc = 0;
1690 nand_read_page_op(chip, page, 0, NULL, 0);
1691 host->use_ecc = false;
1693 if (nandc->props->qpic_v2)
1694 raw_cw = ecc->steps - 1;
1696 clear_bam_transaction(nandc);
1697 set_address(host, host->cw_size * cw, page);
1698 update_rw_regs(host, 1, true, raw_cw);
1699 config_nand_page_read(chip);
1701 data_size1 = mtd->writesize - host->cw_size * (ecc->steps - 1);
1702 oob_size1 = host->bbm_size;
1704 if (qcom_nandc_is_last_cw(ecc, cw)) {
1705 data_size2 = ecc->size - data_size1 -
1706 ((ecc->steps - 1) * 4);
1707 oob_size2 = (ecc->steps * 4) + host->ecc_bytes_hw +
1710 data_size2 = host->cw_data - data_size1;
1711 oob_size2 = host->ecc_bytes_hw + host->spare_bytes;
1714 if (nandc->props->is_bam) {
1715 nandc_set_read_loc(chip, cw, 0, read_loc, data_size1, 0);
1716 read_loc += data_size1;
1718 nandc_set_read_loc(chip, cw, 1, read_loc, oob_size1, 0);
1719 read_loc += oob_size1;
1721 nandc_set_read_loc(chip, cw, 2, read_loc, data_size2, 0);
1722 read_loc += data_size2;
1724 nandc_set_read_loc(chip, cw, 3, read_loc, oob_size2, 1);
1727 config_nand_cw_read(chip, false, raw_cw);
1729 read_data_dma(nandc, reg_off, data_buf, data_size1, 0);
1730 reg_off += data_size1;
1732 read_data_dma(nandc, reg_off, oob_buf, oob_size1, 0);
1733 reg_off += oob_size1;
1735 read_data_dma(nandc, reg_off, data_buf + data_size1, data_size2, 0);
1736 reg_off += data_size2;
1738 read_data_dma(nandc, reg_off, oob_buf + oob_size1, oob_size2, 0);
1740 ret = submit_descs(nandc);
1743 dev_err(nandc->dev, "failure to read raw cw %d\n", cw);
1747 return check_flash_errors(host, 1);
1751 * Bitflips can happen in erased codewords also so this function counts the
1752 * number of 0 in each CW for which ECC engine returns the uncorrectable
1753 * error. The page will be assumed as erased if this count is less than or
1754 * equal to the ecc->strength for each CW.
1756 * 1. Both DATA and OOB need to be checked for number of 0. The
1757 * top-level API can be called with only data buf or OOB buf so use
1758 * chip->data_buf if data buf is null and chip->oob_poi if oob buf
1759 * is null for copying the raw bytes.
1760 * 2. Perform raw read for all the CW which has uncorrectable errors.
1761 * 3. For each CW, check the number of 0 in cw_data and usable OOB bytes.
1762 * The BBM and spare bytes bit flip won’t affect the ECC so don’t check
1763 * the number of bitflips in this area.
1766 check_for_erased_page(struct qcom_nand_host *host, u8 *data_buf,
1767 u8 *oob_buf, unsigned long uncorrectable_cws,
1768 int page, unsigned int max_bitflips)
1770 struct nand_chip *chip = &host->chip;
1771 struct mtd_info *mtd = nand_to_mtd(chip);
1772 struct nand_ecc_ctrl *ecc = &chip->ecc;
1773 u8 *cw_data_buf, *cw_oob_buf;
1774 int cw, data_size, oob_size, ret = 0;
1777 data_buf = nand_get_data_buf(chip);
1780 nand_get_data_buf(chip);
1781 oob_buf = chip->oob_poi;
1784 for_each_set_bit(cw, &uncorrectable_cws, ecc->steps) {
1785 if (qcom_nandc_is_last_cw(ecc, cw)) {
1786 data_size = ecc->size - ((ecc->steps - 1) * 4);
1787 oob_size = (ecc->steps * 4) + host->ecc_bytes_hw;
1789 data_size = host->cw_data;
1790 oob_size = host->ecc_bytes_hw;
1793 /* determine starting buffer address for current CW */
1794 cw_data_buf = data_buf + (cw * host->cw_data);
1795 cw_oob_buf = oob_buf + (cw * ecc->bytes);
1797 ret = qcom_nandc_read_cw_raw(mtd, chip, cw_data_buf,
1798 cw_oob_buf, page, cw);
1803 * make sure it isn't an erased page reported
1804 * as not-erased by HW because of a few bitflips
1806 ret = nand_check_erased_ecc_chunk(cw_data_buf, data_size,
1807 cw_oob_buf + host->bbm_size,
1811 mtd->ecc_stats.failed++;
1813 mtd->ecc_stats.corrected += ret;
1814 max_bitflips = max_t(unsigned int, max_bitflips, ret);
1818 return max_bitflips;
1822 * reads back status registers set by the controller to notify page read
1823 * errors. this is equivalent to what 'ecc->correct()' would do.
1825 static int parse_read_errors(struct qcom_nand_host *host, u8 *data_buf,
1826 u8 *oob_buf, int page)
1828 struct nand_chip *chip = &host->chip;
1829 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1830 struct mtd_info *mtd = nand_to_mtd(chip);
1831 struct nand_ecc_ctrl *ecc = &chip->ecc;
1832 unsigned int max_bitflips = 0, uncorrectable_cws = 0;
1833 struct read_stats *buf;
1834 bool flash_op_err = false, erased;
1836 u8 *data_buf_start = data_buf, *oob_buf_start = oob_buf;
1838 buf = (struct read_stats *)nandc->reg_read_buf;
1839 nandc_read_buffer_sync(nandc, true);
1841 for (i = 0; i < ecc->steps; i++, buf++) {
1842 u32 flash, buffer, erased_cw;
1843 int data_len, oob_len;
1845 if (qcom_nandc_is_last_cw(ecc, i)) {
1846 data_len = ecc->size - ((ecc->steps - 1) << 2);
1847 oob_len = ecc->steps << 2;
1849 data_len = host->cw_data;
1853 flash = le32_to_cpu(buf->flash);
1854 buffer = le32_to_cpu(buf->buffer);
1855 erased_cw = le32_to_cpu(buf->erased_cw);
1858 * Check ECC failure for each codeword. ECC failure can
1859 * happen in either of the following conditions
1860 * 1. If number of bitflips are greater than ECC engine
1862 * 2. If this codeword contains all 0xff for which erased
1863 * codeword detection check will be done.
1865 if ((flash & FS_OP_ERR) && (buffer & BS_UNCORRECTABLE_BIT)) {
1867 * For BCH ECC, ignore erased codeword errors, if
1868 * ERASED_CW bits are set.
1870 if (host->bch_enabled) {
1871 erased = (erased_cw & ERASED_CW) == ERASED_CW;
1873 * For RS ECC, HW reports the erased CW by placing
1874 * special characters at certain offsets in the buffer.
1875 * These special characters will be valid only if
1876 * complete page is read i.e. data_buf is not NULL.
1878 } else if (data_buf) {
1879 erased = erased_chunk_check_and_fixup(data_buf,
1886 uncorrectable_cws |= BIT(i);
1888 * Check if MPU or any other operational error (timeout,
1889 * device failure, etc.) happened for this codeword and
1890 * make flash_op_err true. If flash_op_err is set, then
1891 * EIO will be returned for page read.
1893 } else if (flash & (FS_OP_ERR | FS_MPU_ERR)) {
1894 flash_op_err = true;
1896 * No ECC or operational errors happened. Check the number of
1897 * bits corrected and update the ecc_stats.corrected.
1902 stat = buffer & BS_CORRECTABLE_ERR_MSK;
1903 mtd->ecc_stats.corrected += stat;
1904 max_bitflips = max(max_bitflips, stat);
1908 data_buf += data_len;
1910 oob_buf += oob_len + ecc->bytes;
1916 if (!uncorrectable_cws)
1917 return max_bitflips;
1919 return check_for_erased_page(host, data_buf_start, oob_buf_start,
1920 uncorrectable_cws, page,
1925 * helper to perform the actual page read operation, used by ecc->read_page(),
1928 static int read_page_ecc(struct qcom_nand_host *host, u8 *data_buf,
1929 u8 *oob_buf, int page)
1931 struct nand_chip *chip = &host->chip;
1932 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
1933 struct nand_ecc_ctrl *ecc = &chip->ecc;
1934 u8 *data_buf_start = data_buf, *oob_buf_start = oob_buf;
1937 config_nand_page_read(chip);
1939 /* queue cmd descs for each codeword */
1940 for (i = 0; i < ecc->steps; i++) {
1941 int data_size, oob_size;
1943 if (qcom_nandc_is_last_cw(ecc, i)) {
1944 data_size = ecc->size - ((ecc->steps - 1) << 2);
1945 oob_size = (ecc->steps << 2) + host->ecc_bytes_hw +
1948 data_size = host->cw_data;
1949 oob_size = host->ecc_bytes_hw + host->spare_bytes;
1952 if (nandc->props->is_bam) {
1953 if (data_buf && oob_buf) {
1954 nandc_set_read_loc(chip, i, 0, 0, data_size, 0);
1955 nandc_set_read_loc(chip, i, 1, data_size,
1957 } else if (data_buf) {
1958 nandc_set_read_loc(chip, i, 0, 0, data_size, 1);
1960 nandc_set_read_loc(chip, i, 0, data_size,
1965 config_nand_cw_read(chip, true, i);
1968 read_data_dma(nandc, FLASH_BUF_ACC, data_buf,
1972 * when ecc is enabled, the controller doesn't read the real
1973 * or dummy bad block markers in each chunk. To maintain a
1974 * consistent layout across RAW and ECC reads, we just
1975 * leave the real/dummy BBM offsets empty (i.e, filled with
1981 for (j = 0; j < host->bbm_size; j++)
1984 read_data_dma(nandc, FLASH_BUF_ACC + data_size,
1985 oob_buf, oob_size, 0);
1989 data_buf += data_size;
1991 oob_buf += oob_size;
1994 ret = submit_descs(nandc);
1998 dev_err(nandc->dev, "failure to read page/oob\n");
2002 return parse_read_errors(host, data_buf_start, oob_buf_start, page);
2006 * a helper that copies the last step/codeword of a page (containing free oob)
2007 * into our local buffer
2009 static int copy_last_cw(struct qcom_nand_host *host, int page)
2011 struct nand_chip *chip = &host->chip;
2012 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2013 struct nand_ecc_ctrl *ecc = &chip->ecc;
2017 clear_read_regs(nandc);
2019 size = host->use_ecc ? host->cw_data : host->cw_size;
2021 /* prepare a clean read buffer */
2022 memset(nandc->data_buffer, 0xff, size);
2024 set_address(host, host->cw_size * (ecc->steps - 1), page);
2025 update_rw_regs(host, 1, true, ecc->steps - 1);
2027 config_nand_single_cw_page_read(chip, host->use_ecc, ecc->steps - 1);
2029 read_data_dma(nandc, FLASH_BUF_ACC, nandc->data_buffer, size, 0);
2031 ret = submit_descs(nandc);
2033 dev_err(nandc->dev, "failed to copy last codeword\n");
2040 /* implements ecc->read_page() */
2041 static int qcom_nandc_read_page(struct nand_chip *chip, uint8_t *buf,
2042 int oob_required, int page)
2044 struct qcom_nand_host *host = to_qcom_nand_host(chip);
2045 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2046 u8 *data_buf, *oob_buf = NULL;
2048 nand_read_page_op(chip, page, 0, NULL, 0);
2050 oob_buf = oob_required ? chip->oob_poi : NULL;
2052 clear_bam_transaction(nandc);
2054 return read_page_ecc(host, data_buf, oob_buf, page);
2057 /* implements ecc->read_page_raw() */
2058 static int qcom_nandc_read_page_raw(struct nand_chip *chip, uint8_t *buf,
2059 int oob_required, int page)
2061 struct mtd_info *mtd = nand_to_mtd(chip);
2062 struct qcom_nand_host *host = to_qcom_nand_host(chip);
2063 struct nand_ecc_ctrl *ecc = &chip->ecc;
2065 u8 *data_buf = buf, *oob_buf = chip->oob_poi;
2067 for (cw = 0; cw < ecc->steps; cw++) {
2068 ret = qcom_nandc_read_cw_raw(mtd, chip, data_buf, oob_buf,
2073 data_buf += host->cw_data;
2074 oob_buf += ecc->bytes;
2080 /* implements ecc->read_oob() */
2081 static int qcom_nandc_read_oob(struct nand_chip *chip, int page)
2083 struct qcom_nand_host *host = to_qcom_nand_host(chip);
2084 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2085 struct nand_ecc_ctrl *ecc = &chip->ecc;
2087 clear_read_regs(nandc);
2088 clear_bam_transaction(nandc);
2090 host->use_ecc = true;
2091 set_address(host, 0, page);
2092 update_rw_regs(host, ecc->steps, true, 0);
2094 return read_page_ecc(host, NULL, chip->oob_poi, page);
2097 /* implements ecc->write_page() */
2098 static int qcom_nandc_write_page(struct nand_chip *chip, const uint8_t *buf,
2099 int oob_required, int page)
2101 struct qcom_nand_host *host = to_qcom_nand_host(chip);
2102 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2103 struct nand_ecc_ctrl *ecc = &chip->ecc;
2104 u8 *data_buf, *oob_buf;
2107 nand_prog_page_begin_op(chip, page, 0, NULL, 0);
2109 clear_read_regs(nandc);
2110 clear_bam_transaction(nandc);
2112 data_buf = (u8 *)buf;
2113 oob_buf = chip->oob_poi;
2115 host->use_ecc = true;
2116 update_rw_regs(host, ecc->steps, false, 0);
2117 config_nand_page_write(chip);
2119 for (i = 0; i < ecc->steps; i++) {
2120 int data_size, oob_size;
2122 if (qcom_nandc_is_last_cw(ecc, i)) {
2123 data_size = ecc->size - ((ecc->steps - 1) << 2);
2124 oob_size = (ecc->steps << 2) + host->ecc_bytes_hw +
2127 data_size = host->cw_data;
2128 oob_size = ecc->bytes;
2132 write_data_dma(nandc, FLASH_BUF_ACC, data_buf, data_size,
2133 i == (ecc->steps - 1) ? NAND_BAM_NO_EOT : 0);
2136 * when ECC is enabled, we don't really need to write anything
2137 * to oob for the first n - 1 codewords since these oob regions
2138 * just contain ECC bytes that's written by the controller
2139 * itself. For the last codeword, we skip the bbm positions and
2140 * write to the free oob area.
2142 if (qcom_nandc_is_last_cw(ecc, i)) {
2143 oob_buf += host->bbm_size;
2145 write_data_dma(nandc, FLASH_BUF_ACC + data_size,
2146 oob_buf, oob_size, 0);
2149 config_nand_cw_write(chip);
2151 data_buf += data_size;
2152 oob_buf += oob_size;
2155 ret = submit_descs(nandc);
2157 dev_err(nandc->dev, "failure to write page\n");
2162 ret = nand_prog_page_end_op(chip);
2167 /* implements ecc->write_page_raw() */
2168 static int qcom_nandc_write_page_raw(struct nand_chip *chip,
2169 const uint8_t *buf, int oob_required,
2172 struct mtd_info *mtd = nand_to_mtd(chip);
2173 struct qcom_nand_host *host = to_qcom_nand_host(chip);
2174 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2175 struct nand_ecc_ctrl *ecc = &chip->ecc;
2176 u8 *data_buf, *oob_buf;
2179 nand_prog_page_begin_op(chip, page, 0, NULL, 0);
2180 clear_read_regs(nandc);
2181 clear_bam_transaction(nandc);
2183 data_buf = (u8 *)buf;
2184 oob_buf = chip->oob_poi;
2186 host->use_ecc = false;
2187 update_rw_regs(host, ecc->steps, false, 0);
2188 config_nand_page_write(chip);
2190 for (i = 0; i < ecc->steps; i++) {
2191 int data_size1, data_size2, oob_size1, oob_size2;
2192 int reg_off = FLASH_BUF_ACC;
2194 data_size1 = mtd->writesize - host->cw_size * (ecc->steps - 1);
2195 oob_size1 = host->bbm_size;
2197 if (qcom_nandc_is_last_cw(ecc, i)) {
2198 data_size2 = ecc->size - data_size1 -
2199 ((ecc->steps - 1) << 2);
2200 oob_size2 = (ecc->steps << 2) + host->ecc_bytes_hw +
2203 data_size2 = host->cw_data - data_size1;
2204 oob_size2 = host->ecc_bytes_hw + host->spare_bytes;
2207 write_data_dma(nandc, reg_off, data_buf, data_size1,
2209 reg_off += data_size1;
2210 data_buf += data_size1;
2212 write_data_dma(nandc, reg_off, oob_buf, oob_size1,
2214 reg_off += oob_size1;
2215 oob_buf += oob_size1;
2217 write_data_dma(nandc, reg_off, data_buf, data_size2,
2219 reg_off += data_size2;
2220 data_buf += data_size2;
2222 write_data_dma(nandc, reg_off, oob_buf, oob_size2, 0);
2223 oob_buf += oob_size2;
2225 config_nand_cw_write(chip);
2228 ret = submit_descs(nandc);
2230 dev_err(nandc->dev, "failure to write raw page\n");
2235 ret = nand_prog_page_end_op(chip);
2241 * implements ecc->write_oob()
2243 * the NAND controller cannot write only data or only OOB within a codeword
2244 * since ECC is calculated for the combined codeword. So update the OOB from
2245 * chip->oob_poi, and pad the data area with OxFF before writing.
2247 static int qcom_nandc_write_oob(struct nand_chip *chip, int page)
2249 struct mtd_info *mtd = nand_to_mtd(chip);
2250 struct qcom_nand_host *host = to_qcom_nand_host(chip);
2251 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2252 struct nand_ecc_ctrl *ecc = &chip->ecc;
2253 u8 *oob = chip->oob_poi;
2254 int data_size, oob_size;
2257 host->use_ecc = true;
2258 clear_bam_transaction(nandc);
2260 /* calculate the data and oob size for the last codeword/step */
2261 data_size = ecc->size - ((ecc->steps - 1) << 2);
2262 oob_size = mtd->oobavail;
2264 memset(nandc->data_buffer, 0xff, host->cw_data);
2265 /* override new oob content to last codeword */
2266 mtd_ooblayout_get_databytes(mtd, nandc->data_buffer + data_size, oob,
2269 set_address(host, host->cw_size * (ecc->steps - 1), page);
2270 update_rw_regs(host, 1, false, 0);
2272 config_nand_page_write(chip);
2273 write_data_dma(nandc, FLASH_BUF_ACC,
2274 nandc->data_buffer, data_size + oob_size, 0);
2275 config_nand_cw_write(chip);
2277 ret = submit_descs(nandc);
2282 dev_err(nandc->dev, "failure to write oob\n");
2286 return nand_prog_page_end_op(chip);
2289 static int qcom_nandc_block_bad(struct nand_chip *chip, loff_t ofs)
2291 struct mtd_info *mtd = nand_to_mtd(chip);
2292 struct qcom_nand_host *host = to_qcom_nand_host(chip);
2293 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2294 struct nand_ecc_ctrl *ecc = &chip->ecc;
2295 int page, ret, bbpos, bad = 0;
2297 page = (int)(ofs >> chip->page_shift) & chip->pagemask;
2300 * configure registers for a raw sub page read, the address is set to
2301 * the beginning of the last codeword, we don't care about reading ecc
2302 * portion of oob. we just want the first few bytes from this codeword
2303 * that contains the BBM
2305 host->use_ecc = false;
2307 clear_bam_transaction(nandc);
2308 ret = copy_last_cw(host, page);
2312 if (check_flash_errors(host, 1)) {
2313 dev_warn(nandc->dev, "error when trying to read BBM\n");
2317 bbpos = mtd->writesize - host->cw_size * (ecc->steps - 1);
2319 bad = nandc->data_buffer[bbpos] != 0xff;
2321 if (chip->options & NAND_BUSWIDTH_16)
2322 bad = bad || (nandc->data_buffer[bbpos + 1] != 0xff);
2327 static int qcom_nandc_block_markbad(struct nand_chip *chip, loff_t ofs)
2329 struct qcom_nand_host *host = to_qcom_nand_host(chip);
2330 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2331 struct nand_ecc_ctrl *ecc = &chip->ecc;
2334 clear_read_regs(nandc);
2335 clear_bam_transaction(nandc);
2338 * to mark the BBM as bad, we flash the entire last codeword with 0s.
2339 * we don't care about the rest of the content in the codeword since
2340 * we aren't going to use this block again
2342 memset(nandc->data_buffer, 0x00, host->cw_size);
2344 page = (int)(ofs >> chip->page_shift) & chip->pagemask;
2347 host->use_ecc = false;
2348 set_address(host, host->cw_size * (ecc->steps - 1), page);
2349 update_rw_regs(host, 1, false, ecc->steps - 1);
2351 config_nand_page_write(chip);
2352 write_data_dma(nandc, FLASH_BUF_ACC,
2353 nandc->data_buffer, host->cw_size, 0);
2354 config_nand_cw_write(chip);
2356 ret = submit_descs(nandc);
2361 dev_err(nandc->dev, "failure to update BBM\n");
2365 return nand_prog_page_end_op(chip);
2369 * the three functions below implement chip->legacy.read_byte(),
2370 * chip->legacy.read_buf() and chip->legacy.write_buf() respectively. these
2371 * aren't used for reading/writing page data, they are used for smaller data
2372 * like reading id, status etc
2374 static uint8_t qcom_nandc_read_byte(struct nand_chip *chip)
2376 struct qcom_nand_host *host = to_qcom_nand_host(chip);
2377 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2378 u8 *buf = nandc->data_buffer;
2381 if (host->last_command == NAND_CMD_STATUS) {
2384 host->status = NAND_STATUS_READY | NAND_STATUS_WP;
2389 if (nandc->buf_start < nandc->buf_count)
2390 ret = buf[nandc->buf_start++];
2395 static void qcom_nandc_read_buf(struct nand_chip *chip, uint8_t *buf, int len)
2397 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2398 int real_len = min_t(size_t, len, nandc->buf_count - nandc->buf_start);
2400 memcpy(buf, nandc->data_buffer + nandc->buf_start, real_len);
2401 nandc->buf_start += real_len;
2404 static void qcom_nandc_write_buf(struct nand_chip *chip, const uint8_t *buf,
2407 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2408 int real_len = min_t(size_t, len, nandc->buf_count - nandc->buf_start);
2410 memcpy(nandc->data_buffer + nandc->buf_start, buf, real_len);
2412 nandc->buf_start += real_len;
2415 /* we support only one external chip for now */
2416 static void qcom_nandc_select_chip(struct nand_chip *chip, int chipnr)
2418 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2423 dev_warn(nandc->dev, "invalid chip select\n");
2427 * NAND controller page layout info
2429 * Layout with ECC enabled:
2431 * |----------------------| |---------------------------------|
2432 * | xx.......yy| | *********xx.......yy|
2433 * | DATA xx..ECC..yy| | DATA **SPARE**xx..ECC..yy|
2434 * | (516) xx.......yy| | (516-n*4) **(n*4)**xx.......yy|
2435 * | xx.......yy| | *********xx.......yy|
2436 * |----------------------| |---------------------------------|
2437 * codeword 1,2..n-1 codeword n
2438 * <---(528/532 Bytes)--> <-------(528/532 Bytes)--------->
2440 * n = Number of codewords in the page
2442 * * = Spare/free bytes
2443 * x = Unused byte(s)
2444 * y = Reserved byte(s)
2446 * 2K page: n = 4, spare = 16 bytes
2447 * 4K page: n = 8, spare = 32 bytes
2448 * 8K page: n = 16, spare = 64 bytes
2450 * the qcom nand controller operates at a sub page/codeword level. each
2451 * codeword is 528 and 532 bytes for 4 bit and 8 bit ECC modes respectively.
2452 * the number of ECC bytes vary based on the ECC strength and the bus width.
2454 * the first n - 1 codewords contains 516 bytes of user data, the remaining
2455 * 12/16 bytes consist of ECC and reserved data. The nth codeword contains
2456 * both user data and spare(oobavail) bytes that sum up to 516 bytes.
2458 * When we access a page with ECC enabled, the reserved bytes(s) are not
2459 * accessible at all. When reading, we fill up these unreadable positions
2460 * with 0xffs. When writing, the controller skips writing the inaccessible
2463 * Layout with ECC disabled:
2465 * |------------------------------| |---------------------------------------|
2466 * | yy xx.......| | bb *********xx.......|
2467 * | DATA1 yy DATA2 xx..ECC..| | DATA1 bb DATA2 **SPARE**xx..ECC..|
2468 * | (size1) yy (size2) xx.......| | (size1) bb (size2) **(n*4)**xx.......|
2469 * | yy xx.......| | bb *********xx.......|
2470 * |------------------------------| |---------------------------------------|
2471 * codeword 1,2..n-1 codeword n
2472 * <-------(528/532 Bytes)------> <-----------(528/532 Bytes)----------->
2474 * n = Number of codewords in the page
2476 * * = Spare/free bytes
2477 * x = Unused byte(s)
2478 * y = Dummy Bad Bock byte(s)
2479 * b = Real Bad Block byte(s)
2480 * size1/size2 = function of codeword size and 'n'
2482 * when the ECC block is disabled, one reserved byte (or two for 16 bit bus
2483 * width) is now accessible. For the first n - 1 codewords, these are dummy Bad
2484 * Block Markers. In the last codeword, this position contains the real BBM
2486 * In order to have a consistent layout between RAW and ECC modes, we assume
2487 * the following OOB layout arrangement:
2489 * |-----------| |--------------------|
2490 * |yyxx.......| |bb*********xx.......|
2491 * |yyxx..ECC..| |bb*FREEOOB*xx..ECC..|
2492 * |yyxx.......| |bb*********xx.......|
2493 * |yyxx.......| |bb*********xx.......|
2494 * |-----------| |--------------------|
2495 * first n - 1 nth OOB region
2498 * n = Number of codewords in the page
2500 * * = FREE OOB bytes
2501 * y = Dummy bad block byte(s) (inaccessible when ECC enabled)
2502 * x = Unused byte(s)
2503 * b = Real bad block byte(s) (inaccessible when ECC enabled)
2505 * This layout is read as is when ECC is disabled. When ECC is enabled, the
2506 * inaccessible Bad Block byte(s) are ignored when we write to a page/oob,
2507 * and assumed as 0xffs when we read a page/oob. The ECC, unused and
2508 * dummy/real bad block bytes are grouped as ecc bytes (i.e, ecc->bytes is
2509 * the sum of the three).
2511 static int qcom_nand_ooblayout_ecc(struct mtd_info *mtd, int section,
2512 struct mtd_oob_region *oobregion)
2514 struct nand_chip *chip = mtd_to_nand(mtd);
2515 struct qcom_nand_host *host = to_qcom_nand_host(chip);
2516 struct nand_ecc_ctrl *ecc = &chip->ecc;
2522 oobregion->length = (ecc->bytes * (ecc->steps - 1)) +
2524 oobregion->offset = 0;
2526 oobregion->length = host->ecc_bytes_hw + host->spare_bytes;
2527 oobregion->offset = mtd->oobsize - oobregion->length;
2533 static int qcom_nand_ooblayout_free(struct mtd_info *mtd, int section,
2534 struct mtd_oob_region *oobregion)
2536 struct nand_chip *chip = mtd_to_nand(mtd);
2537 struct qcom_nand_host *host = to_qcom_nand_host(chip);
2538 struct nand_ecc_ctrl *ecc = &chip->ecc;
2543 oobregion->length = ecc->steps * 4;
2544 oobregion->offset = ((ecc->steps - 1) * ecc->bytes) + host->bbm_size;
2549 static const struct mtd_ooblayout_ops qcom_nand_ooblayout_ops = {
2550 .ecc = qcom_nand_ooblayout_ecc,
2551 .free = qcom_nand_ooblayout_free,
2555 qcom_nandc_calc_ecc_bytes(int step_size, int strength)
2557 return strength == 4 ? 12 : 16;
2559 NAND_ECC_CAPS_SINGLE(qcom_nandc_ecc_caps, qcom_nandc_calc_ecc_bytes,
2560 NANDC_STEP_SIZE, 4, 8);
2562 static int qcom_nand_attach_chip(struct nand_chip *chip)
2564 struct mtd_info *mtd = nand_to_mtd(chip);
2565 struct qcom_nand_host *host = to_qcom_nand_host(chip);
2566 struct nand_ecc_ctrl *ecc = &chip->ecc;
2567 struct qcom_nand_controller *nandc = get_qcom_nand_controller(chip);
2568 int cwperpage, bad_block_byte, ret;
2572 /* controller only supports 512 bytes data steps */
2573 ecc->size = NANDC_STEP_SIZE;
2574 wide_bus = chip->options & NAND_BUSWIDTH_16 ? true : false;
2575 cwperpage = mtd->writesize / NANDC_STEP_SIZE;
2578 * Each CW has 4 available OOB bytes which will be protected with ECC
2579 * so remaining bytes can be used for ECC.
2581 ret = nand_ecc_choose_conf(chip, &qcom_nandc_ecc_caps,
2582 mtd->oobsize - (cwperpage * 4));
2584 dev_err(nandc->dev, "No valid ECC settings possible\n");
2588 if (ecc->strength >= 8) {
2589 /* 8 bit ECC defaults to BCH ECC on all platforms */
2590 host->bch_enabled = true;
2594 host->ecc_bytes_hw = 14;
2595 host->spare_bytes = 0;
2598 host->ecc_bytes_hw = 13;
2599 host->spare_bytes = 2;
2604 * if the controller supports BCH for 4 bit ECC, the controller
2605 * uses lesser bytes for ECC. If RS is used, the ECC bytes is
2608 if (nandc->props->ecc_modes & ECC_BCH_4BIT) {
2610 host->bch_enabled = true;
2614 host->ecc_bytes_hw = 8;
2615 host->spare_bytes = 2;
2618 host->ecc_bytes_hw = 7;
2619 host->spare_bytes = 4;
2624 host->ecc_bytes_hw = 10;
2627 host->spare_bytes = 0;
2630 host->spare_bytes = 1;
2637 * we consider ecc->bytes as the sum of all the non-data content in a
2638 * step. It gives us a clean representation of the oob area (even if
2639 * all the bytes aren't used for ECC).It is always 16 bytes for 8 bit
2640 * ECC and 12 bytes for 4 bit ECC
2642 ecc->bytes = host->ecc_bytes_hw + host->spare_bytes + host->bbm_size;
2644 ecc->read_page = qcom_nandc_read_page;
2645 ecc->read_page_raw = qcom_nandc_read_page_raw;
2646 ecc->read_oob = qcom_nandc_read_oob;
2647 ecc->write_page = qcom_nandc_write_page;
2648 ecc->write_page_raw = qcom_nandc_write_page_raw;
2649 ecc->write_oob = qcom_nandc_write_oob;
2651 ecc->engine_type = NAND_ECC_ENGINE_TYPE_ON_HOST;
2653 mtd_set_ooblayout(mtd, &qcom_nand_ooblayout_ops);
2654 /* Free the initially allocated BAM transaction for reading the ONFI params */
2655 if (nandc->props->is_bam)
2656 free_bam_transaction(nandc);
2658 nandc->max_cwperpage = max_t(unsigned int, nandc->max_cwperpage,
2661 /* Now allocate the BAM transaction based on updated max_cwperpage */
2662 if (nandc->props->is_bam) {
2663 nandc->bam_txn = alloc_bam_transaction(nandc);
2664 if (!nandc->bam_txn) {
2666 "failed to allocate bam transaction\n");
2672 * DATA_UD_BYTES varies based on whether the read/write command protects
2673 * spare data with ECC too. We protect spare data by default, so we set
2674 * it to main + spare data, which are 512 and 4 bytes respectively.
2676 host->cw_data = 516;
2679 * total bytes in a step, either 528 bytes for 4 bit ECC, or 532 bytes
2682 host->cw_size = host->cw_data + ecc->bytes;
2683 bad_block_byte = mtd->writesize - host->cw_size * (cwperpage - 1) + 1;
2685 host->cfg0 = (cwperpage - 1) << CW_PER_PAGE
2686 | host->cw_data << UD_SIZE_BYTES
2687 | 0 << DISABLE_STATUS_AFTER_WRITE
2688 | 5 << NUM_ADDR_CYCLES
2689 | host->ecc_bytes_hw << ECC_PARITY_SIZE_BYTES_RS
2690 | 0 << STATUS_BFR_READ
2691 | 1 << SET_RD_MODE_AFTER_STATUS
2692 | host->spare_bytes << SPARE_SIZE_BYTES;
2694 host->cfg1 = 7 << NAND_RECOVERY_CYCLES
2695 | 0 << CS_ACTIVE_BSY
2696 | bad_block_byte << BAD_BLOCK_BYTE_NUM
2697 | 0 << BAD_BLOCK_IN_SPARE_AREA
2698 | 2 << WR_RD_BSY_GAP
2699 | wide_bus << WIDE_FLASH
2700 | host->bch_enabled << ENABLE_BCH_ECC;
2702 host->cfg0_raw = (cwperpage - 1) << CW_PER_PAGE
2703 | host->cw_size << UD_SIZE_BYTES
2704 | 5 << NUM_ADDR_CYCLES
2705 | 0 << SPARE_SIZE_BYTES;
2707 host->cfg1_raw = 7 << NAND_RECOVERY_CYCLES
2708 | 0 << CS_ACTIVE_BSY
2709 | 17 << BAD_BLOCK_BYTE_NUM
2710 | 1 << BAD_BLOCK_IN_SPARE_AREA
2711 | 2 << WR_RD_BSY_GAP
2712 | wide_bus << WIDE_FLASH
2713 | 1 << DEV0_CFG1_ECC_DISABLE;
2715 host->ecc_bch_cfg = !host->bch_enabled << ECC_CFG_ECC_DISABLE
2717 | host->cw_data << ECC_NUM_DATA_BYTES
2718 | 1 << ECC_FORCE_CLK_OPEN
2719 | ecc_mode << ECC_MODE
2720 | host->ecc_bytes_hw << ECC_PARITY_SIZE_BYTES_BCH;
2722 if (!nandc->props->qpic_v2)
2723 host->ecc_buf_cfg = 0x203 << NUM_STEPS;
2725 host->clrflashstatus = FS_READY_BSY_N;
2726 host->clrreadstatus = 0xc0;
2727 nandc->regs->erased_cw_detect_cfg_clr =
2728 cpu_to_le32(CLR_ERASED_PAGE_DET);
2729 nandc->regs->erased_cw_detect_cfg_set =
2730 cpu_to_le32(SET_ERASED_PAGE_DET);
2733 "cfg0 %x cfg1 %x ecc_buf_cfg %x ecc_bch cfg %x cw_size %d cw_data %d strength %d parity_bytes %d steps %d\n",
2734 host->cfg0, host->cfg1, host->ecc_buf_cfg, host->ecc_bch_cfg,
2735 host->cw_size, host->cw_data, ecc->strength, ecc->bytes,
2741 static const struct nand_controller_ops qcom_nandc_ops = {
2742 .attach_chip = qcom_nand_attach_chip,
2745 static void qcom_nandc_unalloc(struct qcom_nand_controller *nandc)
2747 if (nandc->props->is_bam) {
2748 if (!dma_mapping_error(nandc->dev, nandc->reg_read_dma))
2749 dma_unmap_single(nandc->dev, nandc->reg_read_dma,
2751 sizeof(*nandc->reg_read_buf),
2755 dma_release_channel(nandc->tx_chan);
2758 dma_release_channel(nandc->rx_chan);
2760 if (nandc->cmd_chan)
2761 dma_release_channel(nandc->cmd_chan);
2764 dma_release_channel(nandc->chan);
2768 static int qcom_nandc_alloc(struct qcom_nand_controller *nandc)
2772 ret = dma_set_coherent_mask(nandc->dev, DMA_BIT_MASK(32));
2774 dev_err(nandc->dev, "failed to set DMA mask\n");
2779 * we use the internal buffer for reading ONFI params, reading small
2780 * data like ID and status, and preforming read-copy-write operations
2781 * when writing to a codeword partially. 532 is the maximum possible
2782 * size of a codeword for our nand controller
2784 nandc->buf_size = 532;
2786 nandc->data_buffer = devm_kzalloc(nandc->dev, nandc->buf_size,
2788 if (!nandc->data_buffer)
2791 nandc->regs = devm_kzalloc(nandc->dev, sizeof(*nandc->regs),
2796 nandc->reg_read_buf = devm_kcalloc(nandc->dev,
2797 MAX_REG_RD, sizeof(*nandc->reg_read_buf),
2799 if (!nandc->reg_read_buf)
2802 if (nandc->props->is_bam) {
2803 nandc->reg_read_dma =
2804 dma_map_single(nandc->dev, nandc->reg_read_buf,
2806 sizeof(*nandc->reg_read_buf),
2808 if (dma_mapping_error(nandc->dev, nandc->reg_read_dma)) {
2809 dev_err(nandc->dev, "failed to DMA MAP reg buffer\n");
2813 nandc->tx_chan = dma_request_chan(nandc->dev, "tx");
2814 if (IS_ERR(nandc->tx_chan)) {
2815 ret = PTR_ERR(nandc->tx_chan);
2816 nandc->tx_chan = NULL;
2817 dev_err_probe(nandc->dev, ret,
2818 "tx DMA channel request failed\n");
2822 nandc->rx_chan = dma_request_chan(nandc->dev, "rx");
2823 if (IS_ERR(nandc->rx_chan)) {
2824 ret = PTR_ERR(nandc->rx_chan);
2825 nandc->rx_chan = NULL;
2826 dev_err_probe(nandc->dev, ret,
2827 "rx DMA channel request failed\n");
2831 nandc->cmd_chan = dma_request_chan(nandc->dev, "cmd");
2832 if (IS_ERR(nandc->cmd_chan)) {
2833 ret = PTR_ERR(nandc->cmd_chan);
2834 nandc->cmd_chan = NULL;
2835 dev_err_probe(nandc->dev, ret,
2836 "cmd DMA channel request failed\n");
2841 * Initially allocate BAM transaction to read ONFI param page.
2842 * After detecting all the devices, this BAM transaction will
2843 * be freed and the next BAM tranasction will be allocated with
2844 * maximum codeword size
2846 nandc->max_cwperpage = 1;
2847 nandc->bam_txn = alloc_bam_transaction(nandc);
2848 if (!nandc->bam_txn) {
2850 "failed to allocate bam transaction\n");
2855 nandc->chan = dma_request_chan(nandc->dev, "rxtx");
2856 if (IS_ERR(nandc->chan)) {
2857 ret = PTR_ERR(nandc->chan);
2859 dev_err_probe(nandc->dev, ret,
2860 "rxtx DMA channel request failed\n");
2865 INIT_LIST_HEAD(&nandc->desc_list);
2866 INIT_LIST_HEAD(&nandc->host_list);
2868 nand_controller_init(&nandc->controller);
2869 nandc->controller.ops = &qcom_nandc_ops;
2873 qcom_nandc_unalloc(nandc);
2877 /* one time setup of a few nand controller registers */
2878 static int qcom_nandc_setup(struct qcom_nand_controller *nandc)
2883 if (!nandc->props->is_qpic)
2884 nandc_write(nandc, SFLASHC_BURST_CFG, 0);
2886 if (!nandc->props->qpic_v2)
2887 nandc_write(nandc, dev_cmd_reg_addr(nandc, NAND_DEV_CMD_VLD),
2888 NAND_DEV_CMD_VLD_VAL);
2890 /* enable ADM or BAM DMA */
2891 if (nandc->props->is_bam) {
2892 nand_ctrl = nandc_read(nandc, NAND_CTRL);
2895 *NAND_CTRL is an operational registers, and CPU
2896 * access to operational registers are read only
2897 * in BAM mode. So update the NAND_CTRL register
2898 * only if it is not in BAM mode. In most cases BAM
2899 * mode will be enabled in bootloader
2901 if (!(nand_ctrl & BAM_MODE_EN))
2902 nandc_write(nandc, NAND_CTRL, nand_ctrl | BAM_MODE_EN);
2904 nandc_write(nandc, NAND_FLASH_CHIP_SELECT, DM_EN);
2907 /* save the original values of these registers */
2908 if (!nandc->props->qpic_v2) {
2909 nandc->cmd1 = nandc_read(nandc, dev_cmd_reg_addr(nandc, NAND_DEV_CMD1));
2910 nandc->vld = NAND_DEV_CMD_VLD_VAL;
2916 static const char * const probes[] = { "cmdlinepart", "ofpart", "qcomsmem", NULL };
2918 static int qcom_nand_host_init_and_register(struct qcom_nand_controller *nandc,
2919 struct qcom_nand_host *host,
2920 struct device_node *dn)
2922 struct nand_chip *chip = &host->chip;
2923 struct mtd_info *mtd = nand_to_mtd(chip);
2924 struct device *dev = nandc->dev;
2927 ret = of_property_read_u32(dn, "reg", &host->cs);
2929 dev_err(dev, "can't get chip-select\n");
2933 nand_set_flash_node(chip, dn);
2934 mtd->name = devm_kasprintf(dev, GFP_KERNEL, "qcom_nand.%d", host->cs);
2938 mtd->owner = THIS_MODULE;
2939 mtd->dev.parent = dev;
2941 chip->legacy.cmdfunc = qcom_nandc_command;
2942 chip->legacy.select_chip = qcom_nandc_select_chip;
2943 chip->legacy.read_byte = qcom_nandc_read_byte;
2944 chip->legacy.read_buf = qcom_nandc_read_buf;
2945 chip->legacy.write_buf = qcom_nandc_write_buf;
2946 chip->legacy.set_features = nand_get_set_features_notsupp;
2947 chip->legacy.get_features = nand_get_set_features_notsupp;
2950 * the bad block marker is readable only when we read the last codeword
2951 * of a page with ECC disabled. currently, the nand_base and nand_bbt
2952 * helpers don't allow us to read BB from a nand chip with ECC
2953 * disabled (MTD_OPS_PLACE_OOB is set by default). use the block_bad
2954 * and block_markbad helpers until we permanently switch to using
2955 * MTD_OPS_RAW for all drivers (with the help of badblockbits)
2957 chip->legacy.block_bad = qcom_nandc_block_bad;
2958 chip->legacy.block_markbad = qcom_nandc_block_markbad;
2960 chip->controller = &nandc->controller;
2961 chip->options |= NAND_NO_SUBPAGE_WRITE | NAND_USES_DMA |
2964 /* set up initial status value */
2965 host->status = NAND_STATUS_READY | NAND_STATUS_WP;
2967 ret = nand_scan(chip, 1);
2971 ret = mtd_device_parse_register(mtd, probes, NULL, NULL, 0);
2978 static int qcom_probe_nand_devices(struct qcom_nand_controller *nandc)
2980 struct device *dev = nandc->dev;
2981 struct device_node *dn = dev->of_node, *child;
2982 struct qcom_nand_host *host;
2985 for_each_available_child_of_node(dn, child) {
2986 host = devm_kzalloc(dev, sizeof(*host), GFP_KERNEL);
2992 ret = qcom_nand_host_init_and_register(nandc, host, child);
2994 devm_kfree(dev, host);
2998 list_add_tail(&host->node, &nandc->host_list);
3004 /* parse custom DT properties here */
3005 static int qcom_nandc_parse_dt(struct platform_device *pdev)
3007 struct qcom_nand_controller *nandc = platform_get_drvdata(pdev);
3008 struct device_node *np = nandc->dev->of_node;
3011 if (!nandc->props->is_bam) {
3012 ret = of_property_read_u32(np, "qcom,cmd-crci",
3015 dev_err(nandc->dev, "command CRCI unspecified\n");
3019 ret = of_property_read_u32(np, "qcom,data-crci",
3022 dev_err(nandc->dev, "data CRCI unspecified\n");
3030 static int qcom_nandc_probe(struct platform_device *pdev)
3032 struct qcom_nand_controller *nandc;
3033 const void *dev_data;
3034 struct device *dev = &pdev->dev;
3035 struct resource *res;
3038 nandc = devm_kzalloc(&pdev->dev, sizeof(*nandc), GFP_KERNEL);
3042 platform_set_drvdata(pdev, nandc);
3045 dev_data = of_device_get_match_data(dev);
3047 dev_err(&pdev->dev, "failed to get device data\n");
3051 nandc->props = dev_data;
3053 nandc->core_clk = devm_clk_get(dev, "core");
3054 if (IS_ERR(nandc->core_clk))
3055 return PTR_ERR(nandc->core_clk);
3057 nandc->aon_clk = devm_clk_get(dev, "aon");
3058 if (IS_ERR(nandc->aon_clk))
3059 return PTR_ERR(nandc->aon_clk);
3061 ret = qcom_nandc_parse_dt(pdev);
3065 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
3066 nandc->base = devm_ioremap_resource(dev, res);
3067 if (IS_ERR(nandc->base))
3068 return PTR_ERR(nandc->base);
3070 nandc->base_phys = res->start;
3071 nandc->base_dma = dma_map_resource(dev, res->start,
3073 DMA_BIDIRECTIONAL, 0);
3074 if (dma_mapping_error(dev, nandc->base_dma))
3077 ret = clk_prepare_enable(nandc->core_clk);
3081 ret = clk_prepare_enable(nandc->aon_clk);
3085 ret = qcom_nandc_alloc(nandc);
3087 goto err_nandc_alloc;
3089 ret = qcom_nandc_setup(nandc);
3093 ret = qcom_probe_nand_devices(nandc);
3100 qcom_nandc_unalloc(nandc);
3102 clk_disable_unprepare(nandc->aon_clk);
3104 clk_disable_unprepare(nandc->core_clk);
3106 dma_unmap_resource(dev, res->start, resource_size(res),
3107 DMA_BIDIRECTIONAL, 0);
3111 static int qcom_nandc_remove(struct platform_device *pdev)
3113 struct qcom_nand_controller *nandc = platform_get_drvdata(pdev);
3114 struct resource *res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
3115 struct qcom_nand_host *host;
3116 struct nand_chip *chip;
3119 list_for_each_entry(host, &nandc->host_list, node) {
3121 ret = mtd_device_unregister(nand_to_mtd(chip));
3126 qcom_nandc_unalloc(nandc);
3128 clk_disable_unprepare(nandc->aon_clk);
3129 clk_disable_unprepare(nandc->core_clk);
3131 dma_unmap_resource(&pdev->dev, nandc->base_dma, resource_size(res),
3132 DMA_BIDIRECTIONAL, 0);
3137 static const struct qcom_nandc_props ipq806x_nandc_props = {
3138 .ecc_modes = (ECC_RS_4BIT | ECC_BCH_8BIT),
3140 .dev_cmd_reg_start = 0x0,
3143 static const struct qcom_nandc_props ipq4019_nandc_props = {
3144 .ecc_modes = (ECC_BCH_4BIT | ECC_BCH_8BIT),
3147 .dev_cmd_reg_start = 0x0,
3150 static const struct qcom_nandc_props ipq8074_nandc_props = {
3151 .ecc_modes = (ECC_BCH_4BIT | ECC_BCH_8BIT),
3154 .dev_cmd_reg_start = 0x7000,
3157 static const struct qcom_nandc_props sdx55_nandc_props = {
3158 .ecc_modes = (ECC_BCH_4BIT | ECC_BCH_8BIT),
3162 .dev_cmd_reg_start = 0x7000,
3166 * data will hold a struct pointer containing more differences once we support
3167 * more controller variants
3169 static const struct of_device_id qcom_nandc_of_match[] = {
3171 .compatible = "qcom,ipq806x-nand",
3172 .data = &ipq806x_nandc_props,
3175 .compatible = "qcom,ipq4019-nand",
3176 .data = &ipq4019_nandc_props,
3179 .compatible = "qcom,ipq6018-nand",
3180 .data = &ipq8074_nandc_props,
3183 .compatible = "qcom,ipq8074-nand",
3184 .data = &ipq8074_nandc_props,
3187 .compatible = "qcom,sdx55-nand",
3188 .data = &sdx55_nandc_props,
3192 MODULE_DEVICE_TABLE(of, qcom_nandc_of_match);
3194 static struct platform_driver qcom_nandc_driver = {
3196 .name = "qcom-nandc",
3197 .of_match_table = qcom_nandc_of_match,
3199 .probe = qcom_nandc_probe,
3200 .remove = qcom_nandc_remove,
3202 module_platform_driver(qcom_nandc_driver);
3204 MODULE_AUTHOR("Archit Taneja <architt@codeaurora.org>");
3205 MODULE_DESCRIPTION("Qualcomm NAND Controller driver");
3206 MODULE_LICENSE("GPL v2");