mtd: rawnand: marvell: Use nand_cleanup() when the device is not yet registered
[linux-2.6-block.git] / drivers / mtd / nand / raw / marvell_nand.c
CommitLineData
02f26ecf
MR
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Marvell NAND flash controller driver
4 *
5 * Copyright (C) 2017 Marvell
6 * Author: Miquel RAYNAL <miquel.raynal@free-electrons.com>
7 *
33c1c5fe
MR
8 *
9 * This NAND controller driver handles two versions of the hardware,
10 * one is called NFCv1 and is available on PXA SoCs and the other is
11 * called NFCv2 and is available on Armada SoCs.
12 *
13 * The main visible difference is that NFCv1 only has Hamming ECC
14 * capabilities, while NFCv2 also embeds a BCH ECC engine. Also, DMA
15 * is not used with NFCv2.
16 *
17 * The ECC layouts are depicted in details in Marvell AN-379, but here
18 * is a brief description.
19 *
20 * When using Hamming, the data is split in 512B chunks (either 1, 2
21 * or 4) and each chunk will have its own ECC "digest" of 6B at the
22 * beginning of the OOB area and eventually the remaining free OOB
23 * bytes (also called "spare" bytes in the driver). This engine
24 * corrects up to 1 bit per chunk and detects reliably an error if
25 * there are at most 2 bitflips. Here is the page layout used by the
26 * controller when Hamming is chosen:
27 *
28 * +-------------------------------------------------------------+
29 * | Data 1 | ... | Data N | ECC 1 | ... | ECCN | Free OOB bytes |
30 * +-------------------------------------------------------------+
31 *
32 * When using the BCH engine, there are N identical (data + free OOB +
33 * ECC) sections and potentially an extra one to deal with
34 * configurations where the chosen (data + free OOB + ECC) sizes do
35 * not align with the page (data + OOB) size. ECC bytes are always
36 * 30B per ECC chunk. Here is the page layout used by the controller
37 * when BCH is chosen:
38 *
39 * +-----------------------------------------
40 * | Data 1 | Free OOB bytes 1 | ECC 1 | ...
41 * +-----------------------------------------
42 *
43 * -------------------------------------------
44 * ... | Data N | Free OOB bytes N | ECC N |
45 * -------------------------------------------
46 *
47 * --------------------------------------------+
48 * Last Data | Last Free OOB bytes | Last ECC |
49 * --------------------------------------------+
50 *
51 * In both cases, the layout seen by the user is always: all data
52 * first, then all free OOB bytes and finally all ECC bytes. With BCH,
53 * ECC bytes are 30B long and are padded with 0xFF to align on 32
54 * bytes.
55 *
56 * The controller has certain limitations that are handled by the
57 * driver:
58 * - It can only read 2k at a time. To overcome this limitation, the
59 * driver issues data cycles on the bus, without issuing new
60 * CMD + ADDR cycles. The Marvell term is "naked" operations.
61 * - The ECC strength in BCH mode cannot be tuned. It is fixed 16
62 * bits. What can be tuned is the ECC block size as long as it
63 * stays between 512B and 2kiB. It's usually chosen based on the
64 * chip ECC requirements. For instance, using 2kiB ECC chunks
65 * provides 4b/512B correctability.
66 * - The controller will always treat data bytes, free OOB bytes
67 * and ECC bytes in that order, no matter what the real layout is
68 * (which is usually all data then all OOB bytes). The
69 * marvell_nfc_layouts array below contains the currently
70 * supported layouts.
71 * - Because of these weird layouts, the Bad Block Markers can be
72 * located in data section. In this case, the NAND_BBT_NO_OOB_BBM
73 * option must be set to prevent scanning/writing bad block
74 * markers.
02f26ecf
MR
75 */
76
77#include <linux/module.h>
78#include <linux/clk.h>
79#include <linux/mtd/rawnand.h>
80#include <linux/of_platform.h>
81#include <linux/iopoll.h>
82#include <linux/interrupt.h>
83#include <linux/slab.h>
84#include <linux/mfd/syscon.h>
85#include <linux/regmap.h>
86#include <asm/unaligned.h>
87
88#include <linux/dmaengine.h>
89#include <linux/dma-mapping.h>
90#include <linux/dma/pxa-dma.h>
91#include <linux/platform_data/mtd-nand-pxa3xx.h>
92
93/* Data FIFO granularity, FIFO reads/writes must be a multiple of this length */
94#define FIFO_DEPTH 8
95#define FIFO_REP(x) (x / sizeof(u32))
96#define BCH_SEQ_READS (32 / FIFO_DEPTH)
97/* NFC does not support transfers of larger chunks at a time */
98#define MAX_CHUNK_SIZE 2112
99/* NFCv1 cannot read more that 7 bytes of ID */
100#define NFCV1_READID_LEN 7
101/* Polling is done at a pace of POLL_PERIOD us until POLL_TIMEOUT is reached */
102#define POLL_PERIOD 0
103#define POLL_TIMEOUT 100000
104/* Interrupt maximum wait period in ms */
105#define IRQ_TIMEOUT 1000
106/* Latency in clock cycles between SoC pins and NFC logic */
107#define MIN_RD_DEL_CNT 3
108/* Maximum number of contiguous address cycles */
109#define MAX_ADDRESS_CYC_NFCV1 5
110#define MAX_ADDRESS_CYC_NFCV2 7
111/* System control registers/bits to enable the NAND controller on some SoCs */
112#define GENCONF_SOC_DEVICE_MUX 0x208
113#define GENCONF_SOC_DEVICE_MUX_NFC_EN BIT(0)
114#define GENCONF_SOC_DEVICE_MUX_ECC_CLK_RST BIT(20)
115#define GENCONF_SOC_DEVICE_MUX_ECC_CORE_RST BIT(21)
116#define GENCONF_SOC_DEVICE_MUX_NFC_INT_EN BIT(25)
117#define GENCONF_CLK_GATING_CTRL 0x220
118#define GENCONF_CLK_GATING_CTRL_ND_GATE BIT(2)
119#define GENCONF_ND_CLK_CTRL 0x700
120#define GENCONF_ND_CLK_CTRL_EN BIT(0)
121
122/* NAND controller data flash control register */
123#define NDCR 0x00
124#define NDCR_ALL_INT GENMASK(11, 0)
125#define NDCR_CS1_CMDDM BIT(7)
126#define NDCR_CS0_CMDDM BIT(8)
127#define NDCR_RDYM BIT(11)
128#define NDCR_ND_ARB_EN BIT(12)
129#define NDCR_RA_START BIT(15)
130#define NDCR_RD_ID_CNT(x) (min_t(unsigned int, x, 0x7) << 16)
131#define NDCR_PAGE_SZ(x) (x >= 2048 ? BIT(24) : 0)
132#define NDCR_DWIDTH_M BIT(26)
133#define NDCR_DWIDTH_C BIT(27)
134#define NDCR_ND_RUN BIT(28)
135#define NDCR_DMA_EN BIT(29)
136#define NDCR_ECC_EN BIT(30)
137#define NDCR_SPARE_EN BIT(31)
138#define NDCR_GENERIC_FIELDS_MASK (~(NDCR_RA_START | NDCR_PAGE_SZ(2048) | \
139 NDCR_DWIDTH_M | NDCR_DWIDTH_C))
140
141/* NAND interface timing parameter 0 register */
142#define NDTR0 0x04
143#define NDTR0_TRP(x) ((min_t(unsigned int, x, 0xF) & 0x7) << 0)
144#define NDTR0_TRH(x) (min_t(unsigned int, x, 0x7) << 3)
145#define NDTR0_ETRP(x) ((min_t(unsigned int, x, 0xF) & 0x8) << 3)
146#define NDTR0_SEL_NRE_EDGE BIT(7)
147#define NDTR0_TWP(x) (min_t(unsigned int, x, 0x7) << 8)
148#define NDTR0_TWH(x) (min_t(unsigned int, x, 0x7) << 11)
149#define NDTR0_TCS(x) (min_t(unsigned int, x, 0x7) << 16)
150#define NDTR0_TCH(x) (min_t(unsigned int, x, 0x7) << 19)
151#define NDTR0_RD_CNT_DEL(x) (min_t(unsigned int, x, 0xF) << 22)
152#define NDTR0_SELCNTR BIT(26)
153#define NDTR0_TADL(x) (min_t(unsigned int, x, 0x1F) << 27)
154
155/* NAND interface timing parameter 1 register */
156#define NDTR1 0x0C
157#define NDTR1_TAR(x) (min_t(unsigned int, x, 0xF) << 0)
158#define NDTR1_TWHR(x) (min_t(unsigned int, x, 0xF) << 4)
159#define NDTR1_TRHW(x) (min_t(unsigned int, x / 16, 0x3) << 8)
160#define NDTR1_PRESCALE BIT(14)
161#define NDTR1_WAIT_MODE BIT(15)
162#define NDTR1_TR(x) (min_t(unsigned int, x, 0xFFFF) << 16)
163
164/* NAND controller status register */
165#define NDSR 0x14
166#define NDSR_WRCMDREQ BIT(0)
167#define NDSR_RDDREQ BIT(1)
168#define NDSR_WRDREQ BIT(2)
169#define NDSR_CORERR BIT(3)
170#define NDSR_UNCERR BIT(4)
171#define NDSR_CMDD(cs) BIT(8 - cs)
172#define NDSR_RDY(rb) BIT(11 + rb)
173#define NDSR_ERRCNT(x) ((x >> 16) & 0x1F)
174
175/* NAND ECC control register */
176#define NDECCCTRL 0x28
177#define NDECCCTRL_BCH_EN BIT(0)
178
179/* NAND controller data buffer register */
180#define NDDB 0x40
181
182/* NAND controller command buffer 0 register */
183#define NDCB0 0x48
184#define NDCB0_CMD1(x) ((x & 0xFF) << 0)
185#define NDCB0_CMD2(x) ((x & 0xFF) << 8)
186#define NDCB0_ADDR_CYC(x) ((x & 0x7) << 16)
187#define NDCB0_ADDR_GET_NUM_CYC(x) (((x) >> 16) & 0x7)
188#define NDCB0_DBC BIT(19)
189#define NDCB0_CMD_TYPE(x) ((x & 0x7) << 21)
190#define NDCB0_CSEL BIT(24)
191#define NDCB0_RDY_BYP BIT(27)
192#define NDCB0_LEN_OVRD BIT(28)
193#define NDCB0_CMD_XTYPE(x) ((x & 0x7) << 29)
194
195/* NAND controller command buffer 1 register */
196#define NDCB1 0x4C
197#define NDCB1_COLS(x) ((x & 0xFFFF) << 0)
198#define NDCB1_ADDRS_PAGE(x) (x << 16)
199
200/* NAND controller command buffer 2 register */
201#define NDCB2 0x50
202#define NDCB2_ADDR5_PAGE(x) (((x >> 16) & 0xFF) << 0)
203#define NDCB2_ADDR5_CYC(x) ((x & 0xFF) << 0)
204
205/* NAND controller command buffer 3 register */
206#define NDCB3 0x54
207#define NDCB3_ADDR6_CYC(x) ((x & 0xFF) << 16)
208#define NDCB3_ADDR7_CYC(x) ((x & 0xFF) << 24)
209
210/* NAND controller command buffer 0 register 'type' and 'xtype' fields */
211#define TYPE_READ 0
212#define TYPE_WRITE 1
213#define TYPE_ERASE 2
214#define TYPE_READ_ID 3
215#define TYPE_STATUS 4
216#define TYPE_RESET 5
217#define TYPE_NAKED_CMD 6
218#define TYPE_NAKED_ADDR 7
219#define TYPE_MASK 7
220#define XTYPE_MONOLITHIC_RW 0
221#define XTYPE_LAST_NAKED_RW 1
222#define XTYPE_FINAL_COMMAND 3
223#define XTYPE_READ 4
224#define XTYPE_WRITE_DISPATCH 4
225#define XTYPE_NAKED_RW 5
226#define XTYPE_COMMAND_DISPATCH 6
227#define XTYPE_MASK 7
228
229/**
230 * Marvell ECC engine works differently than the others, in order to limit the
231 * size of the IP, hardware engineers chose to set a fixed strength at 16 bits
232 * per subpage, and depending on a the desired strength needed by the NAND chip,
233 * a particular layout mixing data/spare/ecc is defined, with a possible last
234 * chunk smaller that the others.
235 *
236 * @writesize: Full page size on which the layout applies
237 * @chunk: Desired ECC chunk size on which the layout applies
238 * @strength: Desired ECC strength (per chunk size bytes) on which the
239 * layout applies
240 * @nchunks: Total number of chunks
241 * @full_chunk_cnt: Number of full-sized chunks, which is the number of
242 * repetitions of the pattern:
243 * (data_bytes + spare_bytes + ecc_bytes).
244 * @data_bytes: Number of data bytes per chunk
245 * @spare_bytes: Number of spare bytes per chunk
246 * @ecc_bytes: Number of ecc bytes per chunk
247 * @last_data_bytes: Number of data bytes in the last chunk
248 * @last_spare_bytes: Number of spare bytes in the last chunk
249 * @last_ecc_bytes: Number of ecc bytes in the last chunk
250 */
251struct marvell_hw_ecc_layout {
252 /* Constraints */
253 int writesize;
254 int chunk;
255 int strength;
256 /* Corresponding layout */
257 int nchunks;
258 int full_chunk_cnt;
259 int data_bytes;
260 int spare_bytes;
261 int ecc_bytes;
262 int last_data_bytes;
263 int last_spare_bytes;
264 int last_ecc_bytes;
265};
266
267#define MARVELL_LAYOUT(ws, dc, ds, nc, fcc, db, sb, eb, ldb, lsb, leb) \
268 { \
269 .writesize = ws, \
270 .chunk = dc, \
271 .strength = ds, \
272 .nchunks = nc, \
273 .full_chunk_cnt = fcc, \
274 .data_bytes = db, \
275 .spare_bytes = sb, \
276 .ecc_bytes = eb, \
277 .last_data_bytes = ldb, \
278 .last_spare_bytes = lsb, \
279 .last_ecc_bytes = leb, \
280 }
281
282/* Layouts explained in AN-379_Marvell_SoC_NFC_ECC */
283static const struct marvell_hw_ecc_layout marvell_nfc_layouts[] = {
284 MARVELL_LAYOUT( 512, 512, 1, 1, 1, 512, 8, 8, 0, 0, 0),
285 MARVELL_LAYOUT( 2048, 512, 1, 1, 1, 2048, 40, 24, 0, 0, 0),
286 MARVELL_LAYOUT( 2048, 512, 4, 1, 1, 2048, 32, 30, 0, 0, 0),
7fd130f7 287 MARVELL_LAYOUT( 2048, 512, 8, 2, 1, 1024, 0, 30,1024,32, 30),
02f26ecf
MR
288 MARVELL_LAYOUT( 4096, 512, 4, 2, 2, 2048, 32, 30, 0, 0, 0),
289 MARVELL_LAYOUT( 4096, 512, 8, 5, 4, 1024, 0, 30, 0, 64, 30),
e8237bfa
KP
290 MARVELL_LAYOUT( 8192, 512, 4, 4, 4, 2048, 0, 30, 0, 0, 0),
291 MARVELL_LAYOUT( 8192, 512, 8, 9, 8, 1024, 0, 30, 0, 160, 30),
02f26ecf
MR
292};
293
294/**
295 * The Nand Flash Controller has up to 4 CE and 2 RB pins. The CE selection
296 * is made by a field in NDCB0 register, and in another field in NDCB2 register.
297 * The datasheet describes the logic with an error: ADDR5 field is once
298 * declared at the beginning of NDCB2, and another time at its end. Because the
299 * ADDR5 field of NDCB2 may be used by other bytes, it would be more logical
300 * to use the last bit of this field instead of the first ones.
301 *
302 * @cs: Wanted CE lane.
303 * @ndcb0_csel: Value of the NDCB0 register with or without the flag
304 * selecting the wanted CE lane. This is set once when
305 * the Device Tree is probed.
306 * @rb: Ready/Busy pin for the flash chip
307 */
308struct marvell_nand_chip_sel {
309 unsigned int cs;
310 u32 ndcb0_csel;
311 unsigned int rb;
312};
313
314/**
315 * NAND chip structure: stores NAND chip device related information
316 *
317 * @chip: Base NAND chip structure
318 * @node: Used to store NAND chips into a list
319 * @layout NAND layout when using hardware ECC
320 * @ndcr: Controller register value for this NAND chip
321 * @ndtr0: Timing registers 0 value for this NAND chip
322 * @ndtr1: Timing registers 1 value for this NAND chip
323 * @selected_die: Current active CS
324 * @nsels: Number of CS lines required by the NAND chip
325 * @sels: Array of CS lines descriptions
326 */
327struct marvell_nand_chip {
328 struct nand_chip chip;
329 struct list_head node;
330 const struct marvell_hw_ecc_layout *layout;
331 u32 ndcr;
332 u32 ndtr0;
333 u32 ndtr1;
334 int addr_cyc;
335 int selected_die;
336 unsigned int nsels;
49f1c330 337 struct marvell_nand_chip_sel sels[];
02f26ecf
MR
338};
339
340static inline struct marvell_nand_chip *to_marvell_nand(struct nand_chip *chip)
341{
342 return container_of(chip, struct marvell_nand_chip, chip);
343}
344
345static inline struct marvell_nand_chip_sel *to_nand_sel(struct marvell_nand_chip
346 *nand)
347{
348 return &nand->sels[nand->selected_die];
349}
350
351/**
352 * NAND controller capabilities for distinction between compatible strings
353 *
354 * @max_cs_nb: Number of Chip Select lines available
355 * @max_rb_nb: Number of Ready/Busy lines available
356 * @need_system_controller: Indicates if the SoC needs to have access to the
357 * system controller (ie. to enable the NAND controller)
358 * @legacy_of_bindings: Indicates if DT parsing must be done using the old
359 * fashion way
360 * @is_nfcv2: NFCv2 has numerous enhancements compared to NFCv1, ie.
361 * BCH error detection and correction algorithm,
362 * NDCB3 register has been added
363 * @use_dma: Use dma for data transfers
364 */
365struct marvell_nfc_caps {
366 unsigned int max_cs_nb;
367 unsigned int max_rb_nb;
368 bool need_system_controller;
369 bool legacy_of_bindings;
370 bool is_nfcv2;
371 bool use_dma;
372};
373
374/**
375 * NAND controller structure: stores Marvell NAND controller information
376 *
377 * @controller: Base controller structure
378 * @dev: Parent device (used to print error messages)
379 * @regs: NAND controller registers
6b6de654 380 * @core_clk: Core clock
1b489eff 381 * @reg_clk: Registers clock
02f26ecf
MR
382 * @complete: Completion object to wait for NAND controller events
383 * @assigned_cs: Bitmask describing already assigned CS lines
384 * @chips: List containing all the NAND chips attached to
385 * this NAND controller
386 * @caps: NAND controller capabilities for each compatible string
387 * @dma_chan: DMA channel (NFCv1 only)
388 * @dma_buf: 32-bit aligned buffer for DMA transfers (NFCv1 only)
389 */
390struct marvell_nfc {
7da45139 391 struct nand_controller controller;
02f26ecf
MR
392 struct device *dev;
393 void __iomem *regs;
6b6de654 394 struct clk *core_clk;
961ba15c 395 struct clk *reg_clk;
02f26ecf
MR
396 struct completion complete;
397 unsigned long assigned_cs;
398 struct list_head chips;
399 struct nand_chip *selected_chip;
400 const struct marvell_nfc_caps *caps;
401
402 /* DMA (NFCv1 only) */
403 bool use_dma;
404 struct dma_chan *dma_chan;
405 u8 *dma_buf;
406};
407
7da45139 408static inline struct marvell_nfc *to_marvell_nfc(struct nand_controller *ctrl)
02f26ecf
MR
409{
410 return container_of(ctrl, struct marvell_nfc, controller);
411}
412
413/**
414 * NAND controller timings expressed in NAND Controller clock cycles
415 *
416 * @tRP: ND_nRE pulse width
417 * @tRH: ND_nRE high duration
418 * @tWP: ND_nWE pulse time
419 * @tWH: ND_nWE high duration
420 * @tCS: Enable signal setup time
421 * @tCH: Enable signal hold time
422 * @tADL: Address to write data delay
423 * @tAR: ND_ALE low to ND_nRE low delay
424 * @tWHR: ND_nWE high to ND_nRE low for status read
425 * @tRHW: ND_nRE high duration, read to write delay
426 * @tR: ND_nWE high to ND_nRE low for read
427 */
428struct marvell_nfc_timings {
429 /* NDTR0 fields */
430 unsigned int tRP;
431 unsigned int tRH;
432 unsigned int tWP;
433 unsigned int tWH;
434 unsigned int tCS;
435 unsigned int tCH;
436 unsigned int tADL;
437 /* NDTR1 fields */
438 unsigned int tAR;
439 unsigned int tWHR;
440 unsigned int tRHW;
441 unsigned int tR;
442};
443
444/**
445 * Derives a duration in numbers of clock cycles.
446 *
447 * @ps: Duration in pico-seconds
448 * @period_ns: Clock period in nano-seconds
449 *
450 * Convert the duration in nano-seconds, then divide by the period and
451 * return the number of clock periods.
452 */
453#define TO_CYCLES(ps, period_ns) (DIV_ROUND_UP(ps / 1000, period_ns))
07ad5a72
MR
454#define TO_CYCLES64(ps, period_ns) (DIV_ROUND_UP_ULL(div_u64(ps, 1000), \
455 period_ns))
02f26ecf
MR
456
457/**
458 * NAND driver structure filled during the parsing of the ->exec_op() subop
459 * subset of instructions.
460 *
461 * @ndcb: Array of values written to NDCBx registers
462 * @cle_ale_delay_ns: Optional delay after the last CMD or ADDR cycle
463 * @rdy_timeout_ms: Timeout for waits on Ready/Busy pin
464 * @rdy_delay_ns: Optional delay after waiting for the RB pin
465 * @data_delay_ns: Optional delay after the data xfer
466 * @data_instr_idx: Index of the data instruction in the subop
467 * @data_instr: Pointer to the data instruction in the subop
468 */
469struct marvell_nfc_op {
470 u32 ndcb[4];
471 unsigned int cle_ale_delay_ns;
472 unsigned int rdy_timeout_ms;
473 unsigned int rdy_delay_ns;
474 unsigned int data_delay_ns;
475 unsigned int data_instr_idx;
476 const struct nand_op_instr *data_instr;
477};
478
479/*
480 * Internal helper to conditionnally apply a delay (from the above structure,
481 * most of the time).
482 */
483static void cond_delay(unsigned int ns)
484{
485 if (!ns)
486 return;
487
488 if (ns < 10000)
489 ndelay(ns);
490 else
491 udelay(DIV_ROUND_UP(ns, 1000));
492}
493
494/*
495 * The controller has many flags that could generate interrupts, most of them
496 * are disabled and polling is used. For the very slow signals, using interrupts
497 * may relax the CPU charge.
498 */
499static void marvell_nfc_disable_int(struct marvell_nfc *nfc, u32 int_mask)
500{
501 u32 reg;
502
503 /* Writing 1 disables the interrupt */
504 reg = readl_relaxed(nfc->regs + NDCR);
505 writel_relaxed(reg | int_mask, nfc->regs + NDCR);
506}
507
508static void marvell_nfc_enable_int(struct marvell_nfc *nfc, u32 int_mask)
509{
510 u32 reg;
511
512 /* Writing 0 enables the interrupt */
513 reg = readl_relaxed(nfc->regs + NDCR);
514 writel_relaxed(reg & ~int_mask, nfc->regs + NDCR);
515}
516
cafb56dd 517static u32 marvell_nfc_clear_int(struct marvell_nfc *nfc, u32 int_mask)
02f26ecf 518{
cafb56dd
MR
519 u32 reg;
520
521 reg = readl_relaxed(nfc->regs + NDSR);
02f26ecf 522 writel_relaxed(int_mask, nfc->regs + NDSR);
cafb56dd
MR
523
524 return reg & int_mask;
02f26ecf
MR
525}
526
527static void marvell_nfc_force_byte_access(struct nand_chip *chip,
528 bool force_8bit)
529{
530 struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
531 u32 ndcr;
532
533 /*
534 * Callers of this function do not verify if the NAND is using a 16-bit
535 * an 8-bit bus for normal operations, so we need to take care of that
536 * here by leaving the configuration unchanged if the NAND does not have
537 * the NAND_BUSWIDTH_16 flag set.
538 */
539 if (!(chip->options & NAND_BUSWIDTH_16))
540 return;
541
542 ndcr = readl_relaxed(nfc->regs + NDCR);
543
544 if (force_8bit)
545 ndcr &= ~(NDCR_DWIDTH_M | NDCR_DWIDTH_C);
546 else
547 ndcr |= NDCR_DWIDTH_M | NDCR_DWIDTH_C;
548
549 writel_relaxed(ndcr, nfc->regs + NDCR);
550}
551
552static int marvell_nfc_wait_ndrun(struct nand_chip *chip)
553{
554 struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
555 u32 val;
556 int ret;
557
558 /*
559 * The command is being processed, wait for the ND_RUN bit to be
560 * cleared by the NFC. If not, we must clear it by hand.
561 */
562 ret = readl_relaxed_poll_timeout(nfc->regs + NDCR, val,
563 (val & NDCR_ND_RUN) == 0,
564 POLL_PERIOD, POLL_TIMEOUT);
565 if (ret) {
566 dev_err(nfc->dev, "Timeout on NAND controller run mode\n");
567 writel_relaxed(readl(nfc->regs + NDCR) & ~NDCR_ND_RUN,
568 nfc->regs + NDCR);
569 return ret;
570 }
571
572 return 0;
573}
574
575/*
576 * Any time a command has to be sent to the controller, the following sequence
577 * has to be followed:
578 * - call marvell_nfc_prepare_cmd()
579 * -> activate the ND_RUN bit that will kind of 'start a job'
580 * -> wait the signal indicating the NFC is waiting for a command
581 * - send the command (cmd and address cycles)
582 * - enventually send or receive the data
583 * - call marvell_nfc_end_cmd() with the corresponding flag
584 * -> wait the flag to be triggered or cancel the job with a timeout
585 *
586 * The following helpers are here to factorize the code a bit so that
587 * specialized functions responsible for executing the actual NAND
588 * operations do not have to replicate the same code blocks.
589 */
590static int marvell_nfc_prepare_cmd(struct nand_chip *chip)
591{
592 struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
593 u32 ndcr, val;
594 int ret;
595
596 /* Poll ND_RUN and clear NDSR before issuing any command */
597 ret = marvell_nfc_wait_ndrun(chip);
598 if (ret) {
a76497dc 599 dev_err(nfc->dev, "Last operation did not succeed\n");
02f26ecf
MR
600 return ret;
601 }
602
603 ndcr = readl_relaxed(nfc->regs + NDCR);
604 writel_relaxed(readl(nfc->regs + NDSR), nfc->regs + NDSR);
605
606 /* Assert ND_RUN bit and wait the NFC to be ready */
607 writel_relaxed(ndcr | NDCR_ND_RUN, nfc->regs + NDCR);
608 ret = readl_relaxed_poll_timeout(nfc->regs + NDSR, val,
609 val & NDSR_WRCMDREQ,
610 POLL_PERIOD, POLL_TIMEOUT);
611 if (ret) {
612 dev_err(nfc->dev, "Timeout on WRCMDRE\n");
613 return -ETIMEDOUT;
614 }
615
616 /* Command may be written, clear WRCMDREQ status bit */
617 writel_relaxed(NDSR_WRCMDREQ, nfc->regs + NDSR);
618
619 return 0;
620}
621
622static void marvell_nfc_send_cmd(struct nand_chip *chip,
623 struct marvell_nfc_op *nfc_op)
624{
625 struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
626 struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
627
628 dev_dbg(nfc->dev, "\nNDCR: 0x%08x\n"
629 "NDCB0: 0x%08x\nNDCB1: 0x%08x\nNDCB2: 0x%08x\nNDCB3: 0x%08x\n",
630 (u32)readl_relaxed(nfc->regs + NDCR), nfc_op->ndcb[0],
631 nfc_op->ndcb[1], nfc_op->ndcb[2], nfc_op->ndcb[3]);
632
633 writel_relaxed(to_nand_sel(marvell_nand)->ndcb0_csel | nfc_op->ndcb[0],
634 nfc->regs + NDCB0);
635 writel_relaxed(nfc_op->ndcb[1], nfc->regs + NDCB0);
636 writel(nfc_op->ndcb[2], nfc->regs + NDCB0);
637
638 /*
639 * Write NDCB0 four times only if LEN_OVRD is set or if ADDR6 or ADDR7
640 * fields are used (only available on NFCv2).
641 */
642 if (nfc_op->ndcb[0] & NDCB0_LEN_OVRD ||
643 NDCB0_ADDR_GET_NUM_CYC(nfc_op->ndcb[0]) >= 6) {
644 if (!WARN_ON_ONCE(!nfc->caps->is_nfcv2))
645 writel(nfc_op->ndcb[3], nfc->regs + NDCB0);
646 }
647}
648
649static int marvell_nfc_end_cmd(struct nand_chip *chip, int flag,
650 const char *label)
651{
652 struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
653 u32 val;
654 int ret;
655
656 ret = readl_relaxed_poll_timeout(nfc->regs + NDSR, val,
657 val & flag,
658 POLL_PERIOD, POLL_TIMEOUT);
659
660 if (ret) {
661 dev_err(nfc->dev, "Timeout on %s (NDSR: 0x%08x)\n",
662 label, val);
663 if (nfc->dma_chan)
664 dmaengine_terminate_all(nfc->dma_chan);
665 return ret;
666 }
667
668 /*
669 * DMA function uses this helper to poll on CMDD bits without wanting
670 * them to be cleared.
671 */
672 if (nfc->use_dma && (readl_relaxed(nfc->regs + NDCR) & NDCR_DMA_EN))
673 return 0;
674
675 writel_relaxed(flag, nfc->regs + NDSR);
676
677 return 0;
678}
679
680static int marvell_nfc_wait_cmdd(struct nand_chip *chip)
681{
682 struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
683 int cs_flag = NDSR_CMDD(to_nand_sel(marvell_nand)->ndcb0_csel);
684
685 return marvell_nfc_end_cmd(chip, cs_flag, "CMDD");
686}
687
688static int marvell_nfc_wait_op(struct nand_chip *chip, unsigned int timeout_ms)
689{
690 struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
cafb56dd 691 u32 pending;
02f26ecf
MR
692 int ret;
693
694 /* Timeout is expressed in ms */
695 if (!timeout_ms)
696 timeout_ms = IRQ_TIMEOUT;
697
698 init_completion(&nfc->complete);
699
700 marvell_nfc_enable_int(nfc, NDCR_RDYM);
701 ret = wait_for_completion_timeout(&nfc->complete,
702 msecs_to_jiffies(timeout_ms));
703 marvell_nfc_disable_int(nfc, NDCR_RDYM);
cafb56dd
MR
704 pending = marvell_nfc_clear_int(nfc, NDSR_RDY(0) | NDSR_RDY(1));
705
706 /*
707 * In case the interrupt was not served in the required time frame,
708 * check if the ISR was not served or if something went actually wrong.
709 */
c2707577 710 if (!ret && !pending) {
02f26ecf
MR
711 dev_err(nfc->dev, "Timeout waiting for RB signal\n");
712 return -ETIMEDOUT;
713 }
714
715 return 0;
716}
717
b2525141
BB
718static void marvell_nfc_select_target(struct nand_chip *chip,
719 unsigned int die_nr)
02f26ecf 720{
02f26ecf
MR
721 struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
722 struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
723 u32 ndcr_generic;
724
02f26ecf
MR
725 /*
726 * Reset the NDCR register to a clean state for this particular chip,
727 * also clear ND_RUN bit.
728 */
729 ndcr_generic = readl_relaxed(nfc->regs + NDCR) &
730 NDCR_GENERIC_FIELDS_MASK & ~NDCR_ND_RUN;
731 writel_relaxed(ndcr_generic | marvell_nand->ndcr, nfc->regs + NDCR);
732
733 /* Also reset the interrupt status register */
734 marvell_nfc_clear_int(nfc, NDCR_ALL_INT);
735
9a8f612c
MR
736 if (chip == nfc->selected_chip && die_nr == marvell_nand->selected_die)
737 return;
738
739 writel_relaxed(marvell_nand->ndtr0, nfc->regs + NDTR0);
740 writel_relaxed(marvell_nand->ndtr1, nfc->regs + NDTR1);
741
02f26ecf
MR
742 nfc->selected_chip = chip;
743 marvell_nand->selected_die = die_nr;
744}
745
746static irqreturn_t marvell_nfc_isr(int irq, void *dev_id)
747{
748 struct marvell_nfc *nfc = dev_id;
749 u32 st = readl_relaxed(nfc->regs + NDSR);
750 u32 ien = (~readl_relaxed(nfc->regs + NDCR)) & NDCR_ALL_INT;
751
752 /*
753 * RDY interrupt mask is one bit in NDCR while there are two status
754 * bit in NDSR (RDY[cs0/cs2] and RDY[cs1/cs3]).
755 */
756 if (st & NDSR_RDY(1))
757 st |= NDSR_RDY(0);
758
759 if (!(st & ien))
760 return IRQ_NONE;
761
762 marvell_nfc_disable_int(nfc, st & NDCR_ALL_INT);
763
53c83b59 764 if (st & (NDSR_RDY(0) | NDSR_RDY(1)))
02f26ecf
MR
765 complete(&nfc->complete);
766
767 return IRQ_HANDLED;
768}
769
770/* HW ECC related functions */
771static void marvell_nfc_enable_hw_ecc(struct nand_chip *chip)
772{
773 struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
774 u32 ndcr = readl_relaxed(nfc->regs + NDCR);
775
776 if (!(ndcr & NDCR_ECC_EN)) {
777 writel_relaxed(ndcr | NDCR_ECC_EN, nfc->regs + NDCR);
778
779 /*
780 * When enabling BCH, set threshold to 0 to always know the
781 * number of corrected bitflips.
782 */
783 if (chip->ecc.algo == NAND_ECC_BCH)
784 writel_relaxed(NDECCCTRL_BCH_EN, nfc->regs + NDECCCTRL);
785 }
786}
787
788static void marvell_nfc_disable_hw_ecc(struct nand_chip *chip)
789{
790 struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
791 u32 ndcr = readl_relaxed(nfc->regs + NDCR);
792
793 if (ndcr & NDCR_ECC_EN) {
794 writel_relaxed(ndcr & ~NDCR_ECC_EN, nfc->regs + NDCR);
795 if (chip->ecc.algo == NAND_ECC_BCH)
796 writel_relaxed(0, nfc->regs + NDECCCTRL);
797 }
798}
799
800/* DMA related helpers */
801static void marvell_nfc_enable_dma(struct marvell_nfc *nfc)
802{
803 u32 reg;
804
805 reg = readl_relaxed(nfc->regs + NDCR);
806 writel_relaxed(reg | NDCR_DMA_EN, nfc->regs + NDCR);
807}
808
809static void marvell_nfc_disable_dma(struct marvell_nfc *nfc)
810{
811 u32 reg;
812
813 reg = readl_relaxed(nfc->regs + NDCR);
814 writel_relaxed(reg & ~NDCR_DMA_EN, nfc->regs + NDCR);
815}
816
817/* Read/write PIO/DMA accessors */
818static int marvell_nfc_xfer_data_dma(struct marvell_nfc *nfc,
819 enum dma_data_direction direction,
820 unsigned int len)
821{
822 unsigned int dma_len = min_t(int, ALIGN(len, 32), MAX_CHUNK_SIZE);
823 struct dma_async_tx_descriptor *tx;
824 struct scatterlist sg;
825 dma_cookie_t cookie;
826 int ret;
827
828 marvell_nfc_enable_dma(nfc);
829 /* Prepare the DMA transfer */
830 sg_init_one(&sg, nfc->dma_buf, dma_len);
831 dma_map_sg(nfc->dma_chan->device->dev, &sg, 1, direction);
832 tx = dmaengine_prep_slave_sg(nfc->dma_chan, &sg, 1,
833 direction == DMA_FROM_DEVICE ?
834 DMA_DEV_TO_MEM : DMA_MEM_TO_DEV,
835 DMA_PREP_INTERRUPT);
836 if (!tx) {
837 dev_err(nfc->dev, "Could not prepare DMA S/G list\n");
838 return -ENXIO;
839 }
840
841 /* Do the task and wait for it to finish */
842 cookie = dmaengine_submit(tx);
843 ret = dma_submit_error(cookie);
844 if (ret)
845 return -EIO;
846
847 dma_async_issue_pending(nfc->dma_chan);
848 ret = marvell_nfc_wait_cmdd(nfc->selected_chip);
849 dma_unmap_sg(nfc->dma_chan->device->dev, &sg, 1, direction);
850 marvell_nfc_disable_dma(nfc);
851 if (ret) {
852 dev_err(nfc->dev, "Timeout waiting for DMA (status: %d)\n",
853 dmaengine_tx_status(nfc->dma_chan, cookie, NULL));
854 dmaengine_terminate_all(nfc->dma_chan);
855 return -ETIMEDOUT;
856 }
857
858 return 0;
859}
860
861static int marvell_nfc_xfer_data_in_pio(struct marvell_nfc *nfc, u8 *in,
862 unsigned int len)
863{
864 unsigned int last_len = len % FIFO_DEPTH;
865 unsigned int last_full_offset = round_down(len, FIFO_DEPTH);
866 int i;
867
868 for (i = 0; i < last_full_offset; i += FIFO_DEPTH)
869 ioread32_rep(nfc->regs + NDDB, in + i, FIFO_REP(FIFO_DEPTH));
870
871 if (last_len) {
872 u8 tmp_buf[FIFO_DEPTH];
873
874 ioread32_rep(nfc->regs + NDDB, tmp_buf, FIFO_REP(FIFO_DEPTH));
875 memcpy(in + last_full_offset, tmp_buf, last_len);
876 }
877
878 return 0;
879}
880
881static int marvell_nfc_xfer_data_out_pio(struct marvell_nfc *nfc, const u8 *out,
882 unsigned int len)
883{
884 unsigned int last_len = len % FIFO_DEPTH;
885 unsigned int last_full_offset = round_down(len, FIFO_DEPTH);
886 int i;
887
888 for (i = 0; i < last_full_offset; i += FIFO_DEPTH)
889 iowrite32_rep(nfc->regs + NDDB, out + i, FIFO_REP(FIFO_DEPTH));
890
891 if (last_len) {
892 u8 tmp_buf[FIFO_DEPTH];
893
894 memcpy(tmp_buf, out + last_full_offset, last_len);
895 iowrite32_rep(nfc->regs + NDDB, tmp_buf, FIFO_REP(FIFO_DEPTH));
896 }
897
898 return 0;
899}
900
901static void marvell_nfc_check_empty_chunk(struct nand_chip *chip,
902 u8 *data, int data_len,
903 u8 *spare, int spare_len,
904 u8 *ecc, int ecc_len,
905 unsigned int *max_bitflips)
906{
907 struct mtd_info *mtd = nand_to_mtd(chip);
908 int bf;
909
910 /*
911 * Blank pages (all 0xFF) that have not been written may be recognized
912 * as bad if bitflips occur, so whenever an uncorrectable error occurs,
913 * check if the entire page (with ECC bytes) is actually blank or not.
914 */
915 if (!data)
916 data_len = 0;
917 if (!spare)
918 spare_len = 0;
919 if (!ecc)
920 ecc_len = 0;
921
922 bf = nand_check_erased_ecc_chunk(data, data_len, ecc, ecc_len,
923 spare, spare_len, chip->ecc.strength);
924 if (bf < 0) {
925 mtd->ecc_stats.failed++;
926 return;
927 }
928
929 /* Update the stats and max_bitflips */
930 mtd->ecc_stats.corrected += bf;
931 *max_bitflips = max_t(unsigned int, *max_bitflips, bf);
932}
933
934/*
935 * Check a chunk is correct or not according to hardware ECC engine.
936 * mtd->ecc_stats.corrected is updated, as well as max_bitflips, however
937 * mtd->ecc_stats.failure is not, the function will instead return a non-zero
938 * value indicating that a check on the emptyness of the subpage must be
939 * performed before declaring the subpage corrupted.
940 */
941static int marvell_nfc_hw_ecc_correct(struct nand_chip *chip,
942 unsigned int *max_bitflips)
943{
944 struct mtd_info *mtd = nand_to_mtd(chip);
945 struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
946 int bf = 0;
947 u32 ndsr;
948
949 ndsr = readl_relaxed(nfc->regs + NDSR);
950
951 /* Check uncorrectable error flag */
952 if (ndsr & NDSR_UNCERR) {
953 writel_relaxed(ndsr, nfc->regs + NDSR);
954
955 /*
956 * Do not increment ->ecc_stats.failed now, instead, return a
957 * non-zero value to indicate that this chunk was apparently
958 * bad, and it should be check to see if it empty or not. If
959 * the chunk (with ECC bytes) is not declared empty, the calling
960 * function must increment the failure count.
961 */
962 return -EBADMSG;
963 }
964
965 /* Check correctable error flag */
966 if (ndsr & NDSR_CORERR) {
967 writel_relaxed(ndsr, nfc->regs + NDSR);
968
969 if (chip->ecc.algo == NAND_ECC_BCH)
970 bf = NDSR_ERRCNT(ndsr);
971 else
972 bf = 1;
973 }
974
975 /* Update the stats and max_bitflips */
976 mtd->ecc_stats.corrected += bf;
977 *max_bitflips = max_t(unsigned int, *max_bitflips, bf);
978
979 return 0;
980}
981
982/* Hamming read helpers */
983static int marvell_nfc_hw_ecc_hmg_do_read_page(struct nand_chip *chip,
984 u8 *data_buf, u8 *oob_buf,
985 bool raw, int page)
986{
987 struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
988 struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
989 const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
990 struct marvell_nfc_op nfc_op = {
991 .ndcb[0] = NDCB0_CMD_TYPE(TYPE_READ) |
992 NDCB0_ADDR_CYC(marvell_nand->addr_cyc) |
993 NDCB0_DBC |
994 NDCB0_CMD1(NAND_CMD_READ0) |
995 NDCB0_CMD2(NAND_CMD_READSTART),
996 .ndcb[1] = NDCB1_ADDRS_PAGE(page),
997 .ndcb[2] = NDCB2_ADDR5_PAGE(page),
998 };
999 unsigned int oob_bytes = lt->spare_bytes + (raw ? lt->ecc_bytes : 0);
1000 int ret;
1001
1002 /* NFCv2 needs more information about the operation being executed */
1003 if (nfc->caps->is_nfcv2)
1004 nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_MONOLITHIC_RW);
1005
1006 ret = marvell_nfc_prepare_cmd(chip);
1007 if (ret)
1008 return ret;
1009
1010 marvell_nfc_send_cmd(chip, &nfc_op);
1011 ret = marvell_nfc_end_cmd(chip, NDSR_RDDREQ,
1012 "RDDREQ while draining FIFO (data/oob)");
1013 if (ret)
1014 return ret;
1015
1016 /*
1017 * Read the page then the OOB area. Unlike what is shown in current
1018 * documentation, spare bytes are protected by the ECC engine, and must
1019 * be at the beginning of the OOB area or running this driver on legacy
1020 * systems will prevent the discovery of the BBM/BBT.
1021 */
1022 if (nfc->use_dma) {
1023 marvell_nfc_xfer_data_dma(nfc, DMA_FROM_DEVICE,
1024 lt->data_bytes + oob_bytes);
1025 memcpy(data_buf, nfc->dma_buf, lt->data_bytes);
1026 memcpy(oob_buf, nfc->dma_buf + lt->data_bytes, oob_bytes);
1027 } else {
1028 marvell_nfc_xfer_data_in_pio(nfc, data_buf, lt->data_bytes);
1029 marvell_nfc_xfer_data_in_pio(nfc, oob_buf, oob_bytes);
1030 }
1031
1032 ret = marvell_nfc_wait_cmdd(chip);
02f26ecf
MR
1033 return ret;
1034}
1035
b9761687 1036static int marvell_nfc_hw_ecc_hmg_read_page_raw(struct nand_chip *chip, u8 *buf,
02f26ecf
MR
1037 int oob_required, int page)
1038{
b2525141 1039 marvell_nfc_select_target(chip, chip->cur_cs);
02f26ecf
MR
1040 return marvell_nfc_hw_ecc_hmg_do_read_page(chip, buf, chip->oob_poi,
1041 true, page);
1042}
1043
b9761687
BB
1044static int marvell_nfc_hw_ecc_hmg_read_page(struct nand_chip *chip, u8 *buf,
1045 int oob_required, int page)
02f26ecf
MR
1046{
1047 const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1048 unsigned int full_sz = lt->data_bytes + lt->spare_bytes + lt->ecc_bytes;
1049 int max_bitflips = 0, ret;
1050 u8 *raw_buf;
1051
b2525141 1052 marvell_nfc_select_target(chip, chip->cur_cs);
02f26ecf
MR
1053 marvell_nfc_enable_hw_ecc(chip);
1054 marvell_nfc_hw_ecc_hmg_do_read_page(chip, buf, chip->oob_poi, false,
1055 page);
1056 ret = marvell_nfc_hw_ecc_correct(chip, &max_bitflips);
1057 marvell_nfc_disable_hw_ecc(chip);
1058
1059 if (!ret)
1060 return max_bitflips;
1061
1062 /*
1063 * When ECC failures are detected, check if the full page has been
1064 * written or not. Ignore the failure if it is actually empty.
1065 */
1066 raw_buf = kmalloc(full_sz, GFP_KERNEL);
1067 if (!raw_buf)
1068 return -ENOMEM;
1069
1070 marvell_nfc_hw_ecc_hmg_do_read_page(chip, raw_buf, raw_buf +
1071 lt->data_bytes, true, page);
1072 marvell_nfc_check_empty_chunk(chip, raw_buf, full_sz, NULL, 0, NULL, 0,
1073 &max_bitflips);
1074 kfree(raw_buf);
1075
1076 return max_bitflips;
1077}
1078
1079/*
1080 * Spare area in Hamming layouts is not protected by the ECC engine (even if
1081 * it appears before the ECC bytes when reading), the ->read_oob_raw() function
1082 * also stands for ->read_oob().
1083 */
b9761687 1084static int marvell_nfc_hw_ecc_hmg_read_oob_raw(struct nand_chip *chip, int page)
02f26ecf 1085{
eeab7174 1086 u8 *buf = nand_get_data_buf(chip);
02f26ecf 1087
b2525141 1088 marvell_nfc_select_target(chip, chip->cur_cs);
eeab7174
BB
1089 return marvell_nfc_hw_ecc_hmg_do_read_page(chip, buf, chip->oob_poi,
1090 true, page);
02f26ecf
MR
1091}
1092
1093/* Hamming write helpers */
1094static int marvell_nfc_hw_ecc_hmg_do_write_page(struct nand_chip *chip,
1095 const u8 *data_buf,
1096 const u8 *oob_buf, bool raw,
1097 int page)
1098{
1099 struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
1100 struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1101 const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1102 struct marvell_nfc_op nfc_op = {
1103 .ndcb[0] = NDCB0_CMD_TYPE(TYPE_WRITE) |
1104 NDCB0_ADDR_CYC(marvell_nand->addr_cyc) |
1105 NDCB0_CMD1(NAND_CMD_SEQIN) |
1106 NDCB0_CMD2(NAND_CMD_PAGEPROG) |
1107 NDCB0_DBC,
1108 .ndcb[1] = NDCB1_ADDRS_PAGE(page),
1109 .ndcb[2] = NDCB2_ADDR5_PAGE(page),
1110 };
1111 unsigned int oob_bytes = lt->spare_bytes + (raw ? lt->ecc_bytes : 0);
1112 int ret;
1113
1114 /* NFCv2 needs more information about the operation being executed */
1115 if (nfc->caps->is_nfcv2)
1116 nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_MONOLITHIC_RW);
1117
1118 ret = marvell_nfc_prepare_cmd(chip);
1119 if (ret)
1120 return ret;
1121
1122 marvell_nfc_send_cmd(chip, &nfc_op);
1123 ret = marvell_nfc_end_cmd(chip, NDSR_WRDREQ,
1124 "WRDREQ while loading FIFO (data)");
1125 if (ret)
1126 return ret;
1127
1128 /* Write the page then the OOB area */
1129 if (nfc->use_dma) {
1130 memcpy(nfc->dma_buf, data_buf, lt->data_bytes);
1131 memcpy(nfc->dma_buf + lt->data_bytes, oob_buf, oob_bytes);
1132 marvell_nfc_xfer_data_dma(nfc, DMA_TO_DEVICE, lt->data_bytes +
1133 lt->ecc_bytes + lt->spare_bytes);
1134 } else {
1135 marvell_nfc_xfer_data_out_pio(nfc, data_buf, lt->data_bytes);
1136 marvell_nfc_xfer_data_out_pio(nfc, oob_buf, oob_bytes);
1137 }
1138
1139 ret = marvell_nfc_wait_cmdd(chip);
1140 if (ret)
1141 return ret;
1142
1143 ret = marvell_nfc_wait_op(chip,
b76401fc 1144 PSEC_TO_MSEC(chip->data_interface.timings.sdr.tPROG_max));
02f26ecf
MR
1145 return ret;
1146}
1147
767eb6fb 1148static int marvell_nfc_hw_ecc_hmg_write_page_raw(struct nand_chip *chip,
02f26ecf
MR
1149 const u8 *buf,
1150 int oob_required, int page)
1151{
b2525141 1152 marvell_nfc_select_target(chip, chip->cur_cs);
02f26ecf
MR
1153 return marvell_nfc_hw_ecc_hmg_do_write_page(chip, buf, chip->oob_poi,
1154 true, page);
1155}
1156
767eb6fb 1157static int marvell_nfc_hw_ecc_hmg_write_page(struct nand_chip *chip,
02f26ecf
MR
1158 const u8 *buf,
1159 int oob_required, int page)
1160{
1161 int ret;
1162
b2525141 1163 marvell_nfc_select_target(chip, chip->cur_cs);
02f26ecf
MR
1164 marvell_nfc_enable_hw_ecc(chip);
1165 ret = marvell_nfc_hw_ecc_hmg_do_write_page(chip, buf, chip->oob_poi,
1166 false, page);
1167 marvell_nfc_disable_hw_ecc(chip);
1168
1169 return ret;
1170}
1171
1172/*
1173 * Spare area in Hamming layouts is not protected by the ECC engine (even if
1174 * it appears before the ECC bytes when reading), the ->write_oob_raw() function
1175 * also stands for ->write_oob().
1176 */
767eb6fb 1177static int marvell_nfc_hw_ecc_hmg_write_oob_raw(struct nand_chip *chip,
02f26ecf
MR
1178 int page)
1179{
767eb6fb 1180 struct mtd_info *mtd = nand_to_mtd(chip);
eeab7174 1181 u8 *buf = nand_get_data_buf(chip);
767eb6fb 1182
eeab7174 1183 memset(buf, 0xFF, mtd->writesize);
02f26ecf 1184
b2525141 1185 marvell_nfc_select_target(chip, chip->cur_cs);
eeab7174
BB
1186 return marvell_nfc_hw_ecc_hmg_do_write_page(chip, buf, chip->oob_poi,
1187 true, page);
02f26ecf
MR
1188}
1189
1190/* BCH read helpers */
b9761687 1191static int marvell_nfc_hw_ecc_bch_read_page_raw(struct nand_chip *chip, u8 *buf,
02f26ecf
MR
1192 int oob_required, int page)
1193{
b9761687 1194 struct mtd_info *mtd = nand_to_mtd(chip);
02f26ecf
MR
1195 const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1196 u8 *oob = chip->oob_poi;
1197 int chunk_size = lt->data_bytes + lt->spare_bytes + lt->ecc_bytes;
1198 int ecc_offset = (lt->full_chunk_cnt * lt->spare_bytes) +
1199 lt->last_spare_bytes;
1200 int data_len = lt->data_bytes;
1201 int spare_len = lt->spare_bytes;
1202 int ecc_len = lt->ecc_bytes;
1203 int chunk;
1204
b2525141
BB
1205 marvell_nfc_select_target(chip, chip->cur_cs);
1206
02f26ecf
MR
1207 if (oob_required)
1208 memset(chip->oob_poi, 0xFF, mtd->oobsize);
1209
1210 nand_read_page_op(chip, page, 0, NULL, 0);
1211
1212 for (chunk = 0; chunk < lt->nchunks; chunk++) {
1213 /* Update last chunk length */
1214 if (chunk >= lt->full_chunk_cnt) {
1215 data_len = lt->last_data_bytes;
1216 spare_len = lt->last_spare_bytes;
1217 ecc_len = lt->last_ecc_bytes;
1218 }
1219
1220 /* Read data bytes*/
1221 nand_change_read_column_op(chip, chunk * chunk_size,
1222 buf + (lt->data_bytes * chunk),
1223 data_len, false);
1224
1225 /* Read spare bytes */
1226 nand_read_data_op(chip, oob + (lt->spare_bytes * chunk),
1227 spare_len, false);
1228
1229 /* Read ECC bytes */
1230 nand_read_data_op(chip, oob + ecc_offset +
1231 (ALIGN(lt->ecc_bytes, 32) * chunk),
1232 ecc_len, false);
1233 }
1234
1235 return 0;
1236}
1237
1238static void marvell_nfc_hw_ecc_bch_read_chunk(struct nand_chip *chip, int chunk,
1239 u8 *data, unsigned int data_len,
1240 u8 *spare, unsigned int spare_len,
1241 int page)
1242{
1243 struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
1244 struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1245 const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1246 int i, ret;
1247 struct marvell_nfc_op nfc_op = {
1248 .ndcb[0] = NDCB0_CMD_TYPE(TYPE_READ) |
1249 NDCB0_ADDR_CYC(marvell_nand->addr_cyc) |
1250 NDCB0_LEN_OVRD,
1251 .ndcb[1] = NDCB1_ADDRS_PAGE(page),
1252 .ndcb[2] = NDCB2_ADDR5_PAGE(page),
1253 .ndcb[3] = data_len + spare_len,
1254 };
1255
1256 ret = marvell_nfc_prepare_cmd(chip);
1257 if (ret)
1258 return;
1259
1260 if (chunk == 0)
1261 nfc_op.ndcb[0] |= NDCB0_DBC |
1262 NDCB0_CMD1(NAND_CMD_READ0) |
1263 NDCB0_CMD2(NAND_CMD_READSTART);
1264
1265 /*
90d61763
BB
1266 * Trigger the monolithic read on the first chunk, then naked read on
1267 * intermediate chunks and finally a last naked read on the last chunk.
02f26ecf 1268 */
90d61763 1269 if (chunk == 0)
02f26ecf 1270 nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_MONOLITHIC_RW);
90d61763
BB
1271 else if (chunk < lt->nchunks - 1)
1272 nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_NAKED_RW);
02f26ecf
MR
1273 else
1274 nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_LAST_NAKED_RW);
1275
1276 marvell_nfc_send_cmd(chip, &nfc_op);
1277
1278 /*
1279 * According to the datasheet, when reading from NDDB
1280 * with BCH enabled, after each 32 bytes reads, we
1281 * have to make sure that the NDSR.RDDREQ bit is set.
1282 *
1283 * Drain the FIFO, 8 32-bit reads at a time, and skip
1284 * the polling on the last read.
1285 *
1286 * Length is a multiple of 32 bytes, hence it is a multiple of 8 too.
1287 */
1288 for (i = 0; i < data_len; i += FIFO_DEPTH * BCH_SEQ_READS) {
1289 marvell_nfc_end_cmd(chip, NDSR_RDDREQ,
1290 "RDDREQ while draining FIFO (data)");
1291 marvell_nfc_xfer_data_in_pio(nfc, data,
1292 FIFO_DEPTH * BCH_SEQ_READS);
1293 data += FIFO_DEPTH * BCH_SEQ_READS;
1294 }
1295
1296 for (i = 0; i < spare_len; i += FIFO_DEPTH * BCH_SEQ_READS) {
1297 marvell_nfc_end_cmd(chip, NDSR_RDDREQ,
1298 "RDDREQ while draining FIFO (OOB)");
1299 marvell_nfc_xfer_data_in_pio(nfc, spare,
1300 FIFO_DEPTH * BCH_SEQ_READS);
1301 spare += FIFO_DEPTH * BCH_SEQ_READS;
1302 }
1303}
1304
b9761687 1305static int marvell_nfc_hw_ecc_bch_read_page(struct nand_chip *chip,
02f26ecf
MR
1306 u8 *buf, int oob_required,
1307 int page)
1308{
b9761687 1309 struct mtd_info *mtd = nand_to_mtd(chip);
02f26ecf 1310 const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
dbfc6718
MR
1311 int data_len = lt->data_bytes, spare_len = lt->spare_bytes;
1312 u8 *data = buf, *spare = chip->oob_poi;
02f26ecf
MR
1313 int max_bitflips = 0;
1314 u32 failure_mask = 0;
dbfc6718 1315 int chunk, ret;
02f26ecf 1316
b2525141
BB
1317 marvell_nfc_select_target(chip, chip->cur_cs);
1318
02f26ecf
MR
1319 /*
1320 * With BCH, OOB is not fully used (and thus not read entirely), not
1321 * expected bytes could show up at the end of the OOB buffer if not
1322 * explicitly erased.
1323 */
1324 if (oob_required)
1325 memset(chip->oob_poi, 0xFF, mtd->oobsize);
1326
1327 marvell_nfc_enable_hw_ecc(chip);
1328
1329 for (chunk = 0; chunk < lt->nchunks; chunk++) {
1330 /* Update length for the last chunk */
1331 if (chunk >= lt->full_chunk_cnt) {
1332 data_len = lt->last_data_bytes;
1333 spare_len = lt->last_spare_bytes;
1334 }
1335
1336 /* Read the chunk and detect number of bitflips */
1337 marvell_nfc_hw_ecc_bch_read_chunk(chip, chunk, data, data_len,
1338 spare, spare_len, page);
1339 ret = marvell_nfc_hw_ecc_correct(chip, &max_bitflips);
1340 if (ret)
1341 failure_mask |= BIT(chunk);
1342
1343 data += data_len;
1344 spare += spare_len;
1345 }
1346
1347 marvell_nfc_disable_hw_ecc(chip);
1348
1349 if (!failure_mask)
1350 return max_bitflips;
1351
1352 /*
1353 * Please note that dumping the ECC bytes during a normal read with OOB
1354 * area would add a significant overhead as ECC bytes are "consumed" by
1355 * the controller in normal mode and must be re-read in raw mode. To
1356 * avoid dropping the performances, we prefer not to include them. The
1357 * user should re-read the page in raw mode if ECC bytes are required.
dbfc6718
MR
1358 */
1359
1360 /*
1361 * In case there is any subpage read error reported by ->correct(), we
1362 * usually re-read only ECC bytes in raw mode and check if the whole
1363 * page is empty. In this case, it is normal that the ECC check failed
1364 * and we just ignore the error.
02f26ecf 1365 *
7fd130f7
MR
1366 * However, it has been empirically observed that for some layouts (e.g
1367 * 2k page, 8b strength per 512B chunk), the controller tries to correct
1368 * bits and may create itself bitflips in the erased area. To overcome
1369 * this strange behavior, the whole page is re-read in raw mode, not
1370 * only the ECC bytes.
02f26ecf
MR
1371 */
1372 for (chunk = 0; chunk < lt->nchunks; chunk++) {
dbfc6718
MR
1373 int data_off_in_page, spare_off_in_page, ecc_off_in_page;
1374 int data_off, spare_off, ecc_off;
1375 int data_len, spare_len, ecc_len;
1376
02f26ecf
MR
1377 /* No failure reported for this chunk, move to the next one */
1378 if (!(failure_mask & BIT(chunk)))
1379 continue;
1380
dbfc6718
MR
1381 data_off_in_page = chunk * (lt->data_bytes + lt->spare_bytes +
1382 lt->ecc_bytes);
1383 spare_off_in_page = data_off_in_page +
1384 (chunk < lt->full_chunk_cnt ? lt->data_bytes :
1385 lt->last_data_bytes);
1386 ecc_off_in_page = spare_off_in_page +
1387 (chunk < lt->full_chunk_cnt ? lt->spare_bytes :
1388 lt->last_spare_bytes);
1389
1390 data_off = chunk * lt->data_bytes;
1391 spare_off = chunk * lt->spare_bytes;
1392 ecc_off = (lt->full_chunk_cnt * lt->spare_bytes) +
1393 lt->last_spare_bytes +
1394 (chunk * (lt->ecc_bytes + 2));
1395
1396 data_len = chunk < lt->full_chunk_cnt ? lt->data_bytes :
1397 lt->last_data_bytes;
1398 spare_len = chunk < lt->full_chunk_cnt ? lt->spare_bytes :
1399 lt->last_spare_bytes;
1400 ecc_len = chunk < lt->full_chunk_cnt ? lt->ecc_bytes :
1401 lt->last_ecc_bytes;
1402
7fd130f7
MR
1403 /*
1404 * Only re-read the ECC bytes, unless we are using the 2k/8b
1405 * layout which is buggy in the sense that the ECC engine will
1406 * try to correct data bytes anyway, creating bitflips. In this
1407 * case, re-read the entire page.
1408 */
1409 if (lt->writesize == 2048 && lt->strength == 8) {
1410 nand_change_read_column_op(chip, data_off_in_page,
1411 buf + data_off, data_len,
1412 false);
1413 nand_change_read_column_op(chip, spare_off_in_page,
1414 chip->oob_poi + spare_off, spare_len,
1415 false);
1416 }
1417
dbfc6718
MR
1418 nand_change_read_column_op(chip, ecc_off_in_page,
1419 chip->oob_poi + ecc_off, ecc_len,
1420 false);
02f26ecf
MR
1421
1422 /* Check the entire chunk (data + spare + ecc) for emptyness */
dbfc6718
MR
1423 marvell_nfc_check_empty_chunk(chip, buf + data_off, data_len,
1424 chip->oob_poi + spare_off, spare_len,
1425 chip->oob_poi + ecc_off, ecc_len,
02f26ecf
MR
1426 &max_bitflips);
1427 }
1428
1429 return max_bitflips;
1430}
1431
b9761687 1432static int marvell_nfc_hw_ecc_bch_read_oob_raw(struct nand_chip *chip, int page)
02f26ecf 1433{
eeab7174 1434 u8 *buf = nand_get_data_buf(chip);
02f26ecf 1435
eeab7174 1436 return chip->ecc.read_page_raw(chip, buf, true, page);
02f26ecf
MR
1437}
1438
b9761687 1439static int marvell_nfc_hw_ecc_bch_read_oob(struct nand_chip *chip, int page)
02f26ecf 1440{
eeab7174 1441 u8 *buf = nand_get_data_buf(chip);
02f26ecf 1442
eeab7174 1443 return chip->ecc.read_page(chip, buf, true, page);
02f26ecf
MR
1444}
1445
1446/* BCH write helpers */
767eb6fb 1447static int marvell_nfc_hw_ecc_bch_write_page_raw(struct nand_chip *chip,
02f26ecf
MR
1448 const u8 *buf,
1449 int oob_required, int page)
1450{
1451 const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1452 int full_chunk_size = lt->data_bytes + lt->spare_bytes + lt->ecc_bytes;
1453 int data_len = lt->data_bytes;
1454 int spare_len = lt->spare_bytes;
1455 int ecc_len = lt->ecc_bytes;
02f26ecf
MR
1456 int spare_offset = 0;
1457 int ecc_offset = (lt->full_chunk_cnt * lt->spare_bytes) +
1458 lt->last_spare_bytes;
1459 int chunk;
1460
b2525141
BB
1461 marvell_nfc_select_target(chip, chip->cur_cs);
1462
02f26ecf
MR
1463 nand_prog_page_begin_op(chip, page, 0, NULL, 0);
1464
1465 for (chunk = 0; chunk < lt->nchunks; chunk++) {
1466 if (chunk >= lt->full_chunk_cnt) {
1467 data_len = lt->last_data_bytes;
1468 spare_len = lt->last_spare_bytes;
1469 ecc_len = lt->last_ecc_bytes;
02f26ecf
MR
1470 }
1471
1472 /* Point to the column of the next chunk */
1473 nand_change_write_column_op(chip, chunk * full_chunk_size,
1474 NULL, 0, false);
1475
1476 /* Write the data */
1477 nand_write_data_op(chip, buf + (chunk * lt->data_bytes),
1478 data_len, false);
1479
1480 if (!oob_required)
1481 continue;
1482
1483 /* Write the spare bytes */
1484 if (spare_len)
1485 nand_write_data_op(chip, chip->oob_poi + spare_offset,
1486 spare_len, false);
1487
1488 /* Write the ECC bytes */
1489 if (ecc_len)
1490 nand_write_data_op(chip, chip->oob_poi + ecc_offset,
1491 ecc_len, false);
1492
1493 spare_offset += spare_len;
1494 ecc_offset += ALIGN(ecc_len, 32);
1495 }
1496
1497 return nand_prog_page_end_op(chip);
1498}
1499
1500static int
1501marvell_nfc_hw_ecc_bch_write_chunk(struct nand_chip *chip, int chunk,
1502 const u8 *data, unsigned int data_len,
1503 const u8 *spare, unsigned int spare_len,
1504 int page)
1505{
1506 struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
1507 struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1508 const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
a2ee41fd 1509 u32 xtype;
02f26ecf
MR
1510 int ret;
1511 struct marvell_nfc_op nfc_op = {
1512 .ndcb[0] = NDCB0_CMD_TYPE(TYPE_WRITE) | NDCB0_LEN_OVRD,
1513 .ndcb[3] = data_len + spare_len,
1514 };
1515
1516 /*
1517 * First operation dispatches the CMD_SEQIN command, issue the address
1518 * cycles and asks for the first chunk of data.
1519 * All operations in the middle (if any) will issue a naked write and
1520 * also ask for data.
1521 * Last operation (if any) asks for the last chunk of data through a
1522 * last naked write.
1523 */
1524 if (chunk == 0) {
a2ee41fd
MR
1525 if (lt->nchunks == 1)
1526 xtype = XTYPE_MONOLITHIC_RW;
1527 else
1528 xtype = XTYPE_WRITE_DISPATCH;
1529
1530 nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(xtype) |
02f26ecf
MR
1531 NDCB0_ADDR_CYC(marvell_nand->addr_cyc) |
1532 NDCB0_CMD1(NAND_CMD_SEQIN);
1533 nfc_op.ndcb[1] |= NDCB1_ADDRS_PAGE(page);
1534 nfc_op.ndcb[2] |= NDCB2_ADDR5_PAGE(page);
1535 } else if (chunk < lt->nchunks - 1) {
1536 nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_NAKED_RW);
1537 } else {
1538 nfc_op.ndcb[0] |= NDCB0_CMD_XTYPE(XTYPE_LAST_NAKED_RW);
1539 }
1540
1541 /* Always dispatch the PAGEPROG command on the last chunk */
1542 if (chunk == lt->nchunks - 1)
1543 nfc_op.ndcb[0] |= NDCB0_CMD2(NAND_CMD_PAGEPROG) | NDCB0_DBC;
1544
1545 ret = marvell_nfc_prepare_cmd(chip);
1546 if (ret)
1547 return ret;
1548
1549 marvell_nfc_send_cmd(chip, &nfc_op);
1550 ret = marvell_nfc_end_cmd(chip, NDSR_WRDREQ,
1551 "WRDREQ while loading FIFO (data)");
1552 if (ret)
1553 return ret;
1554
1555 /* Transfer the contents */
1556 iowrite32_rep(nfc->regs + NDDB, data, FIFO_REP(data_len));
1557 iowrite32_rep(nfc->regs + NDDB, spare, FIFO_REP(spare_len));
1558
1559 return 0;
1560}
1561
767eb6fb 1562static int marvell_nfc_hw_ecc_bch_write_page(struct nand_chip *chip,
02f26ecf
MR
1563 const u8 *buf,
1564 int oob_required, int page)
1565{
767eb6fb 1566 struct mtd_info *mtd = nand_to_mtd(chip);
02f26ecf
MR
1567 const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
1568 const u8 *data = buf;
1569 const u8 *spare = chip->oob_poi;
1570 int data_len = lt->data_bytes;
1571 int spare_len = lt->spare_bytes;
1572 int chunk, ret;
1573
b2525141
BB
1574 marvell_nfc_select_target(chip, chip->cur_cs);
1575
02f26ecf
MR
1576 /* Spare data will be written anyway, so clear it to avoid garbage */
1577 if (!oob_required)
1578 memset(chip->oob_poi, 0xFF, mtd->oobsize);
1579
1580 marvell_nfc_enable_hw_ecc(chip);
1581
1582 for (chunk = 0; chunk < lt->nchunks; chunk++) {
1583 if (chunk >= lt->full_chunk_cnt) {
1584 data_len = lt->last_data_bytes;
1585 spare_len = lt->last_spare_bytes;
1586 }
1587
1588 marvell_nfc_hw_ecc_bch_write_chunk(chip, chunk, data, data_len,
1589 spare, spare_len, page);
1590 data += data_len;
1591 spare += spare_len;
1592
1593 /*
1594 * Waiting only for CMDD or PAGED is not enough, ECC are
1595 * partially written. No flag is set once the operation is
1596 * really finished but the ND_RUN bit is cleared, so wait for it
1597 * before stepping into the next command.
1598 */
1599 marvell_nfc_wait_ndrun(chip);
1600 }
1601
1602 ret = marvell_nfc_wait_op(chip,
b76401fc 1603 PSEC_TO_MSEC(chip->data_interface.timings.sdr.tPROG_max));
02f26ecf
MR
1604
1605 marvell_nfc_disable_hw_ecc(chip);
1606
1607 if (ret)
1608 return ret;
1609
1610 return 0;
1611}
1612
767eb6fb 1613static int marvell_nfc_hw_ecc_bch_write_oob_raw(struct nand_chip *chip,
02f26ecf
MR
1614 int page)
1615{
767eb6fb 1616 struct mtd_info *mtd = nand_to_mtd(chip);
eeab7174 1617 u8 *buf = nand_get_data_buf(chip);
767eb6fb 1618
eeab7174 1619 memset(buf, 0xFF, mtd->writesize);
02f26ecf 1620
eeab7174 1621 return chip->ecc.write_page_raw(chip, buf, true, page);
02f26ecf
MR
1622}
1623
767eb6fb 1624static int marvell_nfc_hw_ecc_bch_write_oob(struct nand_chip *chip, int page)
02f26ecf 1625{
767eb6fb 1626 struct mtd_info *mtd = nand_to_mtd(chip);
eeab7174 1627 u8 *buf = nand_get_data_buf(chip);
767eb6fb 1628
eeab7174 1629 memset(buf, 0xFF, mtd->writesize);
02f26ecf 1630
eeab7174 1631 return chip->ecc.write_page(chip, buf, true, page);
02f26ecf
MR
1632}
1633
1634/* NAND framework ->exec_op() hooks and related helpers */
1635static void marvell_nfc_parse_instructions(struct nand_chip *chip,
1636 const struct nand_subop *subop,
1637 struct marvell_nfc_op *nfc_op)
1638{
1639 const struct nand_op_instr *instr = NULL;
1640 struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1641 bool first_cmd = true;
1642 unsigned int op_id;
1643 int i;
1644
1645 /* Reset the input structure as most of its fields will be OR'ed */
1646 memset(nfc_op, 0, sizeof(struct marvell_nfc_op));
1647
1648 for (op_id = 0; op_id < subop->ninstrs; op_id++) {
1649 unsigned int offset, naddrs;
1650 const u8 *addrs;
21a26806 1651 int len;
02f26ecf
MR
1652
1653 instr = &subop->instrs[op_id];
1654
1655 switch (instr->type) {
1656 case NAND_OP_CMD_INSTR:
1657 if (first_cmd)
1658 nfc_op->ndcb[0] |=
1659 NDCB0_CMD1(instr->ctx.cmd.opcode);
1660 else
1661 nfc_op->ndcb[0] |=
1662 NDCB0_CMD2(instr->ctx.cmd.opcode) |
1663 NDCB0_DBC;
1664
1665 nfc_op->cle_ale_delay_ns = instr->delay_ns;
1666 first_cmd = false;
1667 break;
1668
1669 case NAND_OP_ADDR_INSTR:
1670 offset = nand_subop_get_addr_start_off(subop, op_id);
1671 naddrs = nand_subop_get_num_addr_cyc(subop, op_id);
1672 addrs = &instr->ctx.addr.addrs[offset];
1673
1674 nfc_op->ndcb[0] |= NDCB0_ADDR_CYC(naddrs);
1675
1676 for (i = 0; i < min_t(unsigned int, 4, naddrs); i++)
1677 nfc_op->ndcb[1] |= addrs[i] << (8 * i);
1678
1679 if (naddrs >= 5)
1680 nfc_op->ndcb[2] |= NDCB2_ADDR5_CYC(addrs[4]);
1681 if (naddrs >= 6)
1682 nfc_op->ndcb[3] |= NDCB3_ADDR6_CYC(addrs[5]);
1683 if (naddrs == 7)
1684 nfc_op->ndcb[3] |= NDCB3_ADDR7_CYC(addrs[6]);
1685
1686 nfc_op->cle_ale_delay_ns = instr->delay_ns;
1687 break;
1688
1689 case NAND_OP_DATA_IN_INSTR:
1690 nfc_op->data_instr = instr;
1691 nfc_op->data_instr_idx = op_id;
1692 nfc_op->ndcb[0] |= NDCB0_CMD_TYPE(TYPE_READ);
1693 if (nfc->caps->is_nfcv2) {
1694 nfc_op->ndcb[0] |=
1695 NDCB0_CMD_XTYPE(XTYPE_MONOLITHIC_RW) |
1696 NDCB0_LEN_OVRD;
21a26806 1697 len = nand_subop_get_data_len(subop, op_id);
02f26ecf
MR
1698 nfc_op->ndcb[3] |= round_up(len, FIFO_DEPTH);
1699 }
1700 nfc_op->data_delay_ns = instr->delay_ns;
1701 break;
1702
1703 case NAND_OP_DATA_OUT_INSTR:
1704 nfc_op->data_instr = instr;
1705 nfc_op->data_instr_idx = op_id;
1706 nfc_op->ndcb[0] |= NDCB0_CMD_TYPE(TYPE_WRITE);
1707 if (nfc->caps->is_nfcv2) {
1708 nfc_op->ndcb[0] |=
1709 NDCB0_CMD_XTYPE(XTYPE_MONOLITHIC_RW) |
1710 NDCB0_LEN_OVRD;
21a26806 1711 len = nand_subop_get_data_len(subop, op_id);
02f26ecf
MR
1712 nfc_op->ndcb[3] |= round_up(len, FIFO_DEPTH);
1713 }
1714 nfc_op->data_delay_ns = instr->delay_ns;
1715 break;
1716
1717 case NAND_OP_WAITRDY_INSTR:
1718 nfc_op->rdy_timeout_ms = instr->ctx.waitrdy.timeout_ms;
1719 nfc_op->rdy_delay_ns = instr->delay_ns;
1720 break;
1721 }
1722 }
1723}
1724
1725static int marvell_nfc_xfer_data_pio(struct nand_chip *chip,
1726 const struct nand_subop *subop,
1727 struct marvell_nfc_op *nfc_op)
1728{
1729 struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1730 const struct nand_op_instr *instr = nfc_op->data_instr;
1731 unsigned int op_id = nfc_op->data_instr_idx;
1732 unsigned int len = nand_subop_get_data_len(subop, op_id);
1733 unsigned int offset = nand_subop_get_data_start_off(subop, op_id);
1734 bool reading = (instr->type == NAND_OP_DATA_IN_INSTR);
1735 int ret;
1736
1737 if (instr->ctx.data.force_8bit)
1738 marvell_nfc_force_byte_access(chip, true);
1739
1740 if (reading) {
1741 u8 *in = instr->ctx.data.buf.in + offset;
1742
1743 ret = marvell_nfc_xfer_data_in_pio(nfc, in, len);
1744 } else {
1745 const u8 *out = instr->ctx.data.buf.out + offset;
1746
1747 ret = marvell_nfc_xfer_data_out_pio(nfc, out, len);
1748 }
1749
1750 if (instr->ctx.data.force_8bit)
1751 marvell_nfc_force_byte_access(chip, false);
1752
1753 return ret;
1754}
1755
1756static int marvell_nfc_monolithic_access_exec(struct nand_chip *chip,
1757 const struct nand_subop *subop)
1758{
1759 struct marvell_nfc_op nfc_op;
1760 bool reading;
1761 int ret;
1762
1763 marvell_nfc_parse_instructions(chip, subop, &nfc_op);
1764 reading = (nfc_op.data_instr->type == NAND_OP_DATA_IN_INSTR);
1765
1766 ret = marvell_nfc_prepare_cmd(chip);
1767 if (ret)
1768 return ret;
1769
1770 marvell_nfc_send_cmd(chip, &nfc_op);
1771 ret = marvell_nfc_end_cmd(chip, NDSR_RDDREQ | NDSR_WRDREQ,
1772 "RDDREQ/WRDREQ while draining raw data");
1773 if (ret)
1774 return ret;
1775
1776 cond_delay(nfc_op.cle_ale_delay_ns);
1777
1778 if (reading) {
1779 if (nfc_op.rdy_timeout_ms) {
1780 ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
1781 if (ret)
1782 return ret;
1783 }
1784
1785 cond_delay(nfc_op.rdy_delay_ns);
1786 }
1787
1788 marvell_nfc_xfer_data_pio(chip, subop, &nfc_op);
1789 ret = marvell_nfc_wait_cmdd(chip);
1790 if (ret)
1791 return ret;
1792
1793 cond_delay(nfc_op.data_delay_ns);
1794
1795 if (!reading) {
1796 if (nfc_op.rdy_timeout_ms) {
1797 ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
1798 if (ret)
1799 return ret;
1800 }
1801
1802 cond_delay(nfc_op.rdy_delay_ns);
1803 }
1804
1805 /*
1806 * NDCR ND_RUN bit should be cleared automatically at the end of each
1807 * operation but experience shows that the behavior is buggy when it
1808 * comes to writes (with LEN_OVRD). Clear it by hand in this case.
1809 */
1810 if (!reading) {
1811 struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1812
1813 writel_relaxed(readl(nfc->regs + NDCR) & ~NDCR_ND_RUN,
1814 nfc->regs + NDCR);
1815 }
1816
1817 return 0;
1818}
1819
1820static int marvell_nfc_naked_access_exec(struct nand_chip *chip,
1821 const struct nand_subop *subop)
1822{
1823 struct marvell_nfc_op nfc_op;
1824 int ret;
1825
1826 marvell_nfc_parse_instructions(chip, subop, &nfc_op);
1827
1828 /*
1829 * Naked access are different in that they need to be flagged as naked
1830 * by the controller. Reset the controller registers fields that inform
1831 * on the type and refill them according to the ongoing operation.
1832 */
1833 nfc_op.ndcb[0] &= ~(NDCB0_CMD_TYPE(TYPE_MASK) |
1834 NDCB0_CMD_XTYPE(XTYPE_MASK));
1835 switch (subop->instrs[0].type) {
1836 case NAND_OP_CMD_INSTR:
1837 nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_NAKED_CMD);
1838 break;
1839 case NAND_OP_ADDR_INSTR:
1840 nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_NAKED_ADDR);
1841 break;
1842 case NAND_OP_DATA_IN_INSTR:
1843 nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_READ) |
1844 NDCB0_CMD_XTYPE(XTYPE_LAST_NAKED_RW);
1845 break;
1846 case NAND_OP_DATA_OUT_INSTR:
1847 nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_WRITE) |
1848 NDCB0_CMD_XTYPE(XTYPE_LAST_NAKED_RW);
1849 break;
1850 default:
1851 /* This should never happen */
1852 break;
1853 }
1854
1855 ret = marvell_nfc_prepare_cmd(chip);
1856 if (ret)
1857 return ret;
1858
1859 marvell_nfc_send_cmd(chip, &nfc_op);
1860
1861 if (!nfc_op.data_instr) {
1862 ret = marvell_nfc_wait_cmdd(chip);
1863 cond_delay(nfc_op.cle_ale_delay_ns);
1864 return ret;
1865 }
1866
1867 ret = marvell_nfc_end_cmd(chip, NDSR_RDDREQ | NDSR_WRDREQ,
1868 "RDDREQ/WRDREQ while draining raw data");
1869 if (ret)
1870 return ret;
1871
1872 marvell_nfc_xfer_data_pio(chip, subop, &nfc_op);
1873 ret = marvell_nfc_wait_cmdd(chip);
1874 if (ret)
1875 return ret;
1876
1877 /*
1878 * NDCR ND_RUN bit should be cleared automatically at the end of each
1879 * operation but experience shows that the behavior is buggy when it
1880 * comes to writes (with LEN_OVRD). Clear it by hand in this case.
1881 */
1882 if (subop->instrs[0].type == NAND_OP_DATA_OUT_INSTR) {
1883 struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
1884
1885 writel_relaxed(readl(nfc->regs + NDCR) & ~NDCR_ND_RUN,
1886 nfc->regs + NDCR);
1887 }
1888
1889 return 0;
1890}
1891
1892static int marvell_nfc_naked_waitrdy_exec(struct nand_chip *chip,
1893 const struct nand_subop *subop)
1894{
1895 struct marvell_nfc_op nfc_op;
1896 int ret;
1897
1898 marvell_nfc_parse_instructions(chip, subop, &nfc_op);
1899
1900 ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
1901 cond_delay(nfc_op.rdy_delay_ns);
1902
1903 return ret;
1904}
1905
1906static int marvell_nfc_read_id_type_exec(struct nand_chip *chip,
1907 const struct nand_subop *subop)
1908{
1909 struct marvell_nfc_op nfc_op;
1910 int ret;
1911
1912 marvell_nfc_parse_instructions(chip, subop, &nfc_op);
1913 nfc_op.ndcb[0] &= ~NDCB0_CMD_TYPE(TYPE_READ);
1914 nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_READ_ID);
1915
1916 ret = marvell_nfc_prepare_cmd(chip);
1917 if (ret)
1918 return ret;
1919
1920 marvell_nfc_send_cmd(chip, &nfc_op);
1921 ret = marvell_nfc_end_cmd(chip, NDSR_RDDREQ,
1922 "RDDREQ while reading ID");
1923 if (ret)
1924 return ret;
1925
1926 cond_delay(nfc_op.cle_ale_delay_ns);
1927
1928 if (nfc_op.rdy_timeout_ms) {
1929 ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
1930 if (ret)
1931 return ret;
1932 }
1933
1934 cond_delay(nfc_op.rdy_delay_ns);
1935
1936 marvell_nfc_xfer_data_pio(chip, subop, &nfc_op);
1937 ret = marvell_nfc_wait_cmdd(chip);
1938 if (ret)
1939 return ret;
1940
1941 cond_delay(nfc_op.data_delay_ns);
1942
1943 return 0;
1944}
1945
1946static int marvell_nfc_read_status_exec(struct nand_chip *chip,
1947 const struct nand_subop *subop)
1948{
1949 struct marvell_nfc_op nfc_op;
1950 int ret;
1951
1952 marvell_nfc_parse_instructions(chip, subop, &nfc_op);
1953 nfc_op.ndcb[0] &= ~NDCB0_CMD_TYPE(TYPE_READ);
1954 nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_STATUS);
1955
1956 ret = marvell_nfc_prepare_cmd(chip);
1957 if (ret)
1958 return ret;
1959
1960 marvell_nfc_send_cmd(chip, &nfc_op);
1961 ret = marvell_nfc_end_cmd(chip, NDSR_RDDREQ,
1962 "RDDREQ while reading status");
1963 if (ret)
1964 return ret;
1965
1966 cond_delay(nfc_op.cle_ale_delay_ns);
1967
1968 if (nfc_op.rdy_timeout_ms) {
1969 ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
1970 if (ret)
1971 return ret;
1972 }
1973
1974 cond_delay(nfc_op.rdy_delay_ns);
1975
1976 marvell_nfc_xfer_data_pio(chip, subop, &nfc_op);
1977 ret = marvell_nfc_wait_cmdd(chip);
1978 if (ret)
1979 return ret;
1980
1981 cond_delay(nfc_op.data_delay_ns);
1982
1983 return 0;
1984}
1985
1986static int marvell_nfc_reset_cmd_type_exec(struct nand_chip *chip,
1987 const struct nand_subop *subop)
1988{
1989 struct marvell_nfc_op nfc_op;
1990 int ret;
1991
1992 marvell_nfc_parse_instructions(chip, subop, &nfc_op);
1993 nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_RESET);
1994
1995 ret = marvell_nfc_prepare_cmd(chip);
1996 if (ret)
1997 return ret;
1998
1999 marvell_nfc_send_cmd(chip, &nfc_op);
2000 ret = marvell_nfc_wait_cmdd(chip);
2001 if (ret)
2002 return ret;
2003
2004 cond_delay(nfc_op.cle_ale_delay_ns);
2005
2006 ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
2007 if (ret)
2008 return ret;
2009
2010 cond_delay(nfc_op.rdy_delay_ns);
2011
2012 return 0;
2013}
2014
2015static int marvell_nfc_erase_cmd_type_exec(struct nand_chip *chip,
2016 const struct nand_subop *subop)
2017{
2018 struct marvell_nfc_op nfc_op;
2019 int ret;
2020
2021 marvell_nfc_parse_instructions(chip, subop, &nfc_op);
2022 nfc_op.ndcb[0] |= NDCB0_CMD_TYPE(TYPE_ERASE);
2023
2024 ret = marvell_nfc_prepare_cmd(chip);
2025 if (ret)
2026 return ret;
2027
2028 marvell_nfc_send_cmd(chip, &nfc_op);
2029 ret = marvell_nfc_wait_cmdd(chip);
2030 if (ret)
2031 return ret;
2032
2033 cond_delay(nfc_op.cle_ale_delay_ns);
2034
2035 ret = marvell_nfc_wait_op(chip, nfc_op.rdy_timeout_ms);
2036 if (ret)
2037 return ret;
2038
2039 cond_delay(nfc_op.rdy_delay_ns);
2040
2041 return 0;
2042}
2043
2044static const struct nand_op_parser marvell_nfcv2_op_parser = NAND_OP_PARSER(
2045 /* Monolithic reads/writes */
2046 NAND_OP_PARSER_PATTERN(
2047 marvell_nfc_monolithic_access_exec,
2048 NAND_OP_PARSER_PAT_CMD_ELEM(false),
2049 NAND_OP_PARSER_PAT_ADDR_ELEM(true, MAX_ADDRESS_CYC_NFCV2),
2050 NAND_OP_PARSER_PAT_CMD_ELEM(true),
2051 NAND_OP_PARSER_PAT_WAITRDY_ELEM(true),
2052 NAND_OP_PARSER_PAT_DATA_IN_ELEM(false, MAX_CHUNK_SIZE)),
2053 NAND_OP_PARSER_PATTERN(
2054 marvell_nfc_monolithic_access_exec,
2055 NAND_OP_PARSER_PAT_CMD_ELEM(false),
2056 NAND_OP_PARSER_PAT_ADDR_ELEM(false, MAX_ADDRESS_CYC_NFCV2),
2057 NAND_OP_PARSER_PAT_DATA_OUT_ELEM(false, MAX_CHUNK_SIZE),
2058 NAND_OP_PARSER_PAT_CMD_ELEM(true),
2059 NAND_OP_PARSER_PAT_WAITRDY_ELEM(true)),
2060 /* Naked commands */
2061 NAND_OP_PARSER_PATTERN(
2062 marvell_nfc_naked_access_exec,
2063 NAND_OP_PARSER_PAT_CMD_ELEM(false)),
2064 NAND_OP_PARSER_PATTERN(
2065 marvell_nfc_naked_access_exec,
2066 NAND_OP_PARSER_PAT_ADDR_ELEM(false, MAX_ADDRESS_CYC_NFCV2)),
2067 NAND_OP_PARSER_PATTERN(
2068 marvell_nfc_naked_access_exec,
2069 NAND_OP_PARSER_PAT_DATA_IN_ELEM(false, MAX_CHUNK_SIZE)),
2070 NAND_OP_PARSER_PATTERN(
2071 marvell_nfc_naked_access_exec,
2072 NAND_OP_PARSER_PAT_DATA_OUT_ELEM(false, MAX_CHUNK_SIZE)),
2073 NAND_OP_PARSER_PATTERN(
2074 marvell_nfc_naked_waitrdy_exec,
2075 NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
2076 );
2077
2078static const struct nand_op_parser marvell_nfcv1_op_parser = NAND_OP_PARSER(
2079 /* Naked commands not supported, use a function for each pattern */
2080 NAND_OP_PARSER_PATTERN(
2081 marvell_nfc_read_id_type_exec,
2082 NAND_OP_PARSER_PAT_CMD_ELEM(false),
2083 NAND_OP_PARSER_PAT_ADDR_ELEM(false, MAX_ADDRESS_CYC_NFCV1),
2084 NAND_OP_PARSER_PAT_DATA_IN_ELEM(false, 8)),
2085 NAND_OP_PARSER_PATTERN(
2086 marvell_nfc_erase_cmd_type_exec,
2087 NAND_OP_PARSER_PAT_CMD_ELEM(false),
2088 NAND_OP_PARSER_PAT_ADDR_ELEM(false, MAX_ADDRESS_CYC_NFCV1),
2089 NAND_OP_PARSER_PAT_CMD_ELEM(false),
2090 NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
2091 NAND_OP_PARSER_PATTERN(
2092 marvell_nfc_read_status_exec,
2093 NAND_OP_PARSER_PAT_CMD_ELEM(false),
2094 NAND_OP_PARSER_PAT_DATA_IN_ELEM(false, 1)),
2095 NAND_OP_PARSER_PATTERN(
2096 marvell_nfc_reset_cmd_type_exec,
2097 NAND_OP_PARSER_PAT_CMD_ELEM(false),
2098 NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
2099 NAND_OP_PARSER_PATTERN(
2100 marvell_nfc_naked_waitrdy_exec,
2101 NAND_OP_PARSER_PAT_WAITRDY_ELEM(false)),
2102 );
2103
2104static int marvell_nfc_exec_op(struct nand_chip *chip,
2105 const struct nand_operation *op,
2106 bool check_only)
2107{
2108 struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
2109
ce446b4b
BB
2110 if (!check_only)
2111 marvell_nfc_select_target(chip, op->cs);
b2525141 2112
02f26ecf
MR
2113 if (nfc->caps->is_nfcv2)
2114 return nand_op_parser_exec_op(chip, &marvell_nfcv2_op_parser,
2115 op, check_only);
2116 else
2117 return nand_op_parser_exec_op(chip, &marvell_nfcv1_op_parser,
2118 op, check_only);
2119}
2120
2121/*
2122 * Layouts were broken in old pxa3xx_nand driver, these are supposed to be
2123 * usable.
2124 */
2125static int marvell_nand_ooblayout_ecc(struct mtd_info *mtd, int section,
2126 struct mtd_oob_region *oobregion)
2127{
2128 struct nand_chip *chip = mtd_to_nand(mtd);
2129 const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
2130
2131 if (section)
2132 return -ERANGE;
2133
2134 oobregion->length = (lt->full_chunk_cnt * lt->ecc_bytes) +
2135 lt->last_ecc_bytes;
2136 oobregion->offset = mtd->oobsize - oobregion->length;
2137
2138 return 0;
2139}
2140
2141static int marvell_nand_ooblayout_free(struct mtd_info *mtd, int section,
2142 struct mtd_oob_region *oobregion)
2143{
2144 struct nand_chip *chip = mtd_to_nand(mtd);
2145 const struct marvell_hw_ecc_layout *lt = to_marvell_nand(chip)->layout;
2146
2147 if (section)
2148 return -ERANGE;
2149
2150 /*
2151 * Bootrom looks in bytes 0 & 5 for bad blocks for the
2152 * 4KB page / 4bit BCH combination.
2153 */
2154 if (mtd->writesize == SZ_4K && lt->data_bytes == SZ_2K)
2155 oobregion->offset = 6;
2156 else
2157 oobregion->offset = 2;
2158
2159 oobregion->length = (lt->full_chunk_cnt * lt->spare_bytes) +
2160 lt->last_spare_bytes - oobregion->offset;
2161
2162 return 0;
2163}
2164
2165static const struct mtd_ooblayout_ops marvell_nand_ooblayout_ops = {
2166 .ecc = marvell_nand_ooblayout_ecc,
2167 .free = marvell_nand_ooblayout_free,
2168};
2169
2170static int marvell_nand_hw_ecc_ctrl_init(struct mtd_info *mtd,
2171 struct nand_ecc_ctrl *ecc)
2172{
2173 struct nand_chip *chip = mtd_to_nand(mtd);
2174 struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
2175 const struct marvell_hw_ecc_layout *l;
2176 int i;
2177
2178 if (!nfc->caps->is_nfcv2 &&
2179 (mtd->writesize + mtd->oobsize > MAX_CHUNK_SIZE)) {
2180 dev_err(nfc->dev,
2181 "NFCv1: writesize (%d) cannot be bigger than a chunk (%d)\n",
2182 mtd->writesize, MAX_CHUNK_SIZE - mtd->oobsize);
2183 return -ENOTSUPP;
2184 }
2185
2186 to_marvell_nand(chip)->layout = NULL;
2187 for (i = 0; i < ARRAY_SIZE(marvell_nfc_layouts); i++) {
2188 l = &marvell_nfc_layouts[i];
2189 if (mtd->writesize == l->writesize &&
2190 ecc->size == l->chunk && ecc->strength == l->strength) {
2191 to_marvell_nand(chip)->layout = l;
2192 break;
2193 }
2194 }
2195
2196 if (!to_marvell_nand(chip)->layout ||
2197 (!nfc->caps->is_nfcv2 && ecc->strength > 1)) {
2198 dev_err(nfc->dev,
2199 "ECC strength %d at page size %d is not supported\n",
2200 ecc->strength, mtd->writesize);
2201 return -ENOTSUPP;
2202 }
2203
7fd130f7
MR
2204 /* Special care for the layout 2k/8-bit/512B */
2205 if (l->writesize == 2048 && l->strength == 8) {
2206 if (mtd->oobsize < 128) {
2207 dev_err(nfc->dev, "Requested layout needs at least 128 OOB bytes\n");
2208 return -ENOTSUPP;
2209 } else {
2210 chip->bbt_options |= NAND_BBT_NO_OOB_BBM;
2211 }
2212 }
2213
02f26ecf
MR
2214 mtd_set_ooblayout(mtd, &marvell_nand_ooblayout_ops);
2215 ecc->steps = l->nchunks;
2216 ecc->size = l->data_bytes;
2217
2218 if (ecc->strength == 1) {
2219 chip->ecc.algo = NAND_ECC_HAMMING;
2220 ecc->read_page_raw = marvell_nfc_hw_ecc_hmg_read_page_raw;
2221 ecc->read_page = marvell_nfc_hw_ecc_hmg_read_page;
2222 ecc->read_oob_raw = marvell_nfc_hw_ecc_hmg_read_oob_raw;
2223 ecc->read_oob = ecc->read_oob_raw;
2224 ecc->write_page_raw = marvell_nfc_hw_ecc_hmg_write_page_raw;
2225 ecc->write_page = marvell_nfc_hw_ecc_hmg_write_page;
2226 ecc->write_oob_raw = marvell_nfc_hw_ecc_hmg_write_oob_raw;
2227 ecc->write_oob = ecc->write_oob_raw;
2228 } else {
2229 chip->ecc.algo = NAND_ECC_BCH;
2230 ecc->strength = 16;
2231 ecc->read_page_raw = marvell_nfc_hw_ecc_bch_read_page_raw;
2232 ecc->read_page = marvell_nfc_hw_ecc_bch_read_page;
2233 ecc->read_oob_raw = marvell_nfc_hw_ecc_bch_read_oob_raw;
2234 ecc->read_oob = marvell_nfc_hw_ecc_bch_read_oob;
2235 ecc->write_page_raw = marvell_nfc_hw_ecc_bch_write_page_raw;
2236 ecc->write_page = marvell_nfc_hw_ecc_bch_write_page;
2237 ecc->write_oob_raw = marvell_nfc_hw_ecc_bch_write_oob_raw;
2238 ecc->write_oob = marvell_nfc_hw_ecc_bch_write_oob;
2239 }
2240
2241 return 0;
2242}
2243
2244static int marvell_nand_ecc_init(struct mtd_info *mtd,
2245 struct nand_ecc_ctrl *ecc)
2246{
2247 struct nand_chip *chip = mtd_to_nand(mtd);
2248 struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
2249 int ret;
2250
2251 if (ecc->mode != NAND_ECC_NONE && (!ecc->size || !ecc->strength)) {
6a1b66d6
BB
2252 if (chip->base.eccreq.step_size && chip->base.eccreq.strength) {
2253 ecc->size = chip->base.eccreq.step_size;
2254 ecc->strength = chip->base.eccreq.strength;
02f26ecf
MR
2255 } else {
2256 dev_info(nfc->dev,
2257 "No minimum ECC strength, using 1b/512B\n");
2258 ecc->size = 512;
2259 ecc->strength = 1;
2260 }
2261 }
2262
2263 switch (ecc->mode) {
2264 case NAND_ECC_HW:
2265 ret = marvell_nand_hw_ecc_ctrl_init(mtd, ecc);
2266 if (ret)
2267 return ret;
2268 break;
2269 case NAND_ECC_NONE:
2270 case NAND_ECC_SOFT:
ed6d0285 2271 case NAND_ECC_ON_DIE:
02f26ecf
MR
2272 if (!nfc->caps->is_nfcv2 && mtd->writesize != SZ_512 &&
2273 mtd->writesize != SZ_2K) {
2274 dev_err(nfc->dev, "NFCv1 cannot write %d bytes pages\n",
2275 mtd->writesize);
2276 return -EINVAL;
2277 }
2278 break;
2279 default:
2280 return -EINVAL;
2281 }
2282
2283 return 0;
2284}
2285
2286static u8 bbt_pattern[] = {'M', 'V', 'B', 'b', 't', '0' };
2287static u8 bbt_mirror_pattern[] = {'1', 't', 'b', 'B', 'V', 'M' };
2288
2289static struct nand_bbt_descr bbt_main_descr = {
2290 .options = NAND_BBT_LASTBLOCK | NAND_BBT_CREATE | NAND_BBT_WRITE |
2291 NAND_BBT_2BIT | NAND_BBT_VERSION,
2292 .offs = 8,
2293 .len = 6,
2294 .veroffs = 14,
2295 .maxblocks = 8, /* Last 8 blocks in each chip */
2296 .pattern = bbt_pattern
2297};
2298
2299static struct nand_bbt_descr bbt_mirror_descr = {
2300 .options = NAND_BBT_LASTBLOCK | NAND_BBT_CREATE | NAND_BBT_WRITE |
2301 NAND_BBT_2BIT | NAND_BBT_VERSION,
2302 .offs = 8,
2303 .len = 6,
2304 .veroffs = 14,
2305 .maxblocks = 8, /* Last 8 blocks in each chip */
2306 .pattern = bbt_mirror_pattern
2307};
2308
858838b8 2309static int marvell_nfc_setup_data_interface(struct nand_chip *chip, int chipnr,
02f26ecf
MR
2310 const struct nand_data_interface
2311 *conf)
2312{
02f26ecf
MR
2313 struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
2314 struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
6b6de654 2315 unsigned int period_ns = 1000000000 / clk_get_rate(nfc->core_clk) * 2;
02f26ecf
MR
2316 const struct nand_sdr_timings *sdr;
2317 struct marvell_nfc_timings nfc_tmg;
2318 int read_delay;
2319
2320 sdr = nand_get_sdr_timings(conf);
2321 if (IS_ERR(sdr))
2322 return PTR_ERR(sdr);
2323
2324 /*
2325 * SDR timings are given in pico-seconds while NFC timings must be
2326 * expressed in NAND controller clock cycles, which is half of the
2327 * frequency of the accessible ECC clock retrieved by clk_get_rate().
2328 * This is not written anywhere in the datasheet but was observed
2329 * with an oscilloscope.
2330 *
2331 * NFC datasheet gives equations from which thoses calculations
2332 * are derived, they tend to be slightly more restrictives than the
2333 * given core timings and may improve the overall speed.
2334 */
2335 nfc_tmg.tRP = TO_CYCLES(DIV_ROUND_UP(sdr->tRC_min, 2), period_ns) - 1;
2336 nfc_tmg.tRH = nfc_tmg.tRP;
2337 nfc_tmg.tWP = TO_CYCLES(DIV_ROUND_UP(sdr->tWC_min, 2), period_ns) - 1;
2338 nfc_tmg.tWH = nfc_tmg.tWP;
2339 nfc_tmg.tCS = TO_CYCLES(sdr->tCS_min, period_ns);
2340 nfc_tmg.tCH = TO_CYCLES(sdr->tCH_min, period_ns) - 1;
2341 nfc_tmg.tADL = TO_CYCLES(sdr->tADL_min, period_ns);
2342 /*
2343 * Read delay is the time of propagation from SoC pins to NFC internal
2344 * logic. With non-EDO timings, this is MIN_RD_DEL_CNT clock cycles. In
2345 * EDO mode, an additional delay of tRH must be taken into account so
2346 * the data is sampled on the falling edge instead of the rising edge.
2347 */
2348 read_delay = sdr->tRC_min >= 30000 ?
2349 MIN_RD_DEL_CNT : MIN_RD_DEL_CNT + nfc_tmg.tRH;
2350
2351 nfc_tmg.tAR = TO_CYCLES(sdr->tAR_min, period_ns);
2352 /*
2353 * tWHR and tRHW are supposed to be read to write delays (and vice
2354 * versa) but in some cases, ie. when doing a change column, they must
2355 * be greater than that to be sure tCCS delay is respected.
2356 */
2357 nfc_tmg.tWHR = TO_CYCLES(max_t(int, sdr->tWHR_min, sdr->tCCS_min),
2358 period_ns) - 2,
2359 nfc_tmg.tRHW = TO_CYCLES(max_t(int, sdr->tRHW_min, sdr->tCCS_min),
2360 period_ns);
2361
07ad5a72
MR
2362 /*
2363 * NFCv2: Use WAIT_MODE (wait for RB line), do not rely only on delays.
2364 * NFCv1: No WAIT_MODE, tR must be maximal.
2365 */
2366 if (nfc->caps->is_nfcv2) {
2367 nfc_tmg.tR = TO_CYCLES(sdr->tWB_max, period_ns);
2368 } else {
2369 nfc_tmg.tR = TO_CYCLES64(sdr->tWB_max + sdr->tR_max,
2370 period_ns);
2371 if (nfc_tmg.tR + 3 > nfc_tmg.tCH)
2372 nfc_tmg.tR = nfc_tmg.tCH - 3;
2373 else
2374 nfc_tmg.tR = 0;
2375 }
02f26ecf
MR
2376
2377 if (chipnr < 0)
2378 return 0;
2379
2380 marvell_nand->ndtr0 =
2381 NDTR0_TRP(nfc_tmg.tRP) |
2382 NDTR0_TRH(nfc_tmg.tRH) |
2383 NDTR0_ETRP(nfc_tmg.tRP) |
2384 NDTR0_TWP(nfc_tmg.tWP) |
2385 NDTR0_TWH(nfc_tmg.tWH) |
2386 NDTR0_TCS(nfc_tmg.tCS) |
07ad5a72 2387 NDTR0_TCH(nfc_tmg.tCH);
02f26ecf
MR
2388
2389 marvell_nand->ndtr1 =
2390 NDTR1_TAR(nfc_tmg.tAR) |
2391 NDTR1_TWHR(nfc_tmg.tWHR) |
02f26ecf
MR
2392 NDTR1_TR(nfc_tmg.tR);
2393
07ad5a72
MR
2394 if (nfc->caps->is_nfcv2) {
2395 marvell_nand->ndtr0 |=
2396 NDTR0_RD_CNT_DEL(read_delay) |
2397 NDTR0_SELCNTR |
2398 NDTR0_TADL(nfc_tmg.tADL);
2399
2400 marvell_nand->ndtr1 |=
2401 NDTR1_TRHW(nfc_tmg.tRHW) |
2402 NDTR1_WAIT_MODE;
2403 }
2404
02f26ecf
MR
2405 return 0;
2406}
2407
8831e48b
MR
2408static int marvell_nand_attach_chip(struct nand_chip *chip)
2409{
2410 struct mtd_info *mtd = nand_to_mtd(chip);
2411 struct marvell_nand_chip *marvell_nand = to_marvell_nand(chip);
2412 struct marvell_nfc *nfc = to_marvell_nfc(chip->controller);
2413 struct pxa3xx_nand_platform_data *pdata = dev_get_platdata(nfc->dev);
2414 int ret;
2415
2416 if (pdata && pdata->flash_bbt)
2417 chip->bbt_options |= NAND_BBT_USE_FLASH;
2418
2419 if (chip->bbt_options & NAND_BBT_USE_FLASH) {
2420 /*
2421 * We'll use a bad block table stored in-flash and don't
2422 * allow writing the bad block marker to the flash.
2423 */
2424 chip->bbt_options |= NAND_BBT_NO_OOB_BBM;
2425 chip->bbt_td = &bbt_main_descr;
2426 chip->bbt_md = &bbt_mirror_descr;
2427 }
2428
2429 /* Save the chip-specific fields of NDCR */
2430 marvell_nand->ndcr = NDCR_PAGE_SZ(mtd->writesize);
2431 if (chip->options & NAND_BUSWIDTH_16)
2432 marvell_nand->ndcr |= NDCR_DWIDTH_M | NDCR_DWIDTH_C;
2433
2434 /*
2435 * On small page NANDs, only one cycle is needed to pass the
2436 * column address.
2437 */
2438 if (mtd->writesize <= 512) {
2439 marvell_nand->addr_cyc = 1;
2440 } else {
2441 marvell_nand->addr_cyc = 2;
2442 marvell_nand->ndcr |= NDCR_RA_START;
2443 }
2444
2445 /*
2446 * Now add the number of cycles needed to pass the row
2447 * address.
2448 *
2449 * Addressing a chip using CS 2 or 3 should also need the third row
2450 * cycle but due to inconsistance in the documentation and lack of
2451 * hardware to test this situation, this case is not supported.
2452 */
2453 if (chip->options & NAND_ROW_ADDR_3)
2454 marvell_nand->addr_cyc += 3;
2455 else
2456 marvell_nand->addr_cyc += 2;
2457
2458 if (pdata) {
2459 chip->ecc.size = pdata->ecc_step_size;
2460 chip->ecc.strength = pdata->ecc_strength;
2461 }
2462
2463 ret = marvell_nand_ecc_init(mtd, &chip->ecc);
2464 if (ret) {
2465 dev_err(nfc->dev, "ECC init failed: %d\n", ret);
2466 return ret;
2467 }
2468
2469 if (chip->ecc.mode == NAND_ECC_HW) {
2470 /*
2471 * Subpage write not available with hardware ECC, prohibit also
2472 * subpage read as in userspace subpage access would still be
2473 * allowed and subpage write, if used, would lead to numerous
2474 * uncorrectable ECC errors.
2475 */
2476 chip->options |= NAND_NO_SUBPAGE_WRITE;
2477 }
2478
2479 if (pdata || nfc->caps->legacy_of_bindings) {
2480 /*
2481 * We keep the MTD name unchanged to avoid breaking platforms
2482 * where the MTD cmdline parser is used and the bootloader
2483 * has not been updated to use the new naming scheme.
2484 */
2485 mtd->name = "pxa3xx_nand-0";
2486 } else if (!mtd->name) {
2487 /*
2488 * If the new bindings are used and the bootloader has not been
2489 * updated to pass a new mtdparts parameter on the cmdline, you
2490 * should define the following property in your NAND node, ie:
2491 *
2492 * label = "main-storage";
2493 *
2494 * This way, mtd->name will be set by the core when
2495 * nand_set_flash_node() is called.
2496 */
2497 mtd->name = devm_kasprintf(nfc->dev, GFP_KERNEL,
2498 "%s:nand.%d", dev_name(nfc->dev),
2499 marvell_nand->sels[0].cs);
2500 if (!mtd->name) {
2501 dev_err(nfc->dev, "Failed to allocate mtd->name\n");
2502 return -ENOMEM;
2503 }
2504 }
2505
2506 return 0;
2507}
2508
2509static const struct nand_controller_ops marvell_nand_controller_ops = {
2510 .attach_chip = marvell_nand_attach_chip,
f2abfeb2 2511 .exec_op = marvell_nfc_exec_op,
7a08dbae 2512 .setup_data_interface = marvell_nfc_setup_data_interface,
8831e48b
MR
2513};
2514
02f26ecf
MR
2515static int marvell_nand_chip_init(struct device *dev, struct marvell_nfc *nfc,
2516 struct device_node *np)
2517{
2518 struct pxa3xx_nand_platform_data *pdata = dev_get_platdata(dev);
2519 struct marvell_nand_chip *marvell_nand;
2520 struct mtd_info *mtd;
2521 struct nand_chip *chip;
2522 int nsels, ret, i;
2523 u32 cs, rb;
2524
2525 /*
2526 * The legacy "num-cs" property indicates the number of CS on the only
2527 * chip connected to the controller (legacy bindings does not support
f6997bec 2528 * more than one chip). The CS and RB pins are always the #0.
02f26ecf
MR
2529 *
2530 * When not using legacy bindings, a couple of "reg" and "nand-rb"
2531 * properties must be filled. For each chip, expressed as a subnode,
2532 * "reg" points to the CS lines and "nand-rb" to the RB line.
2533 */
f6997bec 2534 if (pdata || nfc->caps->legacy_of_bindings) {
02f26ecf 2535 nsels = 1;
f6997bec
MR
2536 } else {
2537 nsels = of_property_count_elems_of_size(np, "reg", sizeof(u32));
2538 if (nsels <= 0) {
2539 dev_err(dev, "missing/invalid reg property\n");
2540 return -EINVAL;
2541 }
02f26ecf
MR
2542 }
2543
2544 /* Alloc the nand chip structure */
7b301965
GS
2545 marvell_nand = devm_kzalloc(dev,
2546 struct_size(marvell_nand, sels, nsels),
02f26ecf
MR
2547 GFP_KERNEL);
2548 if (!marvell_nand) {
2549 dev_err(dev, "could not allocate chip structure\n");
2550 return -ENOMEM;
2551 }
2552
2553 marvell_nand->nsels = nsels;
2554 marvell_nand->selected_die = -1;
2555
2556 for (i = 0; i < nsels; i++) {
2557 if (pdata || nfc->caps->legacy_of_bindings) {
2558 /*
2559 * Legacy bindings use the CS lines in natural
2560 * order (0, 1, ...)
2561 */
2562 cs = i;
2563 } else {
2564 /* Retrieve CS id */
2565 ret = of_property_read_u32_index(np, "reg", i, &cs);
2566 if (ret) {
2567 dev_err(dev, "could not retrieve reg property: %d\n",
2568 ret);
2569 return ret;
2570 }
2571 }
2572
2573 if (cs >= nfc->caps->max_cs_nb) {
2574 dev_err(dev, "invalid reg value: %u (max CS = %d)\n",
2575 cs, nfc->caps->max_cs_nb);
2576 return -EINVAL;
2577 }
2578
2579 if (test_and_set_bit(cs, &nfc->assigned_cs)) {
2580 dev_err(dev, "CS %d already assigned\n", cs);
2581 return -EINVAL;
2582 }
2583
2584 /*
2585 * The cs variable represents the chip select id, which must be
2586 * converted in bit fields for NDCB0 and NDCB2 to select the
2587 * right chip. Unfortunately, due to a lack of information on
2588 * the subject and incoherent documentation, the user should not
2589 * use CS1 and CS3 at all as asserting them is not supported in
2590 * a reliable way (due to multiplexing inside ADDR5 field).
2591 */
2592 marvell_nand->sels[i].cs = cs;
2593 switch (cs) {
2594 case 0:
2595 case 2:
2596 marvell_nand->sels[i].ndcb0_csel = 0;
2597 break;
2598 case 1:
2599 case 3:
2600 marvell_nand->sels[i].ndcb0_csel = NDCB0_CSEL;
2601 break;
2602 default:
2603 return -EINVAL;
2604 }
2605
2606 /* Retrieve RB id */
2607 if (pdata || nfc->caps->legacy_of_bindings) {
2608 /* Legacy bindings always use RB #0 */
2609 rb = 0;
2610 } else {
2611 ret = of_property_read_u32_index(np, "nand-rb", i,
2612 &rb);
2613 if (ret) {
2614 dev_err(dev,
2615 "could not retrieve RB property: %d\n",
2616 ret);
2617 return ret;
2618 }
2619 }
2620
2621 if (rb >= nfc->caps->max_rb_nb) {
2622 dev_err(dev, "invalid reg value: %u (max RB = %d)\n",
2623 rb, nfc->caps->max_rb_nb);
2624 return -EINVAL;
2625 }
2626
2627 marvell_nand->sels[i].rb = rb;
2628 }
2629
2630 chip = &marvell_nand->chip;
2631 chip->controller = &nfc->controller;
2632 nand_set_flash_node(chip, np);
2633
07ad5a72 2634 if (!of_property_read_bool(np, "marvell,nand-keep-config"))
7a08dbae 2635 chip->options |= NAND_KEEP_TIMINGS;
02f26ecf
MR
2636
2637 mtd = nand_to_mtd(chip);
2638 mtd->dev.parent = dev;
2639
2640 /*
2641 * Default to HW ECC engine mode. If the nand-ecc-mode property is given
2642 * in the DT node, this entry will be overwritten in nand_scan_ident().
2643 */
2644 chip->ecc.mode = NAND_ECC_HW;
2645
2646 /*
2647 * Save a reference value for timing registers before
2648 * ->setup_data_interface() is called.
2649 */
2650 marvell_nand->ndtr0 = readl_relaxed(nfc->regs + NDTR0);
2651 marvell_nand->ndtr1 = readl_relaxed(nfc->regs + NDTR1);
2652
2653 chip->options |= NAND_BUSWIDTH_AUTO;
02f26ecf 2654
00ad378f 2655 ret = nand_scan(chip, marvell_nand->nsels);
02f26ecf 2656 if (ret) {
8831e48b 2657 dev_err(dev, "could not scan the nand chip\n");
02f26ecf
MR
2658 return ret;
2659 }
2660
2661 if (pdata)
2662 /* Legacy bindings support only one chip */
7576594c 2663 ret = mtd_device_register(mtd, pdata->parts, pdata->nr_parts);
02f26ecf
MR
2664 else
2665 ret = mtd_device_register(mtd, NULL, 0);
2666 if (ret) {
2667 dev_err(dev, "failed to register mtd device: %d\n", ret);
7a0c18fb 2668 nand_cleanup(chip);
02f26ecf
MR
2669 return ret;
2670 }
2671
2672 list_add_tail(&marvell_nand->node, &nfc->chips);
2673
2674 return 0;
2675}
2676
2677static int marvell_nand_chips_init(struct device *dev, struct marvell_nfc *nfc)
2678{
2679 struct device_node *np = dev->of_node;
2680 struct device_node *nand_np;
2681 int max_cs = nfc->caps->max_cs_nb;
2682 int nchips;
2683 int ret;
2684
2685 if (!np)
2686 nchips = 1;
2687 else
2688 nchips = of_get_child_count(np);
2689
2690 if (nchips > max_cs) {
2691 dev_err(dev, "too many NAND chips: %d (max = %d CS)\n", nchips,
2692 max_cs);
2693 return -EINVAL;
2694 }
2695
2696 /*
2697 * Legacy bindings do not use child nodes to exhibit NAND chip
2698 * properties and layout. Instead, NAND properties are mixed with the
2699 * controller ones, and partitions are defined as direct subnodes of the
2700 * NAND controller node.
2701 */
2702 if (nfc->caps->legacy_of_bindings) {
2703 ret = marvell_nand_chip_init(dev, nfc, np);
2704 return ret;
2705 }
2706
2707 for_each_child_of_node(np, nand_np) {
2708 ret = marvell_nand_chip_init(dev, nfc, nand_np);
2709 if (ret) {
2710 of_node_put(nand_np);
2711 return ret;
2712 }
2713 }
2714
2715 return 0;
2716}
2717
2718static void marvell_nand_chips_cleanup(struct marvell_nfc *nfc)
2719{
2720 struct marvell_nand_chip *entry, *temp;
2721
2722 list_for_each_entry_safe(entry, temp, &nfc->chips, node) {
59ac276f 2723 nand_release(&entry->chip);
02f26ecf
MR
2724 list_del(&entry->node);
2725 }
2726}
2727
2728static int marvell_nfc_init_dma(struct marvell_nfc *nfc)
2729{
2730 struct platform_device *pdev = container_of(nfc->dev,
2731 struct platform_device,
2732 dev);
2733 struct dma_slave_config config = {};
2734 struct resource *r;
02f26ecf
MR
2735 int ret;
2736
2737 if (!IS_ENABLED(CONFIG_PXA_DMA)) {
2738 dev_warn(nfc->dev,
2739 "DMA not enabled in configuration\n");
2740 return -ENOTSUPP;
2741 }
2742
2743 ret = dma_set_mask_and_coherent(nfc->dev, DMA_BIT_MASK(32));
2744 if (ret)
2745 return ret;
2746
cf9e2389
PU
2747 nfc->dma_chan = dma_request_chan(nfc->dev, "data");
2748 if (IS_ERR(nfc->dma_chan)) {
2749 ret = PTR_ERR(nfc->dma_chan);
2750 nfc->dma_chan = NULL;
2751 if (ret != -EPROBE_DEFER)
2752 dev_err(nfc->dev, "DMA channel request failed: %d\n",
2753 ret);
2754 return ret;
02f26ecf
MR
2755 }
2756
2757 r = platform_get_resource(pdev, IORESOURCE_MEM, 0);
aafe30ba
PU
2758 if (!r) {
2759 ret = -ENXIO;
2760 goto release_channel;
2761 }
02f26ecf
MR
2762
2763 config.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
2764 config.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
2765 config.src_addr = r->start + NDDB;
2766 config.dst_addr = r->start + NDDB;
2767 config.src_maxburst = 32;
2768 config.dst_maxburst = 32;
2769 ret = dmaengine_slave_config(nfc->dma_chan, &config);
2770 if (ret < 0) {
2771 dev_err(nfc->dev, "Failed to configure DMA channel\n");
aafe30ba 2772 goto release_channel;
02f26ecf
MR
2773 }
2774
2775 /*
2776 * DMA must act on length multiple of 32 and this length may be
2777 * bigger than the destination buffer. Use this buffer instead
2778 * for DMA transfers and then copy the desired amount of data to
2779 * the provided buffer.
2780 */
c495a927 2781 nfc->dma_buf = kmalloc(MAX_CHUNK_SIZE, GFP_KERNEL | GFP_DMA);
aafe30ba
PU
2782 if (!nfc->dma_buf) {
2783 ret = -ENOMEM;
2784 goto release_channel;
2785 }
02f26ecf
MR
2786
2787 nfc->use_dma = true;
2788
2789 return 0;
aafe30ba
PU
2790
2791release_channel:
2792 dma_release_channel(nfc->dma_chan);
2793 nfc->dma_chan = NULL;
2794
2795 return ret;
02f26ecf
MR
2796}
2797
bd9c3f9b
DM
2798static void marvell_nfc_reset(struct marvell_nfc *nfc)
2799{
2800 /*
2801 * ECC operations and interruptions are only enabled when specifically
2802 * needed. ECC shall not be activated in the early stages (fails probe).
2803 * Arbiter flag, even if marked as "reserved", must be set (empirical).
2804 * SPARE_EN bit must always be set or ECC bytes will not be at the same
2805 * offset in the read page and this will fail the protection.
2806 */
2807 writel_relaxed(NDCR_ALL_INT | NDCR_ND_ARB_EN | NDCR_SPARE_EN |
2808 NDCR_RD_ID_CNT(NFCV1_READID_LEN), nfc->regs + NDCR);
2809 writel_relaxed(0xFFFFFFFF, nfc->regs + NDSR);
2810 writel_relaxed(0, nfc->regs + NDECCCTRL);
2811}
2812
02f26ecf
MR
2813static int marvell_nfc_init(struct marvell_nfc *nfc)
2814{
2815 struct device_node *np = nfc->dev->of_node;
2816
2817 /*
2818 * Some SoCs like A7k/A8k need to enable manually the NAND
2819 * controller, gated clocks and reset bits to avoid being bootloader
2820 * dependent. This is done through the use of the System Functions
2821 * registers.
2822 */
2823 if (nfc->caps->need_system_controller) {
2824 struct regmap *sysctrl_base =
2825 syscon_regmap_lookup_by_phandle(np,
2826 "marvell,system-controller");
02f26ecf
MR
2827
2828 if (IS_ERR(sysctrl_base))
2829 return PTR_ERR(sysctrl_base);
2830
88aa3bbf
TP
2831 regmap_write(sysctrl_base, GENCONF_SOC_DEVICE_MUX,
2832 GENCONF_SOC_DEVICE_MUX_NFC_EN |
2833 GENCONF_SOC_DEVICE_MUX_ECC_CLK_RST |
2834 GENCONF_SOC_DEVICE_MUX_ECC_CORE_RST |
2835 GENCONF_SOC_DEVICE_MUX_NFC_INT_EN);
02f26ecf 2836
88aa3bbf
TP
2837 regmap_update_bits(sysctrl_base, GENCONF_CLK_GATING_CTRL,
2838 GENCONF_CLK_GATING_CTRL_ND_GATE,
2839 GENCONF_CLK_GATING_CTRL_ND_GATE);
02f26ecf 2840
88aa3bbf
TP
2841 regmap_update_bits(sysctrl_base, GENCONF_ND_CLK_CTRL,
2842 GENCONF_ND_CLK_CTRL_EN,
2843 GENCONF_ND_CLK_CTRL_EN);
02f26ecf
MR
2844 }
2845
2846 /* Configure the DMA if appropriate */
2847 if (!nfc->caps->is_nfcv2)
2848 marvell_nfc_init_dma(nfc);
2849
bd9c3f9b 2850 marvell_nfc_reset(nfc);
02f26ecf
MR
2851
2852 return 0;
2853}
2854
2855static int marvell_nfc_probe(struct platform_device *pdev)
2856{
2857 struct device *dev = &pdev->dev;
02f26ecf
MR
2858 struct marvell_nfc *nfc;
2859 int ret;
2860 int irq;
2861
2862 nfc = devm_kzalloc(&pdev->dev, sizeof(struct marvell_nfc),
2863 GFP_KERNEL);
2864 if (!nfc)
2865 return -ENOMEM;
2866
2867 nfc->dev = dev;
7da45139 2868 nand_controller_init(&nfc->controller);
8831e48b 2869 nfc->controller.ops = &marvell_nand_controller_ops;
02f26ecf
MR
2870 INIT_LIST_HEAD(&nfc->chips);
2871
5dcc9976 2872 nfc->regs = devm_platform_ioremap_resource(pdev, 0);
02f26ecf
MR
2873 if (IS_ERR(nfc->regs))
2874 return PTR_ERR(nfc->regs);
2875
2876 irq = platform_get_irq(pdev, 0);
aab478ca 2877 if (irq < 0)
02f26ecf 2878 return irq;
02f26ecf 2879
6b6de654 2880 nfc->core_clk = devm_clk_get(&pdev->dev, "core");
961ba15c
GC
2881
2882 /* Managed the legacy case (when the first clock was not named) */
6b6de654
BB
2883 if (nfc->core_clk == ERR_PTR(-ENOENT))
2884 nfc->core_clk = devm_clk_get(&pdev->dev, NULL);
961ba15c 2885
6b6de654
BB
2886 if (IS_ERR(nfc->core_clk))
2887 return PTR_ERR(nfc->core_clk);
02f26ecf 2888
6b6de654 2889 ret = clk_prepare_enable(nfc->core_clk);
02f26ecf
MR
2890 if (ret)
2891 return ret;
2892
961ba15c 2893 nfc->reg_clk = devm_clk_get(&pdev->dev, "reg");
f9e64d61
DM
2894 if (IS_ERR(nfc->reg_clk)) {
2895 if (PTR_ERR(nfc->reg_clk) != -ENOENT) {
961ba15c 2896 ret = PTR_ERR(nfc->reg_clk);
6b6de654 2897 goto unprepare_core_clk;
961ba15c 2898 }
f9e64d61
DM
2899
2900 nfc->reg_clk = NULL;
961ba15c
GC
2901 }
2902
f9e64d61
DM
2903 ret = clk_prepare_enable(nfc->reg_clk);
2904 if (ret)
2905 goto unprepare_core_clk;
2906
02f26ecf
MR
2907 marvell_nfc_disable_int(nfc, NDCR_ALL_INT);
2908 marvell_nfc_clear_int(nfc, NDCR_ALL_INT);
2909 ret = devm_request_irq(dev, irq, marvell_nfc_isr,
2910 0, "marvell-nfc", nfc);
2911 if (ret)
961ba15c 2912 goto unprepare_reg_clk;
02f26ecf
MR
2913
2914 /* Get NAND controller capabilities */
2915 if (pdev->id_entry)
2916 nfc->caps = (void *)pdev->id_entry->driver_data;
2917 else
2918 nfc->caps = of_device_get_match_data(&pdev->dev);
2919
2920 if (!nfc->caps) {
2921 dev_err(dev, "Could not retrieve NFC caps\n");
2922 ret = -EINVAL;
961ba15c 2923 goto unprepare_reg_clk;
02f26ecf
MR
2924 }
2925
2926 /* Init the controller and then probe the chips */
2927 ret = marvell_nfc_init(nfc);
2928 if (ret)
961ba15c 2929 goto unprepare_reg_clk;
02f26ecf
MR
2930
2931 platform_set_drvdata(pdev, nfc);
2932
2933 ret = marvell_nand_chips_init(dev, nfc);
2934 if (ret)
aafe30ba 2935 goto release_dma;
02f26ecf
MR
2936
2937 return 0;
2938
aafe30ba
PU
2939release_dma:
2940 if (nfc->use_dma)
2941 dma_release_channel(nfc->dma_chan);
961ba15c
GC
2942unprepare_reg_clk:
2943 clk_disable_unprepare(nfc->reg_clk);
6b6de654
BB
2944unprepare_core_clk:
2945 clk_disable_unprepare(nfc->core_clk);
02f26ecf
MR
2946
2947 return ret;
2948}
2949
2950static int marvell_nfc_remove(struct platform_device *pdev)
2951{
2952 struct marvell_nfc *nfc = platform_get_drvdata(pdev);
2953
2954 marvell_nand_chips_cleanup(nfc);
2955
2956 if (nfc->use_dma) {
2957 dmaengine_terminate_all(nfc->dma_chan);
2958 dma_release_channel(nfc->dma_chan);
2959 }
2960
961ba15c 2961 clk_disable_unprepare(nfc->reg_clk);
6b6de654 2962 clk_disable_unprepare(nfc->core_clk);
02f26ecf
MR
2963
2964 return 0;
2965}
2966
bd9c3f9b
DM
2967static int __maybe_unused marvell_nfc_suspend(struct device *dev)
2968{
2969 struct marvell_nfc *nfc = dev_get_drvdata(dev);
2970 struct marvell_nand_chip *chip;
2971
2972 list_for_each_entry(chip, &nfc->chips, node)
2973 marvell_nfc_wait_ndrun(&chip->chip);
2974
2975 clk_disable_unprepare(nfc->reg_clk);
2976 clk_disable_unprepare(nfc->core_clk);
2977
2978 return 0;
2979}
2980
2981static int __maybe_unused marvell_nfc_resume(struct device *dev)
2982{
2983 struct marvell_nfc *nfc = dev_get_drvdata(dev);
2984 int ret;
2985
2986 ret = clk_prepare_enable(nfc->core_clk);
2987 if (ret < 0)
2988 return ret;
2989
f9e64d61
DM
2990 ret = clk_prepare_enable(nfc->reg_clk);
2991 if (ret < 0)
2992 return ret;
bd9c3f9b
DM
2993
2994 /*
2995 * Reset nfc->selected_chip so the next command will cause the timing
2e16dc73 2996 * registers to be restored in marvell_nfc_select_target().
bd9c3f9b
DM
2997 */
2998 nfc->selected_chip = NULL;
2999
3000 /* Reset registers that have lost their contents */
3001 marvell_nfc_reset(nfc);
3002
3003 return 0;
3004}
3005
3006static const struct dev_pm_ops marvell_nfc_pm_ops = {
3007 SET_SYSTEM_SLEEP_PM_OPS(marvell_nfc_suspend, marvell_nfc_resume)
3008};
3009
02f26ecf
MR
3010static const struct marvell_nfc_caps marvell_armada_8k_nfc_caps = {
3011 .max_cs_nb = 4,
3012 .max_rb_nb = 2,
3013 .need_system_controller = true,
3014 .is_nfcv2 = true,
3015};
3016
3017static const struct marvell_nfc_caps marvell_armada370_nfc_caps = {
3018 .max_cs_nb = 4,
3019 .max_rb_nb = 2,
3020 .is_nfcv2 = true,
3021};
3022
3023static const struct marvell_nfc_caps marvell_pxa3xx_nfc_caps = {
3024 .max_cs_nb = 2,
3025 .max_rb_nb = 1,
3026 .use_dma = true,
3027};
3028
3029static const struct marvell_nfc_caps marvell_armada_8k_nfc_legacy_caps = {
3030 .max_cs_nb = 4,
3031 .max_rb_nb = 2,
3032 .need_system_controller = true,
3033 .legacy_of_bindings = true,
3034 .is_nfcv2 = true,
3035};
3036
3037static const struct marvell_nfc_caps marvell_armada370_nfc_legacy_caps = {
3038 .max_cs_nb = 4,
3039 .max_rb_nb = 2,
3040 .legacy_of_bindings = true,
3041 .is_nfcv2 = true,
3042};
3043
3044static const struct marvell_nfc_caps marvell_pxa3xx_nfc_legacy_caps = {
3045 .max_cs_nb = 2,
3046 .max_rb_nb = 1,
3047 .legacy_of_bindings = true,
3048 .use_dma = true,
3049};
3050
3051static const struct platform_device_id marvell_nfc_platform_ids[] = {
3052 {
3053 .name = "pxa3xx-nand",
3054 .driver_data = (kernel_ulong_t)&marvell_pxa3xx_nfc_legacy_caps,
3055 },
3056 { /* sentinel */ },
3057};
3058MODULE_DEVICE_TABLE(platform, marvell_nfc_platform_ids);
3059
3060static const struct of_device_id marvell_nfc_of_ids[] = {
3061 {
3062 .compatible = "marvell,armada-8k-nand-controller",
3063 .data = &marvell_armada_8k_nfc_caps,
3064 },
3065 {
3066 .compatible = "marvell,armada370-nand-controller",
3067 .data = &marvell_armada370_nfc_caps,
3068 },
3069 {
3070 .compatible = "marvell,pxa3xx-nand-controller",
3071 .data = &marvell_pxa3xx_nfc_caps,
3072 },
3073 /* Support for old/deprecated bindings: */
3074 {
3075 .compatible = "marvell,armada-8k-nand",
3076 .data = &marvell_armada_8k_nfc_legacy_caps,
3077 },
3078 {
3079 .compatible = "marvell,armada370-nand",
3080 .data = &marvell_armada370_nfc_legacy_caps,
3081 },
3082 {
3083 .compatible = "marvell,pxa3xx-nand",
3084 .data = &marvell_pxa3xx_nfc_legacy_caps,
3085 },
3086 { /* sentinel */ },
3087};
3088MODULE_DEVICE_TABLE(of, marvell_nfc_of_ids);
3089
3090static struct platform_driver marvell_nfc_driver = {
3091 .driver = {
3092 .name = "marvell-nfc",
3093 .of_match_table = marvell_nfc_of_ids,
bd9c3f9b 3094 .pm = &marvell_nfc_pm_ops,
02f26ecf
MR
3095 },
3096 .id_table = marvell_nfc_platform_ids,
3097 .probe = marvell_nfc_probe,
3098 .remove = marvell_nfc_remove,
3099};
3100module_platform_driver(marvell_nfc_driver);
3101
3102MODULE_LICENSE("GPL");
3103MODULE_DESCRIPTION("Marvell NAND controller driver");