projects / linux-2.6-block.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
sata_mv fix SOC flags, enable NCQ on SOC
[linux-2.6-block.git] / drivers / ata / sata_mv.c
CommitLineData
20f733e7
BR		 1/*
2 * sata_mv.c - Marvell SATA support
3 *
e12bef50 4 * Copyright 2008: Marvell Corporation, all rights reserved.
8b260248 5 * Copyright 2005: EMC Corporation, all rights reserved.
e2b1be56 6 * Copyright 2005 Red Hat, Inc. All rights reserved.
20f733e7
BR		 7 *
8 * Please ALWAYS copy linux-ide@vger.kernel.org on emails.
9 *
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU General Public License as published by
12 * the Free Software Foundation; version 2 of the License.
13 *
14 * This program is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 * GNU General Public License for more details.
18 *
19 * You should have received a copy of the GNU General Public License
20 * along with this program; if not, write to the Free Software
21 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
22 *
23 */
24
4a05e209
JG		 25/*
26 sata_mv TODO list:
27
28 1) Needs a full errata audit for all chipsets. I implemented most
29 of the errata workarounds found in the Marvell vendor driver, but
30 I distinctly remember a couple workarounds (one related to PCI-X)
31 are still needed.
32
1fd2e1c2
ML		 33 2) Improve/fix IRQ and error handling sequences.
34
35 3) ATAPI support (Marvell claims the 60xx/70xx chips can do it).
36
37 4) Think about TCQ support here, and for libata in general
38 with controllers that suppport it via host-queuing hardware
39 (a software-only implementation could be a nightmare).
4a05e209
JG		 40
41 5) Investigate problems with PCI Message Signalled Interrupts (MSI).
42
43 6) Add port multiplier support (intermediate)
44
40f0bc2d
ML		 45 7) Fix/reenable hot plug/unplug (should happen as a side-effect of (2) above).
46
4a05e209
JG		 47 8) Develop a low-power-consumption strategy, and implement it.
48
49 9) [Experiment, low priority] See if ATAPI can be supported using
50 "unknown FIS" or "vendor-specific FIS" support, or something creative
51 like that.
52
53 10) [Experiment, low priority] Investigate interrupt coalescing.
54 Quite often, especially with PCI Message Signalled Interrupts (MSI),
55 the overhead reduced by interrupt mitigation is quite often not
56 worth the latency cost.
57
58 11) [Experiment, Marvell value added] Is it possible to use target
59 mode to cross-connect two Linux boxes with Marvell cards? If so,
60 creating LibATA target mode support would be very interesting.
61
62 Target mode, for those without docs, is the ability to directly
63 connect two SATA controllers.
64
4a05e209
JG		 65*/
66
20f733e7
BR		 67#include <linux/kernel.h>
68#include <linux/module.h>
69#include <linux/pci.h>
70#include <linux/init.h>
71#include <linux/blkdev.h>
72#include <linux/delay.h>
73#include <linux/interrupt.h>
8d8b6004 74#include <linux/dmapool.h>
20f733e7 75#include <linux/dma-mapping.h>
a9524a76 76#include <linux/device.h>
f351b2d6
SB		 77#include <linux/platform_device.h>
78#include <linux/ata_platform.h>
20f733e7 79#include <scsi/scsi_host.h>
193515d5 80#include <scsi/scsi_cmnd.h>
6c08772e 81#include <scsi/scsi_device.h>
20f733e7 82#include <linux/libata.h>
20f733e7
BR		 83
84#define DRV_NAME "sata_mv"
1fd2e1c2 85#define DRV_VERSION "1.20"
20f733e7
BR		 86
87enum {
88 /* BAR's are enumerated in terms of pci_resource_start() terms */
89 MV_PRIMARY_BAR = 0, /* offset 0x10: memory space */
90 MV_IO_BAR = 2, /* offset 0x18: IO space */
91 MV_MISC_BAR = 3, /* offset 0x1c: FLASH, NVRAM, SRAM */
92
93 MV_MAJOR_REG_AREA_SZ = 0x10000, /* 64KB */
94 MV_MINOR_REG_AREA_SZ = 0x2000, /* 8KB */
95
96 MV_PCI_REG_BASE = 0,
97 MV_IRQ_COAL_REG_BASE = 0x18000, /* 6xxx part only */
615ab953
ML		 98 MV_IRQ_COAL_CAUSE = (MV_IRQ_COAL_REG_BASE + 0x08),
99 MV_IRQ_COAL_CAUSE_LO = (MV_IRQ_COAL_REG_BASE + 0x88),
100 MV_IRQ_COAL_CAUSE_HI = (MV_IRQ_COAL_REG_BASE + 0x8c),
101 MV_IRQ_COAL_THRESHOLD = (MV_IRQ_COAL_REG_BASE + 0xcc),
102 MV_IRQ_COAL_TIME_THRESHOLD = (MV_IRQ_COAL_REG_BASE + 0xd0),
103
20f733e7 104 MV_SATAHC0_REG_BASE = 0x20000,
522479fb 105 MV_FLASH_CTL = 0x1046c,
bca1c4eb
JG		 106 MV_GPIO_PORT_CTL = 0x104f0,
107 MV_RESET_CFG = 0x180d8,
20f733e7
BR		 108
109 MV_PCI_REG_SZ = MV_MAJOR_REG_AREA_SZ,
110 MV_SATAHC_REG_SZ = MV_MAJOR_REG_AREA_SZ,
111 MV_SATAHC_ARBTR_REG_SZ = MV_MINOR_REG_AREA_SZ, /* arbiter */
112 MV_PORT_REG_SZ = MV_MINOR_REG_AREA_SZ,
113
31961943
BR		 114 MV_MAX_Q_DEPTH = 32,
115 MV_MAX_Q_DEPTH_MASK = MV_MAX_Q_DEPTH - 1,
116
117 /* CRQB needs alignment on a 1KB boundary. Size == 1KB
118 * CRPB needs alignment on a 256B boundary. Size == 256B
31961943
BR		 119 * ePRD (SG) entries need alignment on a 16B boundary. Size == 16B
120 */
121 MV_CRQB_Q_SZ = (32 * MV_MAX_Q_DEPTH),
122 MV_CRPB_Q_SZ = (8 * MV_MAX_Q_DEPTH),
da2fa9ba 123 MV_MAX_SG_CT = 256,
31961943 124 MV_SG_TBL_SZ = (16 * MV_MAX_SG_CT),
31961943 125
20f733e7
BR		 126 MV_PORTS_PER_HC = 4,
127 /* == (port / MV_PORTS_PER_HC) to determine HC from 0-7 port */
128 MV_PORT_HC_SHIFT = 2,
31961943 129 /* == (port % MV_PORTS_PER_HC) to determine hard port from 0-7 port */
20f733e7
BR		 130 MV_PORT_MASK = 3,
131
132 /* Host Flags */
133 MV_FLAG_DUAL_HC = (1 << 30), /* two SATA Host Controllers */
134 MV_FLAG_IRQ_COALESCE = (1 << 29), /* IRQ coalescing capability */
7bb3c529 135 /* SoC integrated controllers, no PCI interface */
e12bef50 136 MV_FLAG_SOC = (1 << 28),
7bb3c529 137
c5d3e45a 138 MV_COMMON_FLAGS = ATA_FLAG_SATA | ATA_FLAG_NO_LEGACY |
bdd4ddde
JG		 139 ATA_FLAG_MMIO | ATA_FLAG_NO_ATAPI |
140 ATA_FLAG_PIO_POLLING,
47c2b677 141 MV_6XXX_FLAGS = MV_FLAG_IRQ_COALESCE,
20f733e7 142
31961943
BR		 143 CRQB_FLAG_READ = (1 << 0),
144 CRQB_TAG_SHIFT = 1,
c5d3e45a 145 CRQB_IOID_SHIFT = 6, /* CRQB Gen-II/IIE IO Id shift */
e12bef50 146 CRQB_PMP_SHIFT = 12, /* CRQB Gen-II/IIE PMP shift */
c5d3e45a 147 CRQB_HOSTQ_SHIFT = 17, /* CRQB Gen-II/IIE HostQueTag shift */
31961943
BR		 148 CRQB_CMD_ADDR_SHIFT = 8,
149 CRQB_CMD_CS = (0x2 << 11),
150 CRQB_CMD_LAST = (1 << 15),
151
152 CRPB_FLAG_STATUS_SHIFT = 8,
c5d3e45a
JG		 153 CRPB_IOID_SHIFT_6 = 5, /* CRPB Gen-II IO Id shift */
154 CRPB_IOID_SHIFT_7 = 7, /* CRPB Gen-IIE IO Id shift */
31961943
BR		 155
156 EPRD_FLAG_END_OF_TBL = (1 << 31),
157
20f733e7
BR		 158 /* PCI interface registers */
159
31961943
BR		 160 PCI_COMMAND_OFS = 0xc00,
161
20f733e7
BR		 162 PCI_MAIN_CMD_STS_OFS = 0xd30,
163 STOP_PCI_MASTER = (1 << 2),
164 PCI_MASTER_EMPTY = (1 << 3),
165 GLOB_SFT_RST = (1 << 4),
166
522479fb
JG		 167 MV_PCI_MODE = 0xd00,
168 MV_PCI_EXP_ROM_BAR_CTL = 0xd2c,
169 MV_PCI_DISC_TIMER = 0xd04,
170 MV_PCI_MSI_TRIGGER = 0xc38,
171 MV_PCI_SERR_MASK = 0xc28,
172 MV_PCI_XBAR_TMOUT = 0x1d04,
173 MV_PCI_ERR_LOW_ADDRESS = 0x1d40,
174 MV_PCI_ERR_HIGH_ADDRESS = 0x1d44,
175 MV_PCI_ERR_ATTRIBUTE = 0x1d48,
176 MV_PCI_ERR_COMMAND = 0x1d50,
177
02a121da
ML		 178 PCI_IRQ_CAUSE_OFS = 0x1d58,
179 PCI_IRQ_MASK_OFS = 0x1d5c,
20f733e7
BR		 180 PCI_UNMASK_ALL_IRQS = 0x7fffff, /* bits 22-0 */
181
02a121da
ML		 182 PCIE_IRQ_CAUSE_OFS = 0x1900,
183 PCIE_IRQ_MASK_OFS = 0x1910,
646a4da5 184 PCIE_UNMASK_ALL_IRQS = 0x40a, /* assorted bits */
02a121da 185
20f733e7
BR		 186 HC_MAIN_IRQ_CAUSE_OFS = 0x1d60,
187 HC_MAIN_IRQ_MASK_OFS = 0x1d64,
f351b2d6
SB		 188 HC_SOC_MAIN_IRQ_CAUSE_OFS = 0x20020,
189 HC_SOC_MAIN_IRQ_MASK_OFS = 0x20024,
20f733e7
BR		 190 PORT0_ERR = (1 << 0), /* shift by port # */
191 PORT0_DONE = (1 << 1), /* shift by port # */
192 HC0_IRQ_PEND = 0x1ff, /* bits 0-8 = HC0's ports */
193 HC_SHIFT = 9, /* bits 9-17 = HC1's ports */
194 PCI_ERR = (1 << 18),
195 TRAN_LO_DONE = (1 << 19), /* 6xxx: IRQ coalescing */
196 TRAN_HI_DONE = (1 << 20), /* 6xxx: IRQ coalescing */
fb621e2f
JG		 197 PORTS_0_3_COAL_DONE = (1 << 8),
198 PORTS_4_7_COAL_DONE = (1 << 17),
20f733e7
BR		 199 PORTS_0_7_COAL_DONE = (1 << 21), /* 6xxx: IRQ coalescing */
200 GPIO_INT = (1 << 22),
201 SELF_INT = (1 << 23),
202 TWSI_INT = (1 << 24),
203 HC_MAIN_RSVD = (0x7f << 25), /* bits 31-25 */
fb621e2f 204 HC_MAIN_RSVD_5 = (0x1fff << 19), /* bits 31-19 */
e12bef50 205 HC_MAIN_RSVD_SOC = (0x3fffffb << 6), /* bits 31-9, 7-6 */
8b260248 206 HC_MAIN_MASKED_IRQS = (TRAN_LO_DONE | TRAN_HI_DONE |
20f733e7
BR		 207 PORTS_0_7_COAL_DONE | GPIO_INT | TWSI_INT |
208 HC_MAIN_RSVD),
fb621e2f
JG		 209 HC_MAIN_MASKED_IRQS_5 = (PORTS_0_3_COAL_DONE | PORTS_4_7_COAL_DONE |
210 HC_MAIN_RSVD_5),
f351b2d6 211 HC_MAIN_MASKED_IRQS_SOC = (PORTS_0_3_COAL_DONE | HC_MAIN_RSVD_SOC),
20f733e7
BR		 212
213 /* SATAHC registers */
214 HC_CFG_OFS = 0,
215
216 HC_IRQ_CAUSE_OFS = 0x14,
31961943 217 CRPB_DMA_DONE = (1 << 0), /* shift by port # */
20f733e7
BR		 218 HC_IRQ_COAL = (1 << 4), /* IRQ coalescing */
219 DEV_IRQ = (1 << 8), /* shift by port # */
220
221 /* Shadow block registers */
31961943
BR		 222 SHD_BLK_OFS = 0x100,
223 SHD_CTL_AST_OFS = 0x20, /* ofs from SHD_BLK_OFS */
20f733e7
BR		 224
225 /* SATA registers */
226 SATA_STATUS_OFS = 0x300, /* ctrl, err regs follow status */
227 SATA_ACTIVE_OFS = 0x350,
0c58912e 228 SATA_FIS_IRQ_CAUSE_OFS = 0x364,
17c5aab5 229
e12bef50 230 LTMODE_OFS = 0x30c,
17c5aab5
ML		 231 LTMODE_BIT8 = (1 << 8), /* unknown, but necessary */
232
47c2b677 233 PHY_MODE3 = 0x310,
bca1c4eb
JG		 234 PHY_MODE4 = 0x314,
235 PHY_MODE2 = 0x330,
e12bef50
ML		 236 SATA_IFCTL_OFS = 0x344,
237 SATA_IFSTAT_OFS = 0x34c,
238 VENDOR_UNIQUE_FIS_OFS = 0x35c,
17c5aab5 239
e12bef50 240 FIS_CFG_OFS = 0x360,
17c5aab5
ML		 241 FIS_CFG_SINGLE_SYNC = (1 << 16), /* SYNC on DMA activation */
242
c9d39130
JG		 243 MV5_PHY_MODE = 0x74,
244 MV5_LT_MODE = 0x30,
245 MV5_PHY_CTL = 0x0C,
e12bef50 246 SATA_INTERFACE_CFG = 0x050,
bca1c4eb
JG		 247
248 MV_M2_PREAMP_MASK = 0x7e0,
20f733e7
BR		 249
250 /* Port registers */
251 EDMA_CFG_OFS = 0,
0c58912e
ML		 252 EDMA_CFG_Q_DEPTH = 0x1f, /* max device queue depth */
253 EDMA_CFG_NCQ = (1 << 5), /* for R/W FPDMA queued */
254 EDMA_CFG_NCQ_GO_ON_ERR = (1 << 14), /* continue on error */
255 EDMA_CFG_RD_BRST_EXT = (1 << 11), /* read burst 512B */
256 EDMA_CFG_WR_BUFF_LEN = (1 << 13), /* write buffer 512B */
e12bef50
ML		 257 EDMA_CFG_EDMA_FBS = (1 << 16), /* EDMA FIS-Based Switching */
258 EDMA_CFG_FBS = (1 << 26), /* FIS-Based Switching */
20f733e7
BR		 259
260 EDMA_ERR_IRQ_CAUSE_OFS = 0x8,
261 EDMA_ERR_IRQ_MASK_OFS = 0xc,
6c1153e0
JG		 262 EDMA_ERR_D_PAR = (1 << 0), /* UDMA data parity err */
263 EDMA_ERR_PRD_PAR = (1 << 1), /* UDMA PRD parity err */
264 EDMA_ERR_DEV = (1 << 2), /* device error */
265 EDMA_ERR_DEV_DCON = (1 << 3), /* device disconnect */
266 EDMA_ERR_DEV_CON = (1 << 4), /* device connected */
267 EDMA_ERR_SERR = (1 << 5), /* SError bits [WBDST] raised */
c5d3e45a
JG		 268 EDMA_ERR_SELF_DIS = (1 << 7), /* Gen II/IIE self-disable */
269 EDMA_ERR_SELF_DIS_5 = (1 << 8), /* Gen I self-disable */
6c1153e0 270 EDMA_ERR_BIST_ASYNC = (1 << 8), /* BIST FIS or Async Notify */
c5d3e45a 271 EDMA_ERR_TRANS_IRQ_7 = (1 << 8), /* Gen IIE transprt layer irq */
6c1153e0
JG		 272 EDMA_ERR_CRQB_PAR = (1 << 9), /* CRQB parity error */
273 EDMA_ERR_CRPB_PAR = (1 << 10), /* CRPB parity error */
274 EDMA_ERR_INTRL_PAR = (1 << 11), /* internal parity error */
275 EDMA_ERR_IORDY = (1 << 12), /* IORdy timeout */
646a4da5 276
6c1153e0 277 EDMA_ERR_LNK_CTRL_RX = (0xf << 13), /* link ctrl rx error */
646a4da5
ML		 278 EDMA_ERR_LNK_CTRL_RX_0 = (1 << 13), /* transient: CRC err */
279 EDMA_ERR_LNK_CTRL_RX_1 = (1 << 14), /* transient: FIFO err */
280 EDMA_ERR_LNK_CTRL_RX_2 = (1 << 15), /* fatal: caught SYNC */
281 EDMA_ERR_LNK_CTRL_RX_3 = (1 << 16), /* transient: FIS rx err */
282
6c1153e0 283 EDMA_ERR_LNK_DATA_RX = (0xf << 17), /* link data rx error */
646a4da5 284
6c1153e0 285 EDMA_ERR_LNK_CTRL_TX = (0x1f << 21), /* link ctrl tx error */
646a4da5
ML		 286 EDMA_ERR_LNK_CTRL_TX_0 = (1 << 21), /* transient: CRC err */
287 EDMA_ERR_LNK_CTRL_TX_1 = (1 << 22), /* transient: FIFO err */
288 EDMA_ERR_LNK_CTRL_TX_2 = (1 << 23), /* transient: caught SYNC */
289 EDMA_ERR_LNK_CTRL_TX_3 = (1 << 24), /* transient: caught DMAT */
290 EDMA_ERR_LNK_CTRL_TX_4 = (1 << 25), /* transient: FIS collision */
291
6c1153e0 292 EDMA_ERR_LNK_DATA_TX = (0x1f << 26), /* link data tx error */
646a4da5 293
6c1153e0 294 EDMA_ERR_TRANS_PROTO = (1 << 31), /* transport protocol error */
c5d3e45a
JG		 295 EDMA_ERR_OVERRUN_5 = (1 << 5),
296 EDMA_ERR_UNDERRUN_5 = (1 << 6),
646a4da5
ML		 297
298 EDMA_ERR_IRQ_TRANSIENT = EDMA_ERR_LNK_CTRL_RX_0 |
299 EDMA_ERR_LNK_CTRL_RX_1 |
300 EDMA_ERR_LNK_CTRL_RX_3 |
40f0bc2d
ML		 301 EDMA_ERR_LNK_CTRL_TX |
302 /* temporary, until we fix hotplug: */
303 (EDMA_ERR_DEV_DCON | EDMA_ERR_DEV_CON),
646a4da5 304
bdd4ddde
JG		 305 EDMA_EH_FREEZE = EDMA_ERR_D_PAR |
306 EDMA_ERR_PRD_PAR |
307 EDMA_ERR_DEV_DCON |
308 EDMA_ERR_DEV_CON |
309 EDMA_ERR_SERR |
310 EDMA_ERR_SELF_DIS |
6c1153e0 311 EDMA_ERR_CRQB_PAR |
bdd4ddde
JG		 312 EDMA_ERR_CRPB_PAR |
313 EDMA_ERR_INTRL_PAR |
314 EDMA_ERR_IORDY |
315 EDMA_ERR_LNK_CTRL_RX_2 |
316 EDMA_ERR_LNK_DATA_RX |
317 EDMA_ERR_LNK_DATA_TX |
318 EDMA_ERR_TRANS_PROTO,
e12bef50 319
bdd4ddde
JG		 320 EDMA_EH_FREEZE_5 = EDMA_ERR_D_PAR |
321 EDMA_ERR_PRD_PAR |
322 EDMA_ERR_DEV_DCON |
323 EDMA_ERR_DEV_CON |
324 EDMA_ERR_OVERRUN_5 |
325 EDMA_ERR_UNDERRUN_5 |
326 EDMA_ERR_SELF_DIS_5 |
6c1153e0 327 EDMA_ERR_CRQB_PAR |
bdd4ddde
JG		 328 EDMA_ERR_CRPB_PAR |
329 EDMA_ERR_INTRL_PAR |
330 EDMA_ERR_IORDY,
20f733e7 331
31961943
BR		 332 EDMA_REQ_Q_BASE_HI_OFS = 0x10,
333 EDMA_REQ_Q_IN_PTR_OFS = 0x14, /* also contains BASE_LO */
31961943
BR		 334
335 EDMA_REQ_Q_OUT_PTR_OFS = 0x18,
336 EDMA_REQ_Q_PTR_SHIFT = 5,
337
338 EDMA_RSP_Q_BASE_HI_OFS = 0x1c,
339 EDMA_RSP_Q_IN_PTR_OFS = 0x20,
340 EDMA_RSP_Q_OUT_PTR_OFS = 0x24, /* also contains BASE_LO */
31961943
BR		 341 EDMA_RSP_Q_PTR_SHIFT = 3,
342
0ea9e179
JG		 343 EDMA_CMD_OFS = 0x28, /* EDMA command register */
344 EDMA_EN = (1 << 0), /* enable EDMA */
345 EDMA_DS = (1 << 1), /* disable EDMA; self-negated */
346 ATA_RST = (1 << 2), /* reset trans/link/phy */
20f733e7 347
c9d39130 348 EDMA_IORDY_TMOUT = 0x34,
bca1c4eb 349 EDMA_ARB_CFG = 0x38,
bca1c4eb 350
31961943
BR		 351 /* Host private flags (hp_flags) */
352 MV_HP_FLAG_MSI = (1 << 0),
47c2b677
JG		 353 MV_HP_ERRATA_50XXB0 = (1 << 1),
354 MV_HP_ERRATA_50XXB2 = (1 << 2),
355 MV_HP_ERRATA_60X1B2 = (1 << 3),
356 MV_HP_ERRATA_60X1C0 = (1 << 4),
e4e7b892 357 MV_HP_ERRATA_XX42A0 = (1 << 5),
0ea9e179
JG		 358 MV_HP_GEN_I = (1 << 6), /* Generation I: 50xx */
359 MV_HP_GEN_II = (1 << 7), /* Generation II: 60xx */
360 MV_HP_GEN_IIE = (1 << 8), /* Generation IIE: 6042/7042 */
02a121da 361 MV_HP_PCIE = (1 << 9), /* PCIe bus/regs: 7042 */
20f733e7 362
31961943 363 /* Port private flags (pp_flags) */
0ea9e179 364 MV_PP_FLAG_EDMA_EN = (1 << 0), /* is EDMA engine enabled? */
72109168 365 MV_PP_FLAG_NCQ_EN = (1 << 1), /* is EDMA set up for NCQ? */
20f733e7
BR		 366};
367
ee9ccdf7
JG		 368#define IS_GEN_I(hpriv) ((hpriv)->hp_flags & MV_HP_GEN_I)
369#define IS_GEN_II(hpriv) ((hpriv)->hp_flags & MV_HP_GEN_II)
e4e7b892 370#define IS_GEN_IIE(hpriv) ((hpriv)->hp_flags & MV_HP_GEN_IIE)
7bb3c529 371#define HAS_PCI(host) (!((host)->ports[0]->flags & MV_FLAG_SOC))
bca1c4eb 372
095fec88 373enum {
baf14aa1
JG		 374 /* DMA boundary 0xffff is required by the s/g splitting
375 * we need on /length/ in mv_fill-sg().
376 */
377 MV_DMA_BOUNDARY = 0xffffU,
095fec88 378
0ea9e179
JG		 379 /* mask of register bits containing lower 32 bits
380 * of EDMA request queue DMA address
381 */
095fec88
JG		 382 EDMA_REQ_Q_BASE_LO_MASK = 0xfffffc00U,
383
0ea9e179 384 /* ditto, for response queue */
095fec88
JG		 385 EDMA_RSP_Q_BASE_LO_MASK = 0xffffff00U,
386};
387
522479fb
JG		 388enum chip_type {
389 chip_504x,
390 chip_508x,
391 chip_5080,
392 chip_604x,
393 chip_608x,
e4e7b892
JG		 394 chip_6042,
395 chip_7042,
f351b2d6 396 chip_soc,
522479fb
JG		 397};
398
31961943
BR		 399/* Command ReQuest Block: 32B */
400struct mv_crqb {
e1469874
ML		 401 __le32 sg_addr;
402 __le32 sg_addr_hi;
403 __le16 ctrl_flags;
404 __le16 ata_cmd[11];
31961943 405};
20f733e7 406
e4e7b892 407struct mv_crqb_iie {
e1469874
ML		 408 __le32 addr;
409 __le32 addr_hi;
410 __le32 flags;
411 __le32 len;
412 __le32 ata_cmd[4];
e4e7b892
JG		 413};
414
31961943
BR		 415/* Command ResPonse Block: 8B */
416struct mv_crpb {
e1469874
ML		 417 __le16 id;
418 __le16 flags;
419 __le32 tmstmp;
20f733e7
BR		 420};
421
31961943
BR		 422/* EDMA Physical Region Descriptor (ePRD); A.K.A. SG */
423struct mv_sg {
e1469874
ML		 424 __le32 addr;
425 __le32 flags_size;
426 __le32 addr_hi;
427 __le32 reserved;
31961943 428};
20f733e7 429
31961943
BR		 430struct mv_port_priv {
431 struct mv_crqb *crqb;
432 dma_addr_t crqb_dma;
433 struct mv_crpb *crpb;
434 dma_addr_t crpb_dma;
eb73d558
ML		 435 struct mv_sg *sg_tbl[MV_MAX_Q_DEPTH];
436 dma_addr_t sg_tbl_dma[MV_MAX_Q_DEPTH];
bdd4ddde
JG		 437
438 unsigned int req_idx;
439 unsigned int resp_idx;
440
31961943
BR		 441 u32 pp_flags;
442};
443
bca1c4eb
JG		 444struct mv_port_signal {
445 u32 amps;
446 u32 pre;
447};
448
02a121da
ML		 449struct mv_host_priv {
450 u32 hp_flags;
451 struct mv_port_signal signal[8];
452 const struct mv_hw_ops *ops;
f351b2d6
SB		 453 int n_ports;
454 void __iomem *base;
455 void __iomem *main_cause_reg_addr;
456 void __iomem *main_mask_reg_addr;
02a121da
ML		 457 u32 irq_cause_ofs;
458 u32 irq_mask_ofs;
459 u32 unmask_all_irqs;
da2fa9ba
ML		 460 /*
461 * These consistent DMA memory pools give us guaranteed
462 * alignment for hardware-accessed data structures,
463 * and less memory waste in accomplishing the alignment.
464 */
465 struct dma_pool *crqb_pool;
466 struct dma_pool *crpb_pool;
467 struct dma_pool *sg_tbl_pool;
02a121da
ML		 468};
469
47c2b677 470struct mv_hw_ops {
2a47ce06
JG		 471 void (*phy_errata)(struct mv_host_priv *hpriv, void __iomem *mmio,
472 unsigned int port);
47c2b677
JG		 473 void (*enable_leds)(struct mv_host_priv *hpriv, void __iomem *mmio);
474 void (*read_preamp)(struct mv_host_priv *hpriv, int idx,
475 void __iomem *mmio);
c9d39130
JG		 476 int (*reset_hc)(struct mv_host_priv *hpriv, void __iomem *mmio,
477 unsigned int n_hc);
522479fb 478 void (*reset_flash)(struct mv_host_priv *hpriv, void __iomem *mmio);
7bb3c529 479 void (*reset_bus)(struct ata_host *host, void __iomem *mmio);
47c2b677
JG		 480};
481
da3dbb17
TH		 482static int mv_scr_read(struct ata_port *ap, unsigned int sc_reg_in, u32 *val);
483static int mv_scr_write(struct ata_port *ap, unsigned int sc_reg_in, u32 val);
484static int mv5_scr_read(struct ata_port *ap, unsigned int sc_reg_in, u32 *val);
485static int mv5_scr_write(struct ata_port *ap, unsigned int sc_reg_in, u32 val);
31961943
BR		 486static int mv_port_start(struct ata_port *ap);
487static void mv_port_stop(struct ata_port *ap);
488static void mv_qc_prep(struct ata_queued_cmd *qc);
e4e7b892 489static void mv_qc_prep_iie(struct ata_queued_cmd *qc);
9a3d9eb0 490static unsigned int mv_qc_issue(struct ata_queued_cmd *qc);
a1efdaba
TH		 491static int mv_hardreset(struct ata_link *link, unsigned int *class,
492 unsigned long deadline);
bdd4ddde
JG		 493static void mv_eh_freeze(struct ata_port *ap);
494static void mv_eh_thaw(struct ata_port *ap);
f273827e 495static void mv6_dev_config(struct ata_device *dev);
20f733e7 496
2a47ce06
JG		 497static void mv5_phy_errata(struct mv_host_priv *hpriv, void __iomem *mmio,
498 unsigned int port);
47c2b677
JG		 499static void mv5_enable_leds(struct mv_host_priv *hpriv, void __iomem *mmio);
500static void mv5_read_preamp(struct mv_host_priv *hpriv, int idx,
501 void __iomem *mmio);
c9d39130
JG		 502static int mv5_reset_hc(struct mv_host_priv *hpriv, void __iomem *mmio,
503 unsigned int n_hc);
522479fb 504static void mv5_reset_flash(struct mv_host_priv *hpriv, void __iomem *mmio);
7bb3c529 505static void mv5_reset_bus(struct ata_host *host, void __iomem *mmio);
47c2b677 506
2a47ce06
JG		 507static void mv6_phy_errata(struct mv_host_priv *hpriv, void __iomem *mmio,
508 unsigned int port);
47c2b677
JG		 509static void mv6_enable_leds(struct mv_host_priv *hpriv, void __iomem *mmio);
510static void mv6_read_preamp(struct mv_host_priv *hpriv, int idx,
511 void __iomem *mmio);
c9d39130
JG		 512static int mv6_reset_hc(struct mv_host_priv *hpriv, void __iomem *mmio,
513 unsigned int n_hc);
522479fb 514static void mv6_reset_flash(struct mv_host_priv *hpriv, void __iomem *mmio);
f351b2d6
SB		 515static void mv_soc_enable_leds(struct mv_host_priv *hpriv,
516 void __iomem *mmio);
517static void mv_soc_read_preamp(struct mv_host_priv *hpriv, int idx,
518 void __iomem *mmio);
519static int mv_soc_reset_hc(struct mv_host_priv *hpriv,
520 void __iomem *mmio, unsigned int n_hc);
521static void mv_soc_reset_flash(struct mv_host_priv *hpriv,
522 void __iomem *mmio);
523static void mv_soc_reset_bus(struct ata_host *host, void __iomem *mmio);
7bb3c529 524static void mv_reset_pci_bus(struct ata_host *host, void __iomem *mmio);
e12bef50 525static void mv_reset_channel(struct mv_host_priv *hpriv, void __iomem *mmio,
c9d39130 526 unsigned int port_no);
e12bef50 527static int mv_stop_edma(struct ata_port *ap);
b562468c 528static int mv_stop_edma_engine(void __iomem *port_mmio);
e12bef50 529static void mv_edma_cfg(struct ata_port *ap, int want_ncq);
47c2b677 530
eb73d558
ML		 531/* .sg_tablesize is (MV_MAX_SG_CT / 2) in the structures below
532 * because we have to allow room for worst case splitting of
533 * PRDs for 64K boundaries in mv_fill_sg().
534 */
c5d3e45a 535static struct scsi_host_template mv5_sht = {
68d1d07b 536 ATA_BASE_SHT(DRV_NAME),
baf14aa1 537 .sg_tablesize = MV_MAX_SG_CT / 2,
c5d3e45a 538 .dma_boundary = MV_DMA_BOUNDARY,
c5d3e45a
JG		 539};
540
541static struct scsi_host_template mv6_sht = {
68d1d07b 542 ATA_NCQ_SHT(DRV_NAME),
138bfdd0 543 .can_queue = MV_MAX_Q_DEPTH - 1,
baf14aa1 544 .sg_tablesize = MV_MAX_SG_CT / 2,
20f733e7 545 .dma_boundary = MV_DMA_BOUNDARY,
20f733e7
BR		 546};
547
029cfd6b
TH		 548static struct ata_port_operations mv5_ops = {
549 .inherits = &ata_sff_port_ops,
c9d39130 550
c9d39130
JG		 551 .qc_prep = mv_qc_prep,
552 .qc_issue = mv_qc_issue,
553
bdd4ddde
JG		 554 .freeze = mv_eh_freeze,
555 .thaw = mv_eh_thaw,
a1efdaba 556 .hardreset = mv_hardreset,
a1efdaba 557 .error_handler = ata_std_error_handler, /* avoid SFF EH */
029cfd6b 558 .post_internal_cmd = ATA_OP_NULL,
bdd4ddde 559
c9d39130
JG		 560 .scr_read = mv5_scr_read,
561 .scr_write = mv5_scr_write,
562
563 .port_start = mv_port_start,
564 .port_stop = mv_port_stop,
c9d39130
JG		 565};
566
029cfd6b
TH		 567static struct ata_port_operations mv6_ops = {
568 .inherits = &mv5_ops,
138bfdd0 569 .qc_defer = ata_std_qc_defer,
029cfd6b 570 .dev_config = mv6_dev_config,
20f733e7
BR		 571 .scr_read = mv_scr_read,
572 .scr_write = mv_scr_write,
20f733e7
BR		 573};
574
029cfd6b
TH		 575static struct ata_port_operations mv_iie_ops = {
576 .inherits = &mv6_ops,
577 .dev_config = ATA_OP_NULL,
e4e7b892 578 .qc_prep = mv_qc_prep_iie,
e4e7b892
JG		 579};
580
98ac62de 581static const struct ata_port_info mv_port_info[] = {
20f733e7 582 { /* chip_504x */
cca3974e 583 .flags = MV_COMMON_FLAGS,
31961943 584 .pio_mask = 0x1f, /* pio0-4 */
bf6263a8 585 .udma_mask = ATA_UDMA6,
c9d39130 586 .port_ops = &mv5_ops,
20f733e7
BR		 587 },
588 { /* chip_508x */
c5d3e45a 589 .flags = MV_COMMON_FLAGS | MV_FLAG_DUAL_HC,
31961943 590 .pio_mask = 0x1f, /* pio0-4 */
bf6263a8 591 .udma_mask = ATA_UDMA6,
c9d39130 592 .port_ops = &mv5_ops,
20f733e7 593 },
47c2b677 594 { /* chip_5080 */
c5d3e45a 595 .flags = MV_COMMON_FLAGS | MV_FLAG_DUAL_HC,
47c2b677 596 .pio_mask = 0x1f, /* pio0-4 */
bf6263a8 597 .udma_mask = ATA_UDMA6,
c9d39130 598 .port_ops = &mv5_ops,
47c2b677 599 },
20f733e7 600 { /* chip_604x */
138bfdd0
ML		 601 .flags = MV_COMMON_FLAGS | MV_6XXX_FLAGS |
602 ATA_FLAG_NCQ,
31961943 603 .pio_mask = 0x1f, /* pio0-4 */
bf6263a8 604 .udma_mask = ATA_UDMA6,
c9d39130 605 .port_ops = &mv6_ops,
20f733e7
BR		 606 },
607 { /* chip_608x */
c5d3e45a 608 .flags = MV_COMMON_FLAGS | MV_6XXX_FLAGS |
138bfdd0 609 ATA_FLAG_NCQ | MV_FLAG_DUAL_HC,
31961943 610 .pio_mask = 0x1f, /* pio0-4 */
bf6263a8 611 .udma_mask = ATA_UDMA6,
c9d39130 612 .port_ops = &mv6_ops,
20f733e7 613 },
e4e7b892 614 { /* chip_6042 */
138bfdd0
ML		 615 .flags = MV_COMMON_FLAGS | MV_6XXX_FLAGS |
616 ATA_FLAG_NCQ,
e4e7b892 617 .pio_mask = 0x1f, /* pio0-4 */
bf6263a8 618 .udma_mask = ATA_UDMA6,
e4e7b892
JG		 619 .port_ops = &mv_iie_ops,
620 },
621 { /* chip_7042 */
138bfdd0
ML		 622 .flags = MV_COMMON_FLAGS | MV_6XXX_FLAGS |
623 ATA_FLAG_NCQ,
e4e7b892 624 .pio_mask = 0x1f, /* pio0-4 */
bf6263a8 625 .udma_mask = ATA_UDMA6,
e4e7b892
JG		 626 .port_ops = &mv_iie_ops,
627 },
f351b2d6 628 { /* chip_soc */
02c1f32f
ML		 629 .flags = MV_COMMON_FLAGS | MV_6XXX_FLAGS |
630 ATA_FLAG_NCQ | MV_FLAG_SOC,
17c5aab5
ML		 631 .pio_mask = 0x1f, /* pio0-4 */
632 .udma_mask = ATA_UDMA6,
633 .port_ops = &mv_iie_ops,
f351b2d6 634 },
20f733e7
BR		 635};
636
3b7d697d 637static const struct pci_device_id mv_pci_tbl[] = {
2d2744fc
JG		 638 { PCI_VDEVICE(MARVELL, 0x5040), chip_504x },
639 { PCI_VDEVICE(MARVELL, 0x5041), chip_504x },
640 { PCI_VDEVICE(MARVELL, 0x5080), chip_5080 },
641 { PCI_VDEVICE(MARVELL, 0x5081), chip_508x },
cfbf723e
AC		 642 /* RocketRAID 1740/174x have different identifiers */
643 { PCI_VDEVICE(TTI, 0x1740), chip_508x },
644 { PCI_VDEVICE(TTI, 0x1742), chip_508x },
2d2744fc
JG		 645
646 { PCI_VDEVICE(MARVELL, 0x6040), chip_604x },
647 { PCI_VDEVICE(MARVELL, 0x6041), chip_604x },
648 { PCI_VDEVICE(MARVELL, 0x6042), chip_6042 },
649 { PCI_VDEVICE(MARVELL, 0x6080), chip_608x },
650 { PCI_VDEVICE(MARVELL, 0x6081), chip_608x },
651
652 { PCI_VDEVICE(ADAPTEC2, 0x0241), chip_604x },
653
d9f9c6bc
FA		 654 /* Adaptec 1430SA */
655 { PCI_VDEVICE(ADAPTEC2, 0x0243), chip_7042 },
656
02a121da 657 /* Marvell 7042 support */
6a3d586d
MT		 658 { PCI_VDEVICE(MARVELL, 0x7042), chip_7042 },
659
02a121da
ML		 660 /* Highpoint RocketRAID PCIe series */
661 { PCI_VDEVICE(TTI, 0x2300), chip_7042 },
662 { PCI_VDEVICE(TTI, 0x2310), chip_7042 },
663
2d2744fc 664 { } /* terminate list */
20f733e7
BR		 665};
666
47c2b677
JG		 667static const struct mv_hw_ops mv5xxx_ops = {
668 .phy_errata = mv5_phy_errata,
669 .enable_leds = mv5_enable_leds,
670 .read_preamp = mv5_read_preamp,
671 .reset_hc = mv5_reset_hc,
522479fb
JG		 672 .reset_flash = mv5_reset_flash,
673 .reset_bus = mv5_reset_bus,
47c2b677
JG		 674};
675
676static const struct mv_hw_ops mv6xxx_ops = {
677 .phy_errata = mv6_phy_errata,
678 .enable_leds = mv6_enable_leds,
679 .read_preamp = mv6_read_preamp,
680 .reset_hc = mv6_reset_hc,
522479fb
JG		 681 .reset_flash = mv6_reset_flash,
682 .reset_bus = mv_reset_pci_bus,
47c2b677
JG		 683};
684
f351b2d6
SB		 685static const struct mv_hw_ops mv_soc_ops = {
686 .phy_errata = mv6_phy_errata,
687 .enable_leds = mv_soc_enable_leds,
688 .read_preamp = mv_soc_read_preamp,
689 .reset_hc = mv_soc_reset_hc,
690 .reset_flash = mv_soc_reset_flash,
691 .reset_bus = mv_soc_reset_bus,
692};
693
20f733e7
BR		 694/*
695 * Functions
696 */
697
698static inline void writelfl(unsigned long data, void __iomem *addr)
699{
700 writel(data, addr);
701 (void) readl(addr); /* flush to avoid PCI posted write */
702}
703
20f733e7
BR		 704static inline void __iomem *mv_hc_base(void __iomem *base, unsigned int hc)
705{
706 return (base + MV_SATAHC0_REG_BASE + (hc * MV_SATAHC_REG_SZ));
707}
708
c9d39130
JG		 709static inline unsigned int mv_hc_from_port(unsigned int port)
710{
711 return port >> MV_PORT_HC_SHIFT;
712}
713
714static inline unsigned int mv_hardport_from_port(unsigned int port)
715{
716 return port & MV_PORT_MASK;
717}
718
719static inline void __iomem *mv_hc_base_from_port(void __iomem *base,
720 unsigned int port)
721{
722 return mv_hc_base(base, mv_hc_from_port(port));
723}
724
20f733e7
BR		 725static inline void __iomem *mv_port_base(void __iomem *base, unsigned int port)
726{
c9d39130 727 return mv_hc_base_from_port(base, port) +
8b260248 728 MV_SATAHC_ARBTR_REG_SZ +
c9d39130 729 (mv_hardport_from_port(port) * MV_PORT_REG_SZ);
20f733e7
BR		 730}
731
e12bef50
ML		 732static void __iomem *mv5_phy_base(void __iomem *mmio, unsigned int port)
733{
734 void __iomem *hc_mmio = mv_hc_base_from_port(mmio, port);
735 unsigned long ofs = (mv_hardport_from_port(port) + 1) * 0x100UL;
736
737 return hc_mmio + ofs;
738}
739
f351b2d6
SB		 740static inline void __iomem *mv_host_base(struct ata_host *host)
741{
742 struct mv_host_priv *hpriv = host->private_data;
743 return hpriv->base;
744}
745
20f733e7
BR		 746static inline void __iomem *mv_ap_base(struct ata_port *ap)
747{
f351b2d6 748 return mv_port_base(mv_host_base(ap->host), ap->port_no);
20f733e7
BR		 749}
750
cca3974e 751static inline int mv_get_hc_count(unsigned long port_flags)
31961943 752{
cca3974e 753 return ((port_flags & MV_FLAG_DUAL_HC) ? 2 : 1);
31961943
BR		 754}
755
c5d3e45a
JG		 756static void mv_set_edma_ptrs(void __iomem *port_mmio,
757 struct mv_host_priv *hpriv,
758 struct mv_port_priv *pp)
759{
bdd4ddde
JG		 760 u32 index;
761
c5d3e45a
JG		 762 /*
763 * initialize request queue
764 */
bdd4ddde
JG		 765 index = (pp->req_idx & MV_MAX_Q_DEPTH_MASK) << EDMA_REQ_Q_PTR_SHIFT;
766
c5d3e45a
JG		 767 WARN_ON(pp->crqb_dma & 0x3ff);
768 writel((pp->crqb_dma >> 16) >> 16, port_mmio + EDMA_REQ_Q_BASE_HI_OFS);
bdd4ddde 769 writelfl((pp->crqb_dma & EDMA_REQ_Q_BASE_LO_MASK) | index,
c5d3e45a
JG		 770 port_mmio + EDMA_REQ_Q_IN_PTR_OFS);
771
772 if (hpriv->hp_flags & MV_HP_ERRATA_XX42A0)
bdd4ddde 773 writelfl((pp->crqb_dma & 0xffffffff) | index,
c5d3e45a
JG		 774 port_mmio + EDMA_REQ_Q_OUT_PTR_OFS);
775 else
bdd4ddde 776 writelfl(index, port_mmio + EDMA_REQ_Q_OUT_PTR_OFS);
c5d3e45a
JG		 777
778 /*
779 * initialize response queue
780 */
bdd4ddde
JG		 781 index = (pp->resp_idx & MV_MAX_Q_DEPTH_MASK) << EDMA_RSP_Q_PTR_SHIFT;
782
c5d3e45a
JG		 783 WARN_ON(pp->crpb_dma & 0xff);
784 writel((pp->crpb_dma >> 16) >> 16, port_mmio + EDMA_RSP_Q_BASE_HI_OFS);
785
786 if (hpriv->hp_flags & MV_HP_ERRATA_XX42A0)
bdd4ddde 787 writelfl((pp->crpb_dma & 0xffffffff) | index,
c5d3e45a
JG		 788 port_mmio + EDMA_RSP_Q_IN_PTR_OFS);
789 else
bdd4ddde 790 writelfl(index, port_mmio + EDMA_RSP_Q_IN_PTR_OFS);
c5d3e45a 791
bdd4ddde 792 writelfl((pp->crpb_dma & EDMA_RSP_Q_BASE_LO_MASK) | index,
c5d3e45a 793 port_mmio + EDMA_RSP_Q_OUT_PTR_OFS);
c5d3e45a
JG		 794}
795
05b308e1
BR		 796/**
797 * mv_start_dma - Enable eDMA engine
798 * @base: port base address
799 * @pp: port private data
800 *
beec7dbc
TH		 801 * Verify the local cache of the eDMA state is accurate with a
802 * WARN_ON.
05b308e1
BR		 803 *
804 * LOCKING:
805 * Inherited from caller.
806 */
0c58912e 807static void mv_start_dma(struct ata_port *ap, void __iomem *port_mmio,
72109168 808 struct mv_port_priv *pp, u8 protocol)
20f733e7 809{
72109168
ML		 810 int want_ncq = (protocol == ATA_PROT_NCQ);
811
812 if (pp->pp_flags & MV_PP_FLAG_EDMA_EN) {
813 int using_ncq = ((pp->pp_flags & MV_PP_FLAG_NCQ_EN) != 0);
814 if (want_ncq != using_ncq)
b562468c 815 mv_stop_edma(ap);
72109168 816 }
c5d3e45a 817 if (!(pp->pp_flags & MV_PP_FLAG_EDMA_EN)) {
0c58912e
ML		 818 struct mv_host_priv *hpriv = ap->host->private_data;
819 int hard_port = mv_hardport_from_port(ap->port_no);
820 void __iomem *hc_mmio = mv_hc_base_from_port(
0fca0d6f 821 mv_host_base(ap->host), hard_port);
0c58912e
ML		 822 u32 hc_irq_cause, ipending;
823
bdd4ddde 824 /* clear EDMA event indicators, if any */
f630d562 825 writelfl(0, port_mmio + EDMA_ERR_IRQ_CAUSE_OFS);
bdd4ddde 826
0c58912e
ML		 827 /* clear EDMA interrupt indicator, if any */
828 hc_irq_cause = readl(hc_mmio + HC_IRQ_CAUSE_OFS);
829 ipending = (DEV_IRQ << hard_port) |
830 (CRPB_DMA_DONE << hard_port);
831 if (hc_irq_cause & ipending) {
832 writelfl(hc_irq_cause & ~ipending,
833 hc_mmio + HC_IRQ_CAUSE_OFS);
834 }
835
e12bef50 836 mv_edma_cfg(ap, want_ncq);
0c58912e
ML		 837
838 /* clear FIS IRQ Cause */
839 writelfl(0, port_mmio + SATA_FIS_IRQ_CAUSE_OFS);
840
f630d562 841 mv_set_edma_ptrs(port_mmio, hpriv, pp);
bdd4ddde 842
f630d562 843 writelfl(EDMA_EN, port_mmio + EDMA_CMD_OFS);
afb0edd9
BR		 844 pp->pp_flags |= MV_PP_FLAG_EDMA_EN;
845 }
f630d562 846 WARN_ON(!(EDMA_EN & readl(port_mmio + EDMA_CMD_OFS)));
20f733e7
BR		 847}
848
05b308e1 849/**
e12bef50 850 * mv_stop_edma_engine - Disable eDMA engine
b562468c 851 * @port_mmio: io base address
05b308e1
BR		 852 *
853 * LOCKING:
854 * Inherited from caller.
855 */
b562468c 856static int mv_stop_edma_engine(void __iomem *port_mmio)
20f733e7 857{
b562468c 858 int i;
31961943 859
b562468c
ML		 860 /* Disable eDMA. The disable bit auto clears. */
861 writelfl(EDMA_DS, port_mmio + EDMA_CMD_OFS);
8b260248 862
b562468c
ML		 863 /* Wait for the chip to confirm eDMA is off. */
864 for (i = 10000; i > 0; i--) {
865 u32 reg = readl(port_mmio + EDMA_CMD_OFS);
4537deb5 866 if (!(reg & EDMA_EN))
b562468c
ML		 867 return 0;
868 udelay(10);
31961943 869 }
b562468c 870 return -EIO;
20f733e7
BR		 871}
872
e12bef50 873static int mv_stop_edma(struct ata_port *ap)
0ea9e179 874{
b562468c
ML		 875 void __iomem *port_mmio = mv_ap_base(ap);
876 struct mv_port_priv *pp = ap->private_data;
0ea9e179 877
b562468c
ML		 878 if (!(pp->pp_flags & MV_PP_FLAG_EDMA_EN))
879 return 0;
880 pp->pp_flags &= ~MV_PP_FLAG_EDMA_EN;
881 if (mv_stop_edma_engine(port_mmio)) {
882 ata_port_printk(ap, KERN_ERR, "Unable to stop eDMA\n");
883 return -EIO;
884 }
885 return 0;
0ea9e179
JG		 886}
887
8a70f8dc 888#ifdef ATA_DEBUG
31961943 889static void mv_dump_mem(void __iomem *start, unsigned bytes)
20f733e7 890{
31961943
BR		 891 int b, w;
892 for (b = 0; b < bytes; ) {
893 DPRINTK("%p: ", start + b);
894 for (w = 0; b < bytes && w < 4; w++) {
2dcb407e 895 printk("%08x ", readl(start + b));
31961943
BR		 896 b += sizeof(u32);
897 }
898 printk("\n");
899 }
31961943 900}
8a70f8dc
JG		 901#endif
902
31961943
BR		 903static void mv_dump_pci_cfg(struct pci_dev *pdev, unsigned bytes)
904{
905#ifdef ATA_DEBUG
906 int b, w;
907 u32 dw;
908 for (b = 0; b < bytes; ) {
909 DPRINTK("%02x: ", b);
910 for (w = 0; b < bytes && w < 4; w++) {
2dcb407e
JG		 911 (void) pci_read_config_dword(pdev, b, &dw);
912 printk("%08x ", dw);
31961943
BR		 913 b += sizeof(u32);
914 }
915 printk("\n");
916 }
917#endif
918}
919static void mv_dump_all_regs(void __iomem *mmio_base, int port,
920 struct pci_dev *pdev)
921{
922#ifdef ATA_DEBUG
8b260248 923 void __iomem *hc_base = mv_hc_base(mmio_base,
31961943
BR		 924 port >> MV_PORT_HC_SHIFT);
925 void __iomem *port_base;
926 int start_port, num_ports, p, start_hc, num_hcs, hc;
927
928 if (0 > port) {
929 start_hc = start_port = 0;
930 num_ports = 8; /* shld be benign for 4 port devs */
931 num_hcs = 2;
932 } else {
933 start_hc = port >> MV_PORT_HC_SHIFT;
934 start_port = port;
935 num_ports = num_hcs = 1;
936 }
8b260248 937 DPRINTK("All registers for port(s) %u-%u:\n", start_port,
31961943
BR		 938 num_ports > 1 ? num_ports - 1 : start_port);
939
940 if (NULL != pdev) {
941 DPRINTK("PCI config space regs:\n");
942 mv_dump_pci_cfg(pdev, 0x68);
943 }
944 DPRINTK("PCI regs:\n");
945 mv_dump_mem(mmio_base+0xc00, 0x3c);
946 mv_dump_mem(mmio_base+0xd00, 0x34);
947 mv_dump_mem(mmio_base+0xf00, 0x4);
948 mv_dump_mem(mmio_base+0x1d00, 0x6c);
949 for (hc = start_hc; hc < start_hc + num_hcs; hc++) {
d220c37e 950 hc_base = mv_hc_base(mmio_base, hc);
31961943
BR		 951 DPRINTK("HC regs (HC %i):\n", hc);
952 mv_dump_mem(hc_base, 0x1c);
953 }
954 for (p = start_port; p < start_port + num_ports; p++) {
955 port_base = mv_port_base(mmio_base, p);
2dcb407e 956 DPRINTK("EDMA regs (port %i):\n", p);
31961943 957 mv_dump_mem(port_base, 0x54);
2dcb407e 958 DPRINTK("SATA regs (port %i):\n", p);
31961943
BR		 959 mv_dump_mem(port_base+0x300, 0x60);
960 }
961#endif
20f733e7
BR		 962}
963
964static unsigned int mv_scr_offset(unsigned int sc_reg_in)
965{
966 unsigned int ofs;
967
968 switch (sc_reg_in) {
969 case SCR_STATUS:
970 case SCR_CONTROL:
971 case SCR_ERROR:
972 ofs = SATA_STATUS_OFS + (sc_reg_in * sizeof(u32));
973 break;
974 case SCR_ACTIVE:
975 ofs = SATA_ACTIVE_OFS; /* active is not with the others */
976 break;
977 default:
978 ofs = 0xffffffffU;
979 break;
980 }
981 return ofs;
982}
983
da3dbb17 984static int mv_scr_read(struct ata_port *ap, unsigned int sc_reg_in, u32 *val)
20f733e7
BR		 985{
986 unsigned int ofs = mv_scr_offset(sc_reg_in);
987
da3dbb17
TH		 988 if (ofs != 0xffffffffU) {
989 *val = readl(mv_ap_base(ap) + ofs);
990 return 0;
991 } else
992 return -EINVAL;
20f733e7
BR		 993}
994
da3dbb17 995static int mv_scr_write(struct ata_port *ap, unsigned int sc_reg_in, u32 val)
20f733e7
BR		 996{
997 unsigned int ofs = mv_scr_offset(sc_reg_in);
998
da3dbb17 999 if (ofs != 0xffffffffU) {
20f733e7 1000 writelfl(val, mv_ap_base(ap) + ofs);
da3dbb17
TH		 1001 return 0;
1002 } else
1003 return -EINVAL;
20f733e7
BR		 1004}
1005
f273827e
ML		 1006static void mv6_dev_config(struct ata_device *adev)
1007{
1008 /*
1009 * We don't have hob_nsect when doing NCQ commands on Gen-II.
1010 * See mv_qc_prep() for more info.
1011 */
1012 if (adev->flags & ATA_DFLAG_NCQ)
1013 if (adev->max_sectors > ATA_MAX_SECTORS)
1014 adev->max_sectors = ATA_MAX_SECTORS;
1015}
1016
e12bef50 1017static void mv_edma_cfg(struct ata_port *ap, int want_ncq)
e4e7b892 1018{
0c58912e 1019 u32 cfg;
e12bef50
ML		 1020 struct mv_port_priv *pp = ap->private_data;
1021 struct mv_host_priv *hpriv = ap->host->private_data;
1022 void __iomem *port_mmio = mv_ap_base(ap);
e4e7b892
JG		 1023
1024 /* set up non-NCQ EDMA configuration */
0c58912e 1025 cfg = EDMA_CFG_Q_DEPTH; /* always 0x1f for *all* chips */
e4e7b892 1026
0c58912e 1027 if (IS_GEN_I(hpriv))
e4e7b892
JG		 1028 cfg |= (1 << 8); /* enab config burst size mask */
1029
0c58912e 1030 else if (IS_GEN_II(hpriv))
e4e7b892
JG		 1031 cfg |= EDMA_CFG_RD_BRST_EXT | EDMA_CFG_WR_BUFF_LEN;
1032
1033 else if (IS_GEN_IIE(hpriv)) {
e728eabe
JG		 1034 cfg |= (1 << 23); /* do not mask PM field in rx'd FIS */
1035 cfg |= (1 << 22); /* enab 4-entry host queue cache */
e4e7b892 1036 cfg |= (1 << 18); /* enab early completion */
e728eabe 1037 cfg |= (1 << 17); /* enab cut-through (dis stor&forwrd) */
e4e7b892
JG		 1038 }
1039
72109168
ML		 1040 if (want_ncq) {
1041 cfg |= EDMA_CFG_NCQ;
1042 pp->pp_flags |= MV_PP_FLAG_NCQ_EN;
1043 } else
1044 pp->pp_flags &= ~MV_PP_FLAG_NCQ_EN;
1045
e4e7b892
JG		 1046 writelfl(cfg, port_mmio + EDMA_CFG_OFS);
1047}
1048
da2fa9ba
ML		 1049static void mv_port_free_dma_mem(struct ata_port *ap)
1050{
1051 struct mv_host_priv *hpriv = ap->host->private_data;
1052 struct mv_port_priv *pp = ap->private_data;
eb73d558 1053 int tag;
da2fa9ba
ML		 1054
1055 if (pp->crqb) {
1056 dma_pool_free(hpriv->crqb_pool, pp->crqb, pp->crqb_dma);
1057 pp->crqb = NULL;
1058 }
1059 if (pp->crpb) {
1060 dma_pool_free(hpriv->crpb_pool, pp->crpb, pp->crpb_dma);
1061 pp->crpb = NULL;
1062 }
eb73d558
ML		 1063 /*
1064 * For GEN_I, there's no NCQ, so we have only a single sg_tbl.
1065 * For later hardware, we have one unique sg_tbl per NCQ tag.
1066 */
1067 for (tag = 0; tag < MV_MAX_Q_DEPTH; ++tag) {
1068 if (pp->sg_tbl[tag]) {
1069 if (tag == 0 || !IS_GEN_I(hpriv))
1070 dma_pool_free(hpriv->sg_tbl_pool,
1071 pp->sg_tbl[tag],
1072 pp->sg_tbl_dma[tag]);
1073 pp->sg_tbl[tag] = NULL;
1074 }
da2fa9ba
ML		 1075 }
1076}
1077
05b308e1
BR		 1078/**
1079 * mv_port_start - Port specific init/start routine.
1080 * @ap: ATA channel to manipulate
1081 *
1082 * Allocate and point to DMA memory, init port private memory,
1083 * zero indices.
1084 *
1085 * LOCKING:
1086 * Inherited from caller.
1087 */
31961943
BR		 1088static int mv_port_start(struct ata_port *ap)
1089{
cca3974e
JG		 1090 struct device *dev = ap->host->dev;
1091 struct mv_host_priv *hpriv = ap->host->private_data;
31961943
BR		 1092 struct mv_port_priv *pp;
1093 void __iomem *port_mmio = mv_ap_base(ap);
0ea9e179 1094 unsigned long flags;
dde20207 1095 int tag;
31961943 1096
24dc5f33 1097 pp = devm_kzalloc(dev, sizeof(*pp), GFP_KERNEL);
6037d6bb 1098 if (!pp)
24dc5f33 1099 return -ENOMEM;
da2fa9ba 1100 ap->private_data = pp;
31961943 1101
da2fa9ba
ML		 1102 pp->crqb = dma_pool_alloc(hpriv->crqb_pool, GFP_KERNEL, &pp->crqb_dma);
1103 if (!pp->crqb)
1104 return -ENOMEM;
1105 memset(pp->crqb, 0, MV_CRQB_Q_SZ);
31961943 1106
da2fa9ba
ML		 1107 pp->crpb = dma_pool_alloc(hpriv->crpb_pool, GFP_KERNEL, &pp->crpb_dma);
1108 if (!pp->crpb)
1109 goto out_port_free_dma_mem;
1110 memset(pp->crpb, 0, MV_CRPB_Q_SZ);
31961943 1111
eb73d558
ML		 1112 /*
1113 * For GEN_I, there's no NCQ, so we only allocate a single sg_tbl.
1114 * For later hardware, we need one unique sg_tbl per NCQ tag.
1115 */
1116 for (tag = 0; tag < MV_MAX_Q_DEPTH; ++tag) {
1117 if (tag == 0 || !IS_GEN_I(hpriv)) {
1118 pp->sg_tbl[tag] = dma_pool_alloc(hpriv->sg_tbl_pool,
1119 GFP_KERNEL, &pp->sg_tbl_dma[tag]);
1120 if (!pp->sg_tbl[tag])
1121 goto out_port_free_dma_mem;
1122 } else {
1123 pp->sg_tbl[tag] = pp->sg_tbl[0];
1124 pp->sg_tbl_dma[tag] = pp->sg_tbl_dma[0];
1125 }
1126 }
31961943 1127
0ea9e179
JG		 1128 spin_lock_irqsave(&ap->host->lock, flags);
1129
e12bef50 1130 mv_edma_cfg(ap, 0);
c5d3e45a 1131 mv_set_edma_ptrs(port_mmio, hpriv, pp);
31961943 1132
0ea9e179
JG		 1133 spin_unlock_irqrestore(&ap->host->lock, flags);
1134
31961943
BR		 1135 /* Don't turn on EDMA here...do it before DMA commands only. Else
1136 * we'll be unable to send non-data, PIO, etc due to restricted access
1137 * to shadow regs.
1138 */
31961943 1139 return 0;
da2fa9ba
ML		 1140
1141out_port_free_dma_mem:
1142 mv_port_free_dma_mem(ap);
1143 return -ENOMEM;
31961943
BR		 1144}
1145
05b308e1
BR		 1146/**
1147 * mv_port_stop - Port specific cleanup/stop routine.
1148 * @ap: ATA channel to manipulate
1149 *
1150 * Stop DMA, cleanup port memory.
1151 *
1152 * LOCKING:
cca3974e 1153 * This routine uses the host lock to protect the DMA stop.
05b308e1 1154 */
31961943
BR		 1155static void mv_port_stop(struct ata_port *ap)
1156{
e12bef50 1157 mv_stop_edma(ap);
da2fa9ba 1158 mv_port_free_dma_mem(ap);
31961943
BR		 1159}
1160
05b308e1
BR		 1161/**
1162 * mv_fill_sg - Fill out the Marvell ePRD (scatter gather) entries
1163 * @qc: queued command whose SG list to source from
1164 *
1165 * Populate the SG list and mark the last entry.
1166 *
1167 * LOCKING:
1168 * Inherited from caller.
1169 */
6c08772e 1170static void mv_fill_sg(struct ata_queued_cmd *qc)
31961943
BR		 1171{
1172 struct mv_port_priv *pp = qc->ap->private_data;
972c26bd 1173 struct scatterlist *sg;
3be6cbd7 1174 struct mv_sg *mv_sg, *last_sg = NULL;
ff2aeb1e 1175 unsigned int si;
31961943 1176
eb73d558 1177 mv_sg = pp->sg_tbl[qc->tag];
ff2aeb1e 1178 for_each_sg(qc->sg, sg, qc->n_elem, si) {
d88184fb
JG		 1179 dma_addr_t addr = sg_dma_address(sg);
1180 u32 sg_len = sg_dma_len(sg);
22374677 1181
4007b493
OJ		 1182 while (sg_len) {
1183 u32 offset = addr & 0xffff;
1184 u32 len = sg_len;
22374677 1185
4007b493
OJ		 1186 if ((offset + sg_len > 0x10000))
1187 len = 0x10000 - offset;
1188
1189 mv_sg->addr = cpu_to_le32(addr & 0xffffffff);
1190 mv_sg->addr_hi = cpu_to_le32((addr >> 16) >> 16);
6c08772e 1191 mv_sg->flags_size = cpu_to_le32(len & 0xffff);
4007b493
OJ		 1192
1193 sg_len -= len;
1194 addr += len;
1195
3be6cbd7 1196 last_sg = mv_sg;
4007b493 1197 mv_sg++;
4007b493 1198 }
31961943 1199 }
3be6cbd7
JG		 1200
1201 if (likely(last_sg))
1202 last_sg->flags_size |= cpu_to_le32(EPRD_FLAG_END_OF_TBL);
31961943
BR		 1203}
1204
5796d1c4 1205static void mv_crqb_pack_cmd(__le16 *cmdw, u8 data, u8 addr, unsigned last)
31961943 1206{
559eedad 1207 u16 tmp = data | (addr << CRQB_CMD_ADDR_SHIFT) | CRQB_CMD_CS |
31961943 1208 (last ? CRQB_CMD_LAST : 0);
559eedad 1209 *cmdw = cpu_to_le16(tmp);
31961943
BR		 1210}
1211
05b308e1
BR		 1212/**
1213 * mv_qc_prep - Host specific command preparation.
1214 * @qc: queued command to prepare
1215 *
1216 * This routine simply redirects to the general purpose routine
1217 * if command is not DMA. Else, it handles prep of the CRQB
1218 * (command request block), does some sanity checking, and calls
1219 * the SG load routine.
1220 *
1221 * LOCKING:
1222 * Inherited from caller.
1223 */
31961943
BR		 1224static void mv_qc_prep(struct ata_queued_cmd *qc)
1225{
1226 struct ata_port *ap = qc->ap;
1227 struct mv_port_priv *pp = ap->private_data;
e1469874 1228 __le16 *cw;
31961943
BR		 1229 struct ata_taskfile *tf;
1230 u16 flags = 0;
a6432436 1231 unsigned in_index;
31961943 1232
138bfdd0
ML		 1233 if ((qc->tf.protocol != ATA_PROT_DMA) &&
1234 (qc->tf.protocol != ATA_PROT_NCQ))
31961943 1235 return;
20f733e7 1236
31961943
BR		 1237 /* Fill in command request block
1238 */
e4e7b892 1239 if (!(qc->tf.flags & ATA_TFLAG_WRITE))
31961943 1240 flags |= CRQB_FLAG_READ;
beec7dbc 1241 WARN_ON(MV_MAX_Q_DEPTH <= qc->tag);
31961943
BR		 1242 flags |= qc->tag << CRQB_TAG_SHIFT;
1243
bdd4ddde
JG		 1244 /* get current queue index from software */
1245 in_index = pp->req_idx & MV_MAX_Q_DEPTH_MASK;
a6432436
ML		 1246
1247 pp->crqb[in_index].sg_addr =
eb73d558 1248 cpu_to_le32(pp->sg_tbl_dma[qc->tag] & 0xffffffff);
a6432436 1249 pp->crqb[in_index].sg_addr_hi =
eb73d558 1250 cpu_to_le32((pp->sg_tbl_dma[qc->tag] >> 16) >> 16);
a6432436 1251 pp->crqb[in_index].ctrl_flags = cpu_to_le16(flags);
31961943 1252
a6432436 1253 cw = &pp->crqb[in_index].ata_cmd[0];
31961943
BR		 1254 tf = &qc->tf;
1255
1256 /* Sadly, the CRQB cannot accomodate all registers--there are
1257 * only 11 bytes...so we must pick and choose required
1258 * registers based on the command. So, we drop feature and
1259 * hob_feature for [RW] DMA commands, but they are needed for
1260 * NCQ. NCQ will drop hob_nsect.
20f733e7 1261 */
31961943
BR		 1262 switch (tf->command) {
1263 case ATA_CMD_READ:
1264 case ATA_CMD_READ_EXT:
1265 case ATA_CMD_WRITE:
1266 case ATA_CMD_WRITE_EXT:
c15d85c8 1267 case ATA_CMD_WRITE_FUA_EXT:
31961943
BR		 1268 mv_crqb_pack_cmd(cw++, tf->hob_nsect, ATA_REG_NSECT, 0);
1269 break;
31961943
BR		 1270 case ATA_CMD_FPDMA_READ:
1271 case ATA_CMD_FPDMA_WRITE:
8b260248 1272 mv_crqb_pack_cmd(cw++, tf->hob_feature, ATA_REG_FEATURE, 0);
31961943
BR		 1273 mv_crqb_pack_cmd(cw++, tf->feature, ATA_REG_FEATURE, 0);
1274 break;
31961943
BR		 1275 default:
1276 /* The only other commands EDMA supports in non-queued and
1277 * non-NCQ mode are: [RW] STREAM DMA and W DMA FUA EXT, none
1278 * of which are defined/used by Linux. If we get here, this
1279 * driver needs work.
1280 *
1281 * FIXME: modify libata to give qc_prep a return value and
1282 * return error here.
1283 */
1284 BUG_ON(tf->command);
1285 break;
1286 }
1287 mv_crqb_pack_cmd(cw++, tf->nsect, ATA_REG_NSECT, 0);
1288 mv_crqb_pack_cmd(cw++, tf->hob_lbal, ATA_REG_LBAL, 0);
1289 mv_crqb_pack_cmd(cw++, tf->lbal, ATA_REG_LBAL, 0);
1290 mv_crqb_pack_cmd(cw++, tf->hob_lbam, ATA_REG_LBAM, 0);
1291 mv_crqb_pack_cmd(cw++, tf->lbam, ATA_REG_LBAM, 0);
1292 mv_crqb_pack_cmd(cw++, tf->hob_lbah, ATA_REG_LBAH, 0);
1293 mv_crqb_pack_cmd(cw++, tf->lbah, ATA_REG_LBAH, 0);
1294 mv_crqb_pack_cmd(cw++, tf->device, ATA_REG_DEVICE, 0);
1295 mv_crqb_pack_cmd(cw++, tf->command, ATA_REG_CMD, 1); /* last */
1296
e4e7b892
JG		 1297 if (!(qc->flags & ATA_QCFLAG_DMAMAP))
1298 return;
1299 mv_fill_sg(qc);
1300}
1301
1302/**
1303 * mv_qc_prep_iie - Host specific command preparation.
1304 * @qc: queued command to prepare
1305 *
1306 * This routine simply redirects to the general purpose routine
1307 * if command is not DMA. Else, it handles prep of the CRQB
1308 * (command request block), does some sanity checking, and calls
1309 * the SG load routine.
1310 *
1311 * LOCKING:
1312 * Inherited from caller.
1313 */
1314static void mv_qc_prep_iie(struct ata_queued_cmd *qc)
1315{
1316 struct ata_port *ap = qc->ap;
1317 struct mv_port_priv *pp = ap->private_data;
1318 struct mv_crqb_iie *crqb;
1319 struct ata_taskfile *tf;
a6432436 1320 unsigned in_index;
e4e7b892
JG		 1321 u32 flags = 0;
1322
138bfdd0
ML		 1323 if ((qc->tf.protocol != ATA_PROT_DMA) &&
1324 (qc->tf.protocol != ATA_PROT_NCQ))
e4e7b892
JG		 1325 return;
1326
e12bef50 1327 /* Fill in Gen IIE command request block */
e4e7b892
JG		 1328 if (!(qc->tf.flags & ATA_TFLAG_WRITE))
1329 flags |= CRQB_FLAG_READ;
1330
beec7dbc 1331 WARN_ON(MV_MAX_Q_DEPTH <= qc->tag);
e4e7b892 1332 flags |= qc->tag << CRQB_TAG_SHIFT;
8c0aeb4a 1333 flags |= qc->tag << CRQB_HOSTQ_SHIFT;
e4e7b892 1334
bdd4ddde
JG		 1335 /* get current queue index from software */
1336 in_index = pp->req_idx & MV_MAX_Q_DEPTH_MASK;
a6432436
ML		 1337
1338 crqb = (struct mv_crqb_iie *) &pp->crqb[in_index];
eb73d558
ML		 1339 crqb->addr = cpu_to_le32(pp->sg_tbl_dma[qc->tag] & 0xffffffff);
1340 crqb->addr_hi = cpu_to_le32((pp->sg_tbl_dma[qc->tag] >> 16) >> 16);
e4e7b892
JG		 1341 crqb->flags = cpu_to_le32(flags);
1342
1343 tf = &qc->tf;
1344 crqb->ata_cmd[0] = cpu_to_le32(
1345 (tf->command << 16) |
1346 (tf->feature << 24)
1347 );
1348 crqb->ata_cmd[1] = cpu_to_le32(
1349 (tf->lbal << 0) |
1350 (tf->lbam << 8) |
1351 (tf->lbah << 16) |
1352 (tf->device << 24)
1353 );
1354 crqb->ata_cmd[2] = cpu_to_le32(
1355 (tf->hob_lbal << 0) |
1356 (tf->hob_lbam << 8) |
1357 (tf->hob_lbah << 16) |
1358 (tf->hob_feature << 24)
1359 );
1360 crqb->ata_cmd[3] = cpu_to_le32(
1361 (tf->nsect << 0) |
1362 (tf->hob_nsect << 8)
1363 );
1364
1365 if (!(qc->flags & ATA_QCFLAG_DMAMAP))
31961943 1366 return;
31961943
BR		 1367 mv_fill_sg(qc);
1368}
1369
05b308e1
BR		 1370/**
1371 * mv_qc_issue - Initiate a command to the host
1372 * @qc: queued command to start
1373 *
1374 * This routine simply redirects to the general purpose routine
1375 * if command is not DMA. Else, it sanity checks our local
1376 * caches of the request producer/consumer indices then enables
1377 * DMA and bumps the request producer index.
1378 *
1379 * LOCKING:
1380 * Inherited from caller.
1381 */
9a3d9eb0 1382static unsigned int mv_qc_issue(struct ata_queued_cmd *qc)
31961943 1383{
c5d3e45a
JG		 1384 struct ata_port *ap = qc->ap;
1385 void __iomem *port_mmio = mv_ap_base(ap);
1386 struct mv_port_priv *pp = ap->private_data;
bdd4ddde 1387 u32 in_index;
31961943 1388
138bfdd0
ML		 1389 if ((qc->tf.protocol != ATA_PROT_DMA) &&
1390 (qc->tf.protocol != ATA_PROT_NCQ)) {
17c5aab5
ML		 1391 /*
1392 * We're about to send a non-EDMA capable command to the
31961943
BR		 1393 * port. Turn off EDMA so there won't be problems accessing
1394 * shadow block, etc registers.
1395 */
b562468c 1396 mv_stop_edma(ap);
9363c382 1397 return ata_sff_qc_issue(qc);
31961943
BR		 1398 }
1399
72109168 1400 mv_start_dma(ap, port_mmio, pp, qc->tf.protocol);
bdd4ddde 1401
bdd4ddde 1402 pp->req_idx++;
31961943 1403
bdd4ddde 1404 in_index = (pp->req_idx & MV_MAX_Q_DEPTH_MASK) << EDMA_REQ_Q_PTR_SHIFT;
31961943
BR		 1405
1406 /* and write the request in pointer to kick the EDMA to life */
bdd4ddde
JG		 1407 writelfl((pp->crqb_dma & EDMA_REQ_Q_BASE_LO_MASK) | in_index,
1408 port_mmio + EDMA_REQ_Q_IN_PTR_OFS);
31961943
BR		 1409
1410 return 0;
1411}
1412
05b308e1
BR		 1413/**
1414 * mv_err_intr - Handle error interrupts on the port
1415 * @ap: ATA channel to manipulate
9b358e30 1416 * @reset_allowed: bool: 0 == don't trigger from reset here
05b308e1
BR		 1417 *
1418 * In most cases, just clear the interrupt and move on. However,
e12bef50
ML		 1419 * some cases require an eDMA reset, which also performs a COMRESET.
1420 * The SERR case requires a clear of pending errors in the SATA
1421 * SERROR register. Finally, if the port disabled DMA,
1422 * update our cached copy to match.
05b308e1
BR		 1423 *
1424 * LOCKING:
1425 * Inherited from caller.
1426 */
bdd4ddde 1427static void mv_err_intr(struct ata_port *ap, struct ata_queued_cmd *qc)
31961943
BR		 1428{
1429 void __iomem *port_mmio = mv_ap_base(ap);
bdd4ddde
JG		 1430 u32 edma_err_cause, eh_freeze_mask, serr = 0;
1431 struct mv_port_priv *pp = ap->private_data;
1432 struct mv_host_priv *hpriv = ap->host->private_data;
1433 unsigned int edma_enabled = (pp->pp_flags & MV_PP_FLAG_EDMA_EN);
1434 unsigned int action = 0, err_mask = 0;
9af5c9c9 1435 struct ata_eh_info *ehi = &ap->link.eh_info;
20f733e7 1436
bdd4ddde 1437 ata_ehi_clear_desc(ehi);
20f733e7 1438
bdd4ddde
JG		 1439 if (!edma_enabled) {
1440 /* just a guess: do we need to do this? should we
1441 * expand this, and do it in all cases?
1442 */
936fd732
TH		 1443 sata_scr_read(&ap->link, SCR_ERROR, &serr);
1444 sata_scr_write_flush(&ap->link, SCR_ERROR, serr);
20f733e7 1445 }
bdd4ddde
JG		 1446
1447 edma_err_cause = readl(port_mmio + EDMA_ERR_IRQ_CAUSE_OFS);
1448
1449 ata_ehi_push_desc(ehi, "edma_err 0x%08x", edma_err_cause);
1450
1451 /*
1452 * all generations share these EDMA error cause bits
1453 */
1454
1455 if (edma_err_cause & EDMA_ERR_DEV)
1456 err_mask |= AC_ERR_DEV;
1457 if (edma_err_cause & (EDMA_ERR_D_PAR | EDMA_ERR_PRD_PAR |
6c1153e0 1458 EDMA_ERR_CRQB_PAR | EDMA_ERR_CRPB_PAR |
bdd4ddde
JG		 1459 EDMA_ERR_INTRL_PAR)) {
1460 err_mask |= AC_ERR_ATA_BUS;
cf480626 1461 action |= ATA_EH_RESET;
b64bbc39 1462 ata_ehi_push_desc(ehi, "parity error");
bdd4ddde
JG		 1463 }
1464 if (edma_err_cause & (EDMA_ERR_DEV_DCON | EDMA_ERR_DEV_CON)) {
1465 ata_ehi_hotplugged(ehi);
1466 ata_ehi_push_desc(ehi, edma_err_cause & EDMA_ERR_DEV_DCON ?
b64bbc39 1467 "dev disconnect" : "dev connect");
cf480626 1468 action |= ATA_EH_RESET;
bdd4ddde
JG		 1469 }
1470
ee9ccdf7 1471 if (IS_GEN_I(hpriv)) {
bdd4ddde
JG		 1472 eh_freeze_mask = EDMA_EH_FREEZE_5;
1473
1474 if (edma_err_cause & EDMA_ERR_SELF_DIS_5) {
5ab063e3 1475 pp = ap->private_data;
bdd4ddde 1476 pp->pp_flags &= ~MV_PP_FLAG_EDMA_EN;
b64bbc39 1477 ata_ehi_push_desc(ehi, "EDMA self-disable");
bdd4ddde
JG		 1478 }
1479 } else {
1480 eh_freeze_mask = EDMA_EH_FREEZE;
1481
1482 if (edma_err_cause & EDMA_ERR_SELF_DIS) {
5ab063e3 1483 pp = ap->private_data;
bdd4ddde 1484 pp->pp_flags &= ~MV_PP_FLAG_EDMA_EN;
b64bbc39 1485 ata_ehi_push_desc(ehi, "EDMA self-disable");
bdd4ddde
JG		 1486 }
1487
1488 if (edma_err_cause & EDMA_ERR_SERR) {
936fd732
TH		 1489 sata_scr_read(&ap->link, SCR_ERROR, &serr);
1490 sata_scr_write_flush(&ap->link, SCR_ERROR, serr);
bdd4ddde 1491 err_mask = AC_ERR_ATA_BUS;
cf480626 1492 action |= ATA_EH_RESET;
bdd4ddde 1493 }
afb0edd9 1494 }
20f733e7
BR		 1495
1496 /* Clear EDMA now that SERR cleanup done */
3606a380 1497 writelfl(~edma_err_cause, port_mmio + EDMA_ERR_IRQ_CAUSE_OFS);
20f733e7 1498
bdd4ddde
JG		 1499 if (!err_mask) {
1500 err_mask = AC_ERR_OTHER;
cf480626 1501 action |= ATA_EH_RESET;
bdd4ddde
JG		 1502 }
1503
1504 ehi->serror |= serr;
1505 ehi->action |= action;
1506
1507 if (qc)
1508 qc->err_mask |= err_mask;
1509 else
1510 ehi->err_mask |= err_mask;
1511
1512 if (edma_err_cause & eh_freeze_mask)
1513 ata_port_freeze(ap);
1514 else
1515 ata_port_abort(ap);
1516}
1517
1518static void mv_intr_pio(struct ata_port *ap)
1519{
1520 struct ata_queued_cmd *qc;
1521 u8 ata_status;
1522
1523 /* ignore spurious intr if drive still BUSY */
1524 ata_status = readb(ap->ioaddr.status_addr);
1525 if (unlikely(ata_status & ATA_BUSY))
1526 return;
1527
1528 /* get active ATA command */
9af5c9c9 1529 qc = ata_qc_from_tag(ap, ap->link.active_tag);
bdd4ddde
JG		 1530 if (unlikely(!qc)) /* no active tag */
1531 return;
1532 if (qc->tf.flags & ATA_TFLAG_POLLING) /* polling; we don't own qc */
1533 return;
1534
1535 /* and finally, complete the ATA command */
1536 qc->err_mask |= ac_err_mask(ata_status);
1537 ata_qc_complete(qc);
1538}
1539
1540static void mv_intr_edma(struct ata_port *ap)
1541{
1542 void __iomem *port_mmio = mv_ap_base(ap);
1543 struct mv_host_priv *hpriv = ap->host->private_data;
1544 struct mv_port_priv *pp = ap->private_data;
1545 struct ata_queued_cmd *qc;
1546 u32 out_index, in_index;
1547 bool work_done = false;
1548
1549 /* get h/w response queue pointer */
1550 in_index = (readl(port_mmio + EDMA_RSP_Q_IN_PTR_OFS)
1551 >> EDMA_RSP_Q_PTR_SHIFT) & MV_MAX_Q_DEPTH_MASK;
1552
1553 while (1) {
1554 u16 status;
6c1153e0 1555 unsigned int tag;
bdd4ddde
JG		 1556
1557 /* get s/w response queue last-read pointer, and compare */
1558 out_index = pp->resp_idx & MV_MAX_Q_DEPTH_MASK;
1559 if (in_index == out_index)
1560 break;
1561
bdd4ddde 1562 /* 50xx: get active ATA command */
0ea9e179 1563 if (IS_GEN_I(hpriv))
9af5c9c9 1564 tag = ap->link.active_tag;
bdd4ddde 1565
6c1153e0
JG		 1566 /* Gen II/IIE: get active ATA command via tag, to enable
1567 * support for queueing. this works transparently for
1568 * queued and non-queued modes.
bdd4ddde 1569 */
8c0aeb4a
ML		 1570 else
1571 tag = le16_to_cpu(pp->crpb[out_index].id) & 0x1f;
bdd4ddde 1572
6c1153e0 1573 qc = ata_qc_from_tag(ap, tag);
bdd4ddde 1574
cb924419
ML		 1575 /* For non-NCQ mode, the lower 8 bits of status
1576 * are from EDMA_ERR_IRQ_CAUSE_OFS,
1577 * which should be zero if all went well.
bdd4ddde
JG		 1578 */
1579 status = le16_to_cpu(pp->crpb[out_index].flags);
cb924419 1580 if ((status & 0xff) && !(pp->pp_flags & MV_PP_FLAG_NCQ_EN)) {
bdd4ddde
JG		 1581 mv_err_intr(ap, qc);
1582 return;
1583 }
1584
1585 /* and finally, complete the ATA command */
1586 if (qc) {
1587 qc->err_mask |=
1588 ac_err_mask(status >> CRPB_FLAG_STATUS_SHIFT);
1589 ata_qc_complete(qc);
1590 }
1591
0ea9e179 1592 /* advance software response queue pointer, to
bdd4ddde
JG		 1593 * indicate (after the loop completes) to hardware
1594 * that we have consumed a response queue entry.
1595 */
1596 work_done = true;
1597 pp->resp_idx++;
1598 }
1599
1600 if (work_done)
1601 writelfl((pp->crpb_dma & EDMA_RSP_Q_BASE_LO_MASK) |
1602 (out_index << EDMA_RSP_Q_PTR_SHIFT),
1603 port_mmio + EDMA_RSP_Q_OUT_PTR_OFS);
20f733e7
BR		 1604}
1605
05b308e1
BR		 1606/**
1607 * mv_host_intr - Handle all interrupts on the given host controller
cca3974e 1608 * @host: host specific structure
05b308e1
BR		 1609 * @relevant: port error bits relevant to this host controller
1610 * @hc: which host controller we're to look at
1611 *
1612 * Read then write clear the HC interrupt status then walk each
1613 * port connected to the HC and see if it needs servicing. Port
1614 * success ints are reported in the HC interrupt status reg, the
1615 * port error ints are reported in the higher level main
1616 * interrupt status register and thus are passed in via the
1617 * 'relevant' argument.
1618 *
1619 * LOCKING:
1620 * Inherited from caller.
1621 */
cca3974e 1622static void mv_host_intr(struct ata_host *host, u32 relevant, unsigned int hc)
20f733e7 1623{
f351b2d6
SB		 1624 struct mv_host_priv *hpriv = host->private_data;
1625 void __iomem *mmio = hpriv->base;
20f733e7 1626 void __iomem *hc_mmio = mv_hc_base(mmio, hc);
20f733e7 1627 u32 hc_irq_cause;
f351b2d6 1628 int port, port0, last_port;
20f733e7 1629
35177265 1630 if (hc == 0)
20f733e7 1631 port0 = 0;
35177265 1632 else
20f733e7 1633 port0 = MV_PORTS_PER_HC;
20f733e7 1634
f351b2d6
SB		 1635 if (HAS_PCI(host))
1636 last_port = port0 + MV_PORTS_PER_HC;
1637 else
1638 last_port = port0 + hpriv->n_ports;
20f733e7
BR		 1639 /* we'll need the HC success int register in most cases */
1640 hc_irq_cause = readl(hc_mmio + HC_IRQ_CAUSE_OFS);
bdd4ddde
JG		 1641 if (!hc_irq_cause)
1642 return;
1643
1644 writelfl(~hc_irq_cause, hc_mmio + HC_IRQ_CAUSE_OFS);
20f733e7
BR		 1645
1646 VPRINTK("ENTER, hc%u relevant=0x%08x HC IRQ cause=0x%08x\n",
2dcb407e 1647 hc, relevant, hc_irq_cause);
20f733e7 1648
8f71efe2 1649 for (port = port0; port < last_port; port++) {
cca3974e 1650 struct ata_port *ap = host->ports[port];
8f71efe2 1651 struct mv_port_priv *pp;
bdd4ddde 1652 int have_err_bits, hard_port, shift;
55d8ca4f 1653
bdd4ddde 1654 if ((!ap) || (ap->flags & ATA_FLAG_DISABLED))
a2c91a88
JG		 1655 continue;
1656
8f71efe2
YL		 1657 pp = ap->private_data;
1658
31961943 1659 shift = port << 1; /* (port * 2) */
e12bef50 1660 if (port >= MV_PORTS_PER_HC)
20f733e7 1661 shift++; /* skip bit 8 in the HC Main IRQ reg */
e12bef50 1662
bdd4ddde
JG		 1663 have_err_bits = ((PORT0_ERR << shift) & relevant);
1664
1665 if (unlikely(have_err_bits)) {
1666 struct ata_queued_cmd *qc;
8b260248 1667
9af5c9c9 1668 qc = ata_qc_from_tag(ap, ap->link.active_tag);
bdd4ddde
JG		 1669 if (qc && (qc->tf.flags & ATA_TFLAG_POLLING))
1670 continue;
1671
1672 mv_err_intr(ap, qc);
1673 continue;
1674 }
1675
1676 hard_port = mv_hardport_from_port(port); /* range 0..3 */
1677
1678 if (pp->pp_flags & MV_PP_FLAG_EDMA_EN) {
1679 if ((CRPB_DMA_DONE << hard_port) & hc_irq_cause)
1680 mv_intr_edma(ap);
1681 } else {
1682 if ((DEV_IRQ << hard_port) & hc_irq_cause)
1683 mv_intr_pio(ap);
20f733e7
BR		 1684 }
1685 }
1686 VPRINTK("EXIT\n");
1687}
1688
bdd4ddde
JG		 1689static void mv_pci_error(struct ata_host *host, void __iomem *mmio)
1690{
02a121da 1691 struct mv_host_priv *hpriv = host->private_data;
bdd4ddde
JG		 1692 struct ata_port *ap;
1693 struct ata_queued_cmd *qc;
1694 struct ata_eh_info *ehi;
1695 unsigned int i, err_mask, printed = 0;
1696 u32 err_cause;
1697
02a121da 1698 err_cause = readl(mmio + hpriv->irq_cause_ofs);
bdd4ddde
JG		 1699
1700 dev_printk(KERN_ERR, host->dev, "PCI ERROR; PCI IRQ cause=0x%08x\n",
1701 err_cause);
1702
1703 DPRINTK("All regs @ PCI error\n");
1704 mv_dump_all_regs(mmio, -1, to_pci_dev(host->dev));
1705
02a121da 1706 writelfl(0, mmio + hpriv->irq_cause_ofs);
bdd4ddde
JG		 1707
1708 for (i = 0; i < host->n_ports; i++) {
1709 ap = host->ports[i];
936fd732 1710 if (!ata_link_offline(&ap->link)) {
9af5c9c9 1711 ehi = &ap->link.eh_info;
bdd4ddde
JG		 1712 ata_ehi_clear_desc(ehi);
1713 if (!printed++)
1714 ata_ehi_push_desc(ehi,
1715 "PCI err cause 0x%08x", err_cause);
1716 err_mask = AC_ERR_HOST_BUS;
cf480626 1717 ehi->action = ATA_EH_RESET;
9af5c9c9 1718 qc = ata_qc_from_tag(ap, ap->link.active_tag);
bdd4ddde
JG		 1719 if (qc)
1720 qc->err_mask |= err_mask;
1721 else
1722 ehi->err_mask |= err_mask;
1723
1724 ata_port_freeze(ap);
1725 }
1726 }
1727}
1728
05b308e1 1729/**
c5d3e45a 1730 * mv_interrupt - Main interrupt event handler
05b308e1
BR		 1731 * @irq: unused
1732 * @dev_instance: private data; in this case the host structure
05b308e1
BR		 1733 *
1734 * Read the read only register to determine if any host
1735 * controllers have pending interrupts. If so, call lower level
1736 * routine to handle. Also check for PCI errors which are only
1737 * reported here.
1738 *
8b260248 1739 * LOCKING:
cca3974e 1740 * This routine holds the host lock while processing pending
05b308e1
BR		 1741 * interrupts.
1742 */
7d12e780 1743static irqreturn_t mv_interrupt(int irq, void *dev_instance)
20f733e7 1744{
cca3974e 1745 struct ata_host *host = dev_instance;
f351b2d6 1746 struct mv_host_priv *hpriv = host->private_data;
20f733e7 1747 unsigned int hc, handled = 0, n_hcs;
f351b2d6 1748 void __iomem *mmio = hpriv->base;
646a4da5 1749 u32 irq_stat, irq_mask;
20f733e7 1750
e12bef50 1751 /* Note to self: &host->lock == &ap->host->lock == ap->lock */
646a4da5 1752 spin_lock(&host->lock);
f351b2d6
SB		 1753
1754 irq_stat = readl(hpriv->main_cause_reg_addr);
1755 irq_mask = readl(hpriv->main_mask_reg_addr);
20f733e7
BR		 1756
1757 /* check the cases where we either have nothing pending or have read
1758 * a bogus register value which can indicate HW removal or PCI fault
1759 */
646a4da5
ML		 1760 if (!(irq_stat & irq_mask) || (0xffffffffU == irq_stat))
1761 goto out_unlock;
20f733e7 1762
cca3974e 1763 n_hcs = mv_get_hc_count(host->ports[0]->flags);
20f733e7 1764
7bb3c529 1765 if (unlikely((irq_stat & PCI_ERR) && HAS_PCI(host))) {
bdd4ddde
JG		 1766 mv_pci_error(host, mmio);
1767 handled = 1;
1768 goto out_unlock; /* skip all other HC irq handling */
1769 }
1770
20f733e7
BR		 1771 for (hc = 0; hc < n_hcs; hc++) {
1772 u32 relevant = irq_stat & (HC0_IRQ_PEND << (hc * HC_SHIFT));
1773 if (relevant) {
cca3974e 1774 mv_host_intr(host, relevant, hc);
bdd4ddde 1775 handled = 1;
20f733e7
BR		 1776 }
1777 }
615ab953 1778
bdd4ddde 1779out_unlock:
cca3974e 1780 spin_unlock(&host->lock);
20f733e7
BR		 1781
1782 return IRQ_RETVAL(handled);
1783}
1784
c9d39130
JG		 1785static unsigned int mv5_scr_offset(unsigned int sc_reg_in)
1786{
1787 unsigned int ofs;
1788
1789 switch (sc_reg_in) {
1790 case SCR_STATUS:
1791 case SCR_ERROR:
1792 case SCR_CONTROL:
1793 ofs = sc_reg_in * sizeof(u32);
1794 break;
1795 default:
1796 ofs = 0xffffffffU;
1797 break;
1798 }
1799 return ofs;
1800}
1801
da3dbb17 1802static int mv5_scr_read(struct ata_port *ap, unsigned int sc_reg_in, u32 *val)
c9d39130 1803{
f351b2d6
SB		 1804 struct mv_host_priv *hpriv = ap->host->private_data;
1805 void __iomem *mmio = hpriv->base;
0d5ff566 1806 void __iomem *addr = mv5_phy_base(mmio, ap->port_no);
c9d39130
JG		 1807 unsigned int ofs = mv5_scr_offset(sc_reg_in);
1808
da3dbb17
TH		 1809 if (ofs != 0xffffffffU) {
1810 *val = readl(addr + ofs);
1811 return 0;
1812 } else
1813 return -EINVAL;
c9d39130
JG		 1814}
1815
da3dbb17 1816static int mv5_scr_write(struct ata_port *ap, unsigned int sc_reg_in, u32 val)
c9d39130 1817{
f351b2d6
SB		 1818 struct mv_host_priv *hpriv = ap->host->private_data;
1819 void __iomem *mmio = hpriv->base;
0d5ff566 1820 void __iomem *addr = mv5_phy_base(mmio, ap->port_no);
c9d39130
JG		 1821 unsigned int ofs = mv5_scr_offset(sc_reg_in);
1822
da3dbb17 1823 if (ofs != 0xffffffffU) {
0d5ff566 1824 writelfl(val, addr + ofs);
da3dbb17
TH		 1825 return 0;
1826 } else
1827 return -EINVAL;
c9d39130
JG		 1828}
1829
7bb3c529 1830static void mv5_reset_bus(struct ata_host *host, void __iomem *mmio)
522479fb 1831{
7bb3c529 1832 struct pci_dev *pdev = to_pci_dev(host->dev);
522479fb
JG		 1833 int early_5080;
1834
44c10138 1835 early_5080 = (pdev->device == 0x5080) && (pdev->revision == 0);
522479fb
JG		 1836
1837 if (!early_5080) {
1838 u32 tmp = readl(mmio + MV_PCI_EXP_ROM_BAR_CTL);
1839 tmp |= (1 << 0);
1840 writel(tmp, mmio + MV_PCI_EXP_ROM_BAR_CTL);
1841 }
1842
7bb3c529 1843 mv_reset_pci_bus(host, mmio);
522479fb
JG		 1844}
1845
1846static void mv5_reset_flash(struct mv_host_priv *hpriv, void __iomem *mmio)
1847{
1848 writel(0x0fcfffff, mmio + MV_FLASH_CTL);
1849}
1850
47c2b677 1851static void mv5_read_preamp(struct mv_host_priv *hpriv, int idx,
ba3fe8fb
JG		 1852 void __iomem *mmio)
1853{
c9d39130
JG		 1854 void __iomem *phy_mmio = mv5_phy_base(mmio, idx);
1855 u32 tmp;
1856
1857 tmp = readl(phy_mmio + MV5_PHY_MODE);
1858
1859 hpriv->signal[idx].pre = tmp & 0x1800; /* bits 12:11 */
1860 hpriv->signal[idx].amps = tmp & 0xe0; /* bits 7:5 */
ba3fe8fb
JG		 1861}
1862
47c2b677 1863static void mv5_enable_leds(struct mv_host_priv *hpriv, void __iomem *mmio)
ba3fe8fb 1864{
522479fb
JG		 1865 u32 tmp;
1866
1867 writel(0, mmio + MV_GPIO_PORT_CTL);
1868
1869 /* FIXME: handle MV_HP_ERRATA_50XXB2 errata */
1870
1871 tmp = readl(mmio + MV_PCI_EXP_ROM_BAR_CTL);
1872 tmp |= ~(1 << 0);
1873 writel(tmp, mmio + MV_PCI_EXP_ROM_BAR_CTL);
ba3fe8fb
JG		 1874}
1875
2a47ce06
JG		 1876static void mv5_phy_errata(struct mv_host_priv *hpriv, void __iomem *mmio,
1877 unsigned int port)
bca1c4eb 1878{
c9d39130
JG		 1879 void __iomem *phy_mmio = mv5_phy_base(mmio, port);
1880 const u32 mask = (1<<12) | (1<<11) | (1<<7) | (1<<6) | (1<<5);
1881 u32 tmp;
1882 int fix_apm_sq = (hpriv->hp_flags & MV_HP_ERRATA_50XXB0);
1883
1884 if (fix_apm_sq) {
1885 tmp = readl(phy_mmio + MV5_LT_MODE);
1886 tmp |= (1 << 19);
1887 writel(tmp, phy_mmio + MV5_LT_MODE);
1888
1889 tmp = readl(phy_mmio + MV5_PHY_CTL);
1890 tmp &= ~0x3;
1891 tmp |= 0x1;
1892 writel(tmp, phy_mmio + MV5_PHY_CTL);
1893 }
1894
1895 tmp = readl(phy_mmio + MV5_PHY_MODE);
1896 tmp &= ~mask;
1897 tmp |= hpriv->signal[port].pre;
1898 tmp |= hpriv->signal[port].amps;
1899 writel(tmp, phy_mmio + MV5_PHY_MODE);
bca1c4eb
JG		 1900}
1901
c9d39130
JG		 1902
1903#undef ZERO
1904#define ZERO(reg) writel(0, port_mmio + (reg))
1905static void mv5_reset_hc_port(struct mv_host_priv *hpriv, void __iomem *mmio,
1906 unsigned int port)
1907{
1908 void __iomem *port_mmio = mv_port_base(mmio, port);
1909
b562468c
ML		 1910 /*
1911 * The datasheet warns against setting ATA_RST when EDMA is active
1912 * (but doesn't say what the problem might be). So we first try
1913 * to disable the EDMA engine before doing the ATA_RST operation.
1914 */
e12bef50 1915 mv_reset_channel(hpriv, mmio, port);
c9d39130
JG		 1916
1917 ZERO(0x028); /* command */
1918 writel(0x11f, port_mmio + EDMA_CFG_OFS);
1919 ZERO(0x004); /* timer */
1920 ZERO(0x008); /* irq err cause */
1921 ZERO(0x00c); /* irq err mask */
1922 ZERO(0x010); /* rq bah */
1923 ZERO(0x014); /* rq inp */
1924 ZERO(0x018); /* rq outp */
1925 ZERO(0x01c); /* respq bah */
1926 ZERO(0x024); /* respq outp */
1927 ZERO(0x020); /* respq inp */
1928 ZERO(0x02c); /* test control */
1929 writel(0xbc, port_mmio + EDMA_IORDY_TMOUT);
1930}
1931#undef ZERO
1932
1933#define ZERO(reg) writel(0, hc_mmio + (reg))
1934static void mv5_reset_one_hc(struct mv_host_priv *hpriv, void __iomem *mmio,
1935 unsigned int hc)
47c2b677 1936{
c9d39130
JG		 1937 void __iomem *hc_mmio = mv_hc_base(mmio, hc);
1938 u32 tmp;
1939
1940 ZERO(0x00c);
1941 ZERO(0x010);
1942 ZERO(0x014);
1943 ZERO(0x018);
1944
1945 tmp = readl(hc_mmio + 0x20);
1946 tmp &= 0x1c1c1c1c;
1947 tmp |= 0x03030303;
1948 writel(tmp, hc_mmio + 0x20);
1949}
1950#undef ZERO
1951
1952static int mv5_reset_hc(struct mv_host_priv *hpriv, void __iomem *mmio,
1953 unsigned int n_hc)
1954{
1955 unsigned int hc, port;
1956
1957 for (hc = 0; hc < n_hc; hc++) {
1958 for (port = 0; port < MV_PORTS_PER_HC; port++)
1959 mv5_reset_hc_port(hpriv, mmio,
1960 (hc * MV_PORTS_PER_HC) + port);
1961
1962 mv5_reset_one_hc(hpriv, mmio, hc);
1963 }
1964
1965 return 0;
47c2b677
JG		 1966}
1967
101ffae2
JG		 1968#undef ZERO
1969#define ZERO(reg) writel(0, mmio + (reg))
7bb3c529 1970static void mv_reset_pci_bus(struct ata_host *host, void __iomem *mmio)
101ffae2 1971{
02a121da 1972 struct mv_host_priv *hpriv = host->private_data;
101ffae2
JG		 1973 u32 tmp;
1974
1975 tmp = readl(mmio + MV_PCI_MODE);
1976 tmp &= 0xff00ffff;
1977 writel(tmp, mmio + MV_PCI_MODE);
1978
1979 ZERO(MV_PCI_DISC_TIMER);
1980 ZERO(MV_PCI_MSI_TRIGGER);
1981 writel(0x000100ff, mmio + MV_PCI_XBAR_TMOUT);
1982 ZERO(HC_MAIN_IRQ_MASK_OFS);
1983 ZERO(MV_PCI_SERR_MASK);
02a121da
ML		 1984 ZERO(hpriv->irq_cause_ofs);
1985 ZERO(hpriv->irq_mask_ofs);
101ffae2
JG		 1986 ZERO(MV_PCI_ERR_LOW_ADDRESS);
1987 ZERO(MV_PCI_ERR_HIGH_ADDRESS);
1988 ZERO(MV_PCI_ERR_ATTRIBUTE);
1989 ZERO(MV_PCI_ERR_COMMAND);
1990}
1991#undef ZERO
1992
1993static void mv6_reset_flash(struct mv_host_priv *hpriv, void __iomem *mmio)
1994{
1995 u32 tmp;
1996
1997 mv5_reset_flash(hpriv, mmio);
1998
1999 tmp = readl(mmio + MV_GPIO_PORT_CTL);
2000 tmp &= 0x3;
2001 tmp |= (1 << 5) | (1 << 6);
2002 writel(tmp, mmio + MV_GPIO_PORT_CTL);
2003}
2004
2005/**
2006 * mv6_reset_hc - Perform the 6xxx global soft reset
2007 * @mmio: base address of the HBA
2008 *
2009 * This routine only applies to 6xxx parts.
2010 *
2011 * LOCKING:
2012 * Inherited from caller.
2013 */
c9d39130
JG		 2014static int mv6_reset_hc(struct mv_host_priv *hpriv, void __iomem *mmio,
2015 unsigned int n_hc)
101ffae2
JG		 2016{
2017 void __iomem *reg = mmio + PCI_MAIN_CMD_STS_OFS;
2018 int i, rc = 0;
2019 u32 t;
2020
2021 /* Following procedure defined in PCI "main command and status
2022 * register" table.
2023 */
2024 t = readl(reg);
2025 writel(t | STOP_PCI_MASTER, reg);
2026
2027 for (i = 0; i < 1000; i++) {
2028 udelay(1);
2029 t = readl(reg);
2dcb407e 2030 if (PCI_MASTER_EMPTY & t)
101ffae2 2031 break;
101ffae2
JG		 2032 }
2033 if (!(PCI_MASTER_EMPTY & t)) {
2034 printk(KERN_ERR DRV_NAME ": PCI master won't flush\n");
2035 rc = 1;
2036 goto done;
2037 }
2038
2039 /* set reset */
2040 i = 5;
2041 do {
2042 writel(t | GLOB_SFT_RST, reg);
2043 t = readl(reg);
2044 udelay(1);
2045 } while (!(GLOB_SFT_RST & t) && (i-- > 0));
2046
2047 if (!(GLOB_SFT_RST & t)) {
2048 printk(KERN_ERR DRV_NAME ": can't set global reset\n");
2049 rc = 1;
2050 goto done;
2051 }
2052
2053 /* clear reset and *reenable the PCI master* (not mentioned in spec) */
2054 i = 5;
2055 do {
2056 writel(t & ~(GLOB_SFT_RST | STOP_PCI_MASTER), reg);
2057 t = readl(reg);
2058 udelay(1);
2059 } while ((GLOB_SFT_RST & t) && (i-- > 0));
2060
2061 if (GLOB_SFT_RST & t) {
2062 printk(KERN_ERR DRV_NAME ": can't clear global reset\n");
2063 rc = 1;
2064 }
2065done:
2066 return rc;
2067}
2068
47c2b677 2069static void mv6_read_preamp(struct mv_host_priv *hpriv, int idx,
ba3fe8fb
JG		 2070 void __iomem *mmio)
2071{
2072 void __iomem *port_mmio;
2073 u32 tmp;
2074
ba3fe8fb
JG		 2075 tmp = readl(mmio + MV_RESET_CFG);
2076 if ((tmp & (1 << 0)) == 0) {
47c2b677 2077 hpriv->signal[idx].amps = 0x7 << 8;
ba3fe8fb
JG		 2078 hpriv->signal[idx].pre = 0x1 << 5;
2079 return;
2080 }
2081
2082 port_mmio = mv_port_base(mmio, idx);
2083 tmp = readl(port_mmio + PHY_MODE2);
2084
2085 hpriv->signal[idx].amps = tmp & 0x700; /* bits 10:8 */
2086 hpriv->signal[idx].pre = tmp & 0xe0; /* bits 7:5 */
2087}
2088
47c2b677 2089static void mv6_enable_leds(struct mv_host_priv *hpriv, void __iomem *mmio)
ba3fe8fb 2090{
47c2b677 2091 writel(0x00000060, mmio + MV_GPIO_PORT_CTL);
ba3fe8fb
JG		 2092}
2093
c9d39130 2094static void mv6_phy_errata(struct mv_host_priv *hpriv, void __iomem *mmio,
2a47ce06 2095 unsigned int port)
bca1c4eb 2096{
c9d39130
JG		 2097 void __iomem *port_mmio = mv_port_base(mmio, port);
2098
bca1c4eb 2099 u32 hp_flags = hpriv->hp_flags;
47c2b677
JG		 2100 int fix_phy_mode2 =
2101 hp_flags & (MV_HP_ERRATA_60X1B2 | MV_HP_ERRATA_60X1C0);
bca1c4eb 2102 int fix_phy_mode4 =
47c2b677
JG		 2103 hp_flags & (MV_HP_ERRATA_60X1B2 | MV_HP_ERRATA_60X1C0);
2104 u32 m2, tmp;
2105
2106 if (fix_phy_mode2) {
2107 m2 = readl(port_mmio + PHY_MODE2);
2108 m2 &= ~(1 << 16);
2109 m2 |= (1 << 31);
2110 writel(m2, port_mmio + PHY_MODE2);
2111
2112 udelay(200);
2113
2114 m2 = readl(port_mmio + PHY_MODE2);
2115 m2 &= ~((1 << 16) | (1 << 31));
2116 writel(m2, port_mmio + PHY_MODE2);
2117
2118 udelay(200);
2119 }
2120
2121 /* who knows what this magic does */
2122 tmp = readl(port_mmio + PHY_MODE3);
2123 tmp &= ~0x7F800000;
2124 tmp |= 0x2A800000;
2125 writel(tmp, port_mmio + PHY_MODE3);
bca1c4eb
JG		 2126
2127 if (fix_phy_mode4) {
47c2b677 2128 u32 m4;
bca1c4eb
JG		 2129
2130 m4 = readl(port_mmio + PHY_MODE4);
47c2b677
JG		 2131
2132 if (hp_flags & MV_HP_ERRATA_60X1B2)
e12bef50 2133 tmp = readl(port_mmio + PHY_MODE3);
bca1c4eb 2134
e12bef50 2135 /* workaround for errata FEr SATA#10 (part 1) */
bca1c4eb
JG		 2136 m4 = (m4 & ~(1 << 1)) | (1 << 0);
2137
2138 writel(m4, port_mmio + PHY_MODE4);
47c2b677
JG		 2139
2140 if (hp_flags & MV_HP_ERRATA_60X1B2)
e12bef50 2141 writel(tmp, port_mmio + PHY_MODE3);
bca1c4eb
JG		 2142 }
2143
2144 /* Revert values of pre-emphasis and signal amps to the saved ones */
2145 m2 = readl(port_mmio + PHY_MODE2);
2146
2147 m2 &= ~MV_M2_PREAMP_MASK;
2a47ce06
JG		 2148 m2 |= hpriv->signal[port].amps;
2149 m2 |= hpriv->signal[port].pre;
47c2b677 2150 m2 &= ~(1 << 16);
bca1c4eb 2151
e4e7b892
JG		 2152 /* according to mvSata 3.6.1, some IIE values are fixed */
2153 if (IS_GEN_IIE(hpriv)) {
2154 m2 &= ~0xC30FF01F;
2155 m2 |= 0x0000900F;
2156 }
2157
bca1c4eb
JG		 2158 writel(m2, port_mmio + PHY_MODE2);
2159}
2160
f351b2d6
SB		 2161/* TODO: use the generic LED interface to configure the SATA Presence */
2162/* & Acitivy LEDs on the board */
2163static void mv_soc_enable_leds(struct mv_host_priv *hpriv,
2164 void __iomem *mmio)
2165{
2166 return;
2167}
2168
2169static void mv_soc_read_preamp(struct mv_host_priv *hpriv, int idx,
2170 void __iomem *mmio)
2171{
2172 void __iomem *port_mmio;
2173 u32 tmp;
2174
2175 port_mmio = mv_port_base(mmio, idx);
2176 tmp = readl(port_mmio + PHY_MODE2);
2177
2178 hpriv->signal[idx].amps = tmp & 0x700; /* bits 10:8 */
2179 hpriv->signal[idx].pre = tmp & 0xe0; /* bits 7:5 */
2180}
2181
2182#undef ZERO
2183#define ZERO(reg) writel(0, port_mmio + (reg))
2184static void mv_soc_reset_hc_port(struct mv_host_priv *hpriv,
2185 void __iomem *mmio, unsigned int port)
2186{
2187 void __iomem *port_mmio = mv_port_base(mmio, port);
2188
b562468c
ML		 2189 /*
2190 * The datasheet warns against setting ATA_RST when EDMA is active
2191 * (but doesn't say what the problem might be). So we first try
2192 * to disable the EDMA engine before doing the ATA_RST operation.
2193 */
e12bef50 2194 mv_reset_channel(hpriv, mmio, port);
f351b2d6
SB		 2195
2196 ZERO(0x028); /* command */
2197 writel(0x101f, port_mmio + EDMA_CFG_OFS);
2198 ZERO(0x004); /* timer */
2199 ZERO(0x008); /* irq err cause */
2200 ZERO(0x00c); /* irq err mask */
2201 ZERO(0x010); /* rq bah */
2202 ZERO(0x014); /* rq inp */
2203 ZERO(0x018); /* rq outp */
2204 ZERO(0x01c); /* respq bah */
2205 ZERO(0x024); /* respq outp */
2206 ZERO(0x020); /* respq inp */
2207 ZERO(0x02c); /* test control */
2208 writel(0xbc, port_mmio + EDMA_IORDY_TMOUT);
2209}
2210
2211#undef ZERO
2212
2213#define ZERO(reg) writel(0, hc_mmio + (reg))
2214static void mv_soc_reset_one_hc(struct mv_host_priv *hpriv,
2215 void __iomem *mmio)
2216{
2217 void __iomem *hc_mmio = mv_hc_base(mmio, 0);
2218
2219 ZERO(0x00c);
2220 ZERO(0x010);
2221 ZERO(0x014);
2222
2223}
2224
2225#undef ZERO
2226
2227static int mv_soc_reset_hc(struct mv_host_priv *hpriv,
2228 void __iomem *mmio, unsigned int n_hc)
2229{
2230 unsigned int port;
2231
2232 for (port = 0; port < hpriv->n_ports; port++)
2233 mv_soc_reset_hc_port(hpriv, mmio, port);
2234
2235 mv_soc_reset_one_hc(hpriv, mmio);
2236
2237 return 0;
2238}
2239
2240static void mv_soc_reset_flash(struct mv_host_priv *hpriv,
2241 void __iomem *mmio)
2242{
2243 return;
2244}
2245
2246static void mv_soc_reset_bus(struct ata_host *host, void __iomem *mmio)
2247{
2248 return;
2249}
2250
b67a1064
ML		 2251static void mv_setup_ifctl(void __iomem *port_mmio, int want_gen2i)
2252{
2253 u32 ifctl = readl(port_mmio + SATA_INTERFACE_CFG);
2254
2255 ifctl = (ifctl & 0xf7f) | 0x9b1000; /* from chip spec */
2256 if (want_gen2i)
2257 ifctl |= (1 << 7); /* enable gen2i speed */
2258 writelfl(ifctl, port_mmio + SATA_INTERFACE_CFG);
2259}
2260
b562468c
ML		 2261/*
2262 * Caller must ensure that EDMA is not active,
2263 * by first doing mv_stop_edma() where needed.
2264 */
e12bef50 2265static void mv_reset_channel(struct mv_host_priv *hpriv, void __iomem *mmio,
c9d39130
JG		 2266 unsigned int port_no)
2267{
2268 void __iomem *port_mmio = mv_port_base(mmio, port_no);
2269
0d8be5cb 2270 mv_stop_edma_engine(port_mmio);
c9d39130
JG		 2271 writelfl(ATA_RST, port_mmio + EDMA_CMD_OFS);
2272
b67a1064
ML		 2273 if (!IS_GEN_I(hpriv)) {
2274 /* Enable 3.0gb/s link speed */
2275 mv_setup_ifctl(port_mmio, 1);
c9d39130 2276 }
b67a1064
ML		 2277 /*
2278 * Strobing ATA_RST here causes a hard reset of the SATA transport,
2279 * link, and physical layers. It resets all SATA interface registers
2280 * (except for SATA_INTERFACE_CFG), and issues a COMRESET to the dev.
c9d39130 2281 */
b67a1064
ML		 2282 writelfl(ATA_RST, port_mmio + EDMA_CMD_OFS);
2283 udelay(25); /* allow reset propagation */
c9d39130
JG		 2284 writelfl(0, port_mmio + EDMA_CMD_OFS);
2285
2286 hpriv->ops->phy_errata(hpriv, mmio, port_no);
2287
ee9ccdf7 2288 if (IS_GEN_I(hpriv))
c9d39130
JG		 2289 mdelay(1);
2290}
2291
cc0680a5 2292static int mv_hardreset(struct ata_link *link, unsigned int *class,
bdd4ddde 2293 unsigned long deadline)
31961943 2294{
cc0680a5 2295 struct ata_port *ap = link->ap;
bdd4ddde 2296 struct mv_host_priv *hpriv = ap->host->private_data;
b562468c 2297 struct mv_port_priv *pp = ap->private_data;
f351b2d6 2298 void __iomem *mmio = hpriv->base;
0d8be5cb
ML		 2299 int rc, attempts = 0, extra = 0;
2300 u32 sstatus;
2301 bool online;
31961943 2302
e12bef50 2303 mv_reset_channel(hpriv, mmio, ap->port_no);
b562468c 2304 pp->pp_flags &= ~MV_PP_FLAG_EDMA_EN;
bdd4ddde 2305
0d8be5cb
ML		 2306 /* Workaround for errata FEr SATA#10 (part 2) */
2307 do {
17c5aab5
ML		 2308 const unsigned long *timing =
2309 sata_ehc_deb_timing(&link->eh_context);
bdd4ddde 2310
17c5aab5
ML		 2311 rc = sata_link_hardreset(link, timing, deadline + extra,
2312 &online, NULL);
2313 if (rc)
0d8be5cb 2314 return rc;
0d8be5cb
ML		 2315 sata_scr_read(link, SCR_STATUS, &sstatus);
2316 if (!IS_GEN_I(hpriv) && ++attempts >= 5 && sstatus == 0x121) {
2317 /* Force 1.5gb/s link speed and try again */
2318 mv_setup_ifctl(mv_ap_base(ap), 0);
2319 if (time_after(jiffies + HZ, deadline))
2320 extra = HZ; /* only extend it once, max */
2321 }
2322 } while (sstatus != 0x0 && sstatus != 0x113 && sstatus != 0x123);
bdd4ddde 2323
17c5aab5 2324 return rc;
bdd4ddde
JG		 2325}
2326
bdd4ddde
JG		 2327static void mv_eh_freeze(struct ata_port *ap)
2328{
f351b2d6 2329 struct mv_host_priv *hpriv = ap->host->private_data;
bdd4ddde
JG		 2330 unsigned int hc = (ap->port_no > 3) ? 1 : 0;
2331 u32 tmp, mask;
2332 unsigned int shift;
2333
2334 /* FIXME: handle coalescing completion events properly */
2335
2336 shift = ap->port_no * 2;
2337 if (hc > 0)
2338 shift++;
2339
2340 mask = 0x3 << shift;
2341
2342 /* disable assertion of portN err, done events */
f351b2d6
SB		 2343 tmp = readl(hpriv->main_mask_reg_addr);
2344 writelfl(tmp & ~mask, hpriv->main_mask_reg_addr);
bdd4ddde
JG		 2345}
2346
2347static void mv_eh_thaw(struct ata_port *ap)
2348{
f351b2d6
SB		 2349 struct mv_host_priv *hpriv = ap->host->private_data;
2350 void __iomem *mmio = hpriv->base;
bdd4ddde
JG		 2351 unsigned int hc = (ap->port_no > 3) ? 1 : 0;
2352 void __iomem *hc_mmio = mv_hc_base(mmio, hc);
2353 void __iomem *port_mmio = mv_ap_base(ap);
2354 u32 tmp, mask, hc_irq_cause;
2355 unsigned int shift, hc_port_no = ap->port_no;
2356
2357 /* FIXME: handle coalescing completion events properly */
2358
2359 shift = ap->port_no * 2;
2360 if (hc > 0) {
2361 shift++;
2362 hc_port_no -= 4;
2363 }
2364
2365 mask = 0x3 << shift;
2366
2367 /* clear EDMA errors on this port */
2368 writel(0, port_mmio + EDMA_ERR_IRQ_CAUSE_OFS);
2369
2370 /* clear pending irq events */
2371 hc_irq_cause = readl(hc_mmio + HC_IRQ_CAUSE_OFS);
2372 hc_irq_cause &= ~(1 << hc_port_no); /* clear CRPB-done */
2373 hc_irq_cause &= ~(1 << (hc_port_no + 8)); /* clear Device int */
2374 writel(hc_irq_cause, hc_mmio + HC_IRQ_CAUSE_OFS);
2375
2376 /* enable assertion of portN err, done events */
f351b2d6
SB		 2377 tmp = readl(hpriv->main_mask_reg_addr);
2378 writelfl(tmp | mask, hpriv->main_mask_reg_addr);
31961943
BR		 2379}
2380
05b308e1
BR		 2381/**
2382 * mv_port_init - Perform some early initialization on a single port.
2383 * @port: libata data structure storing shadow register addresses
2384 * @port_mmio: base address of the port
2385 *
2386 * Initialize shadow register mmio addresses, clear outstanding
2387 * interrupts on the port, and unmask interrupts for the future
2388 * start of the port.
2389 *
2390 * LOCKING:
2391 * Inherited from caller.
2392 */
31961943 2393static void mv_port_init(struct ata_ioports *port, void __iomem *port_mmio)
20f733e7 2394{
0d5ff566 2395 void __iomem *shd_base = port_mmio + SHD_BLK_OFS;
31961943
BR		 2396 unsigned serr_ofs;
2397
8b260248 2398 /* PIO related setup
31961943
BR		 2399 */
2400 port->data_addr = shd_base + (sizeof(u32) * ATA_REG_DATA);
8b260248 2401 port->error_addr =
31961943
BR		 2402 port->feature_addr = shd_base + (sizeof(u32) * ATA_REG_ERR);
2403 port->nsect_addr = shd_base + (sizeof(u32) * ATA_REG_NSECT);
2404 port->lbal_addr = shd_base + (sizeof(u32) * ATA_REG_LBAL);
2405 port->lbam_addr = shd_base + (sizeof(u32) * ATA_REG_LBAM);
2406 port->lbah_addr = shd_base + (sizeof(u32) * ATA_REG_LBAH);
2407 port->device_addr = shd_base + (sizeof(u32) * ATA_REG_DEVICE);
8b260248 2408 port->status_addr =
31961943
BR		 2409 port->command_addr = shd_base + (sizeof(u32) * ATA_REG_STATUS);
2410 /* special case: control/altstatus doesn't have ATA_REG_ address */
2411 port->altstatus_addr = port->ctl_addr = shd_base + SHD_CTL_AST_OFS;
2412
2413 /* unused: */
8d9db2d2 2414 port->cmd_addr = port->bmdma_addr = port->scr_addr = NULL;
20f733e7 2415
31961943
BR		 2416 /* Clear any currently outstanding port interrupt conditions */
2417 serr_ofs = mv_scr_offset(SCR_ERROR);
2418 writelfl(readl(port_mmio + serr_ofs), port_mmio + serr_ofs);
2419 writelfl(0, port_mmio + EDMA_ERR_IRQ_CAUSE_OFS);
2420
646a4da5
ML		 2421 /* unmask all non-transient EDMA error interrupts */
2422 writelfl(~EDMA_ERR_IRQ_TRANSIENT, port_mmio + EDMA_ERR_IRQ_MASK_OFS);
20f733e7 2423
8b260248 2424 VPRINTK("EDMA cfg=0x%08x EDMA IRQ err cause/mask=0x%08x/0x%08x\n",
31961943
BR		 2425 readl(port_mmio + EDMA_CFG_OFS),
2426 readl(port_mmio + EDMA_ERR_IRQ_CAUSE_OFS),
2427 readl(port_mmio + EDMA_ERR_IRQ_MASK_OFS));
20f733e7
BR		 2428}
2429
4447d351 2430static int mv_chip_id(struct ata_host *host, unsigned int board_idx)
bca1c4eb 2431{
4447d351
TH		 2432 struct pci_dev *pdev = to_pci_dev(host->dev);
2433 struct mv_host_priv *hpriv = host->private_data;
bca1c4eb
JG		 2434 u32 hp_flags = hpriv->hp_flags;
2435
5796d1c4 2436 switch (board_idx) {
47c2b677
JG		 2437 case chip_5080:
2438 hpriv->ops = &mv5xxx_ops;
ee9ccdf7 2439 hp_flags |= MV_HP_GEN_I;
47c2b677 2440
44c10138 2441 switch (pdev->revision) {
47c2b677
JG		 2442 case 0x1:
2443 hp_flags |= MV_HP_ERRATA_50XXB0;
2444 break;
2445 case 0x3:
2446 hp_flags |= MV_HP_ERRATA_50XXB2;
2447 break;
2448 default:
2449 dev_printk(KERN_WARNING, &pdev->dev,
2450 "Applying 50XXB2 workarounds to unknown rev\n");
2451 hp_flags |= MV_HP_ERRATA_50XXB2;
2452 break;
2453 }
2454 break;
2455
bca1c4eb
JG		 2456 case chip_504x:
2457 case chip_508x:
47c2b677 2458 hpriv->ops = &mv5xxx_ops;
ee9ccdf7 2459 hp_flags |= MV_HP_GEN_I;
bca1c4eb 2460
44c10138 2461 switch (pdev->revision) {
47c2b677
JG		 2462 case 0x0:
2463 hp_flags |= MV_HP_ERRATA_50XXB0;
2464 break;
2465 case 0x3:
2466 hp_flags |= MV_HP_ERRATA_50XXB2;
2467 break;
2468 default:
2469 dev_printk(KERN_WARNING, &pdev->dev,
2470 "Applying B2 workarounds to unknown rev\n");
2471 hp_flags |= MV_HP_ERRATA_50XXB2;
2472 break;
bca1c4eb
JG		 2473 }
2474 break;
2475
2476 case chip_604x:
2477 case chip_608x:
47c2b677 2478 hpriv->ops = &mv6xxx_ops;
ee9ccdf7 2479 hp_flags |= MV_HP_GEN_II;
47c2b677 2480
44c10138 2481 switch (pdev->revision) {
47c2b677
JG		 2482 case 0x7:
2483 hp_flags |= MV_HP_ERRATA_60X1B2;
2484 break;
2485 case 0x9:
2486 hp_flags |= MV_HP_ERRATA_60X1C0;
bca1c4eb
JG		 2487 break;
2488 default:
2489 dev_printk(KERN_WARNING, &pdev->dev,
47c2b677
JG		 2490 "Applying B2 workarounds to unknown rev\n");
2491 hp_flags |= MV_HP_ERRATA_60X1B2;
bca1c4eb
JG		 2492 break;
2493 }
2494 break;
2495
e4e7b892 2496 case chip_7042:
02a121da 2497 hp_flags |= MV_HP_PCIE;
306b30f7
ML		 2498 if (pdev->vendor == PCI_VENDOR_ID_TTI &&
2499 (pdev->device == 0x2300 || pdev->device == 0x2310))
2500 {
4e520033
ML		 2501 /*
2502 * Highpoint RocketRAID PCIe 23xx series cards:
2503 *
2504 * Unconfigured drives are treated as "Legacy"
2505 * by the BIOS, and it overwrites sector 8 with
2506 * a "Lgcy" metadata block prior to Linux boot.
2507 *
2508 * Configured drives (RAID or JBOD) leave sector 8
2509 * alone, but instead overwrite a high numbered
2510 * sector for the RAID metadata. This sector can
2511 * be determined exactly, by truncating the physical
2512 * drive capacity to a nice even GB value.
2513 *
2514 * RAID metadata is at: (dev->n_sectors & ~0xfffff)
2515 *
2516 * Warn the user, lest they think we're just buggy.
2517 */
2518 printk(KERN_WARNING DRV_NAME ": Highpoint RocketRAID"
2519 " BIOS CORRUPTS DATA on all attached drives,"
2520 " regardless of if/how they are configured."
2521 " BEWARE!\n");
2522 printk(KERN_WARNING DRV_NAME ": For data safety, do not"
2523 " use sectors 8-9 on \"Legacy\" drives,"
2524 " and avoid the final two gigabytes on"
2525 " all RocketRAID BIOS initialized drives.\n");
306b30f7 2526 }
e4e7b892
JG		 2527 case chip_6042:
2528 hpriv->ops = &mv6xxx_ops;
e4e7b892
JG		 2529 hp_flags |= MV_HP_GEN_IIE;
2530
44c10138 2531 switch (pdev->revision) {
e4e7b892
JG		 2532 case 0x0:
2533 hp_flags |= MV_HP_ERRATA_XX42A0;
2534 break;
2535 case 0x1:
2536 hp_flags |= MV_HP_ERRATA_60X1C0;
2537 break;
2538 default:
2539 dev_printk(KERN_WARNING, &pdev->dev,
2540 "Applying 60X1C0 workarounds to unknown rev\n");
2541 hp_flags |= MV_HP_ERRATA_60X1C0;
2542 break;
2543 }
2544 break;
f351b2d6
SB		 2545 case chip_soc:
2546 hpriv->ops = &mv_soc_ops;
2547 hp_flags |= MV_HP_ERRATA_60X1C0;
2548 break;
e4e7b892 2549
bca1c4eb 2550 default:
f351b2d6 2551 dev_printk(KERN_ERR, host->dev,
5796d1c4 2552 "BUG: invalid board index %u\n", board_idx);
bca1c4eb
JG		 2553 return 1;
2554 }
2555
2556 hpriv->hp_flags = hp_flags;
02a121da
ML		 2557 if (hp_flags & MV_HP_PCIE) {
2558 hpriv->irq_cause_ofs = PCIE_IRQ_CAUSE_OFS;
2559 hpriv->irq_mask_ofs = PCIE_IRQ_MASK_OFS;
2560 hpriv->unmask_all_irqs = PCIE_UNMASK_ALL_IRQS;
2561 } else {
2562 hpriv->irq_cause_ofs = PCI_IRQ_CAUSE_OFS;
2563 hpriv->irq_mask_ofs = PCI_IRQ_MASK_OFS;
2564 hpriv->unmask_all_irqs = PCI_UNMASK_ALL_IRQS;
2565 }
bca1c4eb
JG		 2566
2567 return 0;
2568}
2569
05b308e1 2570/**
47c2b677 2571 * mv_init_host - Perform some early initialization of the host.
4447d351
TH		 2572 * @host: ATA host to initialize
2573 * @board_idx: controller index
05b308e1
BR		 2574 *
2575 * If possible, do an early global reset of the host. Then do
2576 * our port init and clear/unmask all/relevant host interrupts.
2577 *
2578 * LOCKING:
2579 * Inherited from caller.
2580 */
4447d351 2581static int mv_init_host(struct ata_host *host, unsigned int board_idx)
20f733e7
BR		 2582{
2583 int rc = 0, n_hc, port, hc;
4447d351 2584 struct mv_host_priv *hpriv = host->private_data;
f351b2d6 2585 void __iomem *mmio = hpriv->base;
47c2b677 2586
4447d351 2587 rc = mv_chip_id(host, board_idx);
bca1c4eb 2588 if (rc)
f351b2d6
SB		 2589 goto done;
2590
2591 if (HAS_PCI(host)) {
2592 hpriv->main_cause_reg_addr = hpriv->base +
2593 HC_MAIN_IRQ_CAUSE_OFS;
2594 hpriv->main_mask_reg_addr = hpriv->base + HC_MAIN_IRQ_MASK_OFS;
2595 } else {
2596 hpriv->main_cause_reg_addr = hpriv->base +
2597 HC_SOC_MAIN_IRQ_CAUSE_OFS;
2598 hpriv->main_mask_reg_addr = hpriv->base +
2599 HC_SOC_MAIN_IRQ_MASK_OFS;
2600 }
2601 /* global interrupt mask */
2602 writel(0, hpriv->main_mask_reg_addr);
bca1c4eb 2603
4447d351 2604 n_hc = mv_get_hc_count(host->ports[0]->flags);
bca1c4eb 2605
4447d351 2606 for (port = 0; port < host->n_ports; port++)
47c2b677 2607 hpriv->ops->read_preamp(hpriv, port, mmio);
20f733e7 2608
c9d39130 2609 rc = hpriv->ops->reset_hc(hpriv, mmio, n_hc);
47c2b677 2610 if (rc)
20f733e7 2611 goto done;
20f733e7 2612
522479fb 2613 hpriv->ops->reset_flash(hpriv, mmio);
7bb3c529 2614 hpriv->ops->reset_bus(host, mmio);
47c2b677 2615 hpriv->ops->enable_leds(hpriv, mmio);
20f733e7 2616
4447d351 2617 for (port = 0; port < host->n_ports; port++) {
cbcdd875 2618 struct ata_port *ap = host->ports[port];
2a47ce06 2619 void __iomem *port_mmio = mv_port_base(mmio, port);
cbcdd875
TH		 2620
2621 mv_port_init(&ap->ioaddr, port_mmio);
2622
7bb3c529 2623#ifdef CONFIG_PCI
f351b2d6
SB		 2624 if (HAS_PCI(host)) {
2625 unsigned int offset = port_mmio - mmio;
2626 ata_port_pbar_desc(ap, MV_PRIMARY_BAR, -1, "mmio");
2627 ata_port_pbar_desc(ap, MV_PRIMARY_BAR, offset, "port");
2628 }
7bb3c529 2629#endif
20f733e7
BR		 2630 }
2631
2632 for (hc = 0; hc < n_hc; hc++) {
31961943
BR		 2633 void __iomem *hc_mmio = mv_hc_base(mmio, hc);
2634
2635 VPRINTK("HC%i: HC config=0x%08x HC IRQ cause "
2636 "(before clear)=0x%08x\n", hc,
2637 readl(hc_mmio + HC_CFG_OFS),
2638 readl(hc_mmio + HC_IRQ_CAUSE_OFS));
2639
2640 /* Clear any currently outstanding hc interrupt conditions */
2641 writelfl(0, hc_mmio + HC_IRQ_CAUSE_OFS);
20f733e7
BR		 2642 }
2643
f351b2d6
SB		 2644 if (HAS_PCI(host)) {
2645 /* Clear any currently outstanding host interrupt conditions */
2646 writelfl(0, mmio + hpriv->irq_cause_ofs);
31961943 2647
f351b2d6
SB		 2648 /* and unmask interrupt generation for host regs */
2649 writelfl(hpriv->unmask_all_irqs, mmio + hpriv->irq_mask_ofs);
2650 if (IS_GEN_I(hpriv))
2651 writelfl(~HC_MAIN_MASKED_IRQS_5,
2652 hpriv->main_mask_reg_addr);
2653 else
2654 writelfl(~HC_MAIN_MASKED_IRQS,
2655 hpriv->main_mask_reg_addr);
2656
2657 VPRINTK("HC MAIN IRQ cause/mask=0x%08x/0x%08x "
2658 "PCI int cause/mask=0x%08x/0x%08x\n",
2659 readl(hpriv->main_cause_reg_addr),
2660 readl(hpriv->main_mask_reg_addr),
2661 readl(mmio + hpriv->irq_cause_ofs),
2662 readl(mmio + hpriv->irq_mask_ofs));
2663 } else {
2664 writelfl(~HC_MAIN_MASKED_IRQS_SOC,
2665 hpriv->main_mask_reg_addr);
2666 VPRINTK("HC MAIN IRQ cause/mask=0x%08x/0x%08x\n",
2667 readl(hpriv->main_cause_reg_addr),
2668 readl(hpriv->main_mask_reg_addr));
2669 }
2670done:
2671 return rc;
2672}
fb621e2f 2673
fbf14e2f
BB		 2674static int mv_create_dma_pools(struct mv_host_priv *hpriv, struct device *dev)
2675{
2676 hpriv->crqb_pool = dmam_pool_create("crqb_q", dev, MV_CRQB_Q_SZ,
2677 MV_CRQB_Q_SZ, 0);
2678 if (!hpriv->crqb_pool)
2679 return -ENOMEM;
2680
2681 hpriv->crpb_pool = dmam_pool_create("crpb_q", dev, MV_CRPB_Q_SZ,
2682 MV_CRPB_Q_SZ, 0);
2683 if (!hpriv->crpb_pool)
2684 return -ENOMEM;
2685
2686 hpriv->sg_tbl_pool = dmam_pool_create("sg_tbl", dev, MV_SG_TBL_SZ,
2687 MV_SG_TBL_SZ, 0);
2688 if (!hpriv->sg_tbl_pool)
2689 return -ENOMEM;
2690
2691 return 0;
2692}
2693
f351b2d6
SB		 2694/**
2695 * mv_platform_probe - handle a positive probe of an soc Marvell
2696 * host
2697 * @pdev: platform device found
2698 *
2699 * LOCKING:
2700 * Inherited from caller.
2701 */
2702static int mv_platform_probe(struct platform_device *pdev)
2703{
2704 static int printed_version;
2705 const struct mv_sata_platform_data *mv_platform_data;
2706 const struct ata_port_info *ppi[] =
2707 { &mv_port_info[chip_soc], NULL };
2708 struct ata_host *host;
2709 struct mv_host_priv *hpriv;
2710 struct resource *res;
2711 int n_ports, rc;
20f733e7 2712
f351b2d6
SB		 2713 if (!printed_version++)
2714 dev_printk(KERN_INFO, &pdev->dev, "version " DRV_VERSION "\n");
bca1c4eb 2715
f351b2d6
SB		 2716 /*
2717 * Simple resource validation ..
2718 */
2719 if (unlikely(pdev->num_resources != 2)) {
2720 dev_err(&pdev->dev, "invalid number of resources\n");
2721 return -EINVAL;
2722 }
2723
2724 /*
2725 * Get the register base first
2726 */
2727 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2728 if (res == NULL)
2729 return -EINVAL;
2730
2731 /* allocate host */
2732 mv_platform_data = pdev->dev.platform_data;
2733 n_ports = mv_platform_data->n_ports;
2734
2735 host = ata_host_alloc_pinfo(&pdev->dev, ppi, n_ports);
2736 hpriv = devm_kzalloc(&pdev->dev, sizeof(*hpriv), GFP_KERNEL);
2737
2738 if (!host || !hpriv)
2739 return -ENOMEM;
2740 host->private_data = hpriv;
2741 hpriv->n_ports = n_ports;
2742
2743 host->iomap = NULL;
f1cb0ea1
SB		 2744 hpriv->base = devm_ioremap(&pdev->dev, res->start,
2745 res->end - res->start + 1);
f351b2d6
SB		 2746 hpriv->base -= MV_SATAHC0_REG_BASE;
2747
fbf14e2f
BB		 2748 rc = mv_create_dma_pools(hpriv, &pdev->dev);
2749 if (rc)
2750 return rc;
2751
f351b2d6
SB		 2752 /* initialize adapter */
2753 rc = mv_init_host(host, chip_soc);
2754 if (rc)
2755 return rc;
2756
2757 dev_printk(KERN_INFO, &pdev->dev,
2758 "slots %u ports %d\n", (unsigned)MV_MAX_Q_DEPTH,
2759 host->n_ports);
2760
2761 return ata_host_activate(host, platform_get_irq(pdev, 0), mv_interrupt,
2762 IRQF_SHARED, &mv6_sht);
2763}
2764
2765/*
2766 *
2767 * mv_platform_remove - unplug a platform interface
2768 * @pdev: platform device
2769 *
2770 * A platform bus SATA device has been unplugged. Perform the needed
2771 * cleanup. Also called on module unload for any active devices.
2772 */
2773static int __devexit mv_platform_remove(struct platform_device *pdev)
2774{
2775 struct device *dev = &pdev->dev;
2776 struct ata_host *host = dev_get_drvdata(dev);
f351b2d6
SB		 2777
2778 ata_host_detach(host);
f351b2d6 2779 return 0;
20f733e7
BR		 2780}
2781
f351b2d6
SB		 2782static struct platform_driver mv_platform_driver = {
2783 .probe = mv_platform_probe,
2784 .remove = __devexit_p(mv_platform_remove),
2785 .driver = {
2786 .name = DRV_NAME,
2787 .owner = THIS_MODULE,
2788 },
2789};
2790
2791
7bb3c529 2792#ifdef CONFIG_PCI
f351b2d6
SB		 2793static int mv_pci_init_one(struct pci_dev *pdev,
2794 const struct pci_device_id *ent);
2795
7bb3c529
SB		 2796
2797static struct pci_driver mv_pci_driver = {
2798 .name = DRV_NAME,
2799 .id_table = mv_pci_tbl,
f351b2d6 2800 .probe = mv_pci_init_one,
7bb3c529
SB		 2801 .remove = ata_pci_remove_one,
2802};
2803
2804/*
2805 * module options
2806 */
2807static int msi; /* Use PCI msi; either zero (off, default) or non-zero */
2808
2809
2810/* move to PCI layer or libata core? */
2811static int pci_go_64(struct pci_dev *pdev)
2812{
2813 int rc;
2814
2815 if (!pci_set_dma_mask(pdev, DMA_64BIT_MASK)) {
2816 rc = pci_set_consistent_dma_mask(pdev, DMA_64BIT_MASK);
2817 if (rc) {
2818 rc = pci_set_consistent_dma_mask(pdev, DMA_32BIT_MASK);
2819 if (rc) {
2820 dev_printk(KERN_ERR, &pdev->dev,
2821 "64-bit DMA enable failed\n");
2822 return rc;
2823 }
2824 }
2825 } else {
2826 rc = pci_set_dma_mask(pdev, DMA_32BIT_MASK);
2827 if (rc) {
2828 dev_printk(KERN_ERR, &pdev->dev,
2829 "32-bit DMA enable failed\n");
2830 return rc;
2831 }
2832 rc = pci_set_consistent_dma_mask(pdev, DMA_32BIT_MASK);
2833 if (rc) {
2834 dev_printk(KERN_ERR, &pdev->dev,
2835 "32-bit consistent DMA enable failed\n");
2836 return rc;
2837 }
2838 }
2839
2840 return rc;
2841}
2842
05b308e1
BR		 2843/**
2844 * mv_print_info - Dump key info to kernel log for perusal.
4447d351 2845 * @host: ATA host to print info about
05b308e1
BR		 2846 *
2847 * FIXME: complete this.
2848 *
2849 * LOCKING:
2850 * Inherited from caller.
2851 */
4447d351 2852static void mv_print_info(struct ata_host *host)
31961943 2853{
4447d351
TH		 2854 struct pci_dev *pdev = to_pci_dev(host->dev);
2855 struct mv_host_priv *hpriv = host->private_data;
44c10138 2856 u8 scc;
c1e4fe71 2857 const char *scc_s, *gen;
31961943
BR		 2858
2859 /* Use this to determine the HW stepping of the chip so we know
2860 * what errata to workaround
2861 */
31961943
BR		 2862 pci_read_config_byte(pdev, PCI_CLASS_DEVICE, &scc);
2863 if (scc == 0)
2864 scc_s = "SCSI";
2865 else if (scc == 0x01)
2866 scc_s = "RAID";
2867 else
c1e4fe71
JG		 2868 scc_s = "?";
2869
2870 if (IS_GEN_I(hpriv))
2871 gen = "I";
2872 else if (IS_GEN_II(hpriv))
2873 gen = "II";
2874 else if (IS_GEN_IIE(hpriv))
2875 gen = "IIE";
2876 else
2877 gen = "?";
31961943 2878
a9524a76 2879 dev_printk(KERN_INFO, &pdev->dev,
c1e4fe71
JG		 2880 "Gen-%s %u slots %u ports %s mode IRQ via %s\n",
2881 gen, (unsigned)MV_MAX_Q_DEPTH, host->n_ports,
31961943
BR		 2882 scc_s, (MV_HP_FLAG_MSI & hpriv->hp_flags) ? "MSI" : "INTx");
2883}
2884
05b308e1 2885/**
f351b2d6 2886 * mv_pci_init_one - handle a positive probe of a PCI Marvell host
05b308e1
BR		 2887 * @pdev: PCI device found
2888 * @ent: PCI device ID entry for the matched host
2889 *
2890 * LOCKING:
2891 * Inherited from caller.
2892 */
f351b2d6
SB		 2893static int mv_pci_init_one(struct pci_dev *pdev,
2894 const struct pci_device_id *ent)
20f733e7 2895{
2dcb407e 2896 static int printed_version;
20f733e7 2897 unsigned int board_idx = (unsigned int)ent->driver_data;
4447d351
TH		 2898 const struct ata_port_info *ppi[] = { &mv_port_info[board_idx], NULL };
2899 struct ata_host *host;
2900 struct mv_host_priv *hpriv;
2901 int n_ports, rc;
20f733e7 2902
a9524a76
JG		 2903 if (!printed_version++)
2904 dev_printk(KERN_INFO, &pdev->dev, "version " DRV_VERSION "\n");
20f733e7 2905
4447d351
TH		 2906 /* allocate host */
2907 n_ports = mv_get_hc_count(ppi[0]->flags) * MV_PORTS_PER_HC;
2908
2909 host = ata_host_alloc_pinfo(&pdev->dev, ppi, n_ports);
2910 hpriv = devm_kzalloc(&pdev->dev, sizeof(*hpriv), GFP_KERNEL);
2911 if (!host || !hpriv)
2912 return -ENOMEM;
2913 host->private_data = hpriv;
f351b2d6 2914 hpriv->n_ports = n_ports;
4447d351
TH		 2915
2916 /* acquire resources */
24dc5f33
TH		 2917 rc = pcim_enable_device(pdev);
2918 if (rc)
20f733e7 2919 return rc;
20f733e7 2920
0d5ff566
TH		 2921 rc = pcim_iomap_regions(pdev, 1 << MV_PRIMARY_BAR, DRV_NAME);
2922 if (rc == -EBUSY)
24dc5f33 2923 pcim_pin_device(pdev);
0d5ff566 2924 if (rc)
24dc5f33 2925 return rc;
4447d351 2926 host->iomap = pcim_iomap_table(pdev);
f351b2d6 2927 hpriv->base = host->iomap[MV_PRIMARY_BAR];
20f733e7 2928
d88184fb
JG		 2929 rc = pci_go_64(pdev);
2930 if (rc)
2931 return rc;
2932
da2fa9ba
ML		 2933 rc = mv_create_dma_pools(hpriv, &pdev->dev);
2934 if (rc)
2935 return rc;
2936
20f733e7 2937 /* initialize adapter */
4447d351 2938 rc = mv_init_host(host, board_idx);
24dc5f33
TH		 2939 if (rc)
2940 return rc;
20f733e7 2941
31961943 2942 /* Enable interrupts */
6a59dcf8 2943 if (msi && pci_enable_msi(pdev))
31961943 2944 pci_intx(pdev, 1);
20f733e7 2945
31961943 2946 mv_dump_pci_cfg(pdev, 0x68);
4447d351 2947 mv_print_info(host);
20f733e7 2948
4447d351 2949 pci_set_master(pdev);
ea8b4db9 2950 pci_try_set_mwi(pdev);
4447d351 2951 return ata_host_activate(host, pdev->irq, mv_interrupt, IRQF_SHARED,
c5d3e45a 2952 IS_GEN_I(hpriv) ? &mv5_sht : &mv6_sht);
20f733e7 2953}
7bb3c529 2954#endif
20f733e7 2955
f351b2d6
SB		 2956static int mv_platform_probe(struct platform_device *pdev);
2957static int __devexit mv_platform_remove(struct platform_device *pdev);
2958
20f733e7
BR		 2959static int __init mv_init(void)
2960{
7bb3c529
SB		 2961 int rc = -ENODEV;
2962#ifdef CONFIG_PCI
2963 rc = pci_register_driver(&mv_pci_driver);
f351b2d6
SB		 2964 if (rc < 0)
2965 return rc;
2966#endif
2967 rc = platform_driver_register(&mv_platform_driver);
2968
2969#ifdef CONFIG_PCI
2970 if (rc < 0)
2971 pci_unregister_driver(&mv_pci_driver);
7bb3c529
SB		 2972#endif
2973 return rc;
20f733e7
BR		 2974}
2975
2976static void __exit mv_exit(void)
2977{
7bb3c529 2978#ifdef CONFIG_PCI
20f733e7 2979 pci_unregister_driver(&mv_pci_driver);
7bb3c529 2980#endif
f351b2d6 2981 platform_driver_unregister(&mv_platform_driver);
20f733e7
BR		 2982}
2983
2984MODULE_AUTHOR("Brett Russ");
2985MODULE_DESCRIPTION("SCSI low-level driver for Marvell SATA controllers");
2986MODULE_LICENSE("GPL");
2987MODULE_DEVICE_TABLE(pci, mv_pci_tbl);
2988MODULE_VERSION(DRV_VERSION);
17c5aab5 2989MODULE_ALIAS("platform:" DRV_NAME);
20f733e7 2990
7bb3c529 2991#ifdef CONFIG_PCI
ddef9bb3
JG		 2992module_param(msi, int, 0444);
2993MODULE_PARM_DESC(msi, "Enable use of PCI MSI (0=off, 1=on)");
7bb3c529 2994#endif
ddef9bb3 2995
20f733e7
BR		 2996module_init(mv_init);
2997module_exit(mv_exit);
Linux 6.x block layer and io_uring tree(s)